From 66dc501fc43464412c910e16b489e75967aa8417 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 17 Oct 2026 21:32:23 +0200 Subject: [PATCH] Split the unit level from the integration level in cmd, builder, link, catalogue, inventory; cheap fixtures for the archive-entry bound, the registry waits and the catalogue copy (issue 895) --- cmd/mesh-controller/acts_test.go | 2 + .../addresses_integration_test.go | 228 ++++++ cmd/mesh-controller/addresses_test.go | 213 ----- cmd/mesh-controller/adopting_test.go | 2 + .../agent_account_integration_test.go | 161 ++++ cmd/mesh-controller/agent_account_test.go | 145 ---- cmd/mesh-controller/api_test.go | 2 + cmd/mesh-controller/asker_bus_test.go | 2 + .../awaiting_push_integration_test.go | 182 +++++ cmd/mesh-controller/awaiting_push_test.go | 166 ---- .../batches_integration_test.go | 698 +++++++++++++++++ cmd/mesh-controller/batches_test.go | 665 ---------------- cmd/mesh-controller/beside_test.go | 2 + .../bindings_integration_test.go | 102 +++ cmd/mesh-controller/bindings_test.go | 90 --- cmd/mesh-controller/build_integration_test.go | 131 ++++ .../build_source_integration_test.go | 353 +++++++++ cmd/mesh-controller/build_source_test.go | 330 -------- cmd/mesh-controller/build_test.go | 118 --- .../bus_waits_integration_test.go | 70 ++ cmd/mesh-controller/bus_waits_test.go | 60 -- cmd/mesh-controller/busobjects_test.go | 2 + .../calls_verb_integration_test.go | 36 + cmd/mesh-controller/calls_verb_test.go | 28 - .../check_builds_integration_test.go | 102 +++ cmd/mesh-controller/check_builds_test.go | 87 --- .../checks_integration_test.go | 56 ++ cmd/mesh-controller/checks_queue_test.go | 2 + cmd/mesh-controller/checks_test.go | 48 -- cmd/mesh-controller/condition_looks_test.go | 2 + .../conditions_brief_integration_test.go | 43 + cmd/mesh-controller/conditions_brief_test.go | 32 - .../conditions_integration_test.go | 32 + cmd/mesh-controller/conditions_test.go | 23 - .../config_moved_integration_test.go | 467 +++++++++++ cmd/mesh-controller/config_moved_test.go | 449 ----------- .../core_box_integration_test.go | 129 +++ cmd/mesh-controller/core_box_test.go | 113 --- .../credential_on_assign_test.go | 2 + cmd/mesh-controller/data_integration_test.go | 148 ++++ cmd/mesh-controller/data_test.go | 134 ---- .../deadletters_integration_test.go | 88 +++ cmd/mesh-controller/deadletters_test.go | 76 -- .../delivery_conditions_test.go | 2 + .../delivery_integration_test.go | 196 +++++ cmd/mesh-controller/delivery_test.go | 184 ----- .../desk_secret_integration_test.go | 96 +++ cmd/mesh-controller/desk_secret_test.go | 79 -- .../doctor_integration_test.go | 246 ++++++ cmd/mesh-controller/doctor_test.go | 228 ------ .../dry_run_integration_test.go | 20 + cmd/mesh-controller/dry_run_test.go | 9 - cmd/mesh-controller/epoch_send_test.go | 2 + cmd/mesh-controller/facts_test.go | 2 + .../failed_wait_provider_test.go | 2 + .../found_wait_integration_test.go | 127 +++ cmd/mesh-controller/found_wait_test.go | 113 --- cmd/mesh-controller/gate_followup_test.go | 2 + cmd/mesh-controller/gate_integration_test.go | 482 ++++++++++++ cmd/mesh-controller/gate_test.go | 456 ----------- .../grant_follows_binding_integration_test.go | 157 ++++ .../grant_follows_binding_test.go | 145 ---- .../grants_step_integration_test.go | 27 + cmd/mesh-controller/grants_step_store_test.go | 2 + cmd/mesh-controller/grants_step_test.go | 16 - .../healers_integration_test.go | 449 +++++++++++ cmd/mesh-controller/healers_test.go | 439 +---------- .../held_back_integration_test.go | 182 +++++ cmd/mesh-controller/held_back_test.go | 168 ---- cmd/mesh-controller/hold_test.go | 2 + cmd/mesh-controller/holdings_test.go | 2 + .../hosts_behind_integration_test.go | 42 + cmd/mesh-controller/hosts_behind_test.go | 37 - .../identity_bound_integration_test.go | 107 +++ cmd/mesh-controller/identity_bound_test.go | 98 --- .../issue201_integration_test.go | 181 +++++ cmd/mesh-controller/issue201_test.go | 166 ---- .../issue348_integration_test.go | 199 +++++ cmd/mesh-controller/issue348_test.go | 189 ----- .../issue352_integration_test.go | 227 ++++++ cmd/mesh-controller/issue352_test.go | 211 ----- .../issue459_integration_test.go | 293 +++++++ cmd/mesh-controller/issue459_test.go | 275 ------- .../issue611_integration_test.go | 54 ++ cmd/mesh-controller/issue611_test.go | 43 - cmd/mesh-controller/issue648_test.go | 2 + .../issue_204_integration_test.go | 33 + cmd/mesh-controller/issue_204_test.go | 24 - .../issued_integration_test.go | 96 +++ cmd/mesh-controller/issued_test.go | 79 -- cmd/mesh-controller/issued_to_vault_test.go | 2 + .../join_through_the_tunnel_test.go | 2 + .../last_send_integration_test.go | 51 ++ cmd/mesh-controller/last_send_test.go | 39 - .../lean_walk_integration_test.go | 141 ++++ cmd/mesh-controller/lean_walk_test.go | 126 --- cmd/mesh-controller/lease_command_test.go | 2 + cmd/mesh-controller/lease_integration_test.go | 154 ++++ cmd/mesh-controller/lease_test.go | 144 ---- .../left_out_integration_test.go | 127 +++ cmd/mesh-controller/left_out_test.go | 115 --- cmd/mesh-controller/lone_merge_test.go | 2 + .../machine_network_integration_test.go | 266 +++++++ cmd/mesh-controller/machine_network_test.go | 255 ------ .../machine_units_integration_test.go | 125 +++ cmd/mesh-controller/machine_units_test.go | 122 --- .../merge_check_nested_test.go | 2 + .../merge_gate_handing_integration_test.go | 63 ++ .../merge_gate_handing_test.go | 52 -- .../merge_gate_integration_test.go | 528 +++++++++++++ cmd/mesh-controller/merge_gate_test.go | 502 ------------ cmd/mesh-controller/merge_on_the_bus_test.go | 2 + .../merge_window_bus_integration_test.go | 90 +++ cmd/mesh-controller/merge_window_bus_test.go | 78 -- cmd/mesh-controller/mesh_for_test.go | 3 +- .../meshcli_integration_test.go | 193 +++++ .../meshcli_reads_integration_test.go | 66 ++ cmd/mesh-controller/meshcli_reads_test.go | 56 -- cmd/mesh-controller/meshcli_stdin_test.go | 2 + cmd/mesh-controller/meshcli_test.go | 177 ----- .../mirrors_integration_test.go | 127 +++ cmd/mesh-controller/mirrors_test.go | 111 --- cmd/mesh-controller/module_health_test.go | 2 + cmd/mesh-controller/moved_bus_test.go | 2 + cmd/mesh-controller/network_test.go | 2 + .../new_module_on_merge_test.go | 2 + cmd/mesh-controller/nodes_test.go | 2 + .../one_resolution_integration_test.go | 92 +++ cmd/mesh-controller/one_resolution_test.go | 85 -- .../operator_wait_integration_test.go | 133 ++++ cmd/mesh-controller/operator_wait_test.go | 117 --- cmd/mesh-controller/outside_accept_test.go | 2 + cmd/mesh-controller/outside_not_given_test.go | 2 + .../over_budget_integration_test.go | 50 ++ cmd/mesh-controller/over_budget_test.go | 38 - .../pending_integration_test.go | 733 ++++++++++++++++++ cmd/mesh-controller/pending_test.go | 698 ----------------- .../person_wait_integration_test.go | 165 ++++ cmd/mesh-controller/person_wait_test.go | 149 ---- cmd/mesh-controller/placement_moved_test.go | 2 + .../placement_words_moved_integration_test.go | 92 +++ .../placement_words_moved_test.go | 79 -- .../plan_clock_integration_test.go | 52 ++ cmd/mesh-controller/plan_clock_test.go | 41 - .../plan_looks_integration_test.go | 188 +++++ cmd/mesh-controller/plan_looks_test.go | 170 ---- ...lan_moved_every_writer_integration_test.go | 37 + .../plan_moved_every_writer_test.go | 29 - cmd/mesh-controller/planner_records_test.go | 2 + .../plans_follow_integration_test.go | 62 ++ cmd/mesh-controller/plans_follow_test.go | 50 -- .../plans_one_at_a_time_test.go | 2 + .../predefined_integration_test.go | 101 +++ cmd/mesh-controller/predefined_test.go | 90 --- .../proposals_integration_test.go | 135 ++++ cmd/mesh-controller/proposals_test.go | 122 --- cmd/mesh-controller/provider_hold_test.go | 2 + .../provisioner_account_integration_test.go | 79 ++ .../provisioner_account_test.go | 69 -- cmd/mesh-controller/push_recreates_test.go | 2 + cmd/mesh-controller/queue_integration_test.go | 492 ++++++++++++ cmd/mesh-controller/queue_test.go | 466 ----------- .../reconnects_integration_test.go | 140 ++++ cmd/mesh-controller/reconnects_test.go | 126 --- .../recorded_kept_integration_test.go | 144 ++++ cmd/mesh-controller/recorded_kept_test.go | 131 ---- cmd/mesh-controller/recorded_moves_test.go | 2 + .../registry_verbs_integration_test.go | 75 ++ cmd/mesh-controller/registry_verbs_test.go | 61 -- cmd/mesh-controller/rehearse_test.go | 3 +- ...eissue_verified_sender_integration_test.go | 57 ++ .../reissue_verified_sender_test.go | 46 -- .../related_integration_test.go | 52 ++ cmd/mesh-controller/related_test.go | 42 - .../release_provisioned_integration_test.go | 145 ++++ .../release_provisioned_test.go | 133 ---- cmd/mesh-controller/release_test.go | 2 + cmd/mesh-controller/replay164_test.go | 2 + cmd/mesh-controller/replay201_test.go | 2 + cmd/mesh-controller/replay237_test.go | 2 + cmd/mesh-controller/replay239_test.go | 2 + cmd/mesh-controller/replay318_test.go | 2 + cmd/mesh-controller/replay325_test.go | 2 + cmd/mesh-controller/replay327_test.go | 2 + .../replay335_integration_test.go | 68 ++ cmd/mesh-controller/replay335_test.go | 56 -- cmd/mesh-controller/replay336_test.go | 2 + cmd/mesh-controller/replay340_test.go | 2 + cmd/mesh-controller/replay348_test.go | 2 + cmd/mesh-controller/replay349_test.go | 2 + cmd/mesh-controller/replay401_test.go | 2 + cmd/mesh-controller/replay614_test.go | 2 + cmd/mesh-controller/replay867_test.go | 2 + .../replays_integration_test.go | 501 ++++++++++++ cmd/mesh-controller/replays_test.go | 482 ------------ .../rest_at_once_integration_test.go | 293 +++++++ cmd/mesh-controller/rest_at_once_test.go | 272 ------- .../retirement_integration_test.go | 191 +++++ cmd/mesh-controller/retirement_test.go | 177 ----- cmd/mesh-controller/rollout_events_test.go | 2 + .../rollout_silent_bus_integration_test.go | 92 +++ .../rollout_silent_bus_test.go | 81 -- cmd/mesh-controller/rollout_test.go | 2 + cmd/mesh-controller/roster_failure_test.go | 2 + cmd/mesh-controller/runtime_health_test.go | 2 + .../same_program_integration_test.go | 58 ++ cmd/mesh-controller/same_program_test.go | 46 -- cmd/mesh-controller/same_source_test.go | 2 + .../sayable_integration_test.go | 150 ++++ cmd/mesh-controller/sayable_test.go | 135 ---- .../seat_dependencies_integration_test.go | 119 +++ cmd/mesh-controller/seat_dependencies_test.go | 109 --- .../seatverbs_integration_test.go | 164 ++++ .../seatverbs_schema_integration_test.go | 185 +++++ cmd/mesh-controller/seatverbs_schema_test.go | 171 ---- cmd/mesh-controller/seatverbs_test.go | 151 ---- .../secret_integration_test.go | 33 + cmd/mesh-controller/secret_test.go | 24 - .../selfexec_integration_test.go | 138 ++++ cmd/mesh-controller/selfexec_test.go | 125 --- .../sendable_integration_test.go | 354 +++++++++ cmd/mesh-controller/sendable_test.go | 338 -------- .../sent_required_by_integration_test.go | 104 +++ cmd/mesh-controller/sent_required_by_test.go | 94 --- .../sequence_integration_test.go | 41 + cmd/mesh-controller/sequence_test.go | 36 - cmd/mesh-controller/served_own_names_test.go | 2 + .../settings_effective_integration_test.go | 38 + .../settings_effective_test.go | 27 - .../settings_walk_integration_test.go | 439 +++++++++++ cmd/mesh-controller/settings_walk_test.go | 420 ---------- .../standing_integration_test.go | 105 +++ cmd/mesh-controller/standing_test.go | 92 --- cmd/mesh-controller/status_summary_test.go | 2 + cmd/mesh-controller/status_test.go | 2 + .../template_integration_test.go | 148 ++++ cmd/mesh-controller/template_test.go | 135 ---- cmd/mesh-controller/terminal_settings_test.go | 2 + .../tier_send_integration_test.go | 410 ++++++++++ cmd/mesh-controller/tier_send_test.go | 391 ---------- cmd/mesh-controller/try_integration_test.go | 39 + cmd/mesh-controller/try_test.go | 28 - cmd/mesh-controller/unbound_test.go | 2 + .../unheld_report_integration_test.go | 36 + cmd/mesh-controller/unheld_report_test.go | 26 - .../unseen_integration_test.go | 19 + cmd/mesh-controller/unseen_test.go | 11 - .../untaken_integration_test.go | 80 ++ cmd/mesh-controller/untaken_test.go | 73 -- cmd/mesh-controller/urgent_unhealthy_test.go | 2 + cmd/mesh-controller/vault_handed_test.go | 2 + .../vault_secrets_integration_test.go | 229 ++++++ cmd/mesh-controller/vault_secrets_test.go | 213 ----- cmd/mesh-controller/vault_switch_test.go | 2 + cmd/mesh-controller/verb_looks_test.go | 2 + cmd/mesh-controller/waiting_provider_test.go | 2 + .../walk_said_integration_test.go | 266 +++++++ cmd/mesh-controller/walk_said_test.go | 245 ------ .../walk_wait_rebuild_754_integration_test.go | 63 ++ .../walk_wait_rebuild_754_test.go | 52 -- internal/builder/build_source_test.go | 2 + internal/builder/builder_integration_test.go | 362 +++++++++ internal/builder/builder_test.go | 342 -------- internal/builder/bundle_integration_test.go | 166 ++++ internal/builder/bundle_test.go | 153 ---- internal/builder/check_base_test.go | 2 + .../check_credential_integration_test.go | 180 +++++ internal/builder/check_credential_test.go | 169 ---- internal/builder/check_integration_test.go | 399 ++++++++++ internal/builder/check_merge_test.go | 2 + internal/builder/check_test.go | 382 --------- internal/builder/dependencies_test.go | 2 + internal/builder/failures_integration_test.go | 152 ++++ internal/builder/failures_test.go | 136 ---- internal/builder/gosource_integration_test.go | 84 ++ internal/builder/gosource_test.go | 74 -- internal/builder/judge_build_test.go | 2 + internal/builder/kill_integration_test.go | 58 ++ internal/builder/kill_test.go | 48 -- internal/builder/lend_integration_test.go | 26 + internal/builder/lend_test.go | 17 - .../builder/mirror_shared_integration_test.go | 40 + internal/builder/mirror_shared_test.go | 30 - internal/builder/packages_integration_test.go | 135 ++++ internal/builder/packages_test.go | 125 --- internal/builder/predefined_test.go | 2 + .../registry_unbound_integration_test.go | 93 +++ internal/builder/registry_unbound_test.go | 84 -- internal/builder/registry_wait_test.go | 8 +- internal/builder/replay310_test.go | 2 + internal/builder/replay341_test.go | 2 + internal/builder/replay363_test.go | 2 + internal/builder/replay713_test.go | 2 + internal/builder/replay716_test.go | 2 + .../builder/reproducible_integration_test.go | 91 +++ internal/builder/reproducible_test.go | 80 -- internal/builder/reuse_integration_test.go | 336 ++++++++ internal/builder/reuse_test.go | 313 -------- internal/builder/siblings_test.go | 2 + internal/builder/source_integration_test.go | 44 ++ internal/builder/source_test.go | 35 - internal/builder/template_integration_test.go | 103 +++ internal/builder/template_test.go | 92 --- internal/builder/upstream_file.go | 3 +- .../builder/upstream_file_integration_test.go | 180 +++++ internal/builder/upstream_file_test.go | 169 +--- .../builder/xz_framing_integration_test.go | 40 + internal/builder/xz_framing_test.go | 29 - .../environment_into_integration_test.go | 91 +++ internal/catalogue/environment_into_test.go | 81 -- internal/catalogue/mounts_test.go | 3 +- internal/inventory/adoption_test.go | 2 + internal/inventory/agent_account_test.go | 2 + internal/inventory/artifacts_test.go | 2 + internal/inventory/bindings_test.go | 2 + internal/inventory/builds_test.go | 2 + .../inventory/busrecords_integration_test.go | 316 ++++++++ internal/inventory/busrecords_test.go | 299 ------- .../inventory/bususers_integration_test.go | 143 ++++ internal/inventory/bususers_test.go | 129 --- internal/inventory/busy_integration_test.go | 66 ++ internal/inventory/busy_test.go | 57 -- internal/inventory/catalogue_test.go | 2 + internal/inventory/collection_test.go | 2 + internal/inventory/config_move_test.go | 2 + internal/inventory/core_box_test.go | 2 + internal/inventory/data_test.go | 2 + internal/inventory/doing_test.go | 2 + internal/inventory/durations_test.go | 2 + internal/inventory/forget_test.go | 2 + internal/inventory/given_test.go | 2 + internal/inventory/heals_test.go | 2 + internal/inventory/hold_plans_test.go | 2 + internal/inventory/hold_test.go | 2 + internal/inventory/holdings_test.go | 2 + internal/inventory/inventory_test.go | 2 + internal/inventory/issued_test.go | 2 + internal/inventory/looks_test.go | 2 + internal/inventory/mirrored_test.go | 2 + internal/inventory/named_peer_test.go | 2 + internal/inventory/older_build_test.go | 2 + internal/inventory/operator_test.go | 2 + internal/inventory/order_test.go | 2 + internal/inventory/outside_test.go | 2 + internal/inventory/pending_test.go | 2 + internal/inventory/people_test.go | 2 + internal/inventory/pin_completed_test.go | 2 + internal/inventory/placement_test.go | 2 + internal/inventory/placement_words_test.go | 2 + internal/inventory/plan_id_test.go | 2 + internal/inventory/plans_test.go | 2 + internal/inventory/ports_test.go | 2 + internal/inventory/predefined_test.go | 2 + internal/inventory/public_domain_test.go | 2 + .../inventory/reachable_integration_test.go | 33 + internal/inventory/reachable_test.go | 21 - .../record_mirrors_integration_test.go | 37 + internal/inventory/record_mirrors_test.go | 26 - .../registered_held_integration_test.go | 59 ++ internal/inventory/registered_held_test.go | 51 -- internal/inventory/retire_test.go | 2 + internal/inventory/seats_test.go | 2 + internal/inventory/secret_ask_test.go | 2 + .../secret_family_integration_test.go | 58 ++ internal/inventory/secret_family_test.go | 47 -- internal/inventory/secret_move_test.go | 2 + internal/inventory/secrets_test.go | 7 +- internal/inventory/sends_test.go | 2 + internal/inventory/sent_builds_test.go | 2 + internal/inventory/sent_bus_users_test.go | 2 + internal/inventory/sent_plans_test.go | 2 + internal/inventory/settings_judged_test.go | 2 + internal/inventory/store_as_run_test.go | 2 + internal/inventory/stored_silent_file_test.go | 2 + internal/inventory/template_test.go | 2 + internal/inventory/tunnel_test.go | 2 + internal/inventory/unknown_field_test.go | 2 + internal/inventory/unseen_test.go | 2 + .../walkphases_582_integration_test.go | 49 ++ internal/inventory/walkphases_582_test.go | 40 - .../inventory/walkphases_integration_test.go | 77 ++ internal/inventory/walkphases_test.go | 67 -- .../inventory/walksaid_integration_test.go | 50 ++ internal/inventory/walksaid_test.go | 41 - internal/link/advisories_integration_test.go | 59 ++ internal/link/advisories_test.go | 51 -- internal/link/builds_nats_test.go | 2 + internal/link/bus_integration_test.go | 124 +++ internal/link/bus_test.go | 115 --- internal/link/busfaults_integration_test.go | 96 +++ internal/link/busfaults_test.go | 87 --- internal/link/calls_integration_test.go | 38 + internal/link/calls_kept_integration_test.go | 219 ++++++ internal/link/calls_kept_test.go | 203 ----- internal/link/calls_test.go | 27 - .../link/check_builds_integration_test.go | 67 ++ internal/link/check_builds_test.go | 58 -- internal/link/checks_nats_integration_test.go | 512 ++++++++++++ internal/link/checks_nats_test.go | 489 ------------ internal/link/conformance_integration_test.go | 148 ++++ internal/link/conformance_sources_test.go | 2 + internal/link/conformance_test.go | 128 --- internal/link/deadletters_ask_test.go | 2 + internal/link/deadletters_integration_test.go | 279 +++++++ .../link/deadletters_module_stream_test.go | 2 + internal/link/deadletters_restricted_test.go | 2 + internal/link/deadletters_test.go | 261 ------- internal/link/enrol_bus_credential_test.go | 2 + internal/link/enrol_claim_test.go | 2 + internal/link/enrol_reply_probe_test.go | 2 + internal/link/enrol_shape_test.go | 2 + internal/link/enrol_tunnel_key_test.go | 2 + internal/link/handacts_test.go | 2 + internal/link/handover_integration_test.go | 75 ++ internal/link/handover_test.go | 62 -- internal/link/heard_order_integration_test.go | 118 +++ internal/link/heard_order_test.go | 106 --- internal/link/heard_test.go | 2 + internal/link/judge_directories_test.go | 2 + internal/link/meshcli_integration_test.go | 96 +++ internal/link/meshcli_test.go | 82 -- internal/link/movesecret_integration_test.go | 58 ++ internal/link/movesecret_test.go | 46 -- internal/link/pages_integration_test.go | 59 ++ internal/link/pages_test.go | 46 -- internal/link/profile_report_test.go | 2 + internal/link/queue_integration_test.go | 220 ++++++ internal/link/queue_test.go | 207 ----- .../link/receive_nats_integration_test.go | 440 +++++++++++ internal/link/receive_nats_test.go | 419 ---------- internal/link/rekey_test.go | 2 + internal/link/replay314_test.go | 2 + internal/link/replay330_test.go | 2 + internal/link/seat_as_test.go | 2 + internal/link/setuid_search_test.go | 2 + internal/link/standby_nats_test.go | 2 + internal/link/standing_integration_test.go | 48 ++ internal/link/standing_test.go | 37 - 438 files changed, 22014 insertions(+), 19825 deletions(-) create mode 100644 cmd/mesh-controller/addresses_integration_test.go create mode 100644 cmd/mesh-controller/agent_account_integration_test.go create mode 100644 cmd/mesh-controller/awaiting_push_integration_test.go create mode 100644 cmd/mesh-controller/batches_integration_test.go create mode 100644 cmd/mesh-controller/bindings_integration_test.go create mode 100644 cmd/mesh-controller/build_integration_test.go create mode 100644 cmd/mesh-controller/build_source_integration_test.go create mode 100644 cmd/mesh-controller/bus_waits_integration_test.go create mode 100644 cmd/mesh-controller/calls_verb_integration_test.go create mode 100644 cmd/mesh-controller/check_builds_integration_test.go create mode 100644 cmd/mesh-controller/checks_integration_test.go create mode 100644 cmd/mesh-controller/conditions_brief_integration_test.go create mode 100644 cmd/mesh-controller/conditions_integration_test.go create mode 100644 cmd/mesh-controller/config_moved_integration_test.go create mode 100644 cmd/mesh-controller/core_box_integration_test.go create mode 100644 cmd/mesh-controller/data_integration_test.go create mode 100644 cmd/mesh-controller/deadletters_integration_test.go create mode 100644 cmd/mesh-controller/delivery_integration_test.go create mode 100644 cmd/mesh-controller/desk_secret_integration_test.go create mode 100644 cmd/mesh-controller/doctor_integration_test.go create mode 100644 cmd/mesh-controller/dry_run_integration_test.go create mode 100644 cmd/mesh-controller/found_wait_integration_test.go create mode 100644 cmd/mesh-controller/gate_integration_test.go create mode 100644 cmd/mesh-controller/grant_follows_binding_integration_test.go create mode 100644 cmd/mesh-controller/grants_step_integration_test.go create mode 100644 cmd/mesh-controller/healers_integration_test.go create mode 100644 cmd/mesh-controller/held_back_integration_test.go create mode 100644 cmd/mesh-controller/hosts_behind_integration_test.go create mode 100644 cmd/mesh-controller/identity_bound_integration_test.go create mode 100644 cmd/mesh-controller/issue201_integration_test.go create mode 100644 cmd/mesh-controller/issue348_integration_test.go create mode 100644 cmd/mesh-controller/issue352_integration_test.go create mode 100644 cmd/mesh-controller/issue459_integration_test.go create mode 100644 cmd/mesh-controller/issue611_integration_test.go create mode 100644 cmd/mesh-controller/issue_204_integration_test.go create mode 100644 cmd/mesh-controller/issued_integration_test.go create mode 100644 cmd/mesh-controller/last_send_integration_test.go create mode 100644 cmd/mesh-controller/lean_walk_integration_test.go create mode 100644 cmd/mesh-controller/lease_integration_test.go create mode 100644 cmd/mesh-controller/left_out_integration_test.go create mode 100644 cmd/mesh-controller/machine_network_integration_test.go create mode 100644 cmd/mesh-controller/machine_units_integration_test.go create mode 100644 cmd/mesh-controller/merge_gate_handing_integration_test.go create mode 100644 cmd/mesh-controller/merge_gate_integration_test.go create mode 100644 cmd/mesh-controller/merge_window_bus_integration_test.go create mode 100644 cmd/mesh-controller/meshcli_integration_test.go create mode 100644 cmd/mesh-controller/meshcli_reads_integration_test.go create mode 100644 cmd/mesh-controller/mirrors_integration_test.go create mode 100644 cmd/mesh-controller/one_resolution_integration_test.go create mode 100644 cmd/mesh-controller/operator_wait_integration_test.go create mode 100644 cmd/mesh-controller/over_budget_integration_test.go create mode 100644 cmd/mesh-controller/pending_integration_test.go create mode 100644 cmd/mesh-controller/person_wait_integration_test.go create mode 100644 cmd/mesh-controller/placement_words_moved_integration_test.go create mode 100644 cmd/mesh-controller/plan_clock_integration_test.go create mode 100644 cmd/mesh-controller/plan_looks_integration_test.go create mode 100644 cmd/mesh-controller/plan_moved_every_writer_integration_test.go create mode 100644 cmd/mesh-controller/plans_follow_integration_test.go create mode 100644 cmd/mesh-controller/predefined_integration_test.go create mode 100644 cmd/mesh-controller/proposals_integration_test.go create mode 100644 cmd/mesh-controller/provisioner_account_integration_test.go create mode 100644 cmd/mesh-controller/queue_integration_test.go create mode 100644 cmd/mesh-controller/reconnects_integration_test.go create mode 100644 cmd/mesh-controller/recorded_kept_integration_test.go create mode 100644 cmd/mesh-controller/registry_verbs_integration_test.go create mode 100644 cmd/mesh-controller/reissue_verified_sender_integration_test.go create mode 100644 cmd/mesh-controller/related_integration_test.go create mode 100644 cmd/mesh-controller/release_provisioned_integration_test.go create mode 100644 cmd/mesh-controller/replay335_integration_test.go create mode 100644 cmd/mesh-controller/replays_integration_test.go create mode 100644 cmd/mesh-controller/rest_at_once_integration_test.go create mode 100644 cmd/mesh-controller/retirement_integration_test.go create mode 100644 cmd/mesh-controller/rollout_silent_bus_integration_test.go create mode 100644 cmd/mesh-controller/same_program_integration_test.go create mode 100644 cmd/mesh-controller/sayable_integration_test.go create mode 100644 cmd/mesh-controller/seat_dependencies_integration_test.go create mode 100644 cmd/mesh-controller/seatverbs_integration_test.go create mode 100644 cmd/mesh-controller/seatverbs_schema_integration_test.go create mode 100644 cmd/mesh-controller/secret_integration_test.go create mode 100644 cmd/mesh-controller/selfexec_integration_test.go create mode 100644 cmd/mesh-controller/sendable_integration_test.go create mode 100644 cmd/mesh-controller/sent_required_by_integration_test.go create mode 100644 cmd/mesh-controller/sequence_integration_test.go create mode 100644 cmd/mesh-controller/settings_effective_integration_test.go create mode 100644 cmd/mesh-controller/settings_walk_integration_test.go create mode 100644 cmd/mesh-controller/standing_integration_test.go create mode 100644 cmd/mesh-controller/template_integration_test.go create mode 100644 cmd/mesh-controller/tier_send_integration_test.go create mode 100644 cmd/mesh-controller/try_integration_test.go create mode 100644 cmd/mesh-controller/unheld_report_integration_test.go create mode 100644 cmd/mesh-controller/unseen_integration_test.go create mode 100644 cmd/mesh-controller/untaken_integration_test.go create mode 100644 cmd/mesh-controller/vault_secrets_integration_test.go create mode 100644 cmd/mesh-controller/walk_said_integration_test.go create mode 100644 cmd/mesh-controller/walk_wait_rebuild_754_integration_test.go create mode 100644 internal/builder/builder_integration_test.go create mode 100644 internal/builder/bundle_integration_test.go create mode 100644 internal/builder/check_credential_integration_test.go create mode 100644 internal/builder/check_integration_test.go create mode 100644 internal/builder/failures_integration_test.go create mode 100644 internal/builder/gosource_integration_test.go create mode 100644 internal/builder/kill_integration_test.go create mode 100644 internal/builder/lend_integration_test.go create mode 100644 internal/builder/mirror_shared_integration_test.go create mode 100644 internal/builder/packages_integration_test.go create mode 100644 internal/builder/registry_unbound_integration_test.go create mode 100644 internal/builder/reproducible_integration_test.go create mode 100644 internal/builder/reuse_integration_test.go create mode 100644 internal/builder/source_integration_test.go create mode 100644 internal/builder/template_integration_test.go create mode 100644 internal/builder/upstream_file_integration_test.go create mode 100644 internal/builder/xz_framing_integration_test.go create mode 100644 internal/catalogue/environment_into_integration_test.go create mode 100644 internal/inventory/busrecords_integration_test.go create mode 100644 internal/inventory/bususers_integration_test.go create mode 100644 internal/inventory/busy_integration_test.go create mode 100644 internal/inventory/reachable_integration_test.go create mode 100644 internal/inventory/record_mirrors_integration_test.go create mode 100644 internal/inventory/registered_held_integration_test.go create mode 100644 internal/inventory/secret_family_integration_test.go create mode 100644 internal/inventory/walkphases_582_integration_test.go create mode 100644 internal/inventory/walkphases_integration_test.go create mode 100644 internal/inventory/walksaid_integration_test.go create mode 100644 internal/link/advisories_integration_test.go create mode 100644 internal/link/bus_integration_test.go create mode 100644 internal/link/busfaults_integration_test.go create mode 100644 internal/link/calls_integration_test.go create mode 100644 internal/link/calls_kept_integration_test.go create mode 100644 internal/link/check_builds_integration_test.go create mode 100644 internal/link/checks_nats_integration_test.go create mode 100644 internal/link/conformance_integration_test.go create mode 100644 internal/link/deadletters_integration_test.go create mode 100644 internal/link/handover_integration_test.go create mode 100644 internal/link/heard_order_integration_test.go create mode 100644 internal/link/meshcli_integration_test.go create mode 100644 internal/link/movesecret_integration_test.go create mode 100644 internal/link/pages_integration_test.go create mode 100644 internal/link/queue_integration_test.go create mode 100644 internal/link/receive_nats_integration_test.go create mode 100644 internal/link/standing_integration_test.go diff --git a/cmd/mesh-controller/acts_test.go b/cmd/mesh-controller/acts_test.go index a2e99a66..2e8e6261 100644 --- a/cmd/mesh-controller/acts_test.go +++ b/cmd/mesh-controller/acts_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/addresses_integration_test.go b/cmd/mesh-controller/addresses_integration_test.go new file mode 100644 index 00000000..3d85873d --- /dev/null +++ b/cmd/mesh-controller/addresses_integration_test.go @@ -0,0 +1,228 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "os" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/licences" +) + +func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, aStore()) + if _, err := assign(ctx, open, "anchor", "distribution"); err != nil { + t.Fatal(err) + } + if err := open.inventory.SetSettings(ctx, "anchor", "distribution", + map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5101}}); err != nil { + t.Fatal(err) + } + // A module the mesh built, recorded by digest and path, running on the other machine. + if err := open.inventory.RecordBuild(ctx, inventory.Build{ + ID: "b1", Repository: "r", Module: "app", Commit: "abc", + Made: []inventory.Artifact{{Name: "server", Kind: "image", + Reference: catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}, + }); err != nil { + t.Fatal(err) + } + register(t, open, catalogue.Manifest{Module: "app", Version: "1", + Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-app", + "image": catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}}) + if _, err := assign(ctx, open, "laptop", "app"); err != nil { + t.Fatal(err) + } + // The runtime's trust is the runtime's module's to write (novox/hq ADR 0222): a stand-in for it + // asks where this machine reaches the store, as the docker module does. + register(t, open, catalogue.Manifest{Module: "runtime", Version: "1", + Resources: []map[string]any{{"id": "daemon", "type": "file", "path": "/etc/docker/daemon.json", + "into": "json", "content": `{"insecure-registries": ["${seat:mesh-artifact-store:reach}"]}` + "\n"}}}) + if _, err := assign(ctx, open, "laptop", "runtime"); err != nil { + t.Fatal(err) + } + + on := map[string]bool{"anchor": true, "laptop": true} + node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on) + if err != nil || !found || node != "anchor" || port != "5101" { + t.Fatalf("the store is found on %q:%q (%v, %v); the node put it on 5101", node, port, found, err) + } + + var trust string + for _, r := range composed(t, open, "laptop").Resources { + if r["path"] == "/etc/docker/daemon.json" { + trust, _ = r["content"].(string) + } + if r["id"] == "app.server" && r["image"] != "anchor.internal:5101/app/server@"+aDigest { + t.Errorf("the image the mesh built is fetched as %v", r["image"]) + } + } + if !strings.Contains(trust, "anchor.internal:5101") || strings.Contains(trust, ":5000") { + t.Fatalf("the runtime is told to trust %q; the node put the store on 5101", trust) + } + + // And a replay to the catalogue says where the store is now. + announced, err := following{open: open}.Announceable(ctx) + if err != nil { + t.Fatal(err) + } + if len(announced) != 1 || announced[0].Made[0].Reference != "anchor.internal:5101/app/server@"+aDigest { + t.Fatalf("the replay announces %+v", announced) + } +} + +func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + m, err := catalogue.ParseManifest(raw) + if err != nil { + t.Fatal(err) + } + control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "controller", Kind: catalogue.ArtifactBundle, + Reference: "https://registry.example/mesh-controller/controller.tar.gz", Digest: aDigest}}) + if err != nil { + t.Fatal(err) + } + register(t, open, control) + register(t, open, catalogue.Manifest{Module: "postgres", Version: "1", + Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}}, + Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}}, + Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store", + "ports": []any{"5432:5432"}, "image": "pg@" + aDigest}}}) + register(t, open, catalogue.Manifest{Module: "lavinmq", Version: "1", + Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}, + Listens: []catalogue.Listening{{Port: 5671, From: catalogue.FromMesh}, + {Port: 5672, From: catalogue.FromMesh}}, + Guards: []int{15672}, + Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker", + "ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}, "image": "mq@" + aDigest}}}) + // The control plane's own bus user is the installer's, seeded at genesis before the controller + // runs (SeedBusUser); without it a push now refuses the credential nobody issued (issue 203). + if err := open.inventory.SeedBusUser(ctx, inventory.BusUser{Username: "anchor.mesh-controller", + Kind: inventory.BusController, Node: "anchor", Module: "mesh-controller"}, "bootstrap"); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil { + t.Fatal(err) + } + // The store's module is registered and given its port, and NOT assigned: the state genesis + // leaves a given-port node in before the foundation is adopted as modules (04-ISSUES/085). + if err := open.inventory.SetSettings(ctx, "anchor", "postgres", + map[string]any{catalogue.PortsSetting: map[string]any{"5432": 6852}}); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "anchor", "lavinmq"); err != nil { + t.Fatal(err) + } + if err := open.inventory.SetSettings(ctx, "anchor", "lavinmq", + map[string]any{catalogue.PortsSetting: map[string]any{"5672": 5679}}); err != nil { + t.Fatal(err) + } + + var env map[string]any + for _, r := range composed(t, open, "anchor").Resources { + if r["id"] == "mesh-controller.controller" { + env, _ = r["env"].(map[string]any) + } + } + if env == nil { + t.Fatal("the control plane's process is not in its own node's declaration") + } + for key, want := range map[string]string{ + "MESH_STORE_INVENTORY_PORT": "6852", + "MESH_STORE_IDENTITY_PORT": "6852", + "MESH_STORE_LICENCES_PORT": "6852", + "MESH_BROKER_AMQP_PORT": "5679", + // Neither given nor assigned by the mesh: the manifest's own number is NOT the answer, + // because the sealed value beside it carries the port genesis wrote (finding F3). + "MESH_BROKER_ADDRESS_PORT": "", + "MESH_BROKER_MANAGEMENT_PORT": "", + } { + if env[key] != want { + t.Errorf("the control plane is told %s=%v; the node says %q", key, env[key], want) + } + } +} + +func aLoneNode(t *testing.T) *stores { + t.Helper() + inventory.ForTest(t) + licences.ForTest(t) + open, err := openStores(t.Context()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(open.Close) + for _, m := range provided { + if err := open.inventory.Provide(t.Context(), m); err != nil { + t.Fatal(err) + } + } + record, err := open.inventory.AddNode(t.Context(), "anchor") + if err != nil { + t.Fatal(err) + } + reported, _ := json.Marshal(map[string]any{"capabilities": []map[string]any{ + {"name": "container-runtime", "present": true}}}) + var profile map[string]any + _ = json.Unmarshal(reported, &profile) + if err := open.inventory.RecordProfile(t.Context(), record.ID, profile); err != nil { + t.Fatal(err) + } + if err := open.inventory.RecordSealingKey(t.Context(), record.ID, aPublicKey(t)); err != nil { + t.Fatal(err) + } + return open +} + +func TestOnANodeWithNoNetworkTheStoreIsReachedByLoopback(t *testing.T) { + open := aLoneNode(t) + ctx := t.Context() + register(t, open, aStore()) + register(t, open, catalogue.Manifest{Module: "builder", Version: "1", + Requires: []string{catalogue.ArtifactStoreProvision}, + Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-builder", + "image": "registry.example/mesh-builder@" + aDigest}}}) + if err := open.inventory.RecordBuild(ctx, inventory.Build{ + ID: "b1", Repository: "r", Module: "postgres", Commit: "abc", + Made: []inventory.Artifact{{Name: "runtime", Kind: "image", + Reference: catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}, + }); err != nil { + t.Fatal(err) + } + register(t, open, catalogue.Manifest{Module: "postgres", Version: "1", + Resources: []map[string]any{{"id": "runtime", "type": "container", "name": "mesh-postgres", + "image": catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}}) + for _, module := range []string{"distribution", "builder", "postgres"} { + if _, err := assign(ctx, open, "anchor", module); err != nil { + t.Fatal(err) + } + } + if err := open.inventory.SetSettings(ctx, "anchor", "distribution", + map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5100}}); err != nil { + t.Fatal(err) + } + + var image any + for _, r := range composed(t, open, "anchor").Resources { + if r["id"] == "postgres.runtime" { + image = r["image"] + } + } + if image != "127.0.0.1:5100/postgres/runtime@"+aDigest { + t.Fatalf("on the store's own node, off any network, the image is fetched as %v", image) + } + held := heldBy(ctx) + if got := held["postgres/runtime"]; got != "127.0.0.1:5100/postgres/runtime@"+aDigest { + t.Fatalf("a builder beside the store is handed the base %q", got) + } +} diff --git a/cmd/mesh-controller/addresses_test.go b/cmd/mesh-controller/addresses_test.go index bc02b37b..5eb8d617 100644 --- a/cmd/mesh-controller/addresses_test.go +++ b/cmd/mesh-controller/addresses_test.go @@ -2,13 +2,10 @@ package main import ( "encoding/json" - "os" "strings" "testing" "github.com/novox/mesh-controller/internal/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/licences" "github.com/novox/mesh-controller/internal/link" ) @@ -86,150 +83,12 @@ func aStore() catalogue.Manifest { // **The trust a machine writes for the store, and the address every built image is fetched // through, say the port the node gave the store** — not the catalogue's number. -func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, aStore()) - if _, err := assign(ctx, open, "anchor", "distribution"); err != nil { - t.Fatal(err) - } - if err := open.inventory.SetSettings(ctx, "anchor", "distribution", - map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5101}}); err != nil { - t.Fatal(err) - } - // A module the mesh built, recorded by digest and path, running on the other machine. - if err := open.inventory.RecordBuild(ctx, inventory.Build{ - ID: "b1", Repository: "r", Module: "app", Commit: "abc", - Made: []inventory.Artifact{{Name: "server", Kind: "image", - Reference: catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}, - }); err != nil { - t.Fatal(err) - } - register(t, open, catalogue.Manifest{Module: "app", Version: "1", - Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-app", - "image": catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}}) - if _, err := assign(ctx, open, "laptop", "app"); err != nil { - t.Fatal(err) - } - // The runtime's trust is the runtime's module's to write (novox/hq ADR 0222): a stand-in for it - // asks where this machine reaches the store, as the docker module does. - register(t, open, catalogue.Manifest{Module: "runtime", Version: "1", - Resources: []map[string]any{{"id": "daemon", "type": "file", "path": "/etc/docker/daemon.json", - "into": "json", "content": `{"insecure-registries": ["${seat:mesh-artifact-store:reach}"]}` + "\n"}}}) - if _, err := assign(ctx, open, "laptop", "runtime"); err != nil { - t.Fatal(err) - } - - on := map[string]bool{"anchor": true, "laptop": true} - node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on) - if err != nil || !found || node != "anchor" || port != "5101" { - t.Fatalf("the store is found on %q:%q (%v, %v); the node put it on 5101", node, port, found, err) - } - - var trust string - for _, r := range composed(t, open, "laptop").Resources { - if r["path"] == "/etc/docker/daemon.json" { - trust, _ = r["content"].(string) - } - if r["id"] == "app.server" && r["image"] != "anchor.internal:5101/app/server@"+aDigest { - t.Errorf("the image the mesh built is fetched as %v", r["image"]) - } - } - if !strings.Contains(trust, "anchor.internal:5101") || strings.Contains(trust, ":5000") { - t.Fatalf("the runtime is told to trust %q; the node put the store on 5101", trust) - } - - // And a replay to the catalogue says where the store is now. - announced, err := following{open: open}.Announceable(ctx) - if err != nil { - t.Fatal(err) - } - if len(announced) != 1 || announced[0].Made[0].Reference != "anchor.internal:5101/app/server@"+aDigest { - t.Fatalf("the replay announces %+v", announced) - } -} // **The control plane's own connections say the port the node gave the store and the broker.** // // Composed from the control plane's own manifest against a real inventory: the store's module is // given 6852 on this node the way genesis or an operator gives it, and the control plane's // process is told so beside the sealed connection genesis wrote. -func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - raw, err := os.ReadFile("../../module.json") - if err != nil { - t.Fatal(err) - } - m, err := catalogue.ParseManifest(raw) - if err != nil { - t.Fatal(err) - } - control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "controller", Kind: catalogue.ArtifactBundle, - Reference: "https://registry.example/mesh-controller/controller.tar.gz", Digest: aDigest}}) - if err != nil { - t.Fatal(err) - } - register(t, open, control) - register(t, open, catalogue.Manifest{Module: "postgres", Version: "1", - Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}}, - Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}}, - Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store", - "ports": []any{"5432:5432"}, "image": "pg@" + aDigest}}}) - register(t, open, catalogue.Manifest{Module: "lavinmq", Version: "1", - Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}, - Listens: []catalogue.Listening{{Port: 5671, From: catalogue.FromMesh}, - {Port: 5672, From: catalogue.FromMesh}}, - Guards: []int{15672}, - Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker", - "ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}, "image": "mq@" + aDigest}}}) - // The control plane's own bus user is the installer's, seeded at genesis before the controller - // runs (SeedBusUser); without it a push now refuses the credential nobody issued (issue 203). - if err := open.inventory.SeedBusUser(ctx, inventory.BusUser{Username: "anchor.mesh-controller", - Kind: inventory.BusController, Node: "anchor", Module: "mesh-controller"}, "bootstrap"); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil { - t.Fatal(err) - } - // The store's module is registered and given its port, and NOT assigned: the state genesis - // leaves a given-port node in before the foundation is adopted as modules (04-ISSUES/085). - if err := open.inventory.SetSettings(ctx, "anchor", "postgres", - map[string]any{catalogue.PortsSetting: map[string]any{"5432": 6852}}); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "anchor", "lavinmq"); err != nil { - t.Fatal(err) - } - if err := open.inventory.SetSettings(ctx, "anchor", "lavinmq", - map[string]any{catalogue.PortsSetting: map[string]any{"5672": 5679}}); err != nil { - t.Fatal(err) - } - - var env map[string]any - for _, r := range composed(t, open, "anchor").Resources { - if r["id"] == "mesh-controller.controller" { - env, _ = r["env"].(map[string]any) - } - } - if env == nil { - t.Fatal("the control plane's process is not in its own node's declaration") - } - for key, want := range map[string]string{ - "MESH_STORE_INVENTORY_PORT": "6852", - "MESH_STORE_IDENTITY_PORT": "6852", - "MESH_STORE_LICENCES_PORT": "6852", - "MESH_BROKER_AMQP_PORT": "5679", - // Neither given nor assigned by the mesh: the manifest's own number is NOT the answer, - // because the sealed value beside it carries the port genesis wrote (finding F3). - "MESH_BROKER_ADDRESS_PORT": "", - "MESH_BROKER_MANAGEMENT_PORT": "", - } { - if env[key] != want { - t.Errorf("the control plane is told %s=%v; the node says %q", key, env[key], want) - } - } -} // withSeatPorts is the control plane's manifest with the seat placeholders in its environment — // added here until module.json carries them (the manifest lands one commit after the code that @@ -274,79 +133,7 @@ func withSeatPorts(m catalogue.Manifest) catalogue.Manifest { // aLoneNode is one capable machine with nothing placed on any network — the control-node during // genesis, before the "network" step, which is after the store, the broker, the vault and the // catalogue have each been built and pushed (finding F2). -func aLoneNode(t *testing.T) *stores { - t.Helper() - inventory.ForTest(t) - licences.ForTest(t) - open, err := openStores(t.Context()) - if err != nil { - t.Fatal(err) - } - t.Cleanup(open.Close) - for _, m := range provided { - if err := open.inventory.Provide(t.Context(), m); err != nil { - t.Fatal(err) - } - } - record, err := open.inventory.AddNode(t.Context(), "anchor") - if err != nil { - t.Fatal(err) - } - reported, _ := json.Marshal(map[string]any{"capabilities": []map[string]any{ - {"name": "container-runtime", "present": true}}}) - var profile map[string]any - _ = json.Unmarshal(reported, &profile) - if err := open.inventory.RecordProfile(t.Context(), record.ID, profile); err != nil { - t.Fatal(err) - } - if err := open.inventory.RecordSealingKey(t.Context(), record.ID, aPublicKey(t)); err != nil { - t.Fatal(err) - } - return open -} // **Before the network exists, the store's own node reaches it by loopback** — never refused, // never handed the scheme: a genesis pushes the store, the broker, the vault and the catalogue to // a node on no network, and builds the catalogue on a base it must be able to pull. -func TestOnANodeWithNoNetworkTheStoreIsReachedByLoopback(t *testing.T) { - open := aLoneNode(t) - ctx := t.Context() - register(t, open, aStore()) - register(t, open, catalogue.Manifest{Module: "builder", Version: "1", - Requires: []string{catalogue.ArtifactStoreProvision}, - Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-builder", - "image": "registry.example/mesh-builder@" + aDigest}}}) - if err := open.inventory.RecordBuild(ctx, inventory.Build{ - ID: "b1", Repository: "r", Module: "postgres", Commit: "abc", - Made: []inventory.Artifact{{Name: "runtime", Kind: "image", - Reference: catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}, - }); err != nil { - t.Fatal(err) - } - register(t, open, catalogue.Manifest{Module: "postgres", Version: "1", - Resources: []map[string]any{{"id": "runtime", "type": "container", "name": "mesh-postgres", - "image": catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}}) - for _, module := range []string{"distribution", "builder", "postgres"} { - if _, err := assign(ctx, open, "anchor", module); err != nil { - t.Fatal(err) - } - } - if err := open.inventory.SetSettings(ctx, "anchor", "distribution", - map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5100}}); err != nil { - t.Fatal(err) - } - - var image any - for _, r := range composed(t, open, "anchor").Resources { - if r["id"] == "postgres.runtime" { - image = r["image"] - } - } - if image != "127.0.0.1:5100/postgres/runtime@"+aDigest { - t.Fatalf("on the store's own node, off any network, the image is fetched as %v", image) - } - held := heldBy(ctx) - if got := held["postgres/runtime"]; got != "127.0.0.1:5100/postgres/runtime@"+aDigest { - t.Fatalf("a builder beside the store is handed the base %q", got) - } -} diff --git a/cmd/mesh-controller/adopting_test.go b/cmd/mesh-controller/adopting_test.go index 3589fec0..5561bb05 100644 --- a/cmd/mesh-controller/adopting_test.go +++ b/cmd/mesh-controller/adopting_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/agent_account_integration_test.go b/cmd/mesh-controller/agent_account_integration_test.go new file mode 100644 index 00000000..1d615a1c --- /dev/null +++ b/cmd/mesh-controller/agent_account_integration_test.go @@ -0,0 +1,161 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + snapshot "github.com/novox/mesh-controller/internal/facts" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-host/rootsearch" +) + +func TestTheSelfCheckSaysAnAgentAccountThatCanBecomeRoot(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + for _, n := range []string{"anchor", "laptop"} { + if _, err := inv.NodeByName(ctx, n); err != nil { + if _, err := inv.AddNode(ctx, n); err != nil { + t.Fatal(err) + } + } + if err := inv.SetAccount(ctx, n, "ops", ""); err != nil { + t.Fatal(err) + } + } + if err := inv.SetAgentAccount(ctx, "anchor", "agent", ""); err != nil { + t.Fatal(err) + } + d := &doctor{open: open} + found, err := probeAgentAccounts(ctx, d) + if err != nil { + t.Fatal(err) + } + found = onlyMachine(found, "anchor") + if len(found) != 1 || found[0].Machine != "anchor" || found[0].Severity != conditions.Urgent || + !strings.Contains(found[0].Said, "not judged") { + t.Fatalf("a named agent account with no verdict: %+v", found) + } + w := plainWordings[kindAgentCanBecomeRoot](conditions.Observation{Kind: kindAgentCanBecomeRoot, Machine: "anchor"}) + if w.Headline == "" || w.Needs == "" || w.Resolved == "" { + t.Errorf("the condition has no plain words: %+v", w) + } + + healthy := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account", + Kind: link.KindAccount, Target: "agent", State: link.StateHealthy, Root: link.RootNever, Account: "agent"} + if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract, + SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{healthy}}); err != nil { + t.Fatal(err) + } + if found, err = probeAgentAccounts(ctx, d); err != nil || len(onlyMachine(found, "anchor")) != 0 { + t.Fatalf("a judged agent account still fails: %+v %v", found, err) + } + if named, confined, why, err := agentConfined(ctx, inv, "anchor", time.Now()); err != nil || !named || !confined { + t.Fatalf("agentConfined on anchor: %v %v %q %v", named, confined, why, err) + } + if named, _, why, err := agentConfined(ctx, inv, "laptop", time.Now()); err != nil || named || + !strings.Contains(why, "operator account") { + t.Fatalf("agentConfined on a machine naming none: %v %q %v", named, why, err) + } +} + +func TestTheFactsCarryTheAgentAccountAsAPseudonym(t *testing.T) { + open, _ := aMeshWithSecrets(t) + ctx := t.Context() + if err := open.inventory.SetAccount(ctx, "anchor", "keeper", ""); err != nil { + t.Fatal(err) + } + if err := open.inventory.SetAgentAccount(ctx, "anchor", "warden", ""); err != nil { + t.Fatal(err) + } + f, err := gatherFacts(ctx, open, "2.11.17") + if err != nil { + t.Fatal(err) + } + body, _ := f.Encode() + if strings.Contains(string(body), "warden") { + t.Error("the agent account's name is in the snapshot") + } + m, ok := f.Machine(snapshot.Pseudonym("machine", "anchor")) + if !ok || m.AgentAccount != snapshot.Pseudonym("account", "warden") || m.Account == m.AgentAccount { + t.Fatalf("the anchor's agent account reads as %q (operator %q)", m.AgentAccount, m.Account) + } +} + +func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) { + if searchQuietFor != rootsearch.Quiet { + t.Fatalf("the quiet is %s and the node-engine's %s: they are one value", searchQuietFor, rootsearch.Quiet) + } + // A daily search that finishes within the quiet never leaves the account unjudged between two of them. + if rootsearch.FreshFor < rootsearch.Every+rootsearch.Quiet { + t.Fatalf("a complete search judges for %s, less than a day's search (%s) and the quiet (%s)", + rootsearch.FreshFor, rootsearch.Every, rootsearch.Quiet) + } + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + if _, err := inv.NodeByName(ctx, "anchor"); err != nil { + if _, err := inv.AddNode(ctx, "anchor"); err != nil { + t.Fatal(err) + } + } + if err := inv.SetAgentAccount(ctx, "anchor", "agent", ""); err != nil { + t.Fatal(err) + } + d := &doctor{open: open} + say := func(state, reason string) []conditions.Observation { + t.Helper() + // The engine's since is always now: it was just restarted. + v := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account", + Kind: link.KindAccount, Target: "agent", State: state, Reason: reason, Root: link.RootNever, + Account: "agent", Since: time.Now()} + if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract, + SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{v}}); err != nil { + t.Fatal(err) + } + found, err := probeAgentAccounts(ctx, d) + if err != nil { + t.Fatal(err) + } + return onlyMachine(found, "anchor") + } + running := link.ReasonRootPending + ": the search for setuid programs, started at 19:21, has not finished yet" + healthy := func() { + t.Helper() + if found := say(link.StateHealthy, ""); len(found) != 0 { + t.Fatalf("a healthy verdict raised: %+v", found) + } + } + if found := say(link.StateUnknown, running); len(found) != 0 { + t.Fatalf("a search first seen now was raised: %+v", found) + } + if _, confined, why, _ := agentConfined(ctx, inv, "anchor", time.Now()); confined || !strings.Contains(why, "not judged") { + t.Fatalf("an account whose search runs was read as confined: %q", why) + } + // The engine restarted again and again, each statement's own since fresh: the controller's clock runs on. + if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil { + t.Fatal(err) + } + healthy() // a complete verdict forgets when the waiting began … + if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil { + t.Fatal(err) // … and this restart loop began past the bound + } + if found := say(link.StateUnknown, running); len(found) != 1 || found[0].Severity != conditions.Urgent { + t.Fatalf("a search pending past the bound, by the controller's clock, was not urgent: %+v", found) + } + healthy() + incomplete := rootsearch.ReasonIncomplete + ": the search for setuid programs did not finish (last tried at " + + "19:23: timeout); it is tried again later" + if found := say(link.StateUnknown, incomplete); len(found) != 1 || found[0].Severity != conditions.Urgent { + t.Fatalf("a search that did not finish was not urgent: %+v", found) + } + if found := say(link.StateUnhealthy, link.ReasonRoot+": in the group docker; "+running); len(found) != 1 || + found[0].Severity != conditions.Urgent { + t.Fatalf("a way to root found while the search runs was not urgent: %+v", found) + } +} diff --git a/cmd/mesh-controller/agent_account_test.go b/cmd/mesh-controller/agent_account_test.go index ee859684..63d5dff9 100644 --- a/cmd/mesh-controller/agent_account_test.go +++ b/cmd/mesh-controller/agent_account_test.go @@ -1,14 +1,12 @@ package main import ( - "github.com/novox/mesh-host/rootsearch" "strings" "testing" "time" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" - snapshot "github.com/novox/mesh-controller/internal/facts" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -65,55 +63,6 @@ func TestAnAgentAccountIsConfinedOnlyOnAHealthyVerdictJudgedForRoot(t *testing.T // DA raises an urgent condition, with plain words, on a machine whose agent account is not judged unable to // become root; a machine that names none is not its to judge. -func TestTheSelfCheckSaysAnAgentAccountThatCanBecomeRoot(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.NodeByName(ctx, n); err != nil { - if _, err := inv.AddNode(ctx, n); err != nil { - t.Fatal(err) - } - } - if err := inv.SetAccount(ctx, n, "ops", ""); err != nil { - t.Fatal(err) - } - } - if err := inv.SetAgentAccount(ctx, "anchor", "agent", ""); err != nil { - t.Fatal(err) - } - d := &doctor{open: open} - found, err := probeAgentAccounts(ctx, d) - if err != nil { - t.Fatal(err) - } - found = onlyMachine(found, "anchor") - if len(found) != 1 || found[0].Machine != "anchor" || found[0].Severity != conditions.Urgent || - !strings.Contains(found[0].Said, "not judged") { - t.Fatalf("a named agent account with no verdict: %+v", found) - } - w := plainWordings[kindAgentCanBecomeRoot](conditions.Observation{Kind: kindAgentCanBecomeRoot, Machine: "anchor"}) - if w.Headline == "" || w.Needs == "" || w.Resolved == "" { - t.Errorf("the condition has no plain words: %+v", w) - } - - healthy := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account", - Kind: link.KindAccount, Target: "agent", State: link.StateHealthy, Root: link.RootNever, Account: "agent"} - if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract, - SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{healthy}}); err != nil { - t.Fatal(err) - } - if found, err = probeAgentAccounts(ctx, d); err != nil || len(onlyMachine(found, "anchor")) != 0 { - t.Fatalf("a judged agent account still fails: %+v %v", found, err) - } - if named, confined, why, err := agentConfined(ctx, inv, "anchor", time.Now()); err != nil || !named || !confined { - t.Fatalf("agentConfined on anchor: %v %v %q %v", named, confined, why, err) - } - if named, _, why, err := agentConfined(ctx, inv, "laptop", time.Now()); err != nil || named || - !strings.Contains(why, "operator account") { - t.Fatalf("agentConfined on a machine naming none: %v %q %v", named, why, err) - } -} func TestTheAgentRootWordsArePlain(t *testing.T) { w := plainWordings[kindAgentCanBecomeRoot](conditions.Observation{Kind: kindAgentCanBecomeRoot, Machine: "anchor"}) @@ -134,28 +83,6 @@ func onlyMachine(obs []conditions.Observation, machine string) []conditions.Obse // The snapshot a merge check composes from carries the agent account as a pseudonym (novox/hq ADR 0266), // so a change is judged against machines that name one, and the name never leaves. -func TestTheFactsCarryTheAgentAccountAsAPseudonym(t *testing.T) { - open, _ := aMeshWithSecrets(t) - ctx := t.Context() - if err := open.inventory.SetAccount(ctx, "anchor", "keeper", ""); err != nil { - t.Fatal(err) - } - if err := open.inventory.SetAgentAccount(ctx, "anchor", "warden", ""); err != nil { - t.Fatal(err) - } - f, err := gatherFacts(ctx, open, "2.11.17") - if err != nil { - t.Fatal(err) - } - body, _ := f.Encode() - if strings.Contains(string(body), "warden") { - t.Error("the agent account's name is in the snapshot") - } - m, ok := f.Machine(snapshot.Pseudonym("machine", "anchor")) - if !ok || m.AgentAccount != snapshot.Pseudonym("account", "warden") || m.Account == m.AgentAccount { - t.Fatalf("the anchor's agent account reads as %q (operator %q)", m.AgentAccount, m.Account) - } -} // No verb runs a `node` command that sets something: through the generic `command` verb, `node account`, // `node agent-account` and every other `node` subcommand but list and show are refused, naming the terminal. @@ -205,75 +132,3 @@ func TestTheNodeVerbOnlyShows(t *testing.T) { // controller first saw it waiting — never from the engine's own "since", which a restart resets, so an engine // restarted in a loop does not keep it quiet. The account is still not confined, and `node show` still says not // judged; a search that failed, or a way to root found, is urgent at once. -func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) { - if searchQuietFor != rootsearch.Quiet { - t.Fatalf("the quiet is %s and the node-engine's %s: they are one value", searchQuietFor, rootsearch.Quiet) - } - // A daily search that finishes within the quiet never leaves the account unjudged between two of them. - if rootsearch.FreshFor < rootsearch.Every+rootsearch.Quiet { - t.Fatalf("a complete search judges for %s, less than a day's search (%s) and the quiet (%s)", - rootsearch.FreshFor, rootsearch.Every, rootsearch.Quiet) - } - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - if _, err := inv.NodeByName(ctx, "anchor"); err != nil { - if _, err := inv.AddNode(ctx, "anchor"); err != nil { - t.Fatal(err) - } - } - if err := inv.SetAgentAccount(ctx, "anchor", "agent", ""); err != nil { - t.Fatal(err) - } - d := &doctor{open: open} - say := func(state, reason string) []conditions.Observation { - t.Helper() - // The engine's since is always now: it was just restarted. - v := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account", - Kind: link.KindAccount, Target: "agent", State: state, Reason: reason, Root: link.RootNever, - Account: "agent", Since: time.Now()} - if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract, - SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{v}}); err != nil { - t.Fatal(err) - } - found, err := probeAgentAccounts(ctx, d) - if err != nil { - t.Fatal(err) - } - return onlyMachine(found, "anchor") - } - running := link.ReasonRootPending + ": the search for setuid programs, started at 19:21, has not finished yet" - healthy := func() { - t.Helper() - if found := say(link.StateHealthy, ""); len(found) != 0 { - t.Fatalf("a healthy verdict raised: %+v", found) - } - } - if found := say(link.StateUnknown, running); len(found) != 0 { - t.Fatalf("a search first seen now was raised: %+v", found) - } - if _, confined, why, _ := agentConfined(ctx, inv, "anchor", time.Now()); confined || !strings.Contains(why, "not judged") { - t.Fatalf("an account whose search runs was read as confined: %q", why) - } - // The engine restarted again and again, each statement's own since fresh: the controller's clock runs on. - if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil { - t.Fatal(err) - } - healthy() // a complete verdict forgets when the waiting began … - if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil { - t.Fatal(err) // … and this restart loop began past the bound - } - if found := say(link.StateUnknown, running); len(found) != 1 || found[0].Severity != conditions.Urgent { - t.Fatalf("a search pending past the bound, by the controller's clock, was not urgent: %+v", found) - } - healthy() - incomplete := rootsearch.ReasonIncomplete + ": the search for setuid programs did not finish (last tried at " + - "19:23: timeout); it is tried again later" - if found := say(link.StateUnknown, incomplete); len(found) != 1 || found[0].Severity != conditions.Urgent { - t.Fatalf("a search that did not finish was not urgent: %+v", found) - } - if found := say(link.StateUnhealthy, link.ReasonRoot+": in the group docker; "+running); len(found) != 1 || - found[0].Severity != conditions.Urgent { - t.Fatalf("a way to root found while the search runs was not urgent: %+v", found) - } -} diff --git a/cmd/mesh-controller/api_test.go b/cmd/mesh-controller/api_test.go index 876c5a9b..9cc5d99a 100644 --- a/cmd/mesh-controller/api_test.go +++ b/cmd/mesh-controller/api_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/asker_bus_test.go b/cmd/mesh-controller/asker_bus_test.go index 0e22c949..32c53642 100644 --- a/cmd/mesh-controller/asker_bus_test.go +++ b/cmd/mesh-controller/asker_bus_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/awaiting_push_integration_test.go b/cmd/mesh-controller/awaiting_push_integration_test.go new file mode 100644 index 00000000..ee881d98 --- /dev/null +++ b/cmd/mesh-controller/awaiting_push_integration_test.go @@ -0,0 +1,182 @@ +//go:build integration + +package main + +import ( + "errors" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/overlay" +) + +func TestAModuleAssignedAndNotPushedIsAwaitingAPushNotUncomposable(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, aKeeper()) + pushedBy(t, open, "laptop") // the machine was pushed before; then the module is assigned + if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { + t.Fatal(err) + } + + // The read the rest of the mesh asks still refuses it, with the composer's typed error: nothing + // can be compared about a secret that does not exist (status), and nothing here string-matches. + plan, settings, err := planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + gens, err := generators(ctx, open) + if err != nil { + t.Fatal(err) + } + _, err = declarationWith(ctx, open, "laptop", plan, settings, gens, Reading) + var notMade *catalogue.NotMadeError + if !errors.As(err, ¬Made) || notMade.Module != "keeper" || notMade.Name != "repository" { + t.Fatalf("a read composition did not say which secret is not made, typed: %v", err) + } + // Foreseen, it composes, names what the push makes, and made nothing. + foreseen, err := declarationWith(ctx, open, "laptop", plan, settings, gens, Foreseeing) + if err != nil || len(foreseen.foreseen) != 1 || foreseen.foreseen[0] != "keeper/repository" { + t.Fatalf("foreseen: %v %v", foreseen.foreseen, err) + } + if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "keeper", "repository"); err != nil || held { + t.Fatalf("asking ahead of the push made the secret (held %v, %v)", held, err) + } + + // Within the bound: nothing at all — no urgent, no warning, nobody notified. + if got := d1(t, open); len(got) != 0 { + t.Fatalf("a machine waiting for a push raised %+v", got) + } + + // Past the bound: a warning, not urgent, saying what the push will make. + before := awaitingPushBound + awaitingPushBound = -time.Minute + t.Cleanup(func() { awaitingPushBound = before }) + got := d1(t, open) + if len(got) != 1 || got[0].Key() != "machine.laptop.awaiting-push" || got[0].Severity != conditions.Warning || + !strings.Contains(got[0].Summary, "keeper/repository") || !strings.Contains(got[0].Summary, "push laptop") { + t.Fatalf("a machine left un-pushed past the bound: %+v", got) + } + + // Pushed: the secret is made and D1 says nothing. + pushedBy(t, open, "laptop") + if got := d1(t, open); len(got) != 0 { + t.Fatalf("a pushed machine still raised %+v", got) + } +} + +func TestASecretThePushCannotMakeIsStillUncomposable(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, aTalker()) + if _, err := assign(ctx, open, "laptop", "talker"); err != nil { + t.Fatal(err) + } + got := d1(t, open) + if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || got[0].Severity != conditions.Urgent || + !strings.Contains(got[0].Summary, "module issue talker --node laptop") { + t.Fatalf("a push that will be refused was not said urgently: %+v", got) + } + + // And a machine whose composition fails for anything else, with a secret waiting beside it, is + // uncomposable for that — the stand-in hides nothing. + register(t, open, aKeeper()) + if _, err := assign(ctx, open, "anchor", "keeper"); err != nil { + t.Fatal(err) + } + one, two := rivals() + register(t, open, one) + register(t, open, two) + _, _ = assign(ctx, open, "anchor", "rival-one") + _, _ = assign(ctx, open, "anchor", "rival-two") + keys := map[string]conditions.Severity{} + for _, o := range d1(t, open) { + keys[o.Key()] = o.Severity + } + if keys["machine.anchor.uncomposable"] != conditions.Urgent { + t.Fatalf("a real failure beside a waiting secret: %v", keys) + } +} + +func TestAGivenSecretUnderAnOldKeyIsNotForeseen(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, aKeeper()) + if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { + t.Fatal(err) + } + if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "keeper", "repository", "given"); err != nil { + t.Fatal(err) + } + record, err := open.inventory.NodeByName(ctx, "laptop") + if err != nil { + t.Fatal(err) + } + if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil { + t.Fatal(err) + } + got := d1(t, open) + if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "issue it again") { + t.Fatalf("a given secret under an old key: %+v", got) + } +} + +func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, aKeeper()) + pushedBy(t, open, "laptop") + sent := time.Now() + since, err := open.inventory.AwaitingSince(ctx, "laptop") + if err != nil || since.After(sent) { + t.Fatalf("nothing assigned since the send: waiting since %v (%v), the send was before %v", since, err, sent) + } + if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { + t.Fatal(err) + } + since, err = open.inventory.AwaitingSince(ctx, "laptop") + if err != nil || since.Before(sent.Add(-time.Second)) { + t.Fatalf("assigned after the send: waiting since %v (%v), not from the assignment", since, err) + } +} + +func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, aKeeper()) + pushedBy(t, open, "laptop") + if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { + t.Fatal(err) + } + sends, err := readDeliveries(ctx, open.inventory) + if err != nil { + t.Fatal(err) + } + if sends["laptop"].settled("keeper", time.Now()) { + t.Fatal("a holder assigned and not pushed is expected to answer") + } + if !sends["laptop"].settled(overlay.Name, time.Now().Add(time.Hour)) { + t.Fatal("a module the machine was sent long ago is not expected to answer") + } + pushedBy(t, open, "laptop") + sends, err = readDeliveries(ctx, open.inventory) + if err != nil { + t.Fatal(err) + } + if sends["laptop"].settled("keeper", time.Now()) { + t.Fatal("pushed a moment ago and not reported, a holder is already expected") + } + if !sends["laptop"].settled("keeper", time.Now().Add(reportGrace+time.Minute)) { + t.Fatal("pushed past the grace, a holder is not expected") + } + if _, ok := sends["laptop"].carried["keeper"]; !ok { + t.Fatalf("the send's builds do not carry keeper: %v", sends["laptop"].carried) + } + pushedAndApplied(t, open, "laptop") + if sends, err = readDeliveries(ctx, open.inventory); err != nil || !sends["laptop"].settled("keeper", time.Now()) { + t.Fatalf("pushed and reported applied, a holder is not expected to answer (%v)", err) + } +} diff --git a/cmd/mesh-controller/awaiting_push_test.go b/cmd/mesh-controller/awaiting_push_test.go index e0890c6f..0ec32245 100644 --- a/cmd/mesh-controller/awaiting_push_test.go +++ b/cmd/mesh-controller/awaiting_push_test.go @@ -1,15 +1,12 @@ package main import ( - "errors" - "strings" "testing" "time" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/overlay" ) // Between `assign` and `push` a module's own secrets are not made yet: the push makes them. D1 composed @@ -81,140 +78,14 @@ func d1(t *testing.T, open *stores) []conditions.Observation { // **THE WINDOW, REPRODUCED**: assigned and not pushed, a module whose own secret the push makes is // waiting, not uncomposable; past the bound it is a warning naming what the push makes; pushed, nothing. -func TestAModuleAssignedAndNotPushedIsAwaitingAPushNotUncomposable(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, aKeeper()) - pushedBy(t, open, "laptop") // the machine was pushed before; then the module is assigned - if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { - t.Fatal(err) - } - - // The read the rest of the mesh asks still refuses it, with the composer's typed error: nothing - // can be compared about a secret that does not exist (status), and nothing here string-matches. - plan, settings, err := planFor(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - gens, err := generators(ctx, open) - if err != nil { - t.Fatal(err) - } - _, err = declarationWith(ctx, open, "laptop", plan, settings, gens, Reading) - var notMade *catalogue.NotMadeError - if !errors.As(err, ¬Made) || notMade.Module != "keeper" || notMade.Name != "repository" { - t.Fatalf("a read composition did not say which secret is not made, typed: %v", err) - } - // Foreseen, it composes, names what the push makes, and made nothing. - foreseen, err := declarationWith(ctx, open, "laptop", plan, settings, gens, Foreseeing) - if err != nil || len(foreseen.foreseen) != 1 || foreseen.foreseen[0] != "keeper/repository" { - t.Fatalf("foreseen: %v %v", foreseen.foreseen, err) - } - if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "keeper", "repository"); err != nil || held { - t.Fatalf("asking ahead of the push made the secret (held %v, %v)", held, err) - } - - // Within the bound: nothing at all — no urgent, no warning, nobody notified. - if got := d1(t, open); len(got) != 0 { - t.Fatalf("a machine waiting for a push raised %+v", got) - } - - // Past the bound: a warning, not urgent, saying what the push will make. - before := awaitingPushBound - awaitingPushBound = -time.Minute - t.Cleanup(func() { awaitingPushBound = before }) - got := d1(t, open) - if len(got) != 1 || got[0].Key() != "machine.laptop.awaiting-push" || got[0].Severity != conditions.Warning || - !strings.Contains(got[0].Summary, "keeper/repository") || !strings.Contains(got[0].Summary, "push laptop") { - t.Fatalf("a machine left un-pushed past the bound: %+v", got) - } - - // Pushed: the secret is made and D1 says nothing. - pushedBy(t, open, "laptop") - if got := d1(t, open); len(got) != 0 { - t.Fatalf("a pushed machine still raised %+v", got) - } -} // **A REAL FAILURE IS STILL URGENT**: a secret the push would be refused on is not one it will make. // A bus credential nobody issued (issue 203) fails the push, so D1 says it — urgent, in the push's words. -func TestASecretThePushCannotMakeIsStillUncomposable(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, aTalker()) - if _, err := assign(ctx, open, "laptop", "talker"); err != nil { - t.Fatal(err) - } - got := d1(t, open) - if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || got[0].Severity != conditions.Urgent || - !strings.Contains(got[0].Summary, "module issue talker --node laptop") { - t.Fatalf("a push that will be refused was not said urgently: %+v", got) - } - - // And a machine whose composition fails for anything else, with a secret waiting beside it, is - // uncomposable for that — the stand-in hides nothing. - register(t, open, aKeeper()) - if _, err := assign(ctx, open, "anchor", "keeper"); err != nil { - t.Fatal(err) - } - one, two := rivals() - register(t, open, one) - register(t, open, two) - _, _ = assign(ctx, open, "anchor", "rival-one") - _, _ = assign(ctx, open, "anchor", "rival-two") - keys := map[string]conditions.Severity{} - for _, o := range d1(t, open) { - keys[o.Key()] = o.Severity - } - if keys["machine.anchor.uncomposable"] != conditions.Urgent { - t.Fatalf("a real failure beside a waiting secret: %v", keys) - } -} // A given secret sealed to a key the machine no longer has is not the mesh's to make again: the push is // refused on it, so D1 is too. -func TestAGivenSecretUnderAnOldKeyIsNotForeseen(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, aKeeper()) - if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { - t.Fatal(err) - } - if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "keeper", "repository", "given"); err != nil { - t.Fatal(err) - } - record, err := open.inventory.NodeByName(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil { - t.Fatal(err) - } - got := d1(t, open) - if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "issue it again") { - t.Fatalf("a given secret under an old key: %+v", got) - } -} // The bound is read from when the machine began waiting: the oldest assignment since its last send. -func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, aKeeper()) - pushedBy(t, open, "laptop") - sent := time.Now() - since, err := open.inventory.AwaitingSince(ctx, "laptop") - if err != nil || since.After(sent) { - t.Fatalf("nothing assigned since the send: waiting since %v (%v), the send was before %v", since, err, sent) - } - if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { - t.Fatal(err) - } - since, err = open.inventory.AwaitingSince(ctx, "laptop") - if err != nil || since.Before(sent.Add(-time.Second)) { - t.Fatalf("assigned after the send: waiting since %v (%v), not from the assignment", since, err) - } -} // **D3 AND D13 WAIT FOR THE SEND**: a holder is expected to answer on a machine once the machine was sent // it and had time to report — never between assign and push. @@ -242,40 +113,3 @@ func TestAHolderIsExpectedOnlyOnceSentAndReported(t *testing.T) { } // And through the stores: assigned, not in the last send; pushed and reported, carried and settled. -func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, aKeeper()) - pushedBy(t, open, "laptop") - if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { - t.Fatal(err) - } - sends, err := readDeliveries(ctx, open.inventory) - if err != nil { - t.Fatal(err) - } - if sends["laptop"].settled("keeper", time.Now()) { - t.Fatal("a holder assigned and not pushed is expected to answer") - } - if !sends["laptop"].settled(overlay.Name, time.Now().Add(time.Hour)) { - t.Fatal("a module the machine was sent long ago is not expected to answer") - } - pushedBy(t, open, "laptop") - sends, err = readDeliveries(ctx, open.inventory) - if err != nil { - t.Fatal(err) - } - if sends["laptop"].settled("keeper", time.Now()) { - t.Fatal("pushed a moment ago and not reported, a holder is already expected") - } - if !sends["laptop"].settled("keeper", time.Now().Add(reportGrace+time.Minute)) { - t.Fatal("pushed past the grace, a holder is not expected") - } - if _, ok := sends["laptop"].carried["keeper"]; !ok { - t.Fatalf("the send's builds do not carry keeper: %v", sends["laptop"].carried) - } - pushedAndApplied(t, open, "laptop") - if sends, err = readDeliveries(ctx, open.inventory); err != nil || !sends["laptop"].settled("keeper", time.Now()) { - t.Fatalf("pushed and reported applied, a holder is not expected to answer (%v)", err) - } -} diff --git a/cmd/mesh-controller/batches_integration_test.go b/cmd/mesh-controller/batches_integration_test.go new file mode 100644 index 00000000..880e9940 --- /dev/null +++ b/cmd/mesh-controller/batches_integration_test.go @@ -0,0 +1,698 @@ +//go:build integration + +package main + +import ( + "bytes" + "io" + "os" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func windowed(t *testing.T) *stores { + t.Helper() + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + settings := t.TempDir() + "/merge-window.json" + if err := os.WriteFile(settings, []byte(`{"merge-window": "90s", "merge-window-at-most": "10m"}`), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv(mergeWindowFileVar, settings) + noGroups(t) + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, + inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0", + Head: "c0"}); err != nil { + t.Fatal(err) + } + for _, m := range []struct{ module, repository, path string }{ + {"app", "novox/mesh-catalog", "modules/app"}, + {"notes", "novox/mesh-catalog", "modules/notes"}, + {"one", "novox/one", ""}, + {"two", "novox/two", ""}, + {"three", "novox/three", ""}, + } { + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1"}, + inventory.Source{Repository: m.repository, Seat: "git", Path: m.path, Ref: "main", BuiltFrom: "c0", + Head: "c0"}); err != nil { + t.Fatal(err) + } + } + return open +} + +func hear(t *testing.T, open *stores, m link.SourceMoved, now time.Time) { + t.Helper() + if err := (following{open: open}).hearMerge(t.Context(), m, now); err != nil { + t.Fatal(err) + } +} + +func cutAt(t *testing.T, open *stores, now time.Time) { + t.Helper() + if err := cutBatchesHeld(t.Context(), open, now); err != nil { + t.Fatal(err) + } +} + +func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + asked := asksWithPaths(t) + claude := catalogueMerge("553b7191claude", "app", t0) + dunst := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second)) + hear(t, open, claude, t0.Add(time.Second)) + hear(t, open, dunst, t0.Add(18*time.Second)) + if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 { + t.Fatalf("within the window: %d walk(s), %d batch(es)", len(ws), len(bs)) + } + cutAt(t, open, t0.Add(18*time.Second+89*time.Second)) + if ws, _ := walks(t, open); len(ws) != 0 { + t.Fatalf("cut before the window closed: %+v", ws) + } + cutAt(t, open, t0.Add(18*time.Second+90*time.Second)) + ws, bs := walks(t, open) + if len(ws) != 1 || len(bs) != 0 { + t.Fatalf("after the window: %d walk(s), %d batch(es)", len(ws), len(bs)) + } + w := ws[0] + if w.Commit != dunst.Commit || len(w.Commits) != 1 || w.Commits[0].Commit != dunst.Commit { + t.Fatalf("the walk is at %s %+v, not the later commit", w.Commit, w.Commits) + } + for _, m := range []string{"app", "notes"} { + if _, in := w.Modules[m]; !in { + t.Fatalf("the walk does not build %s, which one of its merges moved: %v", m, w.Modules) + } + } + want := []inventory.PlanMerge{{Repository: "novox/mesh-catalog", Commit: claude.Commit, Carried: dunst.Commit}, + {Repository: "novox/mesh-catalog", Commit: dunst.Commit}} + if w.Delivery == nil || !slices.EqualFunc(w.Delivery.Merges, want, sameMerge) { + t.Fatalf("the walk answers %+v, want %+v", w.Delivery, want) + } + if len(*asked) != 2 || (*asked)[0][2] != "main" || (*asked)[1][2] != "main" { + t.Fatalf("the walk did not ask its modules at the branch, which holds the commit it carries: %v", *asked) + } + // Heard again, as the bus may hand it over twice: never a second merge, nor a second walk. + hear(t, open, claude, t0.Add(5*time.Minute)) + if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 { + t.Fatalf("a merge heard twice made %d walk(s) and %d batch(es)", len(ws), len(bs)) + } +} + +func TestThreeRepositoriesInOneWindowAreOneWalk(t *testing.T) { + open := windowed(t) + groupsWaiting(t, "feat/three") + asksRecorded(t) + for i, r := range []string{"one", "two", "three"} { + hear(t, open, fromHead(repoMerge(r, "c-"+r, t0.Add(time.Duration(i)*20*time.Second)), "feat/three"), + t0.Add(time.Duration(i)*20*time.Second+time.Second)) + } + cutAt(t, open, t0.Add(41*time.Second+90*time.Second)) + ws, bs := walks(t, open) + if len(ws) != 1 || len(bs) != 0 { + t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs)) + } + w := ws[0] + if len(w.Commits) != 3 || len(w.Delivery.Merges) != 3 { + t.Fatalf("the walk carries %+v and answers %+v", w.Commits, w.Delivery.Merges) + } + for _, r := range []string{"one", "two", "three"} { + if w.CommitOf("novox/"+r) != "c-"+r { + t.Fatalf("the walk carries %s at %q", r, w.CommitOf("novox/"+r)) + } + if _, in := w.Modules[r]; !in { + t.Fatalf("the walk does not build %s: %v", r, w.Modules) + } + } + // Each module's ask is recorded at its own repository's commit. + for _, r := range []string{"one", "two", "three"} { + q, found, err := open.inventory.BuildRequestByID(t.Context(), w.Modules[r].Build) + if err != nil || !found || q.Commit != "c-"+r { + t.Fatalf("%s's build was not recorded at its own commit: %+v %v %v", r, q, found, err) + } + } +} + +func TestAMergeRestartsTheWindowAndTheMaximumClosesIt(t *testing.T) { + open := windowed(t) + groupsWaiting(t, "feat/both", "feat/busy") + asksWithPaths(t) + hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0) + hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(80*time.Second)), "feat/both"), t0.Add(80*time.Second)) + cutAt(t, open, t0.Add(100*time.Second)) + if ws, _ := walks(t, open); len(ws) != 0 { + t.Fatal("the window closed 90 seconds after its first merge, not after its last") + } + cutAt(t, open, t0.Add(170*time.Second)) + if ws, _ := walks(t, open); len(ws) != 1 || len(ws[0].Commits) != 2 { + t.Fatalf("the window did not close 90 seconds after its last merge: %+v", ws) + } + + busy := windowed(t) + groupsWaiting(t, "feat/both", "feat/busy") + asksWithPaths(t) + var last time.Time + for i := 0; i < 12; i++ { + last = t0.Add(time.Duration(i) * time.Minute) + repo := []string{"one", "two", "three"}[i%3] + hear(t, busy, fromHead(repoMerge(repo, "c"+string(rune('a'+i)), last), "feat/busy"), last) + if ws, _ := walks(t, busy); len(ws) > 0 { + if i != 10 { + t.Fatalf("a busy window closed at merge %d (minute %d), not at its maximum", i, i) + } + break + } + } + ws, _ := walks(t, busy) + if len(ws) != 1 { + t.Fatalf("ten minutes of merges a minute apart were never cut: %d walk(s)", len(ws)) + } +} + +func TestTheAssemblingBatchIsShown(t *testing.T) { + open := windowed(t) + groupsWaiting(t, "pr/553b7191claude") + asksWithPaths(t) + var said []inventory.Plan + was := inventory.PlanSaved + inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { said = append(said, p) } + t.Cleanup(func() { inventory.PlanSaved = was }) + now := time.Now().UTC() + claude := catalogueMerge("553b7191claude", "app", now.Add(-20*time.Second)) + claude.Number, claude.Title = 174, "claude-code: an agent proposes a section" + dunst := catalogueMerge("48bda475dunst", "notes", now.Add(-2*time.Second)) + dunst.Number, dunst.Title = 175, "dunst: the font the operator chose" + hear(t, open, claude, now.Add(-19*time.Second)) + hear(t, open, dunst, now.Add(-time.Second)) + hear(t, open, fromHead(repoMerge("one", "a6bc0931one", now), dunst.Head), now) + out := captured(t, func() error { return plansCommand(t.Context(), nil) }) + lines := strings.Split(out, "\n") + first := lines[0] + for _, want := range []string{"assembling: ", " s left (at the latest ", "grouped: novox/mesh-catalog@48bda475 " + + "(answers 553b7191), novox/one@a6bc0931; plan not yet calculated"} { + if !strings.Contains(first, want) { + t.Fatalf("plans' first line %q does not say %q", first, want) + } + } + // Under it, one line per repository: the pull request, what it answers, what it moves. + if len(lines) < 3 || strings.TrimSpace(lines[1]) != "mesh-catalog #175 dunst: the font the operator chose (answers "+ + "#174 claude-code: an agent proposes a section) · app, notes" || + strings.TrimSpace(lines[2]) != "one a6bc0931 · one" { + t.Fatalf("the grouped list reads %q", lines[1:4]) + } + if len(said) == 0 || said[len(said)-1].State != inventory.PlanAssembling || said[len(said)-1].Delivery.Batch == nil || + len(said[len(said)-1].Delivery.Merges) != 3 { + t.Fatalf("the batch was not said as assembling with its merges: %+v", said) + } +} + +func captured(t *testing.T, run func() error) string { + t.Helper() + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + was := os.Stdout + os.Stdout = w + runErr := run() + os.Stdout = was + _ = w.Close() + var b bytes.Buffer + _, _ = io.Copy(&b, r) + if runErr != nil { + t.Fatal(runErr) + } + return b.String() +} + +func TestAMergeDuringAWalkJoinsTheNextBatch(t *testing.T) { + open := windowed(t) + asksWithPaths(t) + hear(t, open, repoMerge("one", "c1", t0), t0) + cutAt(t, open, t0.Add(90*time.Second)) + ws, _ := walks(t, open) + if len(ws) != 1 || !ws[0].Open() { + t.Fatalf("the first batch was not cut: %+v", ws) + } + first := ws[0] + hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute)) + cutAt(t, open, t0.Add(5*time.Minute)) + ws, bs := walks(t, open) + if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID { + t.Fatalf("the merge during the walk: %d walk(s), batches %+v", len(ws), bs) + } + if got, _ := open.inventory.PlanByID(t.Context(), first.ID); !got.Open() { + t.Fatalf("the started walk was %s by the next merge", got.State) + } + if !strings.HasPrefix(batchWords(bs[0], t0.Add(5*time.Minute)), "queued behind "+first.ID) { + t.Fatalf("a queued batch reads %q", batchWords(bs[0], t0.Add(5*time.Minute))) + } + first.State = inventory.PlanDone + if err := open.inventory.SavePlan(t.Context(), &first); err != nil { + t.Fatal(err) + } + cutAt(t, open, t0.Add(6*time.Minute)) + ws, bs = walks(t, open) + if len(ws) != 2 || len(bs) != 0 || ws[0].Commit != "c2" { + t.Fatalf("the queued batch was not cut when the walk ended: %+v %+v", ws, bs) + } +} + +func TestAWaitingWalkIsFoldedIntoTheNextBatch(t *testing.T) { + open := windowed(t) + asksWithPaths(t) + ctx := t.Context() + if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1", + Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main", + BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil { + t.Fatal(err) + } + hear(t, open, repoMerge("one", "c1", t0), t0) + cutAt(t, open, t0.Add(90*time.Second)) + ws, _ := walks(t, open) + if len(ws) != 1 || !ws[0].Waiting() { + t.Fatalf("the walk does not wait for its word: %+v", ws) + } + waiting := ws[0] + hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute)) + cutAt(t, open, t0.Add(2*time.Minute+90*time.Second)) + ws, bs := walks(t, open) + if len(bs) != 0 || len(ws) != 2 { + t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs)) + } + folded, err := open.inventory.PlanByID(ctx, waiting.ID) + if err != nil { + t.Fatal(err) + } + newer := ws[0] + if folded.State != inventory.PlanSuperseded || folded.Delivery.TakenOverBy != newer.ID { + t.Fatalf("the waiting walk is %s, taken over by %q", folded.State, folded.Delivery.TakenOverBy) + } + if len(newer.Delivery.Merges) != 2 || newer.CommitOf("novox/one") != "c2" { + t.Fatalf("the newer walk answers %+v", newer.Delivery.Merges) + } + if _, in := newer.Modules["one"]; !in { + t.Fatalf("what the folded walk was to build is not built: %v", newer.Modules) + } +} + +func TestALateMergeIsAnsweredByTheWalkOfTheLaterOne(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + asksWithPaths(t) + ctx := t.Context() + later := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second)) + hear(t, open, later, t0.Add(18*time.Second)) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + w := ws[0] + w.State = inventory.PlanDone + if err := open.inventory.SavePlan(ctx, &w); err != nil { + t.Fatal(err) + } + // Moves only notes, which the done walk built from the branch after it: answered there, at once. + hear(t, open, catalogueMerge("553b7191early", "notes", t0), t0.Add(15*time.Minute)) + got, _ := open.inventory.PlanByID(ctx, w.ID) + if got.State != inventory.PlanDone || len(got.Delivery.Merges) != 2 || + !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: "553b7191early", + Carried: later.Commit}) { + t.Fatalf("the late merge is not named by the walk that carried it: %+v", got.Delivery.Merges) + } + // Moves app, which that walk never built: the next batch walks it. + hear(t, open, catalogueMerge("1111aaaaapp", "app", t0.Add(time.Second)), t0.Add(16*time.Minute)) + _, bs := walks(t, open) + if len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "1111aaaaapp" { + t.Fatalf("a late merge moving what the walk never built did not join the next batch: %+v", bs) + } +} + +func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + asked := asksWithPaths(t) + ctx := t.Context() + // Made before the cut marks their modules seen, as merges are: walked again, they are news all the same. + oldest := catalogueMerge("aaaa0001", "app", t0.Add(-3*time.Minute)) + middle := catalogueMerge("bbbb0002", "app", t0.Add(-3*time.Minute+10*time.Second)) + newest := catalogueMerge("cccc0003", "notes", t0.Add(-3*time.Minute+20*time.Second)) + for i, m := range []link.SourceMoved{oldest, middle, newest} { + hear(t, open, m, t0.Add(time.Duration(i)*10*time.Second+time.Second)) + } + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + failed := ws[0] + failed.State, failed.Note = inventory.PlanFailed, "notes failed to build in tier 0" + if err := open.inventory.SavePlan(ctx, &failed); err != nil { + t.Fatal(err) + } + cutAt(t, open, t0.Add(3*time.Minute)) + got, _ := open.inventory.PlanByID(ctx, failed.ID) + if got.State != inventory.PlanFailed || len(got.Delivery.Merges) != 3 { + t.Fatalf("the failed walk's record changed: %s %+v", got.State, got.Delivery.Merges) + } + ws, _ = walks(t, open) + alone := ws[0] + if alone.ID == failed.ID || alone.Commit != middle.Commit || len(alone.Delivery.Merges) != 1 || + !sameMerge(alone.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: middle.Commit}) { + t.Fatalf("the newest earlier merge was not walked alone on its own commit: %+v", alone) + } + if _, in := alone.Modules["app"]; !in || (*asked)[len(*asked)-1] != [3]string{"novox/mesh-catalog", "modules/app", + middle.Commit} { + t.Fatalf("the merge walked alone does not build app at its own commit: %v, asked %v", alone.Modules, *asked) + } + // Retried, the failed walk would name merges the search answers now. + if _, err := retryPlan(ctx, open, failed.ID); err == nil || !strings.Contains(err.Error(), "walked alone") { + t.Fatalf("a failed walk whose merges are searched was retried: %v", err) + } + alone.State = inventory.PlanDone + if err := open.inventory.SavePlan(ctx, &alone); err != nil { + t.Fatal(err) + } + cutAt(t, open, t0.Add(4*time.Minute)) + ws, bs := walks(t, open) + if ws[0].ID != alone.ID || len(bs) != 0 { + t.Fatalf("the search went on after a merge was delivered: %+v", ws[0]) + } + done, _ := open.inventory.PlanByID(ctx, alone.ID) + if len(done.Delivery.Merges) != 2 || !sameMerge(done.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", + Commit: oldest.Commit, Carried: middle.Commit}) { + t.Fatalf("the oldest merge is not answered by the walk that delivered the one after it: %+v", done.Delivery.Merges) + } + // A stopped walk starts no search: a person ended it. + hear(t, open, catalogueMerge("dddd0004", "app", t0.Add(5*time.Minute)), t0.Add(5*time.Minute)) + hear(t, open, catalogueMerge("eeee0005", "notes", t0.Add(5*time.Minute+time.Second)), t0.Add(5*time.Minute+time.Second)) + cutAt(t, open, t0.Add(8*time.Minute)) + ws, _ = walks(t, open) + if _, err := stopWalk(ctx, open.inventory, ws[0].ID, "mesh-delivery for jochen", "not now"); err != nil { + t.Fatal(err) + } + cutAt(t, open, t0.Add(9*time.Minute)) + if again, _ := walks(t, open); again[0].ID != ws[0].ID { + t.Fatalf("a stopped walk's earlier merge was walked alone: %+v", again[0]) + } +} + +func TestAGroupsOrderBecomesTiersInsideTheWalk(t *testing.T) { + for _, grouped := range []bool{true, false} { + open := windowed(t) + everyHeadWaits(t) + asksWithPaths(t) + ctx := t.Context() + for _, m := range []string{"mesh-host"} { + if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, + inventory.Source{Repository: "novox/" + m, Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + } + host := repoMerge("mesh-host", "h1", t0) + ctl := repoMerge("mesh-controller", "k1", t0.Add(time.Second)) + host.Head, ctl.Head = "feat/together", "feat/together" + if !grouped { + ctl.Head = "feat/alone" + } + hear(t, open, ctl, t0.Add(2*time.Second)) + hear(t, open, host, t0.Add(3*time.Second)) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + if len(ws) != 1 { + t.Fatalf("%d walks", len(ws)) + } + tiers := ws[0].Tiers + if grouped { + if len(tiers) != 2 || !slices.Equal(tiers[0], []string{"mesh-host"}) || + !slices.Equal(tiers[1], []string{"mesh-controller"}) { + t.Fatalf("the group's order is not the walk's tiers: %v", tiers) + } + } else if len(tiers) != 1 { + t.Fatalf("two merges of no group were ordered: %v", tiers) + } + } +} + +func TestABatchSurvivesARestart(t *testing.T) { + open := windowed(t) + groupsWaiting(t, "feat/both") + asksWithPaths(t) + hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0) + hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(30*time.Second)), "feat/both"), t0.Add(30*time.Second)) + + // A new controller: nothing of the first one's but the store. + again, err := openStores(t.Context()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(again.Close) + hear(t, again, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0.Add(60*time.Second)) // the bus hands it over again + cutAt(t, again, t0.Add(119*time.Second)) + if ws, bs := walks(t, again); len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 { + t.Fatalf("the restarted controller's batch: %d walk(s), %+v", len(ws), bs) + } + cutAt(t, again, t0.Add(120*time.Second)) + ws, bs := walks(t, again) + if len(ws) != 1 || len(bs) != 0 || len(ws[0].Delivery.Merges) != 2 { + t.Fatalf("the restarted controller did not cut the batch at its window: %+v %+v", ws, bs) + } +} + +func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + asksWithPaths(t) + ctx := t.Context() + c2 := repoMerge("one", "c2", t0.Add(-50*time.Second)) + hear(t, open, c2, t0) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + w := ws[0] + w.State = inventory.PlanDone + if err := open.inventory.SavePlan(ctx, &w); err != nil { + t.Fatal(err) + } + hear(t, open, repoMerge("one", "c1", t0.Add(-60*time.Second)), t0.Add(15*time.Minute)) + got, _ := open.inventory.PlanByID(ctx, w.ID) + if len(got.Delivery.Merges) != 2 || !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/one", + Commit: "c1", Carried: "c2"}) { + t.Fatalf("the late merge was not named by the walk that carried it: %+v", got.Delivery.Merges) + } + + // A walk of two that built only two: a late merge of one is not its to name. + hear(t, open, repoMerge("two", "d2", t0.Add(16*time.Minute)), t0.Add(16*time.Minute)) + cutAt(t, open, t0.Add(18*time.Minute)) + ws, _ = walks(t, open) + w = ws[0] + delete(w.Modules, "two") + w.State = inventory.PlanDone + if err := open.inventory.SavePlan(ctx, &w); err != nil { + t.Fatal(err) + } + hear(t, open, repoMerge("two", "d1", t0.Add(15*time.Minute)), t0.Add(20*time.Minute)) + if got, _ := open.inventory.PlanByID(ctx, w.ID); len(got.Delivery.Merges) != 1 { + t.Fatalf("a walk that never built what the late merge moves named it: %+v", got.Delivery.Merges) + } + if _, bs := walks(t, open); len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "d1" { + t.Fatalf("the late merge did not join the next batch: %+v", bs) + } +} + +func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) { + open := windowed(t) + asked := asksWithPaths(t) + ctx := t.Context() + hear(t, open, repoMerge("one", "c1", t0), t0) + cutAt(t, open, t0.Add(2*time.Minute)) + started, _ := walks(t, open) + waiting := inventory.Plan{ID: "plan-waiting", Repository: "novox/one", Branch: "main", Commit: "d1", Created: t0, + State: inventory.PlanBuilding, Tiers: [][]string{{"one"}}, Modules: map[string]*inventory.PlanModule{"one": {}}, + Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}} + if err := open.inventory.SavePlan(ctx, &waiting); err != nil { + t.Fatal(err) + } + if _, err := letGo(ctx, open.inventory, waiting.ID, catalogue.DeliverySeat, "its turn"); err != nil { + t.Fatalf("the word was refused beside %s: %v", started[0].ID, err) + } + before := len(*asked) + advanceHeld(ctx, open) + got, _ := open.inventory.PlanByID(ctx, waiting.ID) + if len(*asked) != before || !strings.Contains(got.Note, "starts once "+started[0].ID) { + t.Fatalf("a walk let go beside a started one asked (%d → %d) or does not say it waits: %q", before, len(*asked), got.Note) + } + // Read as a wait, an hour on: its note on the line, never LATE, and no tier of it late for S3. + later := time.Now().Add(time.Hour) + if line := planLine(got, later); !strings.Contains(line, "starts once "+started[0].ID) || strings.Contains(line, "LATE") { + t.Fatalf("a deferred walk reads %q", line) + } + facts, _, err := gatherPlans(ctx, open.inventory, later, nil) + if err != nil { + t.Fatal(err) + } + for _, f := range facts { + if f.id == got.ID { + t.Fatalf("a deferred walk is watched as a tier running late: %+v", f) + } + } + done := started[0] + done.State = inventory.PlanDone + if err := open.inventory.SavePlan(ctx, &done); err != nil { + t.Fatal(err) + } + advanceHeld(ctx, open) + if len(*asked) != before+1 { + t.Fatalf("the walk did not start once the started one ended: asked %v", *asked) + } +} + +func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + asked := asksWithPaths(t) + ctx := t.Context() + if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1", + Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main", + BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil { + t.Fatal(err) + } + hear(t, open, catalogueMerge("aaaa0001", "app", t0), t0) + hear(t, open, repoMerge("mesh-controller", "k1", t0.Add(10*time.Second)), t0.Add(10*time.Second)) + hear(t, open, catalogueMerge("bbbb0002", "notes", t0.Add(20*time.Second)), t0.Add(20*time.Second)) + _, bs := walks(t, open) + var ownBatch, catalogueBatch inventory.Plan + for _, b := range bs { + if b.OwnPath() { + ownBatch = b + } else { + catalogueBatch = b + } + } + if len(bs) != 2 || ownBatch.ID == "" || catalogueBatch.ID == "" || len(catalogueBatch.Delivery.Merges) != 2 || + len(ownBatch.Delivery.Merges) != 1 { + t.Fatalf("a controller merge and two catalogue merges in one window: %+v", bs) + } + if !strings.Contains(batchWords(ownBatch, t0.Add(30*time.Second)), "own path") { + t.Fatalf("the own-path batch does not say so: %q", batchWords(ownBatch, t0.Add(30*time.Second))) + } + // Both windows closed, the controller's batch is cut first and starts; the catalogue batch queues behind it, + // is cut when the controller's walk ended, and waits for its word with both merges. + cutAt(t, open, t0.Add(2*time.Minute)) + ws, bs := walks(t, open) + if len(ws) != 1 || ws[0].Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k1" { + t.Fatalf("the controller's batch was not cut first into a started walk: %+v", ws) + } + ownWalk := ws[0] + for _, m := range []string{"app", "notes"} { + if _, in := ownWalk.Modules[m]; in { + t.Fatalf("the controller's walk builds %s, a catalogue module: it skipped its turn", m) + } + } + if len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != ownWalk.ID || bs[0].OwnPath() { + t.Fatalf("the catalogue batch does not queue behind the controller's walk: %+v", bs) + } + ownWalk.State = inventory.PlanDone + if err := open.inventory.SavePlan(ctx, &ownWalk); err != nil { + t.Fatal(err) + } + cutAt(t, open, t0.Add(3*time.Minute)) + ws, bs = walks(t, open) + if len(bs) != 0 || ws[0].ID != catalogueBatch.ID || !ws[0].Waiting() || len(ws[0].Delivery.Merges) != 2 { + t.Fatalf("the catalogue batch was not cut into a waiting walk once the controller's ended: %+v %+v", ws, bs) + } + for _, m := range []string{"app", "notes"} { + if _, in := ws[0].Modules[m]; !in { + t.Fatalf("the catalogue walk does not build %s: %v", m, ws[0].Modules) + } + } + for _, a := range *asked { + if a[1] == "modules/app" || a[1] == "modules/notes" { + t.Fatalf("a catalogue module was asked without the word: %v", *asked) + } + } + + // A catalogue walk waiting for its word when an own-path batch is cut keeps waiting beside the own-path + // walk, is not folded into it, and its word is taken meanwhile: it starts once the own-path walk ended. + catalogueWalk := ws[0] + hear(t, open, repoMerge("mesh-controller", "k2", t0.Add(4*time.Minute)), t0.Add(4*time.Minute)) + cutAt(t, open, t0.Add(6*time.Minute)) + ws, _ = walks(t, open) + kept, _ := open.inventory.PlanByID(ctx, catalogueWalk.ID) + if !kept.Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k2" || ws[0].Waiting() { + t.Fatalf("the waiting catalogue walk was not kept waiting beside the controller's walk: %s %+v", kept.State, ws) + } + if _, in := ws[0].Modules["app"]; in { + t.Fatalf("the controller's walk folded the waiting catalogue walk in: %v", ws[0].Modules) + } + if _, err := letGo(ctx, open.inventory, catalogueWalk.ID, catalogue.DeliverySeat, "its turn"); err != nil { + t.Fatal(err) + } + before := len(*asked) + advanceHeld(ctx, open) + if len(*asked) != before { + t.Fatalf("the catalogue walk started beside the controller's: asked %v", (*asked)[before:]) + } + own2 := ws[0] + own2.State = inventory.PlanDone + if err := open.inventory.SavePlan(ctx, &own2); err != nil { + t.Fatal(err) + } + advanceHeld(ctx, open) + if len(*asked) < before+1 || (*asked)[before][1] != "modules/app" { + t.Fatalf("the catalogue walk did not start once the controller's ended: %v", (*asked)[before:]) + } +} + +func TestALateCatalogueMergeIsNotAnsweredByAnOwnPathWalk(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + asksWithPaths(t) + ctx := t.Context() + for _, m := range []struct { + module string + claims []catalogue.Claim + }{ + {"nats", nil}, + {"mesh-delivery", []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, + } { + if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1", Claims: m.claims}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m.module, Ref: "main", + BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + } + if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil { + t.Fatal(err) + } + mixed := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "m1xed", MergedAt: t0.Format(time.RFC3339Nano), + Paths: []string{"modules/nats/module.json", "modules/app/module.json"}, ModuleDirs: []string{"modules/nats", "modules/app"}, + ModuleDirsSaid: true} + hear(t, open, mixed, t0.Add(time.Second)) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + if len(ws) != 1 || ws[0].Waiting() { + t.Fatalf("the mixed merge's walk waited, or was not cut: %+v", ws) + } + done := ws[0] + done.State = inventory.PlanDone + if err := open.inventory.SavePlan(ctx, &done); err != nil { + t.Fatal(err) + } + hear(t, open, catalogueMerge("ear1ier", "app", t0.Add(-30*time.Second)), t0.Add(3*time.Minute)) + got, _ := open.inventory.PlanByID(ctx, done.ID) + _, bs := walks(t, open) + if len(got.Delivery.Merges) != 1 || len(bs) != 1 || bs[0].OwnPath() || bs[0].Delivery.Merges[0].Commit != "ear1ier" { + t.Fatalf("the late catalogue merge was answered by the own-path walk (%+v) rather than the next catalogue batch (%+v)", + got.Delivery.Merges, bs) + } +} diff --git a/cmd/mesh-controller/batches_test.go b/cmd/mesh-controller/batches_test.go index 90a5ac8a..55c8ca94 100644 --- a/cmd/mesh-controller/batches_test.go +++ b/cmd/mesh-controller/batches_test.go @@ -1,11 +1,9 @@ package main import ( - "bytes" "context" "encoding/json" "errors" - "io" "os" "slices" "strings" @@ -33,37 +31,6 @@ func init() { // windowed is a mesh whose controller's settings give a merge window of 90 seconds and at most 10 minutes, as its // settings file says them, and // modules built from the catalogue (app, notes) and from three repositories of their own. -func windowed(t *testing.T) *stores { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - settings := t.TempDir() + "/merge-window.json" - if err := os.WriteFile(settings, []byte(`{"merge-window": "90s", "merge-window-at-most": "10m"}`), 0o600); err != nil { - t.Fatal(err) - } - t.Setenv(mergeWindowFileVar, settings) - noGroups(t) - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, - inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0", - Head: "c0"}); err != nil { - t.Fatal(err) - } - for _, m := range []struct{ module, repository, path string }{ - {"app", "novox/mesh-catalog", "modules/app"}, - {"notes", "novox/mesh-catalog", "modules/notes"}, - {"one", "novox/one", ""}, - {"two", "novox/two", ""}, - {"three", "novox/three", ""}, - } { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1"}, - inventory.Source{Repository: m.repository, Seat: "git", Path: m.path, Ref: "main", BuiltFrom: "c0", - Head: "c0"}); err != nil { - t.Fatal(err) - } - } - return open -} // t0 is the moment a test's clock starts: merges are dated by it, and heard after it. var t0 = time.Now().UTC().Truncate(time.Second) @@ -152,20 +119,8 @@ func lone(m link.SourceMoved) link.SourceMoved { } // hear is a merge heard at a moment. -func hear(t *testing.T, open *stores, m link.SourceMoved, now time.Time) { - t.Helper() - if err := (following{open: open}).hearMerge(t.Context(), m, now); err != nil { - t.Fatal(err) - } -} // cutAt is the cutter looking at a moment. -func cutAt(t *testing.T, open *stores, now time.Time) { - t.Helper() - if err := cutBatchesHeld(t.Context(), open, now); err != nil { - t.Fatal(err) - } -} // walks is every plan record that is a walk, newest first, and the batches not yet cut. func walks(t *testing.T, open *stores) (walks, batches []inventory.Plan) { @@ -186,426 +141,34 @@ func walks(t *testing.T, open *stores) (walks, batches []inventory.Plan) { // The replay of issue 362: two catalogue merges 18 seconds apart are one batch, cut once into one walk at the // later commit, which names both merges — the earlier carried by the later. -func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asked := asksWithPaths(t) - claude := catalogueMerge("553b7191claude", "app", t0) - dunst := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second)) - hear(t, open, claude, t0.Add(time.Second)) - hear(t, open, dunst, t0.Add(18*time.Second)) - if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 { - t.Fatalf("within the window: %d walk(s), %d batch(es)", len(ws), len(bs)) - } - cutAt(t, open, t0.Add(18*time.Second+89*time.Second)) - if ws, _ := walks(t, open); len(ws) != 0 { - t.Fatalf("cut before the window closed: %+v", ws) - } - cutAt(t, open, t0.Add(18*time.Second+90*time.Second)) - ws, bs := walks(t, open) - if len(ws) != 1 || len(bs) != 0 { - t.Fatalf("after the window: %d walk(s), %d batch(es)", len(ws), len(bs)) - } - w := ws[0] - if w.Commit != dunst.Commit || len(w.Commits) != 1 || w.Commits[0].Commit != dunst.Commit { - t.Fatalf("the walk is at %s %+v, not the later commit", w.Commit, w.Commits) - } - for _, m := range []string{"app", "notes"} { - if _, in := w.Modules[m]; !in { - t.Fatalf("the walk does not build %s, which one of its merges moved: %v", m, w.Modules) - } - } - want := []inventory.PlanMerge{{Repository: "novox/mesh-catalog", Commit: claude.Commit, Carried: dunst.Commit}, - {Repository: "novox/mesh-catalog", Commit: dunst.Commit}} - if w.Delivery == nil || !slices.EqualFunc(w.Delivery.Merges, want, sameMerge) { - t.Fatalf("the walk answers %+v, want %+v", w.Delivery, want) - } - if len(*asked) != 2 || (*asked)[0][2] != "main" || (*asked)[1][2] != "main" { - t.Fatalf("the walk did not ask its modules at the branch, which holds the commit it carries: %v", *asked) - } - // Heard again, as the bus may hand it over twice: never a second merge, nor a second walk. - hear(t, open, claude, t0.Add(5*time.Minute)) - if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 { - t.Fatalf("a merge heard twice made %d walk(s) and %d batch(es)", len(ws), len(bs)) - } -} // Merges of three repositories in one window are one walk, one commit for each. -func TestThreeRepositoriesInOneWindowAreOneWalk(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "feat/three") - asksRecorded(t) - for i, r := range []string{"one", "two", "three"} { - hear(t, open, fromHead(repoMerge(r, "c-"+r, t0.Add(time.Duration(i)*20*time.Second)), "feat/three"), - t0.Add(time.Duration(i)*20*time.Second+time.Second)) - } - cutAt(t, open, t0.Add(41*time.Second+90*time.Second)) - ws, bs := walks(t, open) - if len(ws) != 1 || len(bs) != 0 { - t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs)) - } - w := ws[0] - if len(w.Commits) != 3 || len(w.Delivery.Merges) != 3 { - t.Fatalf("the walk carries %+v and answers %+v", w.Commits, w.Delivery.Merges) - } - for _, r := range []string{"one", "two", "three"} { - if w.CommitOf("novox/"+r) != "c-"+r { - t.Fatalf("the walk carries %s at %q", r, w.CommitOf("novox/"+r)) - } - if _, in := w.Modules[r]; !in { - t.Fatalf("the walk does not build %s: %v", r, w.Modules) - } - } - // Each module's ask is recorded at its own repository's commit. - for _, r := range []string{"one", "two", "three"} { - q, found, err := open.inventory.BuildRequestByID(t.Context(), w.Modules[r].Build) - if err != nil || !found || q.Commit != "c-"+r { - t.Fatalf("%s's build was not recorded at its own commit: %+v %v %v", r, q, found, err) - } - } -} // Each merge restarts the window: a merge at second 80 keeps the batch open until second 170; the window // closes at most ten minutes after its first merge however busy. -func TestAMergeRestartsTheWindowAndTheMaximumClosesIt(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "feat/both", "feat/busy") - asksWithPaths(t) - hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0) - hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(80*time.Second)), "feat/both"), t0.Add(80*time.Second)) - cutAt(t, open, t0.Add(100*time.Second)) - if ws, _ := walks(t, open); len(ws) != 0 { - t.Fatal("the window closed 90 seconds after its first merge, not after its last") - } - cutAt(t, open, t0.Add(170*time.Second)) - if ws, _ := walks(t, open); len(ws) != 1 || len(ws[0].Commits) != 2 { - t.Fatalf("the window did not close 90 seconds after its last merge: %+v", ws) - } - - busy := windowed(t) - groupsWaiting(t, "feat/both", "feat/busy") - asksWithPaths(t) - var last time.Time - for i := 0; i < 12; i++ { - last = t0.Add(time.Duration(i) * time.Minute) - repo := []string{"one", "two", "three"}[i%3] - hear(t, busy, fromHead(repoMerge(repo, "c"+string(rune('a'+i)), last), "feat/busy"), last) - if ws, _ := walks(t, busy); len(ws) > 0 { - if i != 10 { - t.Fatalf("a busy window closed at merge %d (minute %d), not at its maximum", i, i) - } - break - } - } - ws, _ := walks(t, busy) - if len(ws) != 1 { - t.Fatalf("ten minutes of merges a minute apart were never cut: %d walk(s)", len(ws)) - } -} // `plans` lists the batch being assembled first, in the operator's words: how long is left, when at the latest, // what is grouped, and that the plan is not yet calculated. It says so on the bus as plan-moved, too. -func TestTheAssemblingBatchIsShown(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/553b7191claude") - asksWithPaths(t) - var said []inventory.Plan - was := inventory.PlanSaved - inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { said = append(said, p) } - t.Cleanup(func() { inventory.PlanSaved = was }) - now := time.Now().UTC() - claude := catalogueMerge("553b7191claude", "app", now.Add(-20*time.Second)) - claude.Number, claude.Title = 174, "claude-code: an agent proposes a section" - dunst := catalogueMerge("48bda475dunst", "notes", now.Add(-2*time.Second)) - dunst.Number, dunst.Title = 175, "dunst: the font the operator chose" - hear(t, open, claude, now.Add(-19*time.Second)) - hear(t, open, dunst, now.Add(-time.Second)) - hear(t, open, fromHead(repoMerge("one", "a6bc0931one", now), dunst.Head), now) - out := captured(t, func() error { return plansCommand(t.Context(), nil) }) - lines := strings.Split(out, "\n") - first := lines[0] - for _, want := range []string{"assembling: ", " s left (at the latest ", "grouped: novox/mesh-catalog@48bda475 " + - "(answers 553b7191), novox/one@a6bc0931; plan not yet calculated"} { - if !strings.Contains(first, want) { - t.Fatalf("plans' first line %q does not say %q", first, want) - } - } - // Under it, one line per repository: the pull request, what it answers, what it moves. - if len(lines) < 3 || strings.TrimSpace(lines[1]) != "mesh-catalog #175 dunst: the font the operator chose (answers "+ - "#174 claude-code: an agent proposes a section) · app, notes" || - strings.TrimSpace(lines[2]) != "one a6bc0931 · one" { - t.Fatalf("the grouped list reads %q", lines[1:4]) - } - if len(said) == 0 || said[len(said)-1].State != inventory.PlanAssembling || said[len(said)-1].Delivery.Batch == nil || - len(said[len(said)-1].Delivery.Merges) != 3 { - t.Fatalf("the batch was not said as assembling with its merges: %+v", said) - } -} // captured is what a command printed. -func captured(t *testing.T, run func() error) string { - t.Helper() - r, w, err := os.Pipe() - if err != nil { - t.Fatal(err) - } - was := os.Stdout - os.Stdout = w - runErr := run() - os.Stdout = was - _ = w.Close() - var b bytes.Buffer - _, _ = io.Copy(&b, r) - if runErr != nil { - t.Fatal(runErr) - } - return b.String() -} // A merge heard during a walk it is related to (the same repository, issue 973) joins the next batch, which is // cut when the walk ends; the walk that started is never superseded (ADR 0276 decision 4, ADR 0407 decision 4). -func TestAMergeDuringAWalkJoinsTheNextBatch(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - hear(t, open, repoMerge("one", "c1", t0), t0) - cutAt(t, open, t0.Add(90*time.Second)) - ws, _ := walks(t, open) - if len(ws) != 1 || !ws[0].Open() { - t.Fatalf("the first batch was not cut: %+v", ws) - } - first := ws[0] - hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute)) - cutAt(t, open, t0.Add(5*time.Minute)) - ws, bs := walks(t, open) - if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID { - t.Fatalf("the merge during the walk: %d walk(s), batches %+v", len(ws), bs) - } - if got, _ := open.inventory.PlanByID(t.Context(), first.ID); !got.Open() { - t.Fatalf("the started walk was %s by the next merge", got.State) - } - if !strings.HasPrefix(batchWords(bs[0], t0.Add(5*time.Minute)), "queued behind "+first.ID) { - t.Fatalf("a queued batch reads %q", batchWords(bs[0], t0.Add(5*time.Minute))) - } - first.State = inventory.PlanDone - if err := open.inventory.SavePlan(t.Context(), &first); err != nil { - t.Fatal(err) - } - cutAt(t, open, t0.Add(6*time.Minute)) - ws, bs = walks(t, open) - if len(ws) != 2 || len(bs) != 0 || ws[0].Commit != "c2" { - t.Fatalf("the queued batch was not cut when the walk ended: %+v %+v", ws, bs) - } -} // A walk waiting for its delivery's word is folded into the next batch's walk, which answers its merges; it // names the walk that took it over. -func TestAWaitingWalkIsFoldedIntoTheNextBatch(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - ctx := t.Context() - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1", - Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } - hear(t, open, repoMerge("one", "c1", t0), t0) - cutAt(t, open, t0.Add(90*time.Second)) - ws, _ := walks(t, open) - if len(ws) != 1 || !ws[0].Waiting() { - t.Fatalf("the walk does not wait for its word: %+v", ws) - } - waiting := ws[0] - hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute)) - cutAt(t, open, t0.Add(2*time.Minute+90*time.Second)) - ws, bs := walks(t, open) - if len(bs) != 0 || len(ws) != 2 { - t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs)) - } - folded, err := open.inventory.PlanByID(ctx, waiting.ID) - if err != nil { - t.Fatal(err) - } - newer := ws[0] - if folded.State != inventory.PlanSuperseded || folded.Delivery.TakenOverBy != newer.ID { - t.Fatalf("the waiting walk is %s, taken over by %q", folded.State, folded.Delivery.TakenOverBy) - } - if len(newer.Delivery.Merges) != 2 || newer.CommitOf("novox/one") != "c2" { - t.Fatalf("the newer walk answers %+v", newer.Delivery.Merges) - } - if _, in := newer.Modules["one"]; !in { - t.Fatalf("what the folded walk was to build is not built: %v", newer.Modules) - } -} // A merge heard after a later merge of its branch was walked is answered by that walk when it builds all it // moves — named at once on a walk done — and joins the next batch when it does not. -func TestALateMergeIsAnsweredByTheWalkOfTheLaterOne(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - ctx := t.Context() - later := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second)) - hear(t, open, later, t0.Add(18*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - w := ws[0] - w.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &w); err != nil { - t.Fatal(err) - } - // Moves only notes, which the done walk built from the branch after it: answered there, at once. - hear(t, open, catalogueMerge("553b7191early", "notes", t0), t0.Add(15*time.Minute)) - got, _ := open.inventory.PlanByID(ctx, w.ID) - if got.State != inventory.PlanDone || len(got.Delivery.Merges) != 2 || - !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: "553b7191early", - Carried: later.Commit}) { - t.Fatalf("the late merge is not named by the walk that carried it: %+v", got.Delivery.Merges) - } - // Moves app, which that walk never built: the next batch walks it. - hear(t, open, catalogueMerge("1111aaaaapp", "app", t0.Add(time.Second)), t0.Add(16*time.Minute)) - _, bs := walks(t, open) - if len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "1111aaaaapp" { - t.Fatalf("a late merge moving what the walk never built did not join the next batch: %+v", bs) - } -} // A failed walk marks nothing delivered: its earlier merges are walked alone, on their own commit, the newest // first; the first delivered answers the older ones, and the search stops. -func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asked := asksWithPaths(t) - ctx := t.Context() - // Made before the cut marks their modules seen, as merges are: walked again, they are news all the same. - oldest := catalogueMerge("aaaa0001", "app", t0.Add(-3*time.Minute)) - middle := catalogueMerge("bbbb0002", "app", t0.Add(-3*time.Minute+10*time.Second)) - newest := catalogueMerge("cccc0003", "notes", t0.Add(-3*time.Minute+20*time.Second)) - for i, m := range []link.SourceMoved{oldest, middle, newest} { - hear(t, open, m, t0.Add(time.Duration(i)*10*time.Second+time.Second)) - } - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - failed := ws[0] - failed.State, failed.Note = inventory.PlanFailed, "notes failed to build in tier 0" - if err := open.inventory.SavePlan(ctx, &failed); err != nil { - t.Fatal(err) - } - cutAt(t, open, t0.Add(3*time.Minute)) - got, _ := open.inventory.PlanByID(ctx, failed.ID) - if got.State != inventory.PlanFailed || len(got.Delivery.Merges) != 3 { - t.Fatalf("the failed walk's record changed: %s %+v", got.State, got.Delivery.Merges) - } - ws, _ = walks(t, open) - alone := ws[0] - if alone.ID == failed.ID || alone.Commit != middle.Commit || len(alone.Delivery.Merges) != 1 || - !sameMerge(alone.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: middle.Commit}) { - t.Fatalf("the newest earlier merge was not walked alone on its own commit: %+v", alone) - } - if _, in := alone.Modules["app"]; !in || (*asked)[len(*asked)-1] != [3]string{"novox/mesh-catalog", "modules/app", - middle.Commit} { - t.Fatalf("the merge walked alone does not build app at its own commit: %v, asked %v", alone.Modules, *asked) - } - // Retried, the failed walk would name merges the search answers now. - if _, err := retryPlan(ctx, open, failed.ID); err == nil || !strings.Contains(err.Error(), "walked alone") { - t.Fatalf("a failed walk whose merges are searched was retried: %v", err) - } - alone.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &alone); err != nil { - t.Fatal(err) - } - cutAt(t, open, t0.Add(4*time.Minute)) - ws, bs := walks(t, open) - if ws[0].ID != alone.ID || len(bs) != 0 { - t.Fatalf("the search went on after a merge was delivered: %+v", ws[0]) - } - done, _ := open.inventory.PlanByID(ctx, alone.ID) - if len(done.Delivery.Merges) != 2 || !sameMerge(done.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", - Commit: oldest.Commit, Carried: middle.Commit}) { - t.Fatalf("the oldest merge is not answered by the walk that delivered the one after it: %+v", done.Delivery.Merges) - } - // A stopped walk starts no search: a person ended it. - hear(t, open, catalogueMerge("dddd0004", "app", t0.Add(5*time.Minute)), t0.Add(5*time.Minute)) - hear(t, open, catalogueMerge("eeee0005", "notes", t0.Add(5*time.Minute+time.Second)), t0.Add(5*time.Minute+time.Second)) - cutAt(t, open, t0.Add(8*time.Minute)) - ws, _ = walks(t, open) - if _, err := stopWalk(ctx, open.inventory, ws[0].ID, "mesh-delivery for jochen", "not now"); err != nil { - t.Fatal(err) - } - cutAt(t, open, t0.Add(9*time.Minute)) - if again, _ := walks(t, open); again[0].ID != ws[0].ID { - t.Fatalf("a stopped walk's earlier merge was walked alone: %+v", again[0]) - } -} // A delivery group's order holds inside the walk (ADR 0249 unchanged): merges of two repositories from one head // branch name, the node-engine's before the controller's, are tiered so; without the group, one tier. -func TestAGroupsOrderBecomesTiersInsideTheWalk(t *testing.T) { - for _, grouped := range []bool{true, false} { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - ctx := t.Context() - for _, m := range []string{"mesh-host"} { - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, - inventory.Source{Repository: "novox/" + m, Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - } - host := repoMerge("mesh-host", "h1", t0) - ctl := repoMerge("mesh-controller", "k1", t0.Add(time.Second)) - host.Head, ctl.Head = "feat/together", "feat/together" - if !grouped { - ctl.Head = "feat/alone" - } - hear(t, open, ctl, t0.Add(2*time.Second)) - hear(t, open, host, t0.Add(3*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - if len(ws) != 1 { - t.Fatalf("%d walks", len(ws)) - } - tiers := ws[0].Tiers - if grouped { - if len(tiers) != 2 || !slices.Equal(tiers[0], []string{"mesh-host"}) || - !slices.Equal(tiers[1], []string{"mesh-controller"}) { - t.Fatalf("the group's order is not the walk's tiers: %v", tiers) - } - } else if len(tiers) != 1 { - t.Fatalf("two merges of no group were ordered: %v", tiers) - } - } -} // A restarted controller resumes the window where it stood: the batch, its merges and their times are in the // store, a merge handed over again is not doubled, and the batch is cut when its window closes. -func TestABatchSurvivesARestart(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "feat/both") - asksWithPaths(t) - hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0) - hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(30*time.Second)), "feat/both"), t0.Add(30*time.Second)) - - // A new controller: nothing of the first one's but the store. - again, err := openStores(t.Context()) - if err != nil { - t.Fatal(err) - } - t.Cleanup(again.Close) - hear(t, again, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0.Add(60*time.Second)) // the bus hands it over again - cutAt(t, again, t0.Add(119*time.Second)) - if ws, bs := walks(t, again); len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 { - t.Fatalf("the restarted controller's batch: %d walk(s), %+v", len(ws), bs) - } - cutAt(t, again, t0.Add(120*time.Second)) - ws, bs := walks(t, again) - if len(ws) != 1 || len(bs) != 0 || len(ws[0].Delivery.Merges) != 2 { - t.Fatalf("the restarted controller did not cut the batch at its window: %+v %+v", ws, bs) - } -} // The window's settings reach the controller in its settings file, read at every look; one that says no // duration, or no file, is the default. @@ -650,94 +213,9 @@ func TestAWaitingWalkNamesTheMergesItAnswers(t *testing.T) { // A merge heard after a later merge of its branch was cut is named by that walk however its modules read since // the cut, in a repository of one module (review of this change); a walk that never built what it moves names // nothing, and the merge joins the next batch. -func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - ctx := t.Context() - c2 := repoMerge("one", "c2", t0.Add(-50*time.Second)) - hear(t, open, c2, t0) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - w := ws[0] - w.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &w); err != nil { - t.Fatal(err) - } - hear(t, open, repoMerge("one", "c1", t0.Add(-60*time.Second)), t0.Add(15*time.Minute)) - got, _ := open.inventory.PlanByID(ctx, w.ID) - if len(got.Delivery.Merges) != 2 || !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/one", - Commit: "c1", Carried: "c2"}) { - t.Fatalf("the late merge was not named by the walk that carried it: %+v", got.Delivery.Merges) - } - - // A walk of two that built only two: a late merge of one is not its to name. - hear(t, open, repoMerge("two", "d2", t0.Add(16*time.Minute)), t0.Add(16*time.Minute)) - cutAt(t, open, t0.Add(18*time.Minute)) - ws, _ = walks(t, open) - w = ws[0] - delete(w.Modules, "two") - w.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &w); err != nil { - t.Fatal(err) - } - hear(t, open, repoMerge("two", "d1", t0.Add(15*time.Minute)), t0.Add(20*time.Minute)) - if got, _ := open.inventory.PlanByID(ctx, w.ID); len(got.Delivery.Merges) != 1 { - t.Fatalf("a walk that never built what the late merge moves named it: %+v", got.Delivery.Merges) - } - if _, bs := walks(t, open); len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "d1" { - t.Fatalf("the late merge did not join the next batch: %+v", bs) - } -} // A related walk waits for the delivery's word too: a waiting walk let go beside a started one it is related to // (the same repository, issue 973) takes the word and starts once that one ended, asking nothing before. -func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) { - open := windowed(t) - asked := asksWithPaths(t) - ctx := t.Context() - hear(t, open, repoMerge("one", "c1", t0), t0) - cutAt(t, open, t0.Add(2*time.Minute)) - started, _ := walks(t, open) - waiting := inventory.Plan{ID: "plan-waiting", Repository: "novox/one", Branch: "main", Commit: "d1", Created: t0, - State: inventory.PlanBuilding, Tiers: [][]string{{"one"}}, Modules: map[string]*inventory.PlanModule{"one": {}}, - Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}} - if err := open.inventory.SavePlan(ctx, &waiting); err != nil { - t.Fatal(err) - } - if _, err := letGo(ctx, open.inventory, waiting.ID, catalogue.DeliverySeat, "its turn"); err != nil { - t.Fatalf("the word was refused beside %s: %v", started[0].ID, err) - } - before := len(*asked) - advanceHeld(ctx, open) - got, _ := open.inventory.PlanByID(ctx, waiting.ID) - if len(*asked) != before || !strings.Contains(got.Note, "starts once "+started[0].ID) { - t.Fatalf("a walk let go beside a started one asked (%d → %d) or does not say it waits: %q", before, len(*asked), got.Note) - } - // Read as a wait, an hour on: its note on the line, never LATE, and no tier of it late for S3. - later := time.Now().Add(time.Hour) - if line := planLine(got, later); !strings.Contains(line, "starts once "+started[0].ID) || strings.Contains(line, "LATE") { - t.Fatalf("a deferred walk reads %q", line) - } - facts, _, err := gatherPlans(ctx, open.inventory, later, nil) - if err != nil { - t.Fatal(err) - } - for _, f := range facts { - if f.id == got.ID { - t.Fatalf("a deferred walk is watched as a tier running late: %+v", f) - } - } - done := started[0] - done.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &done); err != nil { - t.Fatal(err) - } - advanceHeld(ctx, open) - if len(*asked) != before+1 { - t.Fatalf("the walk did not start once the started one ended: asked %v", *asked) - } -} // A file a merge of the batch removed and a later one brought back is not removed; one removed last is. func TestARemovedFileIsTheLastMergesWord(t *testing.T) { @@ -761,106 +239,6 @@ func TestARemovedFileIsTheLastMergesWord(t *testing.T) { // (decided during the build, 2026-10-10): the two are batches of their own, cut one after the other; a walk on // the own path folds no waiting catalogue walk but batches its merges again behind it; the catalogue's walk // still waits for the word, so no catalogue delivery skips its turn. -func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asked := asksWithPaths(t) - ctx := t.Context() - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1", - Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } - hear(t, open, catalogueMerge("aaaa0001", "app", t0), t0) - hear(t, open, repoMerge("mesh-controller", "k1", t0.Add(10*time.Second)), t0.Add(10*time.Second)) - hear(t, open, catalogueMerge("bbbb0002", "notes", t0.Add(20*time.Second)), t0.Add(20*time.Second)) - _, bs := walks(t, open) - var ownBatch, catalogueBatch inventory.Plan - for _, b := range bs { - if b.OwnPath() { - ownBatch = b - } else { - catalogueBatch = b - } - } - if len(bs) != 2 || ownBatch.ID == "" || catalogueBatch.ID == "" || len(catalogueBatch.Delivery.Merges) != 2 || - len(ownBatch.Delivery.Merges) != 1 { - t.Fatalf("a controller merge and two catalogue merges in one window: %+v", bs) - } - if !strings.Contains(batchWords(ownBatch, t0.Add(30*time.Second)), "own path") { - t.Fatalf("the own-path batch does not say so: %q", batchWords(ownBatch, t0.Add(30*time.Second))) - } - // Both windows closed, the controller's batch is cut first and starts; the catalogue batch queues behind it, - // is cut when the controller's walk ended, and waits for its word with both merges. - cutAt(t, open, t0.Add(2*time.Minute)) - ws, bs := walks(t, open) - if len(ws) != 1 || ws[0].Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k1" { - t.Fatalf("the controller's batch was not cut first into a started walk: %+v", ws) - } - ownWalk := ws[0] - for _, m := range []string{"app", "notes"} { - if _, in := ownWalk.Modules[m]; in { - t.Fatalf("the controller's walk builds %s, a catalogue module: it skipped its turn", m) - } - } - if len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != ownWalk.ID || bs[0].OwnPath() { - t.Fatalf("the catalogue batch does not queue behind the controller's walk: %+v", bs) - } - ownWalk.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &ownWalk); err != nil { - t.Fatal(err) - } - cutAt(t, open, t0.Add(3*time.Minute)) - ws, bs = walks(t, open) - if len(bs) != 0 || ws[0].ID != catalogueBatch.ID || !ws[0].Waiting() || len(ws[0].Delivery.Merges) != 2 { - t.Fatalf("the catalogue batch was not cut into a waiting walk once the controller's ended: %+v %+v", ws, bs) - } - for _, m := range []string{"app", "notes"} { - if _, in := ws[0].Modules[m]; !in { - t.Fatalf("the catalogue walk does not build %s: %v", m, ws[0].Modules) - } - } - for _, a := range *asked { - if a[1] == "modules/app" || a[1] == "modules/notes" { - t.Fatalf("a catalogue module was asked without the word: %v", *asked) - } - } - - // A catalogue walk waiting for its word when an own-path batch is cut keeps waiting beside the own-path - // walk, is not folded into it, and its word is taken meanwhile: it starts once the own-path walk ended. - catalogueWalk := ws[0] - hear(t, open, repoMerge("mesh-controller", "k2", t0.Add(4*time.Minute)), t0.Add(4*time.Minute)) - cutAt(t, open, t0.Add(6*time.Minute)) - ws, _ = walks(t, open) - kept, _ := open.inventory.PlanByID(ctx, catalogueWalk.ID) - if !kept.Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k2" || ws[0].Waiting() { - t.Fatalf("the waiting catalogue walk was not kept waiting beside the controller's walk: %s %+v", kept.State, ws) - } - if _, in := ws[0].Modules["app"]; in { - t.Fatalf("the controller's walk folded the waiting catalogue walk in: %v", ws[0].Modules) - } - if _, err := letGo(ctx, open.inventory, catalogueWalk.ID, catalogue.DeliverySeat, "its turn"); err != nil { - t.Fatal(err) - } - before := len(*asked) - advanceHeld(ctx, open) - if len(*asked) != before { - t.Fatalf("the catalogue walk started beside the controller's: asked %v", (*asked)[before:]) - } - own2 := ws[0] - own2.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &own2); err != nil { - t.Fatal(err) - } - advanceHeld(ctx, open) - if len(*asked) < before+1 || (*asked)[before][1] != "modules/app" { - t.Fatalf("the catalogue walk did not start once the controller's ended: %v", (*asked)[before:]) - } -} // `plans` names a walk by what it moves (asked by the operator, 2026-10-10): the repository's pull request and // title, the modules it moves and the machines running them, then the state words; a record naming no pull @@ -906,46 +284,3 @@ func sameMerge(a, b inventory.PlanMerge) bool { // A catalogue merge heard after a later merge of its branch was walked without the word (a merge that touched // the bus too, on the controller's own path) is not answered by that walk while the delivery seat has a holder: // its delivery would skip its turn. It joins the next catalogue batch. -func TestALateCatalogueMergeIsNotAnsweredByAnOwnPathWalk(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - ctx := t.Context() - for _, m := range []struct { - module string - claims []catalogue.Claim - }{ - {"nats", nil}, - {"mesh-delivery", []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, - } { - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1", Claims: m.claims}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m.module, Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - } - if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } - mixed := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "m1xed", MergedAt: t0.Format(time.RFC3339Nano), - Paths: []string{"modules/nats/module.json", "modules/app/module.json"}, ModuleDirs: []string{"modules/nats", "modules/app"}, - ModuleDirsSaid: true} - hear(t, open, mixed, t0.Add(time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - if len(ws) != 1 || ws[0].Waiting() { - t.Fatalf("the mixed merge's walk waited, or was not cut: %+v", ws) - } - done := ws[0] - done.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &done); err != nil { - t.Fatal(err) - } - hear(t, open, catalogueMerge("ear1ier", "app", t0.Add(-30*time.Second)), t0.Add(3*time.Minute)) - got, _ := open.inventory.PlanByID(ctx, done.ID) - _, bs := walks(t, open) - if len(got.Delivery.Merges) != 1 || len(bs) != 1 || bs[0].OwnPath() || bs[0].Delivery.Merges[0].Commit != "ear1ier" { - t.Fatalf("the late catalogue merge was answered by the own-path walk (%+v) rather than the next catalogue batch (%+v)", - got.Delivery.Merges, bs) - } -} diff --git a/cmd/mesh-controller/beside_test.go b/cmd/mesh-controller/beside_test.go index a4c0237d..0d417830 100644 --- a/cmd/mesh-controller/beside_test.go +++ b/cmd/mesh-controller/beside_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/bindings_integration_test.go b/cmd/mesh-controller/bindings_integration_test.go new file mode 100644 index 00000000..580c1fde --- /dev/null +++ b/cmd/mesh-controller/bindings_integration_test.go @@ -0,0 +1,102 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" +) + +func TestTheIncidentAConsumerStaysBesideItsDataUntilAPersonPinsIt(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { + t.Fatal(err) + } + for _, m := range storeManifests() { + register(t, open, m) + } + assignAll := func(pairs ...[2]string) { + for _, a := range pairs { + if _, err := assign(ctx, open, a[0], a[1]); err != nil { + t.Fatalf("assign %s %s: %v", a[0], a[1], err) + } + } + } + // The anchor's store and resolver hold the mesh's seats, on record; the laptop runs its own of each. + assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"}) + for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} { + if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil { + t.Fatal(err) + } + } + assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"}, + [2]string{"laptop", "board"}) + + plan, _, err := planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" || n.Module != "store" || !n.KeepsData { + t.Fatalf("the consumer was bound to %s/%s (keeps data: %v); its data is beside it", n.From, n.Module, n.KeepsData) + } + if n := need(t, plan, "network", "wildcard-resolution"); n.From != "anchor" || n.KeepsData { + t.Fatalf("the resolver was bound to %s (keeps data: %v); its seat is held on anchor (issue 258)", n.From, n.KeepsData) + } + + // Sent, and recorded: only the binding to data. + bindings := boundToData(plan, nil) + if len(bindings) != 1 || bindings[0].Provider != (catalogue.Chosen{Node: "laptop", Module: "store"}) { + t.Fatalf("recorded %+v", bindings) + } + if err := inv.RecordBindings(ctx, "laptop", bindings); err != nil { + t.Fatal(err) + } + if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 { + t.Fatalf("a mesh bound where it was sent: %+v, %v", found, err) + } + + // The laptop's store taken away: refused, not moved to the anchor's empty one. + if err := inv.Unassign(ctx, "laptop", "store"); err != nil { + t.Fatal(err) + } + if _, _, err := planFor(ctx, open, "laptop"); err == nil || !unresolvable(err) || + !strings.Contains(err.Error(), "board on laptop is bound to laptop/store") || + !strings.Contains(err.Error(), "pin laptop postgres-database anchor store") { + t.Fatalf("the consumer's store went and it was answered elsewhere: %v", err) + } + + // A person pins the anchor's: it moves, said before it is sent, and the record keeps where it was. + if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil { + t.Fatal(err) + } + plan, _, err = planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" { + t.Fatalf("pinned to the anchor and bound to %s", n.From) + } + found, err := probeBindings(ctx, &doctor{open: open}) + if err != nil { + t.Fatal(err) + } + if len(found) != 1 || found[0].Kind != kindBindingMoving || found[0].Severity != conditions.Warning || + !strings.Contains(found[0].Summary, "board's postgres-database moves from laptop/store to anchor/store") { + t.Fatalf("a pinned move is not said before it is sent: %+v", found) + } + if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil { + t.Fatal(err) + } + all, err := inv.Bindings(ctx) + if err != nil || len(all) != 1 || all[0].MovedFrom != "laptop/store" { + t.Fatalf("%+v, %v", all, err) + } + if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 { + t.Fatalf("a move sent is still said: %+v, %v", found, err) + } +} diff --git a/cmd/mesh-controller/bindings_test.go b/cmd/mesh-controller/bindings_test.go index 0a6ed5b5..98cb6b69 100644 --- a/cmd/mesh-controller/bindings_test.go +++ b/cmd/mesh-controller/bindings_test.go @@ -39,96 +39,6 @@ func need(t *testing.T, plan catalogue.Resolution, consumer, provision string) c // The incident through the stores: the laptop runs its own store and a consumer of it, the anchor's // store holds the mesh's seat. The consumer stays beside its data, the resolver follows its seat, the // binding is recorded as sent, and a pin — only a pin — moves it, said before the push that carries it. -func TestTheIncidentAConsumerStaysBesideItsDataUntilAPersonPinsIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { - t.Fatal(err) - } - for _, m := range storeManifests() { - register(t, open, m) - } - assignAll := func(pairs ...[2]string) { - for _, a := range pairs { - if _, err := assign(ctx, open, a[0], a[1]); err != nil { - t.Fatalf("assign %s %s: %v", a[0], a[1], err) - } - } - } - // The anchor's store and resolver hold the mesh's seats, on record; the laptop runs its own of each. - assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"}) - for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} { - if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil { - t.Fatal(err) - } - } - assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"}, - [2]string{"laptop", "board"}) - - plan, _, err := planFor(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" || n.Module != "store" || !n.KeepsData { - t.Fatalf("the consumer was bound to %s/%s (keeps data: %v); its data is beside it", n.From, n.Module, n.KeepsData) - } - if n := need(t, plan, "network", "wildcard-resolution"); n.From != "anchor" || n.KeepsData { - t.Fatalf("the resolver was bound to %s (keeps data: %v); its seat is held on anchor (issue 258)", n.From, n.KeepsData) - } - - // Sent, and recorded: only the binding to data. - bindings := boundToData(plan, nil) - if len(bindings) != 1 || bindings[0].Provider != (catalogue.Chosen{Node: "laptop", Module: "store"}) { - t.Fatalf("recorded %+v", bindings) - } - if err := inv.RecordBindings(ctx, "laptop", bindings); err != nil { - t.Fatal(err) - } - if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 { - t.Fatalf("a mesh bound where it was sent: %+v, %v", found, err) - } - - // The laptop's store taken away: refused, not moved to the anchor's empty one. - if err := inv.Unassign(ctx, "laptop", "store"); err != nil { - t.Fatal(err) - } - if _, _, err := planFor(ctx, open, "laptop"); err == nil || !unresolvable(err) || - !strings.Contains(err.Error(), "board on laptop is bound to laptop/store") || - !strings.Contains(err.Error(), "pin laptop postgres-database anchor store") { - t.Fatalf("the consumer's store went and it was answered elsewhere: %v", err) - } - - // A person pins the anchor's: it moves, said before it is sent, and the record keeps where it was. - if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil { - t.Fatal(err) - } - plan, _, err = planFor(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" { - t.Fatalf("pinned to the anchor and bound to %s", n.From) - } - found, err := probeBindings(ctx, &doctor{open: open}) - if err != nil { - t.Fatal(err) - } - if len(found) != 1 || found[0].Kind != kindBindingMoving || found[0].Severity != conditions.Warning || - !strings.Contains(found[0].Summary, "board's postgres-database moves from laptop/store to anchor/store") { - t.Fatalf("a pinned move is not said before it is sent: %+v", found) - } - if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil { - t.Fatal(err) - } - all, err := inv.Bindings(ctx) - if err != nil || len(all) != 1 || all[0].MovedFrom != "laptop/store" { - t.Fatalf("%+v, %v", all, err) - } - if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 { - t.Fatalf("a move sent is still said: %+v, %v", found, err) - } -} // What one machine's resolution says against its record. func TestBindingFindingsSayAKeptMoveAndAMoveNothingAskedFor(t *testing.T) { diff --git a/cmd/mesh-controller/build_integration_test.go b/cmd/mesh-controller/build_integration_test.go new file mode 100644 index 00000000..c2afe6e7 --- /dev/null +++ b/cmd/mesh-controller/build_integration_test.go @@ -0,0 +1,131 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestABuildHeardIsRecordedAndRegistered(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + manifest, _ := json.Marshal(map[string]any{"module": "shop", "version": "3"}) + m, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, link.BuildResult{ + ID: "b-1", Repository: "http://forge.internal:20000/novox/shop.git", Path: "modules/shop", + Ref: "main", On: "anchor", Commit: "abcdef0123", Manifest: manifest, + Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/shop"}, + })) + if err != nil { + t.Fatal(err) + } + if m.Module != "shop" { + t.Fatalf("registered %q", m.Module) + } + shelf, err := open.inventory.Catalogue(ctx) + if err != nil { + t.Fatal(err) + } + if _, held := shelf["shop"]; !held { + t.Fatal("the module a heard build produced is not in the catalogue") + } + src, err := open.inventory.SourceOf(ctx, "shop") + if err != nil || src.Seat != "git" || src.Repository != "novox/shop" || src.BuiltFrom != "abcdef0123" { + t.Fatalf("the source is the seat and the path, never the URL: %+v %v", src, err) + } + builds, err := open.inventory.Builds(ctx, "shop", 5) + if err != nil || len(builds) != 1 || builds[0].ID != "b-1" { + t.Fatalf("the build is not recorded once: %v %v", builds, err) + } + + named, _ := json.Marshal(map[string]any{"module": "idp", "version": "1", "resources": []any{ + map[string]any{"id": "server", "type": "container", "image": "x@sha256:aa", + "env": map[string]any{"KC_HOSTNAME": "https://login.mesh-one.be"}}}}) + _, _, err = takeIn(ctx, open.inventory, link.BuildResult{ + ID: "b-2", Repository: "/r", On: "anchor", Commit: "0123456789", Manifest: named}) + if err == nil || !strings.Contains(err.Error(), "does not register it") { + t.Fatalf("a definition naming an installation was taken in: %v", err) + } + if shelf, _ := open.inventory.Catalogue(ctx); shelf["idp"].Module != "" { + t.Fatal("the refused module was registered anyway") + } + if builds, _ := open.inventory.Builds(ctx, "idp", 5); len(builds) != 1 { + t.Fatalf("the refused build was not recorded: %v", builds) + } + + _, _, err = takeIn(ctx, open.inventory, link.BuildResult{ID: "b-3", Repository: "/r", On: "anchor", Failed: "no compiler"}) + if err == nil || !strings.Contains(err.Error(), "no compiler") { + t.Fatalf("a failure is said in the builder's words: %v", err) + } +} + +func TestABuildAtACommitKeepsTheBranchTheModuleFollows(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + manifest, _ := json.Marshal(map[string]any{"module": "unifi", "version": "1"}) + result := func(id, ref, commit string) link.BuildResult { + return link.BuildResult{ID: id, Repository: "http://forge.internal:20000/novox/mesh-catalog.git", + Path: "modules/unifi", Ref: ref, On: "anchor", Commit: commit, Manifest: manifest, + Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}} + } + if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, result("b-1", "main", "1111111aaaa"))); err != nil { + t.Fatal(err) + } + if _, _, err := takeIn(ctx, open.inventory, result("b-2", "9c97a8a", "9c97a8a1d2c3")); err != nil { + t.Fatal(err) + } + src, err := open.inventory.SourceOf(ctx, "unifi") + if err != nil { + t.Fatal(err) + } + if src.Ref != "main" || src.BuiltFrom != "9c97a8a1d2c3" { + t.Errorf("after a build at a commit the module follows %q, built from %q; want main, 9c97a8a1d2c3", src.Ref, src.BuiltFrom) + } + + // One new to the catalogue, first built at a commit, follows the default branch. + other, _ := json.Marshal(map[string]any{"module": "letta", "version": "1"}) + r := result("b-3", "deadbeef", "deadbeefcafe") + r.Manifest, r.Path = other, "modules/letta" + if _, _, err := takeIn(ctx, open.inventory, r); err != nil { + t.Fatal(err) + } + if src, _ := open.inventory.SourceOf(ctx, "letta"); src.Ref != "" { + t.Errorf("a module first built at a commit follows %q, want the default branch", src.Ref) + } +} + +func TestAnOlderBuildHeardLaterDoesNotReplaceTheNewer(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + older := time.Date(2026, 10, 3, 21, 33, 45, 0, time.UTC) + newer := time.Date(2026, 10, 3, 21, 51, 57, 0, time.UTC) + result := func(asked time.Time, image string) link.BuildResult { + manifest, _ := json.Marshal(map[string]any{"module": "postgres", "version": image}) + return link.BuildResult{ID: link.NewBuildID(asked), Repository: "http://forge.internal:20000/novox/mesh-catalog.git", + Path: "modules/postgres", Ref: "main", On: "anchor", Commit: "efff5415", Manifest: manifest, + Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}} + } + if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, result(newer, "4bcd5f73"))); err != nil { + t.Fatal(err) + } + _, _, err := takeIn(ctx, open.inventory, result(older, "0ab07fa9")) + if !errors.Is(err, inventory.ErrSuperseded) { + t.Fatalf("the older request's outcome was taken in as current: %v", err) + } + shelf, err := open.inventory.Catalogue(ctx) + if err != nil { + t.Fatal(err) + } + if got := shelf["postgres"].Version; got != "4bcd5f73" { + t.Errorf("postgres is %q; want the newer request's 4bcd5f73", got) + } + if builds, _ := open.inventory.Builds(ctx, "postgres", 5); len(builds) != 2 { + t.Errorf("the late build was not recorded: %v", builds) + } +} diff --git a/cmd/mesh-controller/build_source_integration_test.go b/cmd/mesh-controller/build_source_integration_test.go new file mode 100644 index 00000000..8bc6ded9 --- /dev/null +++ b/cmd/mesh-controller/build_source_integration_test.go @@ -0,0 +1,353 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "slices" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func theCatalogue(t *testing.T) *stores { + t.Helper() + open := aMesh(t) + ctx := t.Context() + for _, b := range []link.BuildResult{ + onTrunk("build-sudo", "novox/mesh-catalog", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "1"}), + onTrunk("build-host", "novox/mesh-host", "git", "", map[string]any{"module": "mesh-host", "version": "1"}), + } { + keptAsked(t, open.inventory, b.ID, b.Source.Repository, b.Path, true) + if _, _, err := takeIn(ctx, open.inventory, b); err != nil { + t.Fatal(err) + } + } + return open +} + +func TestABuildFromAnAgentsRepositoryIsNotRegisteredAsAModuleTheMeshHolds(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + for _, c := range []struct { + name, repository, path, module string + }{ + {"its own repository naming sudo", "agent/sudo", "modules/sudo", "sudo"}, + {"a fork of the catalogue", "agent/mesh-catalog", "modules/sudo", "sudo"}, + {"a fork of the node-engine", "agent/mesh-host", "", "mesh-host"}, + } { + t.Run(c.name, func(t *testing.T) { + id := "build-" + strings.ReplaceAll(c.repository, "/", "-") + keptAsked(t, open.inventory, id, c.repository, c.path, false) // through the build verb + evil := onTrunk(id, c.repository, "git", c.path, map[string]any{"module": c.module, "version": "evil"}) + _, _, err := takeIn(ctx, open.inventory, evil) + if !errors.Is(err, errNotItsSource) { + t.Fatalf("a build of %s was taken in as %s: %v", c.repository, c.module, err) + } + shelf, err := open.inventory.Catalogue(ctx) + if err != nil { + t.Fatal(err) + } + if got := shelf[c.module].Version; got != "1" { + t.Fatalf("%s is now %q, from %s", c.module, got, c.repository) + } + if _, found, _ := open.inventory.BuildByID(ctx, id); !found { + t.Errorf("the refused build %s is not recorded", id) + } + }) + } +} + +func TestANewModuleFromARepositoryTheCatalogueDoesNotBuildFromIsNotRegistered(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + keptAsked(t, open.inventory, "build-new", "agent/tools", "", false) + _, _, err := takeIn(ctx, open.inventory, onTrunk("build-new", "agent/tools", "git", "", + map[string]any{"module": "agent-tools", "version": "1"})) + if !errors.Is(err, errNotItsSource) { + t.Fatalf("a new module from an agent's repository was taken in: %v", err) + } + // And one asked of nobody here — an outcome on the bus no request was kept for — the same. + _, _, err = takeIn(ctx, open.inventory, onTrunk("build-unasked", "agent/tools", "git", "", + map[string]any{"module": "agent-tools", "version": "1"})) + if !errors.Is(err, errNotItsSource) { + t.Fatalf("an outcome nobody asked for was taken in: %v", err) + } + if shelf, _ := open.inventory.Catalogue(ctx); shelf["agent-tools"].Module != "" { + t.Fatal("the refused module is in the catalogue") + } +} + +func TestAtTheTerminalAnotherRepositoryIsRegistered(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + keptAsked(t, open.inventory, "build-moved", "novox/sudo", "", true) + if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-moved", "novox/sudo", "git", "", + map[string]any{"module": "sudo", "version": "2"})); err != nil { + t.Fatalf("a move the operator asked for at the terminal was refused: %v", err) + } + if src, _ := open.inventory.SourceOf(ctx, "sudo"); src.Repository != "novox/sudo" { + t.Fatalf("sudo is built from %q", src.Repository) + } + if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-external", + Repository: "http://forge.internal:20000/someone/app.git", For: "build", AtTerminal: true}); err != nil { + t.Fatal(err) + } + if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-external", "someone/app", "", "", + map[string]any{"module": "app", "version": "1"})); err != nil { + t.Fatalf("a new module the operator asked for at the terminal was refused: %v", err) + } +} + +func TestADeliveryFromTheRegisteredRepositoryIsRegistered(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + rebuilt := onTrunk("build-plan", "novox/mesh-catalog", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "2"}) + if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: rebuilt.ID, + Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/sudo", For: "plan"}); err != nil { + t.Fatal(err) + } + if _, _, err := takeIn(ctx, open.inventory, rebuilt); err != nil { + t.Fatalf("a plan's build of the module's own repository was refused: %v", err) + } + added := onTrunk("build-merge", "novox/mesh-catalog", "git", "modules/zram", map[string]any{"module": "zram", "version": "1"}) + if _, _, err := takeIn(ctx, open.inventory, added); err != nil { + t.Fatalf("a module a merge added to the catalogue repository was refused: %v", err) + } + shelf, _ := open.inventory.Catalogue(ctx) + if shelf["sudo"].Version != "2" || shelf["zram"].Module == "" { + t.Fatalf("not registered: sudo %q, zram %q", shelf["sudo"].Version, shelf["zram"].Module) + } +} + +func TestAVerbAsksNoBuildOfARepositoryTheCatalogueDoesNotBuildFrom(t *testing.T) { + theCatalogue(t) + ctx := t.Context() + t.Setenv(verbVar, "build") + t.Setenv(link.CallerVar, "node-tools.anchor, through the mesh-controller seat") + err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, "http://forge.internal:20000/agent/sudo.git") + var policy *heldAtTheTerminal + if !errors.As(err, &policy) { + t.Fatalf("a verb's build of an agent's repository was asked: %v", err) + } + if err := verbMayAsk(ctx, buildSource{Repository: "novox/mesh-catalog", Seat: "git"}, + "http://forge.internal:20000/novox/mesh-catalog.git"); err != nil { + t.Fatalf("a verb's build of the catalogue repository was refused: %v", err) + } + t.Setenv(verbVar, "") + t.Setenv(link.CallerVar, "") + if err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, ""); err != nil { + t.Fatalf("the terminal was refused: %v", err) + } +} + +func TestARollbackNeverPutsBackABuildFromAnotherRepository(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + inv := open.inventory + fork := onTrunk("build-1791500000000000000", "agent/mesh-catalog", "git", "modules/sudo", + map[string]any{"module": "sudo", "version": "evil"}) + fork.Commit = "c0ffee0123456789" // the commit sudo was registered at + keptAsked(t, inv, fork.ID, "agent/mesh-catalog", "modules/sudo", false) + if _, _, err := takeIn(ctx, inv, fork); !errors.Is(err, errNotItsSource) { + t.Fatalf("the fork's build was taken in: %v", err) + } + // Asked after the registered build, whenever the test runs: an id naming a fixed moment read as older + // than the registered build once the clock passed it (2026-10-10 02:40 UTC), and the test failed on main. + failed := onTrunk(fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()), "novox/mesh-catalog", "git", "modules/sudo", + map[string]any{"module": "sudo", "version": "2"}) + failed.Commit = "badbadbad0123456" + if _, _, err := takeIn(ctx, inv, failed); err != nil { + t.Fatal(err) + } + record, _, err := inv.BuildByID(ctx, failed.ID) + if err != nil { + t.Fatal(err) + } + previous, found, err := inv.PreviousBuild(ctx, "sudo", "c0ffee0123456789", record) + if err != nil { + t.Fatal(err) + } + if found && previous.ID == fork.ID { + t.Fatalf("a rollback would put back the fork's build %s", previous.ID) + } + if !found || previous.ID != "build-sudo" { + t.Fatalf("a rollback puts back %q (found %v), want the registered build-sudo", previous.ID, found) + } +} + +func TestATrunkTheForgeDoesNotProtectRegistersNothing(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + theForge.Store("novox/unguarded", forgeFacts{ID: 7, Why: "the rule main lets a person push to main directly"}) + t.Cleanup(func() { theForge.Delete("novox/unguarded") }) + first := onTrunk("build-unguarded-1", "novox/unguarded", "git", "", map[string]any{"module": "unguarded", "version": "1"}) + keptAsked(t, open.inventory, first.ID, "novox/unguarded", "", true) + if _, _, err := takeIn(ctx, open.inventory, first); err != nil { + t.Fatalf("at the terminal: %v", err) + } + again := onTrunk("build-unguarded-2", "novox/unguarded", "git", "", map[string]any{"module": "unguarded", "version": "2"}) + if _, _, err := takeIn(ctx, open.inventory, again); !errors.Is(err, errNotItsSource) || + !strings.Contains(err.Error(), "push to main directly") { + t.Fatalf("a rebuild from an unprotected trunk was taken in: %v", err) + } + theForge.Store("novox/unguarded", errors.New("nothing serves gitea.gitea_api")) + if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-unguarded-3", "novox/unguarded", "git", "", + map[string]any{"module": "unguarded", "version": "3"})); !errors.Is(err, errNotItsSource) { + t.Fatalf("a forge that could not say was read as a protected trunk: %v", err) + } + if shelf, _ := open.inventory.Catalogue(ctx); shelf["unguarded"].Version != "1" { + t.Fatalf("unguarded is %q", shelf["unguarded"].Version) + } +} + +func TestARepositoryMadeAgainUnderItsNameIsNotTheModulesSource(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + theForge.Store("novox/remade", forgeFacts{ID: 100, Guarded: true}) + t.Cleanup(func() { theForge.Delete("novox/remade") }) + first := onTrunk("build-remade-1", "novox/remade", "git", "", map[string]any{"module": "remade", "version": "1"}) + keptAsked(t, open.inventory, first.ID, "novox/remade", "", true) + if _, _, err := takeIn(ctx, open.inventory, first); err != nil { + t.Fatal(err) + } + if id, _ := open.inventory.SourceIdentity(ctx, "remade"); id != 100 { + t.Fatalf("the forge's id was not recorded: %d", id) + } + theForge.Store("novox/remade", forgeFacts{ID: 101, Guarded: true}) // deleted, and made again by an agent + if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-remade-2", "novox/remade", "git", "", + map[string]any{"module": "remade", "version": "evil"})); !errors.Is(err, errNotItsSource) || + !strings.Contains(err.Error(), "made again") { + t.Fatalf("a repository made again under the name was taken in: %v", err) + } + // And a new module from it, beside the one registered from the first, the same. + if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-remade-3", "novox/remade", "git", "modules/other", + map[string]any{"module": "other", "version": "1"})); !errors.Is(err, errNotItsSource) { + t.Fatalf("a new module from a repository made again was taken in: %v", err) + } +} + +func TestATerminalRequestCoversOnlyWhatItAsked(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-asked", Repository: "novox/app", + Seat: "git", Path: "modules/app", For: "build", AtTerminal: true}); err != nil { + t.Fatal(err) + } + other := onTrunk("build-asked", "agent/sudo", "git", "modules/app", map[string]any{"module": "sudo", "version": "evil"}) + if _, _, err := takeIn(ctx, open.inventory, other); !errors.Is(err, errNotItsSource) { + t.Fatalf("an outcome of another repository under a terminal request's id was taken in: %v", err) + } + elsewhere := onTrunk("build-asked", "novox/app", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "evil"}) + if _, _, err := takeIn(ctx, open.inventory, elsewhere); !errors.Is(err, errNotItsSource) { + t.Fatalf("an outcome of another path under a terminal request's id was taken in: %v", err) + } +} + +func TestTheServingControllerIsNeverTheTerminal(t *testing.T) { + t.Setenv(verbVar, "") + t.Setenv(link.CallerVar, "") + t.Setenv(servedVar, "") + if !startedAtTheTerminal() { + t.Fatal("a process started by hand is not the terminal") + } + markServed() + if startedAtTheTerminal() { + t.Fatal("the serving controller reads as the terminal") + } + child := exec.Command(os.Args[0], "-test.run=^$") + child.Env = os.Environ() + if !slices.Contains(child.Env, servedVar+"=1") { + t.Fatal("what the serving controller starts does not carry its mark") + } +} + +func TestAnUnknownIdentityKeepsTheRecordedOne(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + if err := open.inventory.SetSourceIdentity(ctx, "sudo", 100); err != nil { + t.Fatal(err) + } + if err := open.inventory.SetSourceIdentity(ctx, "sudo", 0); err != nil { + t.Fatal(err) + } + if id, _ := open.inventory.SourceIdentity(ctx, "sudo"); id != 100 { + t.Fatalf("the recorded id became %d", id) + } +} + +func TestAModuleMovesOnTheOperatorsApproveOfTheMergeThatMovedIt(t *testing.T) { + const commit = "c0ffee0123456789" + for _, c := range []struct { + what string + change func(*anApprovedMerge) + unkept bool + request string // who asked: "merge", or "build" through a verb + moved bool + }{ + {what: "the operator's Approve of the merged head", request: "merge", moved: true}, + {what: "no merge mesh-delivery announced", unkept: true, request: "merge"}, + {what: "a warrant for other heads", request: "merge", change: func(a *anApprovedMerge) { + a.merge.Heads = map[string]string{"novox/mesh-host@aaaaaaaaaaaa": "e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5"} + }}, + {what: "a Decline", request: "merge", change: func(a *anApprovedMerge) { + a.warrant.Option, a.warrant.Label, a.merge.Option = "decline", "Decline", "decline" + }}, + {what: "a person who is not the operator", request: "merge", change: func(a *anApprovedMerge) { a.warrant.By.Who = "agent" }}, + {what: "a build asked through a verb", request: "build"}, + } { + open := theCatalogue(t) + ctx := t.Context() + inv := open.inventory + a := approvedMerge(t) + a.merge.Owner, a.merge.Repo, a.merge.Number, a.merge.HeadSHA = "novox", "mesh-host", 7, "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1" + if c.change != nil { + c.change(&a) + } + wasKept, wasAsk := keptDeliveryMerge, routerAskOf + keptDeliveryMerge = func(_ context.Context, owner, repo string, number int, head string) (deliveryMerge, bool, error) { + return a.merge, !c.unkept && owner == "novox" && repo == "mesh-host" && number == 7 && head == a.merge.HeadSHA, nil + } + routerAskOf = func(_ context.Context, _ *inventory.Inventory, id string) (*asks.Ask, *asks.Warrant, error) { + if id != a.merge.AskID { + t.Errorf("%s: the router was asked for %q", c.what, id) + } + return &a.ask, &a.warrant, nil + } + t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk }) + event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-host", "base": "main", "number": 7, + "head_sha": a.merge.HeadSHA, "merge_commit_sha": commit}) + if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-host", Branch: "main", Commit: commit, + Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil { + t.Fatal(err) + } + id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()) + if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-host", Seat: "git", + Path: "modules/sudo", Ref: "main", Commit: commit, For: c.request}); err != nil { + t.Fatal(err) + } + moved := onTrunk(id, "novox/mesh-host", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "2"}) + moved.Commit = commit + _, _, err := takeIn(ctx, inv, moved) + src, _ := inv.SourceOf(ctx, "sudo") + switch { + case c.moved && err != nil: + t.Errorf("%s: the move was refused: %v", c.what, err) + case c.moved && src.Repository != "novox/mesh-host": + t.Errorf("%s: sudo is built from %q", c.what, src.Repository) + case !c.moved && !errors.Is(err, errNotItsSource): + t.Errorf("%s: the move was not refused as ADR 0266 has it: %v", c.what, err) + case !c.moved && src.Repository != "novox/mesh-catalog": + t.Errorf("%s: a refused move changed sudo's source to %q", c.what, src.Repository) + } + } +} diff --git a/cmd/mesh-controller/build_source_test.go b/cmd/mesh-controller/build_source_test.go index 03ce2bd0..1f692d8b 100644 --- a/cmd/mesh-controller/build_source_test.go +++ b/cmd/mesh-controller/build_source_test.go @@ -3,18 +3,10 @@ package main import ( "context" "encoding/json" - "errors" - "fmt" "hash/fnv" - "os" - "os/exec" - "slices" "strings" "sync" "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/asks" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" @@ -47,141 +39,14 @@ func keptAsked(t *testing.T, inv *inventory.Inventory, id, repository, path stri } // theCatalogue is a mesh whose sudo is built from the catalogue repository and whose node-engine from its own. -func theCatalogue(t *testing.T) *stores { - t.Helper() - open := aMesh(t) - ctx := t.Context() - for _, b := range []link.BuildResult{ - onTrunk("build-sudo", "novox/mesh-catalog", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "1"}), - onTrunk("build-host", "novox/mesh-host", "git", "", map[string]any{"module": "mesh-host", "version": "1"}), - } { - keptAsked(t, open.inventory, b.ID, b.Source.Repository, b.Path, true) - if _, _, err := takeIn(ctx, open.inventory, b); err != nil { - t.Fatal(err) - } - } - return open -} - -func TestABuildFromAnAgentsRepositoryIsNotRegisteredAsAModuleTheMeshHolds(t *testing.T) { - open := theCatalogue(t) - ctx := t.Context() - for _, c := range []struct { - name, repository, path, module string - }{ - {"its own repository naming sudo", "agent/sudo", "modules/sudo", "sudo"}, - {"a fork of the catalogue", "agent/mesh-catalog", "modules/sudo", "sudo"}, - {"a fork of the node-engine", "agent/mesh-host", "", "mesh-host"}, - } { - t.Run(c.name, func(t *testing.T) { - id := "build-" + strings.ReplaceAll(c.repository, "/", "-") - keptAsked(t, open.inventory, id, c.repository, c.path, false) // through the build verb - evil := onTrunk(id, c.repository, "git", c.path, map[string]any{"module": c.module, "version": "evil"}) - _, _, err := takeIn(ctx, open.inventory, evil) - if !errors.Is(err, errNotItsSource) { - t.Fatalf("a build of %s was taken in as %s: %v", c.repository, c.module, err) - } - shelf, err := open.inventory.Catalogue(ctx) - if err != nil { - t.Fatal(err) - } - if got := shelf[c.module].Version; got != "1" { - t.Fatalf("%s is now %q, from %s", c.module, got, c.repository) - } - if _, found, _ := open.inventory.BuildByID(ctx, id); !found { - t.Errorf("the refused build %s is not recorded", id) - } - }) - } -} - -func TestANewModuleFromARepositoryTheCatalogueDoesNotBuildFromIsNotRegistered(t *testing.T) { - open := theCatalogue(t) - ctx := t.Context() - keptAsked(t, open.inventory, "build-new", "agent/tools", "", false) - _, _, err := takeIn(ctx, open.inventory, onTrunk("build-new", "agent/tools", "git", "", - map[string]any{"module": "agent-tools", "version": "1"})) - if !errors.Is(err, errNotItsSource) { - t.Fatalf("a new module from an agent's repository was taken in: %v", err) - } - // And one asked of nobody here — an outcome on the bus no request was kept for — the same. - _, _, err = takeIn(ctx, open.inventory, onTrunk("build-unasked", "agent/tools", "git", "", - map[string]any{"module": "agent-tools", "version": "1"})) - if !errors.Is(err, errNotItsSource) { - t.Fatalf("an outcome nobody asked for was taken in: %v", err) - } - if shelf, _ := open.inventory.Catalogue(ctx); shelf["agent-tools"].Module != "" { - t.Fatal("the refused module is in the catalogue") - } -} // The operator at the terminal may still move a module, or add one from a new repository. -func TestAtTheTerminalAnotherRepositoryIsRegistered(t *testing.T) { - open := theCatalogue(t) - ctx := t.Context() - keptAsked(t, open.inventory, "build-moved", "novox/sudo", "", true) - if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-moved", "novox/sudo", "git", "", - map[string]any{"module": "sudo", "version": "2"})); err != nil { - t.Fatalf("a move the operator asked for at the terminal was refused: %v", err) - } - if src, _ := open.inventory.SourceOf(ctx, "sudo"); src.Repository != "novox/sudo" { - t.Fatalf("sudo is built from %q", src.Repository) - } - if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-external", - Repository: "http://forge.internal:20000/someone/app.git", For: "build", AtTerminal: true}); err != nil { - t.Fatal(err) - } - if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-external", "someone/app", "", "", - map[string]any{"module": "app", "version": "1"})); err != nil { - t.Fatalf("a new module the operator asked for at the terminal was refused: %v", err) - } -} // The delivery's flow is untouched: a merge's rebuild of a module from its own repository, and a merge adding // a module to a repository the catalogue builds from (novox/hq issue 300), are registered with no terminal. -func TestADeliveryFromTheRegisteredRepositoryIsRegistered(t *testing.T) { - open := theCatalogue(t) - ctx := t.Context() - rebuilt := onTrunk("build-plan", "novox/mesh-catalog", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "2"}) - if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: rebuilt.ID, - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/sudo", For: "plan"}); err != nil { - t.Fatal(err) - } - if _, _, err := takeIn(ctx, open.inventory, rebuilt); err != nil { - t.Fatalf("a plan's build of the module's own repository was refused: %v", err) - } - added := onTrunk("build-merge", "novox/mesh-catalog", "git", "modules/zram", map[string]any{"module": "zram", "version": "1"}) - if _, _, err := takeIn(ctx, open.inventory, added); err != nil { - t.Fatalf("a module a merge added to the catalogue repository was refused: %v", err) - } - shelf, _ := open.inventory.Catalogue(ctx) - if shelf["sudo"].Version != "2" || shelf["zram"].Module == "" { - t.Fatalf("not registered: sudo %q, zram %q", shelf["sudo"].Version, shelf["zram"].Module) - } -} // Through a verb, a build of a repository the catalogue builds nothing from is not even asked: the build node // would run what the agent wrote. -func TestAVerbAsksNoBuildOfARepositoryTheCatalogueDoesNotBuildFrom(t *testing.T) { - theCatalogue(t) - ctx := t.Context() - t.Setenv(verbVar, "build") - t.Setenv(link.CallerVar, "node-tools.anchor, through the mesh-controller seat") - err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, "http://forge.internal:20000/agent/sudo.git") - var policy *heldAtTheTerminal - if !errors.As(err, &policy) { - t.Fatalf("a verb's build of an agent's repository was asked: %v", err) - } - if err := verbMayAsk(ctx, buildSource{Repository: "novox/mesh-catalog", Seat: "git"}, - "http://forge.internal:20000/novox/mesh-catalog.git"); err != nil { - t.Fatalf("a verb's build of the catalogue repository was refused: %v", err) - } - t.Setenv(verbVar, "") - t.Setenv(link.CallerVar, "") - if err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, ""); err != nil { - t.Fatalf("the terminal was refused: %v", err) - } -} // asTheOperator keeps a build as asked at the controller's terminal, as the operator's first build of a module // from a repository the catalogue does not yet build from is (novox/hq ADR 0266), and hands it back. @@ -201,40 +66,6 @@ func asTheOperator(t *testing.T, inv *inventory.Inventory, b link.BuildResult) l // A rollback puts back only a build of the module's own repository: an agent's build of the module's name, // recorded and refused, at the very commit the machine ran before (a fork carries it), is never registered by // the back door of a failed gate. -func TestARollbackNeverPutsBackABuildFromAnotherRepository(t *testing.T) { - open := theCatalogue(t) - ctx := t.Context() - inv := open.inventory - fork := onTrunk("build-1791500000000000000", "agent/mesh-catalog", "git", "modules/sudo", - map[string]any{"module": "sudo", "version": "evil"}) - fork.Commit = "c0ffee0123456789" // the commit sudo was registered at - keptAsked(t, inv, fork.ID, "agent/mesh-catalog", "modules/sudo", false) - if _, _, err := takeIn(ctx, inv, fork); !errors.Is(err, errNotItsSource) { - t.Fatalf("the fork's build was taken in: %v", err) - } - // Asked after the registered build, whenever the test runs: an id naming a fixed moment read as older - // than the registered build once the clock passed it (2026-10-10 02:40 UTC), and the test failed on main. - failed := onTrunk(fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()), "novox/mesh-catalog", "git", "modules/sudo", - map[string]any{"module": "sudo", "version": "2"}) - failed.Commit = "badbadbad0123456" - if _, _, err := takeIn(ctx, inv, failed); err != nil { - t.Fatal(err) - } - record, _, err := inv.BuildByID(ctx, failed.ID) - if err != nil { - t.Fatal(err) - } - previous, found, err := inv.PreviousBuild(ctx, "sudo", "c0ffee0123456789", record) - if err != nil { - t.Fatal(err) - } - if found && previous.ID == fork.ID { - t.Fatalf("a rollback would put back the fork's build %s", previous.ID) - } - if !found || previous.ID != "build-sudo" { - t.Fatalf("a rollback puts back %q (found %v), want the registered build-sudo", previous.ID, found) - } -} // theForge is what the forge says in a test, by owner/name: a repository not named here is protected as a // trunk must be, with an id of its own. @@ -258,97 +89,15 @@ func init() { // A module's trunk the forge does not protect — direct pushes, no required status — or a forge that cannot say, // registers nothing through a verb: the trunk rule means nothing on a branch anyone pushes to. -func TestATrunkTheForgeDoesNotProtectRegistersNothing(t *testing.T) { - open := theCatalogue(t) - ctx := t.Context() - theForge.Store("novox/unguarded", forgeFacts{ID: 7, Why: "the rule main lets a person push to main directly"}) - t.Cleanup(func() { theForge.Delete("novox/unguarded") }) - first := onTrunk("build-unguarded-1", "novox/unguarded", "git", "", map[string]any{"module": "unguarded", "version": "1"}) - keptAsked(t, open.inventory, first.ID, "novox/unguarded", "", true) - if _, _, err := takeIn(ctx, open.inventory, first); err != nil { - t.Fatalf("at the terminal: %v", err) - } - again := onTrunk("build-unguarded-2", "novox/unguarded", "git", "", map[string]any{"module": "unguarded", "version": "2"}) - if _, _, err := takeIn(ctx, open.inventory, again); !errors.Is(err, errNotItsSource) || - !strings.Contains(err.Error(), "push to main directly") { - t.Fatalf("a rebuild from an unprotected trunk was taken in: %v", err) - } - theForge.Store("novox/unguarded", errors.New("nothing serves gitea.gitea_api")) - if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-unguarded-3", "novox/unguarded", "git", "", - map[string]any{"module": "unguarded", "version": "3"})); !errors.Is(err, errNotItsSource) { - t.Fatalf("a forge that could not say was read as a protected trunk: %v", err) - } - if shelf, _ := open.inventory.Catalogue(ctx); shelf["unguarded"].Version != "1" { - t.Fatalf("unguarded is %q", shelf["unguarded"].Version) - } -} // A repository deleted and made again under the module's repository's name is another repository: the forge's // id, recorded at registration, tells them apart. -func TestARepositoryMadeAgainUnderItsNameIsNotTheModulesSource(t *testing.T) { - open := theCatalogue(t) - ctx := t.Context() - theForge.Store("novox/remade", forgeFacts{ID: 100, Guarded: true}) - t.Cleanup(func() { theForge.Delete("novox/remade") }) - first := onTrunk("build-remade-1", "novox/remade", "git", "", map[string]any{"module": "remade", "version": "1"}) - keptAsked(t, open.inventory, first.ID, "novox/remade", "", true) - if _, _, err := takeIn(ctx, open.inventory, first); err != nil { - t.Fatal(err) - } - if id, _ := open.inventory.SourceIdentity(ctx, "remade"); id != 100 { - t.Fatalf("the forge's id was not recorded: %d", id) - } - theForge.Store("novox/remade", forgeFacts{ID: 101, Guarded: true}) // deleted, and made again by an agent - if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-remade-2", "novox/remade", "git", "", - map[string]any{"module": "remade", "version": "evil"})); !errors.Is(err, errNotItsSource) || - !strings.Contains(err.Error(), "made again") { - t.Fatalf("a repository made again under the name was taken in: %v", err) - } - // And a new module from it, beside the one registered from the first, the same. - if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-remade-3", "novox/remade", "git", "modules/other", - map[string]any{"module": "other", "version": "1"})); !errors.Is(err, errNotItsSource) { - t.Fatalf("a new module from a repository made again was taken in: %v", err) - } -} // The terminal's mark is the operator's for the repository and path they asked: an outcome of another, under that // build's id, is not theirs. -func TestATerminalRequestCoversOnlyWhatItAsked(t *testing.T) { - open := theCatalogue(t) - ctx := t.Context() - if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-asked", Repository: "novox/app", - Seat: "git", Path: "modules/app", For: "build", AtTerminal: true}); err != nil { - t.Fatal(err) - } - other := onTrunk("build-asked", "agent/sudo", "git", "modules/app", map[string]any{"module": "sudo", "version": "evil"}) - if _, _, err := takeIn(ctx, open.inventory, other); !errors.Is(err, errNotItsSource) { - t.Fatalf("an outcome of another repository under a terminal request's id was taken in: %v", err) - } - elsewhere := onTrunk("build-asked", "novox/app", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "evil"}) - if _, _, err := takeIn(ctx, open.inventory, elsewhere); !errors.Is(err, errNotItsSource) { - t.Fatalf("an outcome of another path under a terminal request's id was taken in: %v", err) - } -} // The serving controller, and everything it starts, is never the terminal: a build asked in it reads as asked // through the mesh even when no verb and no caller is named. -func TestTheServingControllerIsNeverTheTerminal(t *testing.T) { - t.Setenv(verbVar, "") - t.Setenv(link.CallerVar, "") - t.Setenv(servedVar, "") - if !startedAtTheTerminal() { - t.Fatal("a process started by hand is not the terminal") - } - markServed() - if startedAtTheTerminal() { - t.Fatal("the serving controller reads as the terminal") - } - child := exec.Command(os.Args[0], "-test.run=^$") - child.Env = os.Environ() - if !slices.Contains(child.Env, servedVar+"=1") { - t.Fatal("what the serving controller starts does not carry its mark") - } -} // The forge applies one rule to a branch: the rule named for it, else the first glob that covers it. A stronger // rule later in the list is not what guards the branch, and is not read as if it were (the confirmation review). @@ -371,88 +120,9 @@ func TestTheRuleJudgedIsTheOneTheForgeApplies(t *testing.T) { } // An id the forge could not give keeps the id already recorded. -func TestAnUnknownIdentityKeepsTheRecordedOne(t *testing.T) { - open := theCatalogue(t) - ctx := t.Context() - if err := open.inventory.SetSourceIdentity(ctx, "sudo", 100); err != nil { - t.Fatal(err) - } - if err := open.inventory.SetSourceIdentity(ctx, "sudo", 0); err != nil { - t.Fatal(err) - } - if id, _ := open.inventory.SourceIdentity(ctx, "sudo"); id != 100 { - t.Fatalf("the recorded id became %d", id) - } -} // **The operator's Approve of a reviewed group registers a moved module from its new repository** (novox/hq ADR 0420 §7, // issue 1025). sudo is built from the catalogue; a merge into the repository of node-engine's module, announced by // mesh-delivery with the ask it merged on, brings its build from there. Accepted with the operator's Approve of the // merged head; refused with no kept merge, a warrant for other heads, a Decline, a person who is not the operator, and // through a verb. -func TestAModuleMovesOnTheOperatorsApproveOfTheMergeThatMovedIt(t *testing.T) { - const commit = "c0ffee0123456789" - for _, c := range []struct { - what string - change func(*anApprovedMerge) - unkept bool - request string // who asked: "merge", or "build" through a verb - moved bool - }{ - {what: "the operator's Approve of the merged head", request: "merge", moved: true}, - {what: "no merge mesh-delivery announced", unkept: true, request: "merge"}, - {what: "a warrant for other heads", request: "merge", change: func(a *anApprovedMerge) { - a.merge.Heads = map[string]string{"novox/mesh-host@aaaaaaaaaaaa": "e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5"} - }}, - {what: "a Decline", request: "merge", change: func(a *anApprovedMerge) { - a.warrant.Option, a.warrant.Label, a.merge.Option = "decline", "Decline", "decline" - }}, - {what: "a person who is not the operator", request: "merge", change: func(a *anApprovedMerge) { a.warrant.By.Who = "agent" }}, - {what: "a build asked through a verb", request: "build"}, - } { - open := theCatalogue(t) - ctx := t.Context() - inv := open.inventory - a := approvedMerge(t) - a.merge.Owner, a.merge.Repo, a.merge.Number, a.merge.HeadSHA = "novox", "mesh-host", 7, "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1" - if c.change != nil { - c.change(&a) - } - wasKept, wasAsk := keptDeliveryMerge, routerAskOf - keptDeliveryMerge = func(_ context.Context, owner, repo string, number int, head string) (deliveryMerge, bool, error) { - return a.merge, !c.unkept && owner == "novox" && repo == "mesh-host" && number == 7 && head == a.merge.HeadSHA, nil - } - routerAskOf = func(_ context.Context, _ *inventory.Inventory, id string) (*asks.Ask, *asks.Warrant, error) { - if id != a.merge.AskID { - t.Errorf("%s: the router was asked for %q", c.what, id) - } - return &a.ask, &a.warrant, nil - } - t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk }) - event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-host", "base": "main", "number": 7, - "head_sha": a.merge.HeadSHA, "merge_commit_sha": commit}) - if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-host", Branch: "main", Commit: commit, - Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil { - t.Fatal(err) - } - id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()) - if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-host", Seat: "git", - Path: "modules/sudo", Ref: "main", Commit: commit, For: c.request}); err != nil { - t.Fatal(err) - } - moved := onTrunk(id, "novox/mesh-host", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "2"}) - moved.Commit = commit - _, _, err := takeIn(ctx, inv, moved) - src, _ := inv.SourceOf(ctx, "sudo") - switch { - case c.moved && err != nil: - t.Errorf("%s: the move was refused: %v", c.what, err) - case c.moved && src.Repository != "novox/mesh-host": - t.Errorf("%s: sudo is built from %q", c.what, src.Repository) - case !c.moved && !errors.Is(err, errNotItsSource): - t.Errorf("%s: the move was not refused as ADR 0266 has it: %v", c.what, err) - case !c.moved && src.Repository != "novox/mesh-catalog": - t.Errorf("%s: a refused move changed sudo's source to %q", c.what, src.Repository) - } - } -} diff --git a/cmd/mesh-controller/build_test.go b/cmd/mesh-controller/build_test.go index aa70ff81..f7fb0490 100644 --- a/cmd/mesh-controller/build_test.go +++ b/cmd/mesh-controller/build_test.go @@ -1,13 +1,9 @@ package main import ( - "encoding/json" - "errors" - "strings" "testing" "time" - "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -15,126 +11,12 @@ import ( // the module registered with its source as the seat and path when the request said so — never the // URL. A definition naming an installation is recorded and not registered; a failure is recorded // and said. -func TestABuildHeardIsRecordedAndRegistered(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - manifest, _ := json.Marshal(map[string]any{"module": "shop", "version": "3"}) - m, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, link.BuildResult{ - ID: "b-1", Repository: "http://forge.internal:20000/novox/shop.git", Path: "modules/shop", - Ref: "main", On: "anchor", Commit: "abcdef0123", Manifest: manifest, - Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/shop"}, - })) - if err != nil { - t.Fatal(err) - } - if m.Module != "shop" { - t.Fatalf("registered %q", m.Module) - } - shelf, err := open.inventory.Catalogue(ctx) - if err != nil { - t.Fatal(err) - } - if _, held := shelf["shop"]; !held { - t.Fatal("the module a heard build produced is not in the catalogue") - } - src, err := open.inventory.SourceOf(ctx, "shop") - if err != nil || src.Seat != "git" || src.Repository != "novox/shop" || src.BuiltFrom != "abcdef0123" { - t.Fatalf("the source is the seat and the path, never the URL: %+v %v", src, err) - } - builds, err := open.inventory.Builds(ctx, "shop", 5) - if err != nil || len(builds) != 1 || builds[0].ID != "b-1" { - t.Fatalf("the build is not recorded once: %v %v", builds, err) - } - - named, _ := json.Marshal(map[string]any{"module": "idp", "version": "1", "resources": []any{ - map[string]any{"id": "server", "type": "container", "image": "x@sha256:aa", - "env": map[string]any{"KC_HOSTNAME": "https://login.mesh-one.be"}}}}) - _, _, err = takeIn(ctx, open.inventory, link.BuildResult{ - ID: "b-2", Repository: "/r", On: "anchor", Commit: "0123456789", Manifest: named}) - if err == nil || !strings.Contains(err.Error(), "does not register it") { - t.Fatalf("a definition naming an installation was taken in: %v", err) - } - if shelf, _ := open.inventory.Catalogue(ctx); shelf["idp"].Module != "" { - t.Fatal("the refused module was registered anyway") - } - if builds, _ := open.inventory.Builds(ctx, "idp", 5); len(builds) != 1 { - t.Fatalf("the refused build was not recorded: %v", builds) - } - - _, _, err = takeIn(ctx, open.inventory, link.BuildResult{ID: "b-3", Repository: "/r", On: "anchor", Failed: "no compiler"}) - if err == nil || !strings.Contains(err.Error(), "no compiler") { - t.Fatalf("a failure is said in the builder's words: %v", err) - } -} // novox/hq 04-ISSUES/215: a build asked at a commit is recorded as built from that commit, and the // module keeps following the branch it followed — a new one, the default branch. -func TestABuildAtACommitKeepsTheBranchTheModuleFollows(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - manifest, _ := json.Marshal(map[string]any{"module": "unifi", "version": "1"}) - result := func(id, ref, commit string) link.BuildResult { - return link.BuildResult{ID: id, Repository: "http://forge.internal:20000/novox/mesh-catalog.git", - Path: "modules/unifi", Ref: ref, On: "anchor", Commit: commit, Manifest: manifest, - Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}} - } - if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, result("b-1", "main", "1111111aaaa"))); err != nil { - t.Fatal(err) - } - if _, _, err := takeIn(ctx, open.inventory, result("b-2", "9c97a8a", "9c97a8a1d2c3")); err != nil { - t.Fatal(err) - } - src, err := open.inventory.SourceOf(ctx, "unifi") - if err != nil { - t.Fatal(err) - } - if src.Ref != "main" || src.BuiltFrom != "9c97a8a1d2c3" { - t.Errorf("after a build at a commit the module follows %q, built from %q; want main, 9c97a8a1d2c3", src.Ref, src.BuiltFrom) - } - - // One new to the catalogue, first built at a commit, follows the default branch. - other, _ := json.Marshal(map[string]any{"module": "letta", "version": "1"}) - r := result("b-3", "deadbeef", "deadbeefcafe") - r.Manifest, r.Path = other, "modules/letta" - if _, _, err := takeIn(ctx, open.inventory, r); err != nil { - t.Fatal(err) - } - if src, _ := open.inventory.SourceOf(ctx, "letta"); src.Ref != "" { - t.Errorf("a module first built at a commit follows %q, want the default branch", src.Ref) - } -} // novox/hq 04-ISSUES/219: an older request heard after a newer one is recorded and not registered, // so a push sends what the newer request built. -func TestAnOlderBuildHeardLaterDoesNotReplaceTheNewer(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - older := time.Date(2026, 10, 3, 21, 33, 45, 0, time.UTC) - newer := time.Date(2026, 10, 3, 21, 51, 57, 0, time.UTC) - result := func(asked time.Time, image string) link.BuildResult { - manifest, _ := json.Marshal(map[string]any{"module": "postgres", "version": image}) - return link.BuildResult{ID: link.NewBuildID(asked), Repository: "http://forge.internal:20000/novox/mesh-catalog.git", - Path: "modules/postgres", Ref: "main", On: "anchor", Commit: "efff5415", Manifest: manifest, - Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}} - } - if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, result(newer, "4bcd5f73"))); err != nil { - t.Fatal(err) - } - _, _, err := takeIn(ctx, open.inventory, result(older, "0ab07fa9")) - if !errors.Is(err, inventory.ErrSuperseded) { - t.Fatalf("the older request's outcome was taken in as current: %v", err) - } - shelf, err := open.inventory.Catalogue(ctx) - if err != nil { - t.Fatal(err) - } - if got := shelf["postgres"].Version; got != "4bcd5f73" { - t.Errorf("postgres is %q; want the newer request's 4bcd5f73", got) - } - if builds, _ := open.inventory.Builds(ctx, "postgres", 5); len(builds) != 2 { - t.Errorf("the late build was not recorded: %v", builds) - } -} func TestABuildIDSaysWhenItWasAsked(t *testing.T) { at := time.Date(2026, 10, 3, 21, 51, 57, 392539762, time.UTC) diff --git a/cmd/mesh-controller/bus_waits_integration_test.go b/cmd/mesh-controller/bus_waits_integration_test.go new file mode 100644 index 00000000..6fec986c --- /dev/null +++ b/cmd/mesh-controller/bus_waits_integration_test.go @@ -0,0 +1,70 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestASendHeldForTheBusStepIsFoundFromTheWalksItHolds(t *testing.T) { + now := time.Date(2026, 10, 8, 18, 30, 0, 0, time.UTC) + b := aBusOnAnchor() + walk := func(id, repo, note string, updated time.Time) inventory.Plan { + return inventory.Plan{ID: id, Repository: repo, Commit: "c0ffee001122", State: inventory.PlanRolling, Tier: 0, + Tiers: [][]string{{"mesh-host"}}, Modules: map[string]*inventory.PlanModule{"mesh-host": {}}, Note: note, + Updated: updated} + } + for _, c := range []struct { + name string + bus busPending + plans []inventory.Plan + first map[string]time.Time + want []string + since time.Time + }{ + {"a walk refused for the bus", b, + []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-27*time.Minute))}, nil, + []string{"plan-1"}, now.Add(-27 * time.Minute)}, + {"refused earlier than its last save, as this controller saw it", b, + []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-5*time.Minute))}, + map[string]time.Time{"plan-1": now.Add(-28 * time.Minute)}, []string{"plan-1"}, now.Add(-28 * time.Minute)}, + {"a walk refused for another reason", b, + []inventory.Plan{walk("plan-1", "novox/mesh-host", "tier 0: the build seat is paused — tried again", now)}, nil, + nil, time.Time{}}, + {"refused for an older bus build than the one held now", func() busPending { o := b; o.to = "99999999cccc"; return o }(), + []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}}, + {"the bus already runs the build held: its step started", func() busPending { + o := aBusOnAnchor() + o.from = map[string]string{"anchor": o.to} + return o + }(), []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}}, + {"a walk waiting for its delivery's word asks no send", b, func() []inventory.Plan { + p := walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now) + p.Delivery = &inventory.PlanDelivery{Awaits: "mesh-delivery"} + return []inventory.Plan{p} + }(), nil, nil, time.Time{}}, + } { + t.Run(c.name, func(t *testing.T) { + got := busWaitsOf(c.plans, c.bus, func(id string) time.Time { return c.first[id] }) + var ids []string + for _, w := range got.waits { + ids = append(ids, w.plan) + } + if strings.Join(ids, ",") != strings.Join(c.want, ",") { + t.Fatalf("held for the bus: %v, want %v", ids, c.want) + } + if len(c.want) > 0 { + if !got.waits[0].since.Equal(c.since) { + t.Errorf("waiting since %s, want %s", got.waits[0].since, c.since) + } + if strings.Join(got.machines, ",") != "anchor" || got.to != b.to || got.module != "nats" { + t.Errorf("behind %+v", got) + } + } + }) + } +} diff --git a/cmd/mesh-controller/bus_waits_test.go b/cmd/mesh-controller/bus_waits_test.go index 2a462b21..668d7cff 100644 --- a/cmd/mesh-controller/bus_waits_test.go +++ b/cmd/mesh-controller/bus_waits_test.go @@ -7,7 +7,6 @@ import ( "time" "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" ) // novox/hq issue 336: a send held because it would replace the bus outside its planned step waits for a @@ -27,65 +26,6 @@ func refusedNote(b busPending, tier int) string { return "tier " + string(rune('0'+tier)) + ": " + errBusWaits.Error() + ": " + held + " — tried again" } -func TestASendHeldForTheBusStepIsFoundFromTheWalksItHolds(t *testing.T) { - now := time.Date(2026, 10, 8, 18, 30, 0, 0, time.UTC) - b := aBusOnAnchor() - walk := func(id, repo, note string, updated time.Time) inventory.Plan { - return inventory.Plan{ID: id, Repository: repo, Commit: "c0ffee001122", State: inventory.PlanRolling, Tier: 0, - Tiers: [][]string{{"mesh-host"}}, Modules: map[string]*inventory.PlanModule{"mesh-host": {}}, Note: note, - Updated: updated} - } - for _, c := range []struct { - name string - bus busPending - plans []inventory.Plan - first map[string]time.Time - want []string - since time.Time - }{ - {"a walk refused for the bus", b, - []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-27*time.Minute))}, nil, - []string{"plan-1"}, now.Add(-27 * time.Minute)}, - {"refused earlier than its last save, as this controller saw it", b, - []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-5*time.Minute))}, - map[string]time.Time{"plan-1": now.Add(-28 * time.Minute)}, []string{"plan-1"}, now.Add(-28 * time.Minute)}, - {"a walk refused for another reason", b, - []inventory.Plan{walk("plan-1", "novox/mesh-host", "tier 0: the build seat is paused — tried again", now)}, nil, - nil, time.Time{}}, - {"refused for an older bus build than the one held now", func() busPending { o := b; o.to = "99999999cccc"; return o }(), - []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}}, - {"the bus already runs the build held: its step started", func() busPending { - o := aBusOnAnchor() - o.from = map[string]string{"anchor": o.to} - return o - }(), []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}}, - {"a walk waiting for its delivery's word asks no send", b, func() []inventory.Plan { - p := walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now) - p.Delivery = &inventory.PlanDelivery{Awaits: "mesh-delivery"} - return []inventory.Plan{p} - }(), nil, nil, time.Time{}}, - } { - t.Run(c.name, func(t *testing.T) { - got := busWaitsOf(c.plans, c.bus, func(id string) time.Time { return c.first[id] }) - var ids []string - for _, w := range got.waits { - ids = append(ids, w.plan) - } - if strings.Join(ids, ",") != strings.Join(c.want, ",") { - t.Fatalf("held for the bus: %v, want %v", ids, c.want) - } - if len(c.want) > 0 { - if !got.waits[0].since.Equal(c.since) { - t.Errorf("waiting since %s, want %s", got.waits[0].since, c.since) - } - if strings.Join(got.machines, ",") != "anchor" || got.to != b.to || got.module != "nats" { - t.Errorf("behind %+v", got) - } - } - }) - } -} - // **Said at once, not as lateness, and cleared when the step starts**: the walk held only for the bus raises // the bus's condition on the first tick, before any tier bound, naming what waits, the bus build from and to, // since when and `bus upgrade`, for the operator; S3 says nothing of it even past its bound; and the diff --git a/cmd/mesh-controller/busobjects_test.go b/cmd/mesh-controller/busobjects_test.go index 962a4659..349ab225 100644 --- a/cmd/mesh-controller/busobjects_test.go +++ b/cmd/mesh-controller/busobjects_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/calls_verb_integration_test.go b/cmd/mesh-controller/calls_verb_integration_test.go new file mode 100644 index 00000000..3efc354a --- /dev/null +++ b/cmd/mesh-controller/calls_verb_integration_test.go @@ -0,0 +1,36 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" +) + +func TestCallsIsServedAndSaysWhatItKeeps(t *testing.T) { + handlers, behind, err := seatToolHandlers() + if err != nil || len(behind) != 0 { + t.Fatalf("%v %v", behind, err) + } + calls, ok := handlers["calls"] + if !ok { + t.Fatal("calls is not served") + } + if _, err := calls(context.Background(), json.RawMessage(`{"node":"anchor"}`)); err == nil || + !strings.Contains(err.Error(), `"node"`) { + t.Errorf("calls took an argument it does not declare: %v", err) + } + if _, err := calls(context.Background(), json.RawMessage(`{"call":"call-0-0"}`)); err == nil || + !strings.Contains(err.Error(), "not across a restart") { + t.Errorf("an unknown call was not said plainly: %v", err) + } + got, err := calls(context.Background(), json.RawMessage(`{}`)) + if err != nil { + t.Fatal(err) + } + if _, listed := got.(map[string]any)["calls"]; !listed { + t.Errorf("calls answered %v", got) + } +} diff --git a/cmd/mesh-controller/calls_verb_test.go b/cmd/mesh-controller/calls_verb_test.go index 606a53f8..4ff06dcb 100644 --- a/cmd/mesh-controller/calls_verb_test.go +++ b/cmd/mesh-controller/calls_verb_test.go @@ -1,9 +1,6 @@ package main import ( - "context" - "encoding/json" - "strings" "testing" "time" @@ -52,28 +49,3 @@ func TestAPushAnswersBeforeItSends(t *testing.T) { } // `calls` is served, takes a call's id and nothing else, and says plainly when it holds no such call. -func TestCallsIsServedAndSaysWhatItKeeps(t *testing.T) { - handlers, behind, err := seatToolHandlers() - if err != nil || len(behind) != 0 { - t.Fatalf("%v %v", behind, err) - } - calls, ok := handlers["calls"] - if !ok { - t.Fatal("calls is not served") - } - if _, err := calls(context.Background(), json.RawMessage(`{"node":"anchor"}`)); err == nil || - !strings.Contains(err.Error(), `"node"`) { - t.Errorf("calls took an argument it does not declare: %v", err) - } - if _, err := calls(context.Background(), json.RawMessage(`{"call":"call-0-0"}`)); err == nil || - !strings.Contains(err.Error(), "not across a restart") { - t.Errorf("an unknown call was not said plainly: %v", err) - } - got, err := calls(context.Background(), json.RawMessage(`{}`)) - if err != nil { - t.Fatal(err) - } - if _, listed := got.(map[string]any)["calls"]; !listed { - t.Errorf("calls answered %v", got) - } -} diff --git a/cmd/mesh-controller/check_builds_integration_test.go b/cmd/mesh-controller/check_builds_integration_test.go new file mode 100644 index 00000000..37f33a6a --- /dev/null +++ b/cmd/mesh-controller/check_builds_integration_test.go @@ -0,0 +1,102 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestOnlyAWalksAskTakesAChecksBuildAndADryRunNever(t *testing.T) { + p := &inventory.Plan{Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{ + {Repository: "novox/shop", Number: 7}, {Repository: "http://forge.invalid/novox/web.git", Number: 3}, + {Repository: "novox/shop", Number: 7}, {Repository: "novox/shop"}}}} + asked := reuseAskOf(p) + if want := []link.PullRef{{Owner: "novox", Repo: "shop", Number: 7}, {Owner: "novox", Repo: "web", Number: 3}}; !reflect.DeepEqual(asked.Pulls, want) { + t.Fatalf("the walk names %+v", asked.Pulls) + } + walk := withReuse(t.Context(), asked) + if reuseAsked(walk, false) != asked { + t.Fatal("a walk's ask does not carry its leave") + } + if reuseAsked(walk, true) != nil { + t.Fatal("a dry run carries a leave to take a check's build") + } + if reuseAsked(t.Context(), false) != nil { + t.Fatal("an ask that is not a walk's carries a leave") + } +} + +func TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne(t *testing.T) { + withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) + g := aGateMesh(t) + ctx := t.Context() + inv := g.open.inventory + gateEvery = 0 + + digest := "sha256:" + strings.Repeat("c", 64) + reused := func(id, reference string) link.BuildResult { + manifest := mustJSON(t, catalogue.Manifest{Module: "app", Version: "c3", Resources: []map[string]any{ + {"id": "svc", "type": "container", "name": "app", "image": reference}}}) + return link.BuildResult{ID: id, Repository: "novox/mesh-catalog", Path: "modules/app", Ref: "main", On: "anchor", + Commit: "c3", Manifest: manifest, Trunk: "main", OnTrunk: true, Branches: []string{"main"}, + Made: []link.MadeArtifact{{Name: "server", Kind: "image", Reference: reference}}, + Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}, + Reused: "reused from the check of novox/mesh-catalog#9 at head abcdefabcdef (merged into trunk 123456123456): server " + digest, + ReusedFrom: &link.ReusedFrom{Pull: link.PullRef{Owner: "novox", Repo: "mesh-catalog", Number: 9}, + Head: "abcdefabcdef", Trunk: "123456123456", Key: "r1:" + strings.Repeat("1", 64)}} + } + if _, _, err := takeIn(ctx, inv, reused(link.NewBuildID(time.Now()), "store.internal:5000/scratch/app/server@"+digest)); err == nil || + !strings.Contains(err.Error(), "scratch") { + t.Fatalf("a build in the scratch namespace was registered: %v", err) + } + id := link.NewBuildID(time.Now().Add(time.Second)) + result := reused(id, "store.internal:5000/app/server@"+digest) + if _, _, err := takeIn(ctx, inv, result); err != nil { + t.Fatal(err) + } + current, err := inv.CurrentBuilds(ctx) + if err != nil || current["app"].Commit != "c3" { + t.Fatalf("the reused build is not what app is registered at: %+v %v", current["app"], err) + } + shelf, err := inv.Catalogue(ctx) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(shelf["app"].Resources) + if !strings.Contains(string(raw), "app/server@"+digest) { + t.Fatalf("app is registered as %s, not by the check's digest in its own namespace", raw) + } + + // The walk asked it: the walk keeps what its build did, and its gate judges the registered build. + asked := time.Now().UTC() + p := g.plan(t) + p.Commit = "c3" + p.Modules["app"] = &inventory.PlanModule{State: "asked", AskedAt: &asked, Build: id} + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + planBuiltSaying(ctx, g.open, "app", "c3", "", asked, id, result.MadeWords()) + p = g.plan(t) + if m := p.Modules["app"]; m.State != "built" || !strings.HasPrefix(m.Made, "reused from the check of novox/mesh-catalog#9") { + t.Fatalf("the walk keeps %+v", m) + } + for i := 0; i < 6; i++ { + advancePlans(ctx, g.open) + } + p = g.plan(t) + gate := p.Modules["app"].Gate + if gate == nil || gate.To != "c3" || gate.Verdict != inventory.GatePassed { + t.Fatalf("the gate judged %+v, not the reused build", gate) + } + if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c3" { + t.Fatalf("the first machine was sent %v, not the reused build", sent) + } +} diff --git a/cmd/mesh-controller/check_builds_test.go b/cmd/mesh-controller/check_builds_test.go index 0327b85a..7a339b51 100644 --- a/cmd/mesh-controller/check_builds_test.go +++ b/cmd/mesh-controller/check_builds_test.go @@ -2,12 +2,10 @@ package main import ( "context" - "encoding/json" "errors" "reflect" "strings" "testing" - "time" "github.com/novox/mesh-controller/internal/builder" "github.com/novox/mesh-controller/internal/catalogue" @@ -46,25 +44,6 @@ func TestACheckIsAskedToBuildWhatTheMergeWouldMove(t *testing.T) { // A walk's ask carries its leave to take a check's build, naming the pull requests of the merges it answers; a dry // run never does, and neither does an ask that is not a walk's. -func TestOnlyAWalksAskTakesAChecksBuildAndADryRunNever(t *testing.T) { - p := &inventory.Plan{Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{ - {Repository: "novox/shop", Number: 7}, {Repository: "http://forge.invalid/novox/web.git", Number: 3}, - {Repository: "novox/shop", Number: 7}, {Repository: "novox/shop"}}}} - asked := reuseAskOf(p) - if want := []link.PullRef{{Owner: "novox", Repo: "shop", Number: 7}, {Owner: "novox", Repo: "web", Number: 3}}; !reflect.DeepEqual(asked.Pulls, want) { - t.Fatalf("the walk names %+v", asked.Pulls) - } - walk := withReuse(t.Context(), asked) - if reuseAsked(walk, false) != asked { - t.Fatal("a walk's ask does not carry its leave") - } - if reuseAsked(walk, true) != nil { - t.Fatal("a dry run carries a leave to take a check's build") - } - if reuseAsked(t.Context(), false) != nil { - t.Fatal("an ask that is not a walk's carries a leave") - } -} // A walk ended lets go of its merges' check builds — only once it ended, after it took what it reuses. func TestAWalkEndedLetsGoOfItsMergesCheckBuilds(t *testing.T) { @@ -124,72 +103,6 @@ func TestAWalksBuildSaysWhetherItReusedOrBuilt(t *testing.T) { // module's namespace — and the walk keeps "reused from …" for it; its first-node gate judges that registered // build as it judges one the walk built. A scratch reference is refused. Needs the store (MESH_TEST_POSTGRES), and // is named in testdata/store-tests. -func TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne(t *testing.T) { - withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) - g := aGateMesh(t) - ctx := t.Context() - inv := g.open.inventory - gateEvery = 0 - - digest := "sha256:" + strings.Repeat("c", 64) - reused := func(id, reference string) link.BuildResult { - manifest := mustJSON(t, catalogue.Manifest{Module: "app", Version: "c3", Resources: []map[string]any{ - {"id": "svc", "type": "container", "name": "app", "image": reference}}}) - return link.BuildResult{ID: id, Repository: "novox/mesh-catalog", Path: "modules/app", Ref: "main", On: "anchor", - Commit: "c3", Manifest: manifest, Trunk: "main", OnTrunk: true, Branches: []string{"main"}, - Made: []link.MadeArtifact{{Name: "server", Kind: "image", Reference: reference}}, - Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}, - Reused: "reused from the check of novox/mesh-catalog#9 at head abcdefabcdef (merged into trunk 123456123456): server " + digest, - ReusedFrom: &link.ReusedFrom{Pull: link.PullRef{Owner: "novox", Repo: "mesh-catalog", Number: 9}, - Head: "abcdefabcdef", Trunk: "123456123456", Key: "r1:" + strings.Repeat("1", 64)}} - } - if _, _, err := takeIn(ctx, inv, reused(link.NewBuildID(time.Now()), "store.internal:5000/scratch/app/server@"+digest)); err == nil || - !strings.Contains(err.Error(), "scratch") { - t.Fatalf("a build in the scratch namespace was registered: %v", err) - } - id := link.NewBuildID(time.Now().Add(time.Second)) - result := reused(id, "store.internal:5000/app/server@"+digest) - if _, _, err := takeIn(ctx, inv, result); err != nil { - t.Fatal(err) - } - current, err := inv.CurrentBuilds(ctx) - if err != nil || current["app"].Commit != "c3" { - t.Fatalf("the reused build is not what app is registered at: %+v %v", current["app"], err) - } - shelf, err := inv.Catalogue(ctx) - if err != nil { - t.Fatal(err) - } - raw, _ := json.Marshal(shelf["app"].Resources) - if !strings.Contains(string(raw), "app/server@"+digest) { - t.Fatalf("app is registered as %s, not by the check's digest in its own namespace", raw) - } - - // The walk asked it: the walk keeps what its build did, and its gate judges the registered build. - asked := time.Now().UTC() - p := g.plan(t) - p.Commit = "c3" - p.Modules["app"] = &inventory.PlanModule{State: "asked", AskedAt: &asked, Build: id} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - planBuiltSaying(ctx, g.open, "app", "c3", "", asked, id, result.MadeWords()) - p = g.plan(t) - if m := p.Modules["app"]; m.State != "built" || !strings.HasPrefix(m.Made, "reused from the check of novox/mesh-catalog#9") { - t.Fatalf("the walk keeps %+v", m) - } - for i := 0; i < 6; i++ { - advancePlans(ctx, g.open) - } - p = g.plan(t) - gate := p.Modules["app"].Gate - if gate == nil || gate.To != "c3" || gate.Verdict != inventory.GatePassed { - t.Fatalf("the gate judged %+v, not the reused build", gate) - } - if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c3" { - t.Fatalf("the first machine was sent %v, not the reused build", sent) - } -} // A walk's leave names the builds its own pull requests' verdicts made at the head each merge merged — never an // earlier head's — read once per walk, and none when they cannot be read: the builder takes nothing a verdict does diff --git a/cmd/mesh-controller/checks_integration_test.go b/cmd/mesh-controller/checks_integration_test.go new file mode 100644 index 00000000..447c4e2e --- /dev/null +++ b/cmd/mesh-controller/checks_integration_test.go @@ -0,0 +1,56 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestACheckStoppedByItsKeyIsAnnouncedAsNoVerdict(t *testing.T) { + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + legacy, err := js.Conn().JetStream() + if err != nil { + t.Fatal(err) + } + was := checkEvents + checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy} + t.Cleanup(func() { checkEvents = was }) + heard := make(chan *nats.Msg, 4) + sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyChecked), heard) + if err != nil { + t.Fatal(err) + } + defer func() { _ = sub.Unsubscribe() }() + asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 3} + checked(t.Context(), link.BuildResult{ID: "stopped", Ref: "abc", Checked: asked, Failed: link.SupersededByKey}) + checked(t.Context(), link.BuildResult{ID: "broken", Ref: "abc", Checked: asked, Failed: "the facts snapshot cannot be read"}) + select { + case m := <-heard: + var c link.Checked + _ = json.Unmarshal(m.Data, &c) + if c.ID != "broken" || c.Verdict != "error" { + t.Fatalf("the first verdict said is %+v; want only the broken check's error", c) + } + case <-time.After(5 * time.Second): + t.Fatal("nothing was said") + } + select { + case m := <-heard: + t.Fatalf("a second verdict was said: %s", m.Data) + case <-time.After(300 * time.Millisecond): + } +} diff --git a/cmd/mesh-controller/checks_queue_test.go b/cmd/mesh-controller/checks_queue_test.go index 98dfb31a..d79c6207 100644 --- a/cmd/mesh-controller/checks_queue_test.go +++ b/cmd/mesh-controller/checks_queue_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/checks_test.go b/cmd/mesh-controller/checks_test.go index dd6a3b8e..24e37f3d 100644 --- a/cmd/mesh-controller/checks_test.go +++ b/cmd/mesh-controller/checks_test.go @@ -1,16 +1,9 @@ package main import ( - "encoding/json" "errors" "strings" "testing" - "time" - - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/testbus" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" @@ -298,44 +291,3 @@ func TestARecheckedHeadsDirectVerdictNamesItsTrunk(t *testing.T) { // A check stopped by its pull request's key moving on says no verdict (issue 974, review 2): an error said for it would name the // live head, and the newer request is the one that judges. A check that failed for any other reason still says its error. -func TestACheckStoppedByItsKeyIsAnnouncedAsNoVerdict(t *testing.T) { - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - legacy, err := js.Conn().JetStream() - if err != nil { - t.Fatal(err) - } - was := checkEvents - checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy} - t.Cleanup(func() { checkEvents = was }) - heard := make(chan *nats.Msg, 4) - sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyChecked), heard) - if err != nil { - t.Fatal(err) - } - defer func() { _ = sub.Unsubscribe() }() - asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 3} - checked(t.Context(), link.BuildResult{ID: "stopped", Ref: "abc", Checked: asked, Failed: link.SupersededByKey}) - checked(t.Context(), link.BuildResult{ID: "broken", Ref: "abc", Checked: asked, Failed: "the facts snapshot cannot be read"}) - select { - case m := <-heard: - var c link.Checked - _ = json.Unmarshal(m.Data, &c) - if c.ID != "broken" || c.Verdict != "error" { - t.Fatalf("the first verdict said is %+v; want only the broken check's error", c) - } - case <-time.After(5 * time.Second): - t.Fatal("nothing was said") - } - select { - case m := <-heard: - t.Fatalf("a second verdict was said: %s", m.Data) - case <-time.After(300 * time.Millisecond): - } -} diff --git a/cmd/mesh-controller/condition_looks_test.go b/cmd/mesh-controller/condition_looks_test.go index 1c79015a..9295188c 100644 --- a/cmd/mesh-controller/condition_looks_test.go +++ b/cmd/mesh-controller/condition_looks_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/conditions_brief_integration_test.go b/cmd/mesh-controller/conditions_brief_integration_test.go new file mode 100644 index 00000000..5edd1090 --- /dev/null +++ b/cmd/mesh-controller/conditions_brief_integration_test.go @@ -0,0 +1,43 @@ +//go:build integration + +package main + +import ( + "fmt" + "strings" + "testing" + "time" +) + +func TestAVerdictCarriesAProbesFindingsInBrief(t *testing.T) { + var found []string + for i := 0; i < 689; i++ { + found = append(found, fmt.Sprintf("delivery d-%012x has been checking for 13h, past its bound of 45m", i)) + } + run := doctorRun{Run: "run-1", Probes: []probeVerdict{{ID: "D14", Verdict: "failed", Found: found}, + {ID: "D1", Verdict: "passed"}}} + answer, err := verdictAnswer(run, time.Now(), "") + if err != nil { + t.Fatal(err) + } + brief := answer["run"].(doctorRun).Probes[0] + if len(brief.Found) != overviewFindings || brief.FoundCount != 689 || !strings.Contains(brief.More, "probe=D14") { + t.Fatalf("the verdict carries %d findings of D14, count %d, more %q", len(brief.Found), brief.FoundCount, brief.More) + } + if len(run.Probes[0].Found) != 689 { + t.Fatal("the overview cut the run it was made from") + } + whole, err := verdictAnswer(run, time.Now(), "d14") + if err != nil { + t.Fatal(err) + } + if p := whole["run"].(doctorRun).Probes; len(p) != 1 || len(p[0].Found) != 689 || p[0].More != "" { + t.Fatalf("doctor probe=D14 answered %+v", p) + } + if _, err := verdictAnswer(run, time.Now(), "D99"); err == nil { + t.Fatal("a probe the run does not have was answered") + } + if !strings.Contains(doctorText(answer), "probe=D14` gives them all") { + t.Fatalf("the text does not say how to read them all:\n%s", doctorText(answer)) + } +} diff --git a/cmd/mesh-controller/conditions_brief_test.go b/cmd/mesh-controller/conditions_brief_test.go index 8fae4898..135c6336 100644 --- a/cmd/mesh-controller/conditions_brief_test.go +++ b/cmd/mesh-controller/conditions_brief_test.go @@ -74,35 +74,3 @@ func TestAListOfConditionsCarriesEachInBrief(t *testing.T) { // The self-check's verdict carries at most overviewFindings of one probe's findings, with their count and // how to read them all; `doctor probe=` gives that probe whole (novox/hq issue 314). -func TestAVerdictCarriesAProbesFindingsInBrief(t *testing.T) { - var found []string - for i := 0; i < 689; i++ { - found = append(found, fmt.Sprintf("delivery d-%012x has been checking for 13h, past its bound of 45m", i)) - } - run := doctorRun{Run: "run-1", Probes: []probeVerdict{{ID: "D14", Verdict: "failed", Found: found}, - {ID: "D1", Verdict: "passed"}}} - answer, err := verdictAnswer(run, time.Now(), "") - if err != nil { - t.Fatal(err) - } - brief := answer["run"].(doctorRun).Probes[0] - if len(brief.Found) != overviewFindings || brief.FoundCount != 689 || !strings.Contains(brief.More, "probe=D14") { - t.Fatalf("the verdict carries %d findings of D14, count %d, more %q", len(brief.Found), brief.FoundCount, brief.More) - } - if len(run.Probes[0].Found) != 689 { - t.Fatal("the overview cut the run it was made from") - } - whole, err := verdictAnswer(run, time.Now(), "d14") - if err != nil { - t.Fatal(err) - } - if p := whole["run"].(doctorRun).Probes; len(p) != 1 || len(p[0].Found) != 689 || p[0].More != "" { - t.Fatalf("doctor probe=D14 answered %+v", p) - } - if _, err := verdictAnswer(run, time.Now(), "D99"); err == nil { - t.Fatal("a probe the run does not have was answered") - } - if !strings.Contains(doctorText(answer), "probe=D14` gives them all") { - t.Fatalf("the text does not say how to read them all:\n%s", doctorText(answer)) - } -} diff --git a/cmd/mesh-controller/conditions_integration_test.go b/cmd/mesh-controller/conditions_integration_test.go new file mode 100644 index 00000000..a2fb25f0 --- /dev/null +++ b/cmd/mesh-controller/conditions_integration_test.go @@ -0,0 +1,32 @@ +//go:build integration + +package main + +import ( + "errors" + "strings" + "testing" +) + +func TestUnreadableConditionsAreNotAWellMesh(t *testing.T) { + open := aMesh(t) + _, store := withConditionsInMemory(t) + store.SetFail(errors.New("the bus is away")) + asked, err := theThreeQuestions(t.Context(), open) + if err != nil { + t.Fatal(err) + } + if asked.well() || asked.conditionsUnread == "" { + t.Fatalf("well with its conditions unread: %+v", asked.conditionsUnread) + } + said := printed(t, func() error { return printStatus(asked) }) + if !strings.HasPrefix(said, "the open conditions could NOT be read") || strings.Contains(said, "no open conditions") { + t.Fatalf("%s", said) + } + store.SetFail(nil) + asked, _ = theThreeQuestions(t.Context(), open) + said = printed(t, func() error { return printStatus(asked) }) + if asked.well() && !strings.Contains(said, "no open conditions;") { + t.Fatalf("the all-well sentence does not say no conditions are open:\n%s", said) + } +} diff --git a/cmd/mesh-controller/conditions_test.go b/cmd/mesh-controller/conditions_test.go index 35bdefe2..24e1ee7b 100644 --- a/cmd/mesh-controller/conditions_test.go +++ b/cmd/mesh-controller/conditions_test.go @@ -1,7 +1,6 @@ package main import ( - "errors" "strings" "testing" @@ -83,25 +82,3 @@ func TestASilenceThroughTheVerbHoldsAndTheConditionStaysOpen(t *testing.T) { } // **Conditions that cannot be read are not none open**: status says so, and is not well. -func TestUnreadableConditionsAreNotAWellMesh(t *testing.T) { - open := aMesh(t) - _, store := withConditionsInMemory(t) - store.SetFail(errors.New("the bus is away")) - asked, err := theThreeQuestions(t.Context(), open) - if err != nil { - t.Fatal(err) - } - if asked.well() || asked.conditionsUnread == "" { - t.Fatalf("well with its conditions unread: %+v", asked.conditionsUnread) - } - said := printed(t, func() error { return printStatus(asked) }) - if !strings.HasPrefix(said, "the open conditions could NOT be read") || strings.Contains(said, "no open conditions") { - t.Fatalf("%s", said) - } - store.SetFail(nil) - asked, _ = theThreeQuestions(t.Context(), open) - said = printed(t, func() error { return printStatus(asked) }) - if asked.well() && !strings.Contains(said, "no open conditions;") { - t.Fatalf("the all-well sentence does not say no conditions are open:\n%s", said) - } -} diff --git a/cmd/mesh-controller/config_moved_integration_test.go b/cmd/mesh-controller/config_moved_integration_test.go new file mode 100644 index 00000000..7c930d12 --- /dev/null +++ b/cmd/mesh-controller/config_moved_integration_test.go @@ -0,0 +1,467 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/overlay" +) + +func aConfiguredMesh(t *testing.T) (*stores, configuredMesh, map[string]map[string]any) { + t.Helper() + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + for i, name := range []string{"workstation", "desktop"} { + record, err := inv.AddNode(ctx, name) + if err != nil { + t.Fatal(err) + } + if err := inv.SetPlace(ctx, name, name+".example:51820", "here", false, fmt.Sprintf("10.77.0.%d", i+3)); err != nil { + t.Fatal(err) + } + if err := inv.RecordProfile(ctx, record.ID, map[string]any{"capabilities": []any{ + map[string]any{"name": "container-runtime", "present": true}, + map[string]any{"name": "systemd", "present": true}}}); err != nil { + t.Fatal(err) + } + if err := inv.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil { + t.Fatal(err) + } + if err := inv.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, name, overlay.Name); err != nil { + t.Fatal(err) + } + } + raw, err := os.ReadFile(filepath.Join("testdata", "config-moved", "mesh.json")) + if err != nil { + t.Fatal(err) + } + var w configuredMesh + if err := json.Unmarshal(raw, &w); err != nil { + t.Fatal(err) + } + list, err := catalogue.ParsePredefined(w.Predefined) + if err != nil { + t.Fatal(err) + } + catalogue.UsePredefined(list) + served := map[string]map[string]any{} + for _, text := range w.Modules { + var shape map[string]json.RawMessage + if err := json.Unmarshal(text, &shape); err != nil { + t.Fatal(err) + } + var name string + _ = json.Unmarshal(shape["module"], &name) + if s, ok := shape["serves-checked"]; ok { + var serves map[string]any + if err := json.Unmarshal(s, &serves); err != nil { + t.Fatal(err) + } + served[name] = serves + delete(shape, "serves-checked") + } + text, err := json.Marshal(shape) + if err != nil { + t.Fatal(err) + } + m, err := catalogue.ParseManifest(text) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + register(t, open, m) + } + for _, l := range w.Layers { + values, err := json.Marshal(l.Values) + if err != nil { + t.Fatal(err) + } + if l.Node == "" { + inv.ExecForTest(t, `insert into settings (node, module, values) values (null, $1, $2)`, l.Module, string(values)) + continue + } + inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`, + l.Node, l.Module, string(values)) + } + for _, module := range movedKeys(w.On) { + for _, node := range w.On[module] { + if _, err := assign(ctx, open, node, module); err != nil { + t.Fatalf("assign %s %s: %v", node, module, err) + } + } + } + return open, w, served +} + +func TestEveryMachineIsToldTheSameAfterTheMoveOfConfig(t *testing.T) { + open, w, served := aConfiguredMesh(t) + ctx := t.Context() + readBackTries = 0 + t.Cleanup(func() { readBackTries = 15 }) + inv := open.inventory + config := &standInMeshConfig{open: open, values: map[string]any{}} + inv.ReadsConfigFrom(config.laid) + before := toldEveryMachine(t, open, served, w.On) + listedBefore := effectiveEverywhere(t, open, w) + factsBefore, err := gatherFacts(ctx, open, "2.11.17") + if err != nil { + t.Fatal(err) + } + + asked, err := runConfigMove(ctx, inv, config, false, "the move of config, run by a test", "") + if err != nil || !asked.DryRun || asked.Switched || len(asked.Problems) > 0 || len(config.sets) > 0 { + t.Fatalf("the move, not confirmed, says %+v (%v) and set %v", asked, err, config.sets) + } + // nodered's live layer says the file's name for its time zone: set under the config key (issue 751). + if s := setAt(asked, "module.anchor.nodered.time-zone"); len(s) != 1 || s[0].Value != "Europe/Brussels" || + !strings.HasSuffix(s[0].From[0], ": timeZone") { + t.Errorf("nodered's time zone moves as %+v", s) + } + for _, s := range asked.Sets { + if s.Key == "timeZone" { + t.Errorf("a key is set under the file's name: %+v", s) + } + } + // claude-code's whole-mesh managed settings become mesh config for agent-managed-settings (ADR 0348's + // consequences), and its whole-mesh layer sets no module value. + if s := setAt(asked, "mesh.agent-managed-settings"); len(s) != 1 || s[0].From[0] != "claude-code on the whole mesh: managed_settings" { + t.Errorf("claude-code's managed settings move as %+v", s) + } + for _, node := range fourMachines { + if s := setAt(asked, "module."+node+".claude-code.managed_settings"); len(s) > 0 { + t.Errorf("claude-code's managed settings are also set on %s: %+v", node, s) + } + } + // A whole-mesh layer of a value no predefined key holds goes to each machine it is assigned to, under that + // machine's layer: power's on the workstation is the workstation's, its own elsewhere. + if s := setAt(asked, "module.workstation.power.handle-power-key"); len(s) != 1 || s[0].Value != "suspend" { + t.Errorf("power's key on the workstation moves as %+v", s) + } + if s := setAt(asked, "module.anchor.power.handle-power-key"); len(s) != 1 || s[0].Value != "poweroff" { + t.Errorf("power's key on the anchor moves as %+v", s) + } + for _, s := range asked.Sets { + if s.Level == "node" { + t.Errorf("a machine's layer became node config: %+v", s) + } + } + var stays []string + for _, s := range asked.Stays { + stays = append(stays, strings.Fields(s)[0]) + } + sort.Strings(stays) + if want := []string{"bazarr", "bazarr", "bookshelf", "jackett", "kernel", "kometa", "lidarr", "nzbget", "ombi", + "plex", "qbittorrent", "radarr", "sonarr", "tautulli"}; len(stays) < 10 || !containsString(stays, "kernel") { + t.Errorf("the layers that stay are %v, want those of %v", stays, want) + } + if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) { + t.Fatal("a dry run changed what a machine is told") + } + + moved, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "release 57 is delivered") + if err != nil || moved.DryRun || !moved.Switched { + t.Fatalf("the move says %+v (%v)", moved, err) + } + if len(config.sets) != len(asked.Sets) { + t.Fatalf("the move set %d values, and said it would set %d", len(config.sets), len(asked.Sets)) + } + if held := inv.QueryForTest(t, `select module from settings s where values <> '{}' + and module in (select name from module where manifest ? 'config')`); len(held) != 0 { + t.Fatalf("a layer of a module in the config form still holds values: %v", held) + } + after := toldEveryMachine(t, open, served, w.On) + for _, node := range fourMachines { + if after[node] != before[node] { + t.Errorf("%s is told something else after the move:\n%s\nwas:\n%s", node, after[node], before[node]) + } + } + if listedAfter := effectiveEverywhere(t, open, w); !reflect.DeepEqual(listedAfter, listedBefore) { + t.Errorf("settings lists other values after the move:\n%v\nwas:\n%v", listedAfter, listedBefore) + } + // And config lists what settings listed: each module's resolved value on each machine is the value settings + // said before the move. + resolved, err := config.Resolved(ctx) + if err != nil { + t.Fatal(err) + } + for _, r := range resolved { + said, listed := listedBefore[r.Node+"."+r.Module+"."+r.Key] + if r.Source == "unset" { + if listed && said != nil { + t.Errorf("%s of %s on %s is unset in config, and settings said %v", r.Key, r.Module, r.Node, said) + } + continue + } + var v any + _ = json.Unmarshal(r.Value, &v) + if !listed || !reflect.DeepEqual(v, said) { + t.Errorf("config resolves %s of %s on %s as %v, and settings said %v", r.Key, r.Module, r.Node, v, said) + } + } + + // The facts snapshot carries mesh-config's values for the merge gate, as each machine's layer of each module, and + // each machine's declaration composes to the digest it did before the move. + factsAfter, err := gatherFacts(ctx, open, "2.11.17") + if err != nil { + t.Fatal(err) + } + // Machines are named in a snapshot by stand-ins of its own; each is found by what it composes. + digests := func(f snapshotFacts) ([]string, string) { + var out []string + zone := "" + for _, m := range f.Machines { + out = append(out, m.Declaration.Digest) + for _, l := range m.Settings { + if l.Module == "nodered" { + zone = compactJSON(l.Values["time-zone"]) + } + } + } + sort.Strings(out) + return out, zone + } + was, _ := digests(factsBefore) + now, zone := digests(factsAfter) + if zone != `"Europe/Brussels"` { + t.Errorf("the facts snapshot does not carry nodered's time zone from mesh-config: %s", zone) + } + if len(now) != len(fourMachines) || now[0] == "" || !reflect.DeepEqual(now, was) { + t.Errorf("the declarations in the facts snapshot are %v after the move, %v before", now, was) + } + for _, s := range factsAfter.Settings { + if _, isConfig := map[string]bool{"betterbird": true, "claude-code": true, "power": true, "xdg": true}[s.Module]; isConfig && + len(s.Values) > 0 { + t.Errorf("the facts snapshot still carries the whole mesh's layer of %s: %v", s.Module, s.Values) + } + } + + config.sets = nil + again, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "again") + if err != nil || len(config.sets) != 0 || len(again.Moved) != 0 { + t.Fatalf("asked again, the move set %v and says %+v (%v)", config.sets, again, err) + } + if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) { + t.Fatal("the move asked again changed what a machine is told") + } +} + +func TestTheMoveOfConfigRefusesWhatWouldChangeAMachine(t *testing.T) { + open, _, _ := aConfiguredMesh(t) + ctx := t.Context() + readBackTries = 0 + t.Cleanup(func() { readBackTries = 15 }) + inv := open.inventory + config := &standInMeshConfig{open: open, values: map[string]any{}} + inv.ReadsConfigFrom(config.laid) + + inv.ExecForTest(t, `update settings set values = values || '{"endpoints": {"web": {"reach": "public"}}}' + where module = 'nodered'`) + inv.ExecForTest(t, `update settings set values = values || '{"colour": "blue"}' where module = 'unifi'`) + inv.ExecForTest(t, `update settings set values = '{"font-size": 900}' where module = 'dunst'`) + answer, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered") + if err == nil || answer.Switched || len(config.sets) > 0 { + t.Fatalf("the move went ahead: %+v (%v), set %v", answer, err, config.sets) + } + for _, want := range []string{"nodered on anchor still holds placement (endpoints): run move-placement first", + `unifi on anchor says "colour", which unifi does not declare`, "font-size at dunst's own value on workstation"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal does not say %q: %v", want, err) + } + } + inv.ExecForTest(t, `update settings set values = values - 'endpoints' where module = 'nodered'`) + inv.ExecForTest(t, `update settings set values = values - 'colour' where module = 'unifi'`) + inv.ExecForTest(t, `update settings set values = '{"font-size": 13, "width": 500}' where module = 'dunst'`) + + // Set in mesh-config before the move, on a module whose layers say nothing of it: the switch would lay it. + config.values["module.anchor.grafana."+movedKeys(func() map[string]catalogue.SettingDeclaration { + shelf, _ := inv.Catalogue(ctx) + return shelf["grafana"].Settings + }())[0]] = "set before the move" + answer, err = runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered") + if err == nil || !strings.Contains(err.Error(), "grafana on anchor resolves") || answer.Switched { + t.Fatalf("a value no layer says was laid by the switch: %+v (%v)", answer, err) + } + if moved, _ := inv.ConfigMoved(ctx); moved.Moved { + t.Fatal("the switch is on after a refused move") + } + if held := inv.QueryForTest(t, `select count(*) from settings where values <> '{}'`); held[0][0] == "0" { + t.Fatal("a refused move emptied the layers") + } +} + +func TestTheSettingsVerbForwardsToConfigOnceTheValuesMoved(t *testing.T) { + open, w, served := aConfiguredMesh(t) + ctx := t.Context() + inv := open.inventory + readBackTries = 0 + t.Cleanup(func() { readBackTries = 15 }) + config := &standInMeshConfig{open: open, values: map[string]any{}} + inv.ReadsConfigFrom(config.laid) + savedRead, savedAsk := readConfigValues, configAsk + readConfigValues, configAsk = config.laid, config.ask + t.Cleanup(func() { readConfigValues, configAsk = savedRead, savedAsk }) + if _, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil { + t.Fatal(err) + } + + shown := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "nodered", "--node", "anchor"}) }) + for _, want := range []string{`"time-zone": "Europe/Brussels"`, `time-zone = "Europe/Brussels" (module)`, + "mesh-config.config", `"mqtt": {`} { + if !strings.Contains(shown, want) { + t.Errorf("settings show does not say %q:\n%s", want, shown) + } + } + if strings.Contains(shown, "timeZone") { + t.Errorf("settings show still says the file's name:\n%s", shown) + } + + config.asked = nil + said := stdoutOf(t, func() error { + return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 500}`, "--node", "workstation"}) + }) + if config.values["module.workstation.dunst.font-size"] != float64(12) || !strings.Contains(said, "~ font-size") || + containsString(config.asked, "set module.workstation.dunst.width") { + t.Fatalf("settings set reached mesh-config as %v (%v):\n%s", config.values, config.asked, said) + } + told := toldEveryMachine(t, open, served, w.On)["workstation"] + if !strings.Contains(told, "12") { + t.Error("the workstation is not told the font size set through settings") + } + if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`, "--node", "workstation"}); err == nil || + !strings.Contains(err.Error(), "would no longer set width") { + t.Fatalf("a set dropping a value without --replace: %v", err) + } + if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`}); err == nil || + !strings.Contains(err.Error(), "no own value for the whole mesh") { + t.Fatalf("a module value for the whole mesh: %v", err) + } + + t.Setenv(verbVar, "settings") + for _, c := range []struct{ module, node, values, key, held, why string }{ + // A file root reads: refused as before the move (novox/hq issue 339). + {"hostname", "anchor", `{"hostname": "elsewhere"}`, "module.anchor.hostname.hostname", "Ace", "issue 339"}, + {"messenger", "laptop", `{"desktop-machines": ["g14", "shanks"], "time-zone": "UTC"}`, + "module.laptop.messenger.time-zone", "Europe/Brussels", "issue 339"}, + } { + err := settingsCommand(ctx, []string{"set", c.module, c.values, "--node", c.node}) + if err == nil || !strings.Contains(err.Error(), c.why) || config.values[c.key] != c.held { + t.Fatalf("a trusted value of %s through a verb: %v, holds %v", c.module, err, config.values[c.key]) + } + } + // And a trusted value no file makes the terminal's is refused as trusted, since mesh-config takes the controller's + // word for it (ADR 0348 §8). + if err := refuseTrustedThroughAVerb(ctx, inv, "messenger", "laptop", []string{"time-zone"}); err == nil || + !strings.Contains(err.Error(), "ADR 0348 §8") { + t.Fatalf("a trusted value through a verb: %v", err) + } + if err := refuseTrustedThroughAVerb(ctx, inv, "dunst", "workstation", []string{"width"}); err != nil { + t.Fatalf("a value nobody trusts through a verb: %v", err) + } + stdoutOf(t, func() error { + return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 600}`, "--node", "workstation"}) + }) + if config.values["module.workstation.dunst.width"] != float64(600) { + t.Fatalf("a value nobody trusts was not set through a verb: %v", config.values["module.workstation.dunst.width"]) + } + t.Setenv(verbVar, "") + + stdoutOf(t, func() error { return settingsCommand(ctx, []string{"clear", "dunst", "--node", "workstation"}) }) + for _, k := range []string{"font-size", "width"} { + if _, held := config.values["module.workstation.dunst."+k]; held { + t.Errorf("settings clear left dunst's %s on the workstation", k) + } + } + + config.asked = nil + proposed := stdoutOf(t, func() error { + return settingsCommand(ctx, []string{"propose", "hostname", `{"hostname": "elsewhere"}`, "--node", "anchor"}) + }) + if !containsString(config.asked, "propose module.anchor.hostname.hostname") || !strings.Contains(proposed, "p-1") { + t.Fatalf("settings propose reached mesh-config as %v:\n%s", config.asked, proposed) + } + + config.asked = nil + kept := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "kernel", "--node", "laptop"}) }) + if !strings.Contains(kept, "transparent-huge-pages") || len(config.asked) > 0 { + t.Fatalf("a module in the settings form is shown as %q, asking mesh-config %v", kept, config.asked) + } +} + +func TestAnApprovalRaisedBeforeTheMoveOfConfigIsRefusedAfterIt(t *testing.T) { + open, _, _ := aConfiguredMesh(t) + ctx := t.Context() + readBackTries = 0 + t.Cleanup(func() { readBackTries = 15 }) + inv := open.inventory + config := &standInMeshConfig{open: open, values: map[string]any{}} + inv.ReadsConfigFrom(config.laid) + shelf, err := inv.Catalogue(ctx) + if err != nil { + t.Fatal(err) + } + key := movedKeys(func() map[string]catalogue.SettingDeclaration { return shelf["grafana"].Settings }())[0] + values := map[string]any{key: "raised before the move"} + p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(), + Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} + if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has { + t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err) + } + if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched { + t.Fatalf("the move says %+v (%v)", moved, err) + } + if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads { + t.Fatalf("grafana does not read config after the move: %v %v", reads, err) + } + _, err = setLayerIn(open)(ctx, p, "s30", "approved by the operator") + if err == nil || !strings.Contains(err.Error(), "moved to mesh-config since the operator was shown the change") || + !strings.Contains(err.Error(), "propose it again") { + t.Fatalf("the stale approval: %v", err) + } + if layer, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has && len(layer) > 0 { + t.Errorf("the refused approval wrote the layer: %v %v %v", layer, has, err) + } + if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads { + t.Errorf("the refused approval turned ReadsConfig off: %v %v", reads, err) + } +} + +func TestAPlacementOnlyApprovalRaisedBeforeTheMoveIsSetAfterIt(t *testing.T) { + open, _, _ := aConfiguredMesh(t) + ctx := t.Context() + readBackTries = 0 + t.Cleanup(func() { readBackTries = 15 }) + inv := open.inventory + config := &standInMeshConfig{open: open, values: map[string]any{}} + inv.ReadsConfigFrom(config.laid) + values := map[string]any{catalogue.PlacesSetting: map[string]any{"data": "/srv/grafana-data"}} + p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(), + Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} + if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has { + t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err) + } + if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched { + t.Fatalf("the move says %+v (%v)", moved, err) + } + if _, err := setLayerIn(open)(ctx, p, "s31", "approved by the operator"); err != nil { + t.Fatalf("the placement-only approval after the move: %v", err) + } + if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || !has { + t.Errorf("the approval did not set the layer: %v %v", has, err) + } + if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads { + t.Errorf("the approval turned ReadsConfig off: %v %v", reads, err) + } +} diff --git a/cmd/mesh-controller/config_moved_test.go b/cmd/mesh-controller/config_moved_test.go index 75657f8b..390a51e7 100644 --- a/cmd/mesh-controller/config_moved_test.go +++ b/cmd/mesh-controller/config_moved_test.go @@ -4,18 +4,12 @@ import ( "context" "encoding/json" "fmt" - "os" - "path/filepath" - "reflect" - "sort" "strings" "testing" - "time" "github.com/novox/mesh-controller/internal/catalogue" snapshot "github.com/novox/mesh-controller/internal/facts" "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/overlay" ) // configuredMesh is every live settings layer (issue 664) — read with the settings verb on 2026-10-15, on the four @@ -42,94 +36,6 @@ var fourMachines = []string{"anchor", "laptop", "workstation", "desktop"} // aConfiguredMesh is aMesh with two machines more, every module of the configured mesh registered and assigned, and // its layers held as the store holds them once move-placement has run. -func aConfiguredMesh(t *testing.T) (*stores, configuredMesh, map[string]map[string]any) { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - for i, name := range []string{"workstation", "desktop"} { - record, err := inv.AddNode(ctx, name) - if err != nil { - t.Fatal(err) - } - if err := inv.SetPlace(ctx, name, name+".example:51820", "here", false, fmt.Sprintf("10.77.0.%d", i+3)); err != nil { - t.Fatal(err) - } - if err := inv.RecordProfile(ctx, record.ID, map[string]any{"capabilities": []any{ - map[string]any{"name": "container-runtime", "present": true}, - map[string]any{"name": "systemd", "present": true}}}); err != nil { - t.Fatal(err) - } - if err := inv.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil { - t.Fatal(err) - } - if err := inv.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, name, overlay.Name); err != nil { - t.Fatal(err) - } - } - raw, err := os.ReadFile(filepath.Join("testdata", "config-moved", "mesh.json")) - if err != nil { - t.Fatal(err) - } - var w configuredMesh - if err := json.Unmarshal(raw, &w); err != nil { - t.Fatal(err) - } - list, err := catalogue.ParsePredefined(w.Predefined) - if err != nil { - t.Fatal(err) - } - catalogue.UsePredefined(list) - served := map[string]map[string]any{} - for _, text := range w.Modules { - var shape map[string]json.RawMessage - if err := json.Unmarshal(text, &shape); err != nil { - t.Fatal(err) - } - var name string - _ = json.Unmarshal(shape["module"], &name) - if s, ok := shape["serves-checked"]; ok { - var serves map[string]any - if err := json.Unmarshal(s, &serves); err != nil { - t.Fatal(err) - } - served[name] = serves - delete(shape, "serves-checked") - } - text, err := json.Marshal(shape) - if err != nil { - t.Fatal(err) - } - m, err := catalogue.ParseManifest(text) - if err != nil { - t.Fatalf("%s: %v", name, err) - } - register(t, open, m) - } - for _, l := range w.Layers { - values, err := json.Marshal(l.Values) - if err != nil { - t.Fatal(err) - } - if l.Node == "" { - inv.ExecForTest(t, `insert into settings (node, module, values) values (null, $1, $2)`, l.Module, string(values)) - continue - } - inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`, - l.Node, l.Module, string(values)) - } - for _, module := range movedKeys(w.On) { - for _, node := range w.On[module] { - if _, err := assign(ctx, open, node, module); err != nil { - t.Fatalf("assign %s %s: %v", node, module, err) - } - } - } - return open, w, served -} // standInMeshConfig is mesh-config as this test holds it: each value at its level, and resolved from the catalogue // the store holds as mesh-config resolves (novox/hq ADR 0348 §2): the module's own value on the node, else node @@ -330,205 +236,11 @@ func setAt(a configMoveAnswer, key string) []configSet { // and where from, and sets nothing; confirmed, it sets them in mesh-config, reads them back, empties the layers it // moved and switches. Each machine is then told byte for byte what it was told before, every module serves the same, // and `settings` lists the same values `config` resolves. Asked again, it sets nothing. -func TestEveryMachineIsToldTheSameAfterTheMoveOfConfig(t *testing.T) { - open, w, served := aConfiguredMesh(t) - ctx := t.Context() - readBackTries = 0 - t.Cleanup(func() { readBackTries = 15 }) - inv := open.inventory - config := &standInMeshConfig{open: open, values: map[string]any{}} - inv.ReadsConfigFrom(config.laid) - before := toldEveryMachine(t, open, served, w.On) - listedBefore := effectiveEverywhere(t, open, w) - factsBefore, err := gatherFacts(ctx, open, "2.11.17") - if err != nil { - t.Fatal(err) - } - - asked, err := runConfigMove(ctx, inv, config, false, "the move of config, run by a test", "") - if err != nil || !asked.DryRun || asked.Switched || len(asked.Problems) > 0 || len(config.sets) > 0 { - t.Fatalf("the move, not confirmed, says %+v (%v) and set %v", asked, err, config.sets) - } - // nodered's live layer says the file's name for its time zone: set under the config key (issue 751). - if s := setAt(asked, "module.anchor.nodered.time-zone"); len(s) != 1 || s[0].Value != "Europe/Brussels" || - !strings.HasSuffix(s[0].From[0], ": timeZone") { - t.Errorf("nodered's time zone moves as %+v", s) - } - for _, s := range asked.Sets { - if s.Key == "timeZone" { - t.Errorf("a key is set under the file's name: %+v", s) - } - } - // claude-code's whole-mesh managed settings become mesh config for agent-managed-settings (ADR 0348's - // consequences), and its whole-mesh layer sets no module value. - if s := setAt(asked, "mesh.agent-managed-settings"); len(s) != 1 || s[0].From[0] != "claude-code on the whole mesh: managed_settings" { - t.Errorf("claude-code's managed settings move as %+v", s) - } - for _, node := range fourMachines { - if s := setAt(asked, "module."+node+".claude-code.managed_settings"); len(s) > 0 { - t.Errorf("claude-code's managed settings are also set on %s: %+v", node, s) - } - } - // A whole-mesh layer of a value no predefined key holds goes to each machine it is assigned to, under that - // machine's layer: power's on the workstation is the workstation's, its own elsewhere. - if s := setAt(asked, "module.workstation.power.handle-power-key"); len(s) != 1 || s[0].Value != "suspend" { - t.Errorf("power's key on the workstation moves as %+v", s) - } - if s := setAt(asked, "module.anchor.power.handle-power-key"); len(s) != 1 || s[0].Value != "poweroff" { - t.Errorf("power's key on the anchor moves as %+v", s) - } - for _, s := range asked.Sets { - if s.Level == "node" { - t.Errorf("a machine's layer became node config: %+v", s) - } - } - var stays []string - for _, s := range asked.Stays { - stays = append(stays, strings.Fields(s)[0]) - } - sort.Strings(stays) - if want := []string{"bazarr", "bazarr", "bookshelf", "jackett", "kernel", "kometa", "lidarr", "nzbget", "ombi", - "plex", "qbittorrent", "radarr", "sonarr", "tautulli"}; len(stays) < 10 || !containsString(stays, "kernel") { - t.Errorf("the layers that stay are %v, want those of %v", stays, want) - } - if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) { - t.Fatal("a dry run changed what a machine is told") - } - - moved, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "release 57 is delivered") - if err != nil || moved.DryRun || !moved.Switched { - t.Fatalf("the move says %+v (%v)", moved, err) - } - if len(config.sets) != len(asked.Sets) { - t.Fatalf("the move set %d values, and said it would set %d", len(config.sets), len(asked.Sets)) - } - if held := inv.QueryForTest(t, `select module from settings s where values <> '{}' - and module in (select name from module where manifest ? 'config')`); len(held) != 0 { - t.Fatalf("a layer of a module in the config form still holds values: %v", held) - } - after := toldEveryMachine(t, open, served, w.On) - for _, node := range fourMachines { - if after[node] != before[node] { - t.Errorf("%s is told something else after the move:\n%s\nwas:\n%s", node, after[node], before[node]) - } - } - if listedAfter := effectiveEverywhere(t, open, w); !reflect.DeepEqual(listedAfter, listedBefore) { - t.Errorf("settings lists other values after the move:\n%v\nwas:\n%v", listedAfter, listedBefore) - } - // And config lists what settings listed: each module's resolved value on each machine is the value settings - // said before the move. - resolved, err := config.Resolved(ctx) - if err != nil { - t.Fatal(err) - } - for _, r := range resolved { - said, listed := listedBefore[r.Node+"."+r.Module+"."+r.Key] - if r.Source == "unset" { - if listed && said != nil { - t.Errorf("%s of %s on %s is unset in config, and settings said %v", r.Key, r.Module, r.Node, said) - } - continue - } - var v any - _ = json.Unmarshal(r.Value, &v) - if !listed || !reflect.DeepEqual(v, said) { - t.Errorf("config resolves %s of %s on %s as %v, and settings said %v", r.Key, r.Module, r.Node, v, said) - } - } - - // The facts snapshot carries mesh-config's values for the merge gate, as each machine's layer of each module, and - // each machine's declaration composes to the digest it did before the move. - factsAfter, err := gatherFacts(ctx, open, "2.11.17") - if err != nil { - t.Fatal(err) - } - // Machines are named in a snapshot by stand-ins of its own; each is found by what it composes. - digests := func(f snapshotFacts) ([]string, string) { - var out []string - zone := "" - for _, m := range f.Machines { - out = append(out, m.Declaration.Digest) - for _, l := range m.Settings { - if l.Module == "nodered" { - zone = compactJSON(l.Values["time-zone"]) - } - } - } - sort.Strings(out) - return out, zone - } - was, _ := digests(factsBefore) - now, zone := digests(factsAfter) - if zone != `"Europe/Brussels"` { - t.Errorf("the facts snapshot does not carry nodered's time zone from mesh-config: %s", zone) - } - if len(now) != len(fourMachines) || now[0] == "" || !reflect.DeepEqual(now, was) { - t.Errorf("the declarations in the facts snapshot are %v after the move, %v before", now, was) - } - for _, s := range factsAfter.Settings { - if _, isConfig := map[string]bool{"betterbird": true, "claude-code": true, "power": true, "xdg": true}[s.Module]; isConfig && - len(s.Values) > 0 { - t.Errorf("the facts snapshot still carries the whole mesh's layer of %s: %v", s.Module, s.Values) - } - } - - config.sets = nil - again, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "again") - if err != nil || len(config.sets) != 0 || len(again.Moved) != 0 { - t.Fatalf("asked again, the move set %v and says %+v (%v)", config.sets, again, err) - } - if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) { - t.Fatal("the move asked again changed what a machine is told") - } -} // **The move refuses what it cannot move as it composes, and sets nothing** (issue 664): a layer that still holds // placement (move-placement has not run), a key its module does not declare, and a value its declaration refuses — // each said by name; and a value mesh-config resolves from a level no layer says (it would be laid once the switch // is on) leaves every layer and the switch as they were. -func TestTheMoveOfConfigRefusesWhatWouldChangeAMachine(t *testing.T) { - open, _, _ := aConfiguredMesh(t) - ctx := t.Context() - readBackTries = 0 - t.Cleanup(func() { readBackTries = 15 }) - inv := open.inventory - config := &standInMeshConfig{open: open, values: map[string]any{}} - inv.ReadsConfigFrom(config.laid) - - inv.ExecForTest(t, `update settings set values = values || '{"endpoints": {"web": {"reach": "public"}}}' - where module = 'nodered'`) - inv.ExecForTest(t, `update settings set values = values || '{"colour": "blue"}' where module = 'unifi'`) - inv.ExecForTest(t, `update settings set values = '{"font-size": 900}' where module = 'dunst'`) - answer, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered") - if err == nil || answer.Switched || len(config.sets) > 0 { - t.Fatalf("the move went ahead: %+v (%v), set %v", answer, err, config.sets) - } - for _, want := range []string{"nodered on anchor still holds placement (endpoints): run move-placement first", - `unifi on anchor says "colour", which unifi does not declare`, "font-size at dunst's own value on workstation"} { - if !strings.Contains(err.Error(), want) { - t.Errorf("the refusal does not say %q: %v", want, err) - } - } - inv.ExecForTest(t, `update settings set values = values - 'endpoints' where module = 'nodered'`) - inv.ExecForTest(t, `update settings set values = values - 'colour' where module = 'unifi'`) - inv.ExecForTest(t, `update settings set values = '{"font-size": 13, "width": 500}' where module = 'dunst'`) - - // Set in mesh-config before the move, on a module whose layers say nothing of it: the switch would lay it. - config.values["module.anchor.grafana."+movedKeys(func() map[string]catalogue.SettingDeclaration { - shelf, _ := inv.Catalogue(ctx) - return shelf["grafana"].Settings - }())[0]] = "set before the move" - answer, err = runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered") - if err == nil || !strings.Contains(err.Error(), "grafana on anchor resolves") || answer.Switched { - t.Fatalf("a value no layer says was laid by the switch: %+v (%v)", answer, err) - } - if moved, _ := inv.ConfigMoved(ctx); moved.Moved { - t.Fatal("the switch is on after a refused move") - } - if held := inv.QueryForTest(t, `select count(*) from settings where values <> '{}'`); held[0][0] == "0" { - t.Fatal("a refused move emptied the layers") - } -} // The verb's command line: a dry run unless confirmed, and a confirmed move says why, recorded as a person's step of // the release, never a repair (issue 664). @@ -614,172 +326,11 @@ func (c *standInMeshConfig) ask(ctx context.Context, args map[string]any) (json. // has run, a module in the config form is shown from mesh-config, its values set and cleared there as its own on the // machine (placement still beside the assignment), and proposed through it; a trusted value is refused through a // verb, a module value for the whole mesh is refused, and a module still in the settings form keeps today's verb. -func TestTheSettingsVerbForwardsToConfigOnceTheValuesMoved(t *testing.T) { - open, w, served := aConfiguredMesh(t) - ctx := t.Context() - inv := open.inventory - readBackTries = 0 - t.Cleanup(func() { readBackTries = 15 }) - config := &standInMeshConfig{open: open, values: map[string]any{}} - inv.ReadsConfigFrom(config.laid) - savedRead, savedAsk := readConfigValues, configAsk - readConfigValues, configAsk = config.laid, config.ask - t.Cleanup(func() { readConfigValues, configAsk = savedRead, savedAsk }) - if _, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil { - t.Fatal(err) - } - - shown := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "nodered", "--node", "anchor"}) }) - for _, want := range []string{`"time-zone": "Europe/Brussels"`, `time-zone = "Europe/Brussels" (module)`, - "mesh-config.config", `"mqtt": {`} { - if !strings.Contains(shown, want) { - t.Errorf("settings show does not say %q:\n%s", want, shown) - } - } - if strings.Contains(shown, "timeZone") { - t.Errorf("settings show still says the file's name:\n%s", shown) - } - - config.asked = nil - said := stdoutOf(t, func() error { - return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 500}`, "--node", "workstation"}) - }) - if config.values["module.workstation.dunst.font-size"] != float64(12) || !strings.Contains(said, "~ font-size") || - containsString(config.asked, "set module.workstation.dunst.width") { - t.Fatalf("settings set reached mesh-config as %v (%v):\n%s", config.values, config.asked, said) - } - told := toldEveryMachine(t, open, served, w.On)["workstation"] - if !strings.Contains(told, "12") { - t.Error("the workstation is not told the font size set through settings") - } - if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`, "--node", "workstation"}); err == nil || - !strings.Contains(err.Error(), "would no longer set width") { - t.Fatalf("a set dropping a value without --replace: %v", err) - } - if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`}); err == nil || - !strings.Contains(err.Error(), "no own value for the whole mesh") { - t.Fatalf("a module value for the whole mesh: %v", err) - } - - t.Setenv(verbVar, "settings") - for _, c := range []struct{ module, node, values, key, held, why string }{ - // A file root reads: refused as before the move (novox/hq issue 339). - {"hostname", "anchor", `{"hostname": "elsewhere"}`, "module.anchor.hostname.hostname", "Ace", "issue 339"}, - {"messenger", "laptop", `{"desktop-machines": ["g14", "shanks"], "time-zone": "UTC"}`, - "module.laptop.messenger.time-zone", "Europe/Brussels", "issue 339"}, - } { - err := settingsCommand(ctx, []string{"set", c.module, c.values, "--node", c.node}) - if err == nil || !strings.Contains(err.Error(), c.why) || config.values[c.key] != c.held { - t.Fatalf("a trusted value of %s through a verb: %v, holds %v", c.module, err, config.values[c.key]) - } - } - // And a trusted value no file makes the terminal's is refused as trusted, since mesh-config takes the controller's - // word for it (ADR 0348 §8). - if err := refuseTrustedThroughAVerb(ctx, inv, "messenger", "laptop", []string{"time-zone"}); err == nil || - !strings.Contains(err.Error(), "ADR 0348 §8") { - t.Fatalf("a trusted value through a verb: %v", err) - } - if err := refuseTrustedThroughAVerb(ctx, inv, "dunst", "workstation", []string{"width"}); err != nil { - t.Fatalf("a value nobody trusts through a verb: %v", err) - } - stdoutOf(t, func() error { - return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 600}`, "--node", "workstation"}) - }) - if config.values["module.workstation.dunst.width"] != float64(600) { - t.Fatalf("a value nobody trusts was not set through a verb: %v", config.values["module.workstation.dunst.width"]) - } - t.Setenv(verbVar, "") - - stdoutOf(t, func() error { return settingsCommand(ctx, []string{"clear", "dunst", "--node", "workstation"}) }) - for _, k := range []string{"font-size", "width"} { - if _, held := config.values["module.workstation.dunst."+k]; held { - t.Errorf("settings clear left dunst's %s on the workstation", k) - } - } - - config.asked = nil - proposed := stdoutOf(t, func() error { - return settingsCommand(ctx, []string{"propose", "hostname", `{"hostname": "elsewhere"}`, "--node", "anchor"}) - }) - if !containsString(config.asked, "propose module.anchor.hostname.hostname") || !strings.Contains(proposed, "p-1") { - t.Fatalf("settings propose reached mesh-config as %v:\n%s", config.asked, proposed) - } - - config.asked = nil - kept := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "kernel", "--node", "laptop"}) }) - if !strings.Contains(kept, "transparent-huge-pages") || len(config.asked) > 0 { - t.Fatalf("a module in the settings form is shown as %q, asking mesh-config %v", kept, config.asked) - } -} // **An approval for a module whose config has moved is refused, and writes nothing** (issue 830, novox/hq ADR 0348): // a proposal raised while grafana read its layer (it had none, so the layer is as the operator was shown it) and // approved after the move would put its values back into the layer, ReadsConfig would turn false, and the module // would compose from emptied layers everywhere. -func TestAnApprovalRaisedBeforeTheMoveOfConfigIsRefusedAfterIt(t *testing.T) { - open, _, _ := aConfiguredMesh(t) - ctx := t.Context() - readBackTries = 0 - t.Cleanup(func() { readBackTries = 15 }) - inv := open.inventory - config := &standInMeshConfig{open: open, values: map[string]any{}} - inv.ReadsConfigFrom(config.laid) - shelf, err := inv.Catalogue(ctx) - if err != nil { - t.Fatal(err) - } - key := movedKeys(func() map[string]catalogue.SettingDeclaration { return shelf["grafana"].Settings }())[0] - values := map[string]any{key: "raised before the move"} - p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(), - Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} - if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has { - t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err) - } - if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched { - t.Fatalf("the move says %+v (%v)", moved, err) - } - if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads { - t.Fatalf("grafana does not read config after the move: %v %v", reads, err) - } - _, err = setLayerIn(open)(ctx, p, "s30", "approved by the operator") - if err == nil || !strings.Contains(err.Error(), "moved to mesh-config since the operator was shown the change") || - !strings.Contains(err.Error(), "propose it again") { - t.Fatalf("the stale approval: %v", err) - } - if layer, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has && len(layer) > 0 { - t.Errorf("the refused approval wrote the layer: %v %v %v", layer, has, err) - } - if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads { - t.Errorf("the refused approval turned ReadsConfig off: %v %v", reads, err) - } -} // **A proposal of placement only is set after the move, as before** (issue 830): forwardPropose sends it down the // old path on purpose, so refusing it would loop the operator (refused, proposed again, refused). -func TestAPlacementOnlyApprovalRaisedBeforeTheMoveIsSetAfterIt(t *testing.T) { - open, _, _ := aConfiguredMesh(t) - ctx := t.Context() - readBackTries = 0 - t.Cleanup(func() { readBackTries = 15 }) - inv := open.inventory - config := &standInMeshConfig{open: open, values: map[string]any{}} - inv.ReadsConfigFrom(config.laid) - values := map[string]any{catalogue.PlacesSetting: map[string]any{"data": "/srv/grafana-data"}} - p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(), - Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} - if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has { - t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err) - } - if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched { - t.Fatalf("the move says %+v (%v)", moved, err) - } - if _, err := setLayerIn(open)(ctx, p, "s31", "approved by the operator"); err != nil { - t.Fatalf("the placement-only approval after the move: %v", err) - } - if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || !has { - t.Errorf("the approval did not set the layer: %v %v", has, err) - } - if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads { - t.Errorf("the approval turned ReadsConfig off: %v %v", reads, err) - } -} diff --git a/cmd/mesh-controller/core_box_integration_test.go b/cmd/mesh-controller/core_box_integration_test.go new file mode 100644 index 00000000..55660821 --- /dev/null +++ b/cmd/mesh-controller/core_box_integration_test.go @@ -0,0 +1,129 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func aCoreMesh(t *testing.T, contract int) (*stores, *coreStandIn) { + t.Helper() + open := aMesh(t) + ctx := t.Context() + client := aClientModule() + register(t, open, client) + if _, err := assign(ctx, open, "laptop", client.Module); err != nil { + t.Fatal(err) + } + if err := issueProvisionerClient(ctx, open.inventory, client, "laptop", broker.Broker{Fingerprint: "ab"}, + "10.0.0.1:4222"); err != nil { + t.Fatal(err) + } + if _, err := open.inventory.RecordHealth(ctx, inventory.NodeHealth{Node: "laptop", Contract: contract, + SaidAt: time.Now()}); err != nil { + t.Fatal(err) + } + v := &coreStandIn{generation: 1} + withVault(t, v) + return open, v +} + +func TestTheSendCarriesTheVaultsBoxForACoreCredentialAndKeepsIt(t *testing.T) { + open, vault := aCoreMesh(t, link.VaultBoxContract) + key, err := open.inventory.SealingKeyOf(t.Context(), "laptop") + if err != nil { + t.Fatal(err) + } + f := clientsBroker(sentTo(t, open, "laptop")) + if f == nil || f["sealed"] != nil || f[catalogue.VaultSealedField] == nil { + t.Fatalf("the core credential is sent as %v: the vault's box, never the controller's seal", f) + } + if want := []string{"own/laptop/mesh-provisioner-client/" + broker.ProvisionerSecret}; !reflect.DeepEqual(vault.asked, want) { + t.Fatalf("the vault was asked %v", vault.asked) + } + if want, _ := sealingKeyPEM(key); vault.keys[0] != want { + t.Fatalf("the vault was asked to seal to %q, not the machine's key", vault.keys[0]) + } + first, _ := json.Marshal(f[catalogue.VaultSealedField]) + + // The vault answers afresh with the same generation: the kept box is carried again, so the declaration is stable. + again, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField]) + if string(again) != string(first) { + t.Fatalf("a fresh box of the same generation changed the declaration: %s, then %s", first, again) + } + // A rotation in the vault (a newer generation) is carried. + vault.generation = 2 + rotated, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField]) + if string(rotated) == string(first) { + t.Fatal("a newer generation was not carried") + } + if _, found := vaultSilence(t, "laptop"); found { + t.Fatal("a send the vault answered raised its silence") + } +} + +func TestASilentVaultHoldsTheKeptBoxAndSaysSo(t *testing.T) { + open, vault := aCoreMesh(t, link.VaultBoxContract) + first, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField]) + vault.silent = errors.New("no responders") + held := clientsBroker(sentTo(t, open, "laptop")) + got, _ := json.Marshal(held[catalogue.VaultSealedField]) + if held == nil || held["sealed"] != nil || string(got) != string(first) { + t.Fatalf("a silent vault's credential is sent as %v: the box kept, never the controller's seal", held) + } + c, found := vaultSilence(t, "laptop") + if !found || !strings.Contains(c.Summary, "vault") || !strings.Contains(c.Summary, "laptop") { + t.Fatalf("no condition names the vault and the machine: %+v", c) + } + // The vault answers again: the condition clears. + vault.silent = nil + sentTo(t, open, "laptop") + if _, found := vaultSilence(t, "laptop"); found { + t.Fatal("the condition stayed after the vault answered") + } +} + +func TestACoreModuleFirstPlacedWhileTheVaultIsSilentIsSealedByTheController(t *testing.T) { + open, vault := aCoreMesh(t, link.VaultBoxContract) + vault.silent = errors.New("no responders") + plan, settings, err := planFor(t.Context(), open, "laptop") + if err != nil { + t.Fatal(err) + } + declared, err := declarationFor(t.Context(), open, "laptop", plan, settings) + if err != nil { + t.Fatal(err) + } + f := clientsBroker(declared.Resources) + if f == nil || f["sealed"] == nil || f[catalogue.VaultSealedField] != nil { + t.Fatalf("the core credential is composed as %v: the controller's own seal, until the vault holds it", f) + } + if why, left := declared.leftOutWhy["mesh-provisioner-client"]; left { + t.Fatalf("the core module was left out: %v", why) + } + c, found := vaultSilence(t, "laptop") + if !found || !strings.Contains(c.Summary, "sealed by the controller until the vault holds it") { + t.Fatalf("the condition does not say why: %+v", c) + } +} + +func TestAnEngineBelowTheContractIsSentTheControllersSealAsBefore(t *testing.T) { + open, vault := aCoreMesh(t, link.HandedContract) + f := clientsBroker(sentTo(t, open, "laptop")) + if f == nil || f["sealed"] == nil || f[catalogue.VaultSealedField] != nil { + t.Fatalf("an older engine is sent %v", f) + } + if len(vault.asked) != 0 { + t.Fatalf("the vault was asked %v for an older engine", vault.asked) + } +} diff --git a/cmd/mesh-controller/core_box_test.go b/cmd/mesh-controller/core_box_test.go index c73264aa..4a6872de 100644 --- a/cmd/mesh-controller/core_box_test.go +++ b/cmd/mesh-controller/core_box_test.go @@ -7,13 +7,11 @@ import ( "reflect" "strings" "testing" - "time" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" ) // The core's own credentials travel sealed by the vault to their machine (novox/hq ADR 0349 §16, issue 755; the lead's @@ -46,28 +44,6 @@ func withVault(t *testing.T, v *coreStandIn) { t.Cleanup(func() { coreVault = before }) } -func aCoreMesh(t *testing.T, contract int) (*stores, *coreStandIn) { - t.Helper() - open := aMesh(t) - ctx := t.Context() - client := aClientModule() - register(t, open, client) - if _, err := assign(ctx, open, "laptop", client.Module); err != nil { - t.Fatal(err) - } - if err := issueProvisionerClient(ctx, open.inventory, client, "laptop", broker.Broker{Fingerprint: "ab"}, - "10.0.0.1:4222"); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.RecordHealth(ctx, inventory.NodeHealth{Node: "laptop", Contract: contract, - SaidAt: time.Now()}); err != nil { - t.Fatal(err) - } - v := &coreStandIn{generation: 1} - withVault(t, v) - return open, v -} - func clientsBroker(resources []map[string]any) map[string]any { for _, r := range resources { if r["id"] == "mesh-provisioner-client.needs-"+broker.ProvisionerSecret { @@ -86,85 +62,6 @@ func vaultSilence(t *testing.T, node string) (conditions.Condition, bool) { return c, found } -func TestTheSendCarriesTheVaultsBoxForACoreCredentialAndKeepsIt(t *testing.T) { - open, vault := aCoreMesh(t, link.VaultBoxContract) - key, err := open.inventory.SealingKeyOf(t.Context(), "laptop") - if err != nil { - t.Fatal(err) - } - f := clientsBroker(sentTo(t, open, "laptop")) - if f == nil || f["sealed"] != nil || f[catalogue.VaultSealedField] == nil { - t.Fatalf("the core credential is sent as %v: the vault's box, never the controller's seal", f) - } - if want := []string{"own/laptop/mesh-provisioner-client/" + broker.ProvisionerSecret}; !reflect.DeepEqual(vault.asked, want) { - t.Fatalf("the vault was asked %v", vault.asked) - } - if want, _ := sealingKeyPEM(key); vault.keys[0] != want { - t.Fatalf("the vault was asked to seal to %q, not the machine's key", vault.keys[0]) - } - first, _ := json.Marshal(f[catalogue.VaultSealedField]) - - // The vault answers afresh with the same generation: the kept box is carried again, so the declaration is stable. - again, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField]) - if string(again) != string(first) { - t.Fatalf("a fresh box of the same generation changed the declaration: %s, then %s", first, again) - } - // A rotation in the vault (a newer generation) is carried. - vault.generation = 2 - rotated, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField]) - if string(rotated) == string(first) { - t.Fatal("a newer generation was not carried") - } - if _, found := vaultSilence(t, "laptop"); found { - t.Fatal("a send the vault answered raised its silence") - } -} - -func TestASilentVaultHoldsTheKeptBoxAndSaysSo(t *testing.T) { - open, vault := aCoreMesh(t, link.VaultBoxContract) - first, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField]) - vault.silent = errors.New("no responders") - held := clientsBroker(sentTo(t, open, "laptop")) - got, _ := json.Marshal(held[catalogue.VaultSealedField]) - if held == nil || held["sealed"] != nil || string(got) != string(first) { - t.Fatalf("a silent vault's credential is sent as %v: the box kept, never the controller's seal", held) - } - c, found := vaultSilence(t, "laptop") - if !found || !strings.Contains(c.Summary, "vault") || !strings.Contains(c.Summary, "laptop") { - t.Fatalf("no condition names the vault and the machine: %+v", c) - } - // The vault answers again: the condition clears. - vault.silent = nil - sentTo(t, open, "laptop") - if _, found := vaultSilence(t, "laptop"); found { - t.Fatal("the condition stayed after the vault answered") - } -} - -func TestACoreModuleFirstPlacedWhileTheVaultIsSilentIsSealedByTheController(t *testing.T) { - open, vault := aCoreMesh(t, link.VaultBoxContract) - vault.silent = errors.New("no responders") - plan, settings, err := planFor(t.Context(), open, "laptop") - if err != nil { - t.Fatal(err) - } - declared, err := declarationFor(t.Context(), open, "laptop", plan, settings) - if err != nil { - t.Fatal(err) - } - f := clientsBroker(declared.Resources) - if f == nil || f["sealed"] == nil || f[catalogue.VaultSealedField] != nil { - t.Fatalf("the core credential is composed as %v: the controller's own seal, until the vault holds it", f) - } - if why, left := declared.leftOutWhy["mesh-provisioner-client"]; left { - t.Fatalf("the core module was left out: %v", why) - } - c, found := vaultSilence(t, "laptop") - if !found || !strings.Contains(c.Summary, "sealed by the controller until the vault holds it") { - t.Fatalf("the condition does not say why: %+v", c) - } -} - // Issue 908: a machine at the vault-box contract, the vault silent, no kept box: nats is composed with the // controller's own seal, never left out, and the vault-silent condition names it. func TestReplay908(t *testing.T) { @@ -191,16 +88,6 @@ func TestReplay908(t *testing.T) { } // An engine below VaultBoxContract is sent the controller's own seal, as before, and the vault is not asked. -func TestAnEngineBelowTheContractIsSentTheControllersSealAsBefore(t *testing.T) { - open, vault := aCoreMesh(t, link.HandedContract) - f := clientsBroker(sentTo(t, open, "laptop")) - if f == nil || f["sealed"] == nil || f[catalogue.VaultSealedField] != nil { - t.Fatalf("an older engine is sent %v", f) - } - if len(vault.asked) != 0 { - t.Fatalf("the vault was asked %v for an older engine", vault.asked) - } -} // Every question but a send reads the kept box and asks nothing; with none kept, a stand-in the next send asks for. func TestOnlyASendAsksTheVault(t *testing.T) { diff --git a/cmd/mesh-controller/credential_on_assign_test.go b/cmd/mesh-controller/credential_on_assign_test.go index 959ef073..9c8133a0 100644 --- a/cmd/mesh-controller/credential_on_assign_test.go +++ b/cmd/mesh-controller/credential_on_assign_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/data_integration_test.go b/cmd/mesh-controller/data_integration_test.go new file mode 100644 index 00000000..c1142df8 --- /dev/null +++ b/cmd/mesh-controller/data_integration_test.go @@ -0,0 +1,148 @@ +//go:build integration + +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestAMovedPathIsNoShrinkAndAShrinkThereStillIs(t *testing.T) { + inv := inventory.ForTest(t) + ctx := t.Context() + shelf := shelfFor(t, houseManifest) + declared := []inventory.DeclaredData{{Module: "house", Item: "config", Class: "irreplaceable", Owned: true}} + start := time.Now().Add(-6 * time.Hour) + measure := func(at time.Time, path string, size int64) []conditions.Observation { + t.Helper() + if _, err := inv.RecordData(ctx, "home", declared, map[string]map[string]inventory.Measurement{"house": { + "config": {Path: path, Size: bytesOf(size), MeasuredAt: when(at), LastWrite: when(at), + LastBackup: when(at)}}}, "", at); err != nil { + t.Fatal(err) + } + records, err := inv.Data(ctx) + if err != nil { + t.Fatal(err) + } + peaks, err := inv.DataPeaks(ctx, at.Add(-shrinkWindow)) + if err != nil { + t.Fatal(err) + } + return dataFindings(records, peaks, shelf, nil, nil, at) + } + measure(start, "/home/operator/.claude", 94_700_000) + if got := findingsByKind(measure(start.Add(10*time.Minute), "/home/agent/.claude", 490)); got[kindDataShrank].Kind != "" { + t.Fatalf("a moved path raised a shrink: %+v", got[kindDataShrank]) + } + measure(start.Add(2*time.Hour), "/home/agent/.claude", 300<<20) + got := findingsByKind(measure(start.Add(4*time.Hour), "/home/agent/.claude", 1<<20))[kindDataShrank] + if got.Severity != conditions.Urgent || !strings.Contains(got.Summary, "shrank") { + t.Fatalf("a genuine shrink at the new path was not raised: %+v", got) + } +} + +func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { + t.Fatal(err) + } + register(t, open, catalogue.Manifest{Module: "keeper", Version: "1", + Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}}) + house, err := catalogue.ParseManifest([]byte(houseManifest)) + if err != nil { + t.Fatal(err) + } + register(t, open, house) + if _, err := assign(ctx, open, "laptop", "house"); err == nil { + t.Fatal("irreplaceable data was assigned to a machine with nothing to back it up") + } + for _, node := range []string{"laptop", "anchor"} { + if _, err := assign(ctx, open, node, "keeper"); err != nil { + t.Fatal(err) + } + } + if _, err := assign(ctx, open, "laptop", "house"); err != nil { + t.Fatal(err) + } + // Sent, and applied: a holder is asked only once it has been (novox/hq issue 275). + for _, node := range []string{"laptop", "anchor"} { + pushedAndApplied(t, open, node) + } + + conn := onATestBus(t) + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + laptop, anchor := &fakeHolder{}, &fakeHolder{} + laptop.serve(t, conn, "laptop") + anchor.serve(t, conn, "anchor") + now := time.Now() + heard := &watchdogs{last: &signalFacts{now: now, machines: []machineFacts{ + {name: "laptop", lastHeard: now}, {name: "anchor", lastHeard: now}}}} + d := &doctor{open: open, js: js, watchdogs: heard} + var d13 probe + for _, p := range probeRegistry { + if p.ID == probeDataID { + d13 = p + } + } + run := func() []conditions.Observation { + t.Helper() + probing := context.WithValue(ctx, probeAsksKey{}, d13) + obs, err := probeData(probing, d) + if err != nil { + t.Fatal(err) + } + return obs + } + + laptop.set(measuredHouse("/var/lib/house/config", 2<<30, now)) + if obs := run(); len(obs) != 0 { + t.Fatalf("a measured, backed-up item raised %+v", obs) + } + r, err := inv.DataOf(ctx, "laptop", "house", "config") + if err != nil || r.Path != "/var/lib/house/config" || r.Size == nil || *r.Size != 2<<30 || r.LastBackup == nil { + t.Fatalf("what the holder measured was not kept: %+v, %v", r, err) + } + + said, err := unassign(ctx, open, "laptop", "house") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(said, "house's config (/var/lib/house/config, 2.0 GB) stays where it is") { + t.Fatalf("the unassignment does not say the data stays:\n%s", said) + } + laptop.set() + run() + r, err = inv.DataOf(ctx, "laptop", "house", "config") + if err != nil || !r.Retired() || r.Path != "/var/lib/house/config" { + t.Fatalf("unassigned, the irreplaceable item is not kept retired: %+v, %v", r, err) + } + records, _ := inv.Data(ctx) + if rows := retiredData(records, time.Now()); len(rows) != 1 || rows[0].Path != "/var/lib/house/config" { + t.Fatalf("cleanup list: %+v", rows) + } + + // Assigned on the anchor, onto an empty directory. + if _, err := assign(ctx, open, "anchor", "house"); err != nil { + t.Fatal(err) + } + anchor.set(measuredHouse("/var/lib/house/config", 300<<10, time.Now())) + obs := findingsByKind(run()) + o, ok := obs[kindEmptyReplacement] + if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "laptop" { + t.Fatalf("an empty replacement of a module's data was not urgent: %+v", obs) + } +} diff --git a/cmd/mesh-controller/data_test.go b/cmd/mesh-controller/data_test.go index 00324b19..15dcf45e 100644 --- a/cmd/mesh-controller/data_test.go +++ b/cmd/mesh-controller/data_test.go @@ -1,7 +1,6 @@ package main import ( - "context" "encoding/json" "strings" "sync" @@ -10,14 +9,11 @@ import ( "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/link" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" - - "github.com/novox/mesh-controller/internal/testbus" ) func bytesOf(n int64) *int64 { return &n } @@ -173,39 +169,6 @@ func TestAShrinkOfMoreThanHalfIsUrgent(t *testing.T) { // operator's own home (94.7 MB) to the agent account's fresh one (490 B), and `data-shrank` was raised // for data that was never lost. A moved item is read against its new path only, so nothing is raised — // and a genuine shrink at the new path, a week of history later, still is. -func TestAMovedPathIsNoShrinkAndAShrinkThereStillIs(t *testing.T) { - inv := inventory.ForTest(t) - ctx := t.Context() - shelf := shelfFor(t, houseManifest) - declared := []inventory.DeclaredData{{Module: "house", Item: "config", Class: "irreplaceable", Owned: true}} - start := time.Now().Add(-6 * time.Hour) - measure := func(at time.Time, path string, size int64) []conditions.Observation { - t.Helper() - if _, err := inv.RecordData(ctx, "home", declared, map[string]map[string]inventory.Measurement{"house": { - "config": {Path: path, Size: bytesOf(size), MeasuredAt: when(at), LastWrite: when(at), - LastBackup: when(at)}}}, "", at); err != nil { - t.Fatal(err) - } - records, err := inv.Data(ctx) - if err != nil { - t.Fatal(err) - } - peaks, err := inv.DataPeaks(ctx, at.Add(-shrinkWindow)) - if err != nil { - t.Fatal(err) - } - return dataFindings(records, peaks, shelf, nil, nil, at) - } - measure(start, "/home/operator/.claude", 94_700_000) - if got := findingsByKind(measure(start.Add(10*time.Minute), "/home/agent/.claude", 490)); got[kindDataShrank].Kind != "" { - t.Fatalf("a moved path raised a shrink: %+v", got[kindDataShrank]) - } - measure(start.Add(2*time.Hour), "/home/agent/.claude", 300<<20) - got := findingsByKind(measure(start.Add(4*time.Hour), "/home/agent/.claude", 1<<20))[kindDataShrank] - if got.Severity != conditions.Urgent || !strings.Contains(got.Summary, "shrank") { - t.Fatalf("a genuine shrink at the new path was not raised: %+v", got) - } -} // Data said to be written all the time and not written; data with no backup or an old one — urgent when // irreplaceable, a warning when valuable; and a new item given its bound before it is said. @@ -313,103 +276,6 @@ func measuredHouse(path string, size int64, at time.Time) map[string]any { // directory is an empty replacement — through the real stores and a real bus. The unassignment says the // data stays; the self-check's next run retires it (kept, listed by cleanup), and when the module comes // up on another machine with an empty directory while the full one waits retired, that is urgent. -func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { - t.Fatal(err) - } - register(t, open, catalogue.Manifest{Module: "keeper", Version: "1", - Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}}) - house, err := catalogue.ParseManifest([]byte(houseManifest)) - if err != nil { - t.Fatal(err) - } - register(t, open, house) - if _, err := assign(ctx, open, "laptop", "house"); err == nil { - t.Fatal("irreplaceable data was assigned to a machine with nothing to back it up") - } - for _, node := range []string{"laptop", "anchor"} { - if _, err := assign(ctx, open, node, "keeper"); err != nil { - t.Fatal(err) - } - } - if _, err := assign(ctx, open, "laptop", "house"); err != nil { - t.Fatal(err) - } - // Sent, and applied: a holder is asked only once it has been (novox/hq issue 275). - for _, node := range []string{"laptop", "anchor"} { - pushedAndApplied(t, open, node) - } - - conn := onATestBus(t) - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - laptop, anchor := &fakeHolder{}, &fakeHolder{} - laptop.serve(t, conn, "laptop") - anchor.serve(t, conn, "anchor") - now := time.Now() - heard := &watchdogs{last: &signalFacts{now: now, machines: []machineFacts{ - {name: "laptop", lastHeard: now}, {name: "anchor", lastHeard: now}}}} - d := &doctor{open: open, js: js, watchdogs: heard} - var d13 probe - for _, p := range probeRegistry { - if p.ID == probeDataID { - d13 = p - } - } - run := func() []conditions.Observation { - t.Helper() - probing := context.WithValue(ctx, probeAsksKey{}, d13) - obs, err := probeData(probing, d) - if err != nil { - t.Fatal(err) - } - return obs - } - - laptop.set(measuredHouse("/var/lib/house/config", 2<<30, now)) - if obs := run(); len(obs) != 0 { - t.Fatalf("a measured, backed-up item raised %+v", obs) - } - r, err := inv.DataOf(ctx, "laptop", "house", "config") - if err != nil || r.Path != "/var/lib/house/config" || r.Size == nil || *r.Size != 2<<30 || r.LastBackup == nil { - t.Fatalf("what the holder measured was not kept: %+v, %v", r, err) - } - - said, err := unassign(ctx, open, "laptop", "house") - if err != nil { - t.Fatal(err) - } - if !strings.Contains(said, "house's config (/var/lib/house/config, 2.0 GB) stays where it is") { - t.Fatalf("the unassignment does not say the data stays:\n%s", said) - } - laptop.set() - run() - r, err = inv.DataOf(ctx, "laptop", "house", "config") - if err != nil || !r.Retired() || r.Path != "/var/lib/house/config" { - t.Fatalf("unassigned, the irreplaceable item is not kept retired: %+v, %v", r, err) - } - records, _ := inv.Data(ctx) - if rows := retiredData(records, time.Now()); len(rows) != 1 || rows[0].Path != "/var/lib/house/config" { - t.Fatalf("cleanup list: %+v", rows) - } - - // Assigned on the anchor, onto an empty directory. - if _, err := assign(ctx, open, "anchor", "house"); err != nil { - t.Fatal(err) - } - anchor.set(measuredHouse("/var/lib/house/config", 300<<10, time.Now())) - obs := findingsByKind(run()) - o, ok := obs[kindEmptyReplacement] - if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "laptop" { - t.Fatalf("an empty replacement of a module's data was not urgent: %+v", obs) - } -} // Data on redundant storage: the array it is on is watched, one condition per array as loud as the most // precious item on it; an item said to be on redundancy and found on plain storage is said; an item diff --git a/cmd/mesh-controller/deadletters_integration_test.go b/cmd/mesh-controller/deadletters_integration_test.go new file mode 100644 index 00000000..d550393d --- /dev/null +++ b/cmd/mesh-controller/deadletters_integration_test.go @@ -0,0 +1,88 @@ +//go:build integration + +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func servingDeadLetters(t *testing.T) *broker.JetStream { + t.Helper() + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + before := servingBus.Load() + servingBus.Store(js) + t.Cleanup(func() { servingBus.Store(before) }) + return js +} + +func TestDeadLettersListsDropsAndRecordsWhy(t *testing.T) { + js := servingDeadLetters(t) + if _, err := js.Context().Publish("mesh.mod.gitea.event.pull.merged", []byte(`{"n":1}`)); err != nil { + t.Fatal(err) + } + d, err := link.KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":1,"deliveries":5}`)) + if err != nil { + t.Fatal(err) + } + + answer, err := askDeadLetters(t, map[string]any{}) + if err != nil { + t.Fatal(err) + } + listed := answer.(map[string]any) + if listed["held"] != 1 || len(listed["dead_letters"].([]link.DeadLetter)) != 1 { + t.Fatalf("listed %v", listed) + } + + // Refused before anything is done: an act without why, an argument the shape passes over, both acts. + for _, args := range []map[string]any{ + {"drop": "1"}, + {"drop": "1", "why": "x", "limit": "3"}, + {"drop": "1", "deliver": "1", "why": "x"}, + {"why": "x"}, + {"id": "nought"}, + } { + if _, err := askDeadLetters(t, args); err == nil { + t.Errorf("%v was done", args) + } + } + if _, total, _ := link.DeadLetters(js.Context(), "", 0); total != 1 { + t.Fatalf("a refused call changed what is kept: %d left", total) + } + + done, err := askDeadLetters(t, map[string]any{"drop": "1", "why": "the media server took the download in by hand"}) + if err != nil { + t.Fatal(err) + } + if said := done.(map[string]any); said["recorded"] == nil || !strings.Contains(said["done"].(string), "dropped") { + t.Fatalf("answered %v", said) + } + acts, err := link.HandActs(context.Background(), js.Conn(), time.Now().Add(-time.Minute)) + if err != nil || len(acts) != 1 || acts[0].Verb != "dead-letters drop" || acts[0].Cause != causeDeadLetter || + !strings.Contains(acts[0].Condition, "media_sonarr") { + t.Fatalf("recorded %+v (%v)", acts, err) + } + if !personsDecision(acts[0]) { + t.Error("dropping a dead letter is counted as a repair, so S15 would want a healer for it") + } + if _, err := askDeadLetters(t, map[string]any{"id": "1"}); err == nil { + t.Errorf("dead letter %d is still answered after it was dropped", d.ID) + } +} diff --git a/cmd/mesh-controller/deadletters_test.go b/cmd/mesh-controller/deadletters_test.go index e714456f..81d7e9e2 100644 --- a/cmd/mesh-controller/deadletters_test.go +++ b/cmd/mesh-controller/deadletters_test.go @@ -6,31 +6,10 @@ import ( "testing" "time" - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" ) // The serving controller's bus, with the mesh's streams, for the verb to read and act on. -func servingDeadLetters(t *testing.T) *broker.JetStream { - t.Helper() - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - before := servingBus.Load() - servingBus.Store(js) - t.Cleanup(func() { servingBus.Store(before) }) - return js -} func askDeadLetters(t *testing.T, args map[string]any) (any, error) { t.Helper() @@ -41,61 +20,6 @@ func askDeadLetters(t *testing.T, args map[string]any) (any, error) { return deadLettersAnswer(context.Background(), a) } -func TestDeadLettersListsDropsAndRecordsWhy(t *testing.T) { - js := servingDeadLetters(t) - if _, err := js.Context().Publish("mesh.mod.gitea.event.pull.merged", []byte(`{"n":1}`)); err != nil { - t.Fatal(err) - } - d, err := link.KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":1,"deliveries":5}`)) - if err != nil { - t.Fatal(err) - } - - answer, err := askDeadLetters(t, map[string]any{}) - if err != nil { - t.Fatal(err) - } - listed := answer.(map[string]any) - if listed["held"] != 1 || len(listed["dead_letters"].([]link.DeadLetter)) != 1 { - t.Fatalf("listed %v", listed) - } - - // Refused before anything is done: an act without why, an argument the shape passes over, both acts. - for _, args := range []map[string]any{ - {"drop": "1"}, - {"drop": "1", "why": "x", "limit": "3"}, - {"drop": "1", "deliver": "1", "why": "x"}, - {"why": "x"}, - {"id": "nought"}, - } { - if _, err := askDeadLetters(t, args); err == nil { - t.Errorf("%v was done", args) - } - } - if _, total, _ := link.DeadLetters(js.Context(), "", 0); total != 1 { - t.Fatalf("a refused call changed what is kept: %d left", total) - } - - done, err := askDeadLetters(t, map[string]any{"drop": "1", "why": "the media server took the download in by hand"}) - if err != nil { - t.Fatal(err) - } - if said := done.(map[string]any); said["recorded"] == nil || !strings.Contains(said["done"].(string), "dropped") { - t.Fatalf("answered %v", said) - } - acts, err := link.HandActs(context.Background(), js.Conn(), time.Now().Add(-time.Minute)) - if err != nil || len(acts) != 1 || acts[0].Verb != "dead-letters drop" || acts[0].Cause != causeDeadLetter || - !strings.Contains(acts[0].Condition, "media_sonarr") { - t.Fatalf("recorded %+v (%v)", acts, err) - } - if !personsDecision(acts[0]) { - t.Error("dropping a dead letter is counted as a repair, so S15 would want a healer for it") - } - if _, err := askDeadLetters(t, map[string]any{"id": "1"}); err == nil { - t.Errorf("dead letter %d is still answered after it was dropped", d.ID) - } -} - // Open while DEAD_LETTERS holds a message for the consumer, in words the operator reads in one pass: // what is held, and where to act. func TestAConsumersDeadLettersAreSaidUntilActedOn(t *testing.T) { diff --git a/cmd/mesh-controller/delivery_conditions_test.go b/cmd/mesh-controller/delivery_conditions_test.go index 916b3ca3..cb3176fa 100644 --- a/cmd/mesh-controller/delivery_conditions_test.go +++ b/cmd/mesh-controller/delivery_conditions_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/delivery_integration_test.go b/cmd/mesh-controller/delivery_integration_test.go new file mode 100644 index 00000000..ed85ec2f --- /dev/null +++ b/cmd/mesh-controller/delivery_integration_test.go @@ -0,0 +1,196 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + asked := asksRecorded(t) + withConditionsInMemory(t) + for _, name := range []string{"app", "mesh-delivery"} { + m := catalogue.Manifest{Module: name, Version: "1"} + if name == "mesh-delivery" { + m.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}} + } + if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", + Path: "modules/" + name, Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + } + merge := func(commit string, paths ...string) inventory.Plan { + t.Helper() + m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, Paths: paths, + ModuleDirs: []string{"modules/app", "modules/mesh-delivery"}, ModuleDirsSaid: true} + if err := (following{open: open}).SourceMoved(ctx, m); err != nil { + t.Fatal(err) + } + recent, err := inv.RecentPlans(ctx, 1) + if err != nil || len(recent) != 1 || recent[0].Commit != commit { + t.Fatalf("no plan for %s: %v %v", commit, recent, err) + } + return recent[0] + } + + // finish ends a walk as done, so the next merge has nothing of it to take over. + finish := func(id string) { + t.Helper() + w, err := inv.PlanByID(ctx, id) + if err != nil { + t.Fatal(err) + } + w.State = inventory.PlanDone + if err := inv.SavePlan(ctx, &w); err != nil { + t.Fatal(err) + } + } + + // No holder on record: the merge starts its walk, as before. + p := merge("c1aaaaaaaa", "modules/app/index.ts") + if p.Waiting() || len(*asked) != 1 { + t.Fatalf("with no holder on record the walk waited (%v) or asked %v", p.Waiting(), *asked) + } + + // The holder on record: the next merge waits, asking nothing, and an advance asks nothing either. One walk + // is open at a time (novox/hq ADR 0276): the first ends before the next merge's batch is cut. + finish(p.ID) + if _, err := inv.Assign(ctx, "anchor", "mesh-delivery"); err != nil { + t.Fatal(err) + } + p = merge("c2bbbbbbbb", "modules/app/index.ts") + if !p.Waiting() || len(*asked) != 1 || !strings.Contains(p.Note, "plans go "+p.ID) { + t.Fatalf("the walk did not wait for its word: waiting %v, asked %v, note %q", p.Waiting(), *asked, p.Note) + } + advanceHeld(ctx, open) + if len(*asked) != 1 { + t.Fatalf("a waiting walk was advanced into asking: %v", *asked) + } + if line := planLine(p, p.Created); strings.Contains(line, "LATE") || !strings.Contains(line, "waits for") { + t.Errorf("a waiting walk reads %q", line) + } + + // The delivery's word starts it. + if err := deliveryCommand(ctx, []string{"go", p.ID, "--by", catalogue.DeliverySeat, "--why", "its turn"}); err != nil { + t.Fatal(err) + } + if err := deliveryCommand(ctx, []string{"go", p.ID}); err == nil { + t.Fatal("a walk was let go twice") + } + advanceHeld(ctx, open) + if len(*asked) != 2 { + t.Fatalf("the word did not start the walk: %v", *asked) + } + got, err := inv.PlanByID(ctx, p.ID) + if err != nil || got.Delivery == nil || got.Delivery.By != catalogue.DeliverySeat || got.Delivery.Why != "its turn" { + t.Fatalf("the word was not kept: %+v %v", got.Delivery, err) + } + + // The delivery's owner's own merge never waits for it. A walk that started is never taken over (novox/hq + // ADR 0276): the merge's batch is cut once it ended. + finish(p.ID) + p = merge("c3cccccccc", "modules/mesh-delivery/main.go") + if p.Waiting() || len(*asked) != 3 { + t.Fatalf("mesh-delivery's own walk waited for mesh-delivery: %v %v", p.Waiting(), *asked) + } + + // The owner down: a person starts a waiting walk, with why. + finish(p.ID) + p = merge("c4dddddddd", "modules/app/index.ts") + if !p.Waiting() { + t.Fatal("the walk did not wait") + } + if err := plansCommand(ctx, []string{"go", p.ID}); err == nil || !strings.Contains(err.Error(), "--why") { + t.Fatalf("a walk was started by hand without why: %v", err) + } + if err := plansCommand(ctx, []string{"go", p.ID, "--why", "mesh-delivery is down"}); err != nil { + t.Fatal(err) + } + advanceHeld(ctx, open) + got, _ = inv.PlanByID(ctx, p.ID) + if got.Waiting() || !strings.HasPrefix(got.Delivery.By, "a person") || len(*asked) < 4 { + t.Fatalf("a person's word did not start the walk: %+v, asked %v", got.Delivery, *asked) + } + + // The owner's stop: failed, said as stopped by it. + finish(p.ID) + p = merge("c5eeeeeeee", "modules/app/index.ts") + if err := deliveryCommand(ctx, []string{"stop", p.ID, "--why", "the operator stopped it", "--by", + "mesh-delivery for jochen"}); err != nil { + t.Fatal(err) + } + got, _ = inv.PlanByID(ctx, p.ID) + if got.State != inventory.PlanFailed || got.Delivery.Stopped != "mesh-delivery for jochen" || + !strings.Contains(got.Note, "the operator stopped it") { + t.Fatalf("the stop was kept as %s %+v %q", got.State, got.Delivery, got.Note) + } + + // Unassigned: the mesh is back on the controller's own path. + if err := inv.Unassign(ctx, "anchor", "mesh-delivery"); err != nil { + t.Fatal(err) + } + asks := len(*asked) + if p = merge("c6ffffffff", "modules/app/index.ts"); p.Waiting() || len(*asked) != asks+1 { + t.Fatalf("with the holder gone the walk waited: %v", p.Waiting()) + } +} + +func TestAWalkLetGoByAVerbIsSaidAsPlanMoved(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + url := testbus.URL(t) + js, err := broker.Dial(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + before := handActConn + handActConn = js.Conn() + t.Cleanup(func() { handActConn = before }) + heard := make(chan *nats.Msg, 4) + sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), heard) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + + waiting := inventory.Plan{ID: "plan-waits", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c7c7c7c7", + Created: time.Now().UTC(), State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, + Modules: map[string]*inventory.PlanModule{"app": {}}, + Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}} + if err := open.inventory.SavePlan(ctx, &waiting); err != nil { + t.Fatal(err) + } + if err := deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}); err != nil { + t.Fatal(err) + } + select { + case m := <-heard: + var said inventory.Plan + if err := json.Unmarshal(m.Data, &said); err != nil || said.ID != waiting.ID || said.Delivery == nil || + said.Delivery.Go == nil { + t.Fatalf("plan-moved said %s (%v)", m.Data, err) + } + case <-time.After(5 * time.Second): + t.Fatal("a walk let go by a verb was not said") + } + if checkEvents != nil || inventory.PlanSaved != nil { + t.Fatal("the command's own bus was left in place") + } +} diff --git a/cmd/mesh-controller/delivery_test.go b/cmd/mesh-controller/delivery_test.go index 271cd648..d7f1b0df 100644 --- a/cmd/mesh-controller/delivery_test.go +++ b/cmd/mesh-controller/delivery_test.go @@ -1,19 +1,12 @@ package main import ( - "encoding/json" "reflect" "strings" "testing" - "time" - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" ) // novox/hq ADR 0419: a group's order, from the module graph and the manifests' change alone, the same on every @@ -128,137 +121,6 @@ func TestAWalkWaitsOnlyWhileTheDeliverySeatIsHeldAndNeverForTheCore(t *testing.T // novox/hq ADR 0239: with mesh-delivery on record, a merge opens its walk and asks nothing until the // delivery's word; the word starts it; the core's own merge never waits; a person starts a waiting walk by // hand when the owner is down, and the owner's stop ends one as stopped. -func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - asked := asksRecorded(t) - withConditionsInMemory(t) - for _, name := range []string{"app", "mesh-delivery"} { - m := catalogue.Manifest{Module: name, Version: "1"} - if name == "mesh-delivery" { - m.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}} - } - if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", - Path: "modules/" + name, Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - } - merge := func(commit string, paths ...string) inventory.Plan { - t.Helper() - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, Paths: paths, - ModuleDirs: []string{"modules/app", "modules/mesh-delivery"}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - recent, err := inv.RecentPlans(ctx, 1) - if err != nil || len(recent) != 1 || recent[0].Commit != commit { - t.Fatalf("no plan for %s: %v %v", commit, recent, err) - } - return recent[0] - } - - // finish ends a walk as done, so the next merge has nothing of it to take over. - finish := func(id string) { - t.Helper() - w, err := inv.PlanByID(ctx, id) - if err != nil { - t.Fatal(err) - } - w.State = inventory.PlanDone - if err := inv.SavePlan(ctx, &w); err != nil { - t.Fatal(err) - } - } - - // No holder on record: the merge starts its walk, as before. - p := merge("c1aaaaaaaa", "modules/app/index.ts") - if p.Waiting() || len(*asked) != 1 { - t.Fatalf("with no holder on record the walk waited (%v) or asked %v", p.Waiting(), *asked) - } - - // The holder on record: the next merge waits, asking nothing, and an advance asks nothing either. One walk - // is open at a time (novox/hq ADR 0276): the first ends before the next merge's batch is cut. - finish(p.ID) - if _, err := inv.Assign(ctx, "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } - p = merge("c2bbbbbbbb", "modules/app/index.ts") - if !p.Waiting() || len(*asked) != 1 || !strings.Contains(p.Note, "plans go "+p.ID) { - t.Fatalf("the walk did not wait for its word: waiting %v, asked %v, note %q", p.Waiting(), *asked, p.Note) - } - advanceHeld(ctx, open) - if len(*asked) != 1 { - t.Fatalf("a waiting walk was advanced into asking: %v", *asked) - } - if line := planLine(p, p.Created); strings.Contains(line, "LATE") || !strings.Contains(line, "waits for") { - t.Errorf("a waiting walk reads %q", line) - } - - // The delivery's word starts it. - if err := deliveryCommand(ctx, []string{"go", p.ID, "--by", catalogue.DeliverySeat, "--why", "its turn"}); err != nil { - t.Fatal(err) - } - if err := deliveryCommand(ctx, []string{"go", p.ID}); err == nil { - t.Fatal("a walk was let go twice") - } - advanceHeld(ctx, open) - if len(*asked) != 2 { - t.Fatalf("the word did not start the walk: %v", *asked) - } - got, err := inv.PlanByID(ctx, p.ID) - if err != nil || got.Delivery == nil || got.Delivery.By != catalogue.DeliverySeat || got.Delivery.Why != "its turn" { - t.Fatalf("the word was not kept: %+v %v", got.Delivery, err) - } - - // The delivery's owner's own merge never waits for it. A walk that started is never taken over (novox/hq - // ADR 0276): the merge's batch is cut once it ended. - finish(p.ID) - p = merge("c3cccccccc", "modules/mesh-delivery/main.go") - if p.Waiting() || len(*asked) != 3 { - t.Fatalf("mesh-delivery's own walk waited for mesh-delivery: %v %v", p.Waiting(), *asked) - } - - // The owner down: a person starts a waiting walk, with why. - finish(p.ID) - p = merge("c4dddddddd", "modules/app/index.ts") - if !p.Waiting() { - t.Fatal("the walk did not wait") - } - if err := plansCommand(ctx, []string{"go", p.ID}); err == nil || !strings.Contains(err.Error(), "--why") { - t.Fatalf("a walk was started by hand without why: %v", err) - } - if err := plansCommand(ctx, []string{"go", p.ID, "--why", "mesh-delivery is down"}); err != nil { - t.Fatal(err) - } - advanceHeld(ctx, open) - got, _ = inv.PlanByID(ctx, p.ID) - if got.Waiting() || !strings.HasPrefix(got.Delivery.By, "a person") || len(*asked) < 4 { - t.Fatalf("a person's word did not start the walk: %+v, asked %v", got.Delivery, *asked) - } - - // The owner's stop: failed, said as stopped by it. - finish(p.ID) - p = merge("c5eeeeeeee", "modules/app/index.ts") - if err := deliveryCommand(ctx, []string{"stop", p.ID, "--why", "the operator stopped it", "--by", - "mesh-delivery for jochen"}); err != nil { - t.Fatal(err) - } - got, _ = inv.PlanByID(ctx, p.ID) - if got.State != inventory.PlanFailed || got.Delivery.Stopped != "mesh-delivery for jochen" || - !strings.Contains(got.Note, "the operator stopped it") { - t.Fatalf("the stop was kept as %s %+v %q", got.State, got.Delivery, got.Note) - } - - // Unassigned: the mesh is back on the controller's own path. - if err := inv.Unassign(ctx, "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } - asks := len(*asked) - if p = merge("c6ffffffff", "modules/app/index.ts"); p.Waiting() || len(*asked) != asks+1 { - t.Fatalf("with the holder gone the walk waited: %v", p.Waiting()) - } -} // The verbs mesh-delivery asks with become the commands they name, and nothing a caller sends is passed over. func TestTheDeliveryVerbsComposeTheirCommands(t *testing.T) { @@ -295,49 +157,3 @@ func TestTheDeliveryVerbsComposeTheirCommands(t *testing.T) { // A verb runs as a command of its own: what it keeps of a walk is still said as plan-moved, on a bus of the // command's own, so the delivery's owner hears it at once and not only when it reads the walks back. -func TestAWalkLetGoByAVerbIsSaidAsPlanMoved(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - url := testbus.URL(t) - js, err := broker.Dial(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - before := handActConn - handActConn = js.Conn() - t.Cleanup(func() { handActConn = before }) - heard := make(chan *nats.Msg, 4) - sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), heard) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = sub.Unsubscribe() }) - - waiting := inventory.Plan{ID: "plan-waits", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c7c7c7c7", - Created: time.Now().UTC(), State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {}}, - Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}} - if err := open.inventory.SavePlan(ctx, &waiting); err != nil { - t.Fatal(err) - } - if err := deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}); err != nil { - t.Fatal(err) - } - select { - case m := <-heard: - var said inventory.Plan - if err := json.Unmarshal(m.Data, &said); err != nil || said.ID != waiting.ID || said.Delivery == nil || - said.Delivery.Go == nil { - t.Fatalf("plan-moved said %s (%v)", m.Data, err) - } - case <-time.After(5 * time.Second): - t.Fatal("a walk let go by a verb was not said") - } - if checkEvents != nil || inventory.PlanSaved != nil { - t.Fatal("the command's own bus was left in place") - } -} diff --git a/cmd/mesh-controller/desk_secret_integration_test.go b/cmd/mesh-controller/desk_secret_integration_test.go new file mode 100644 index 00000000..39ba3b72 --- /dev/null +++ b/cmd/mesh-controller/desk_secret_integration_test.go @@ -0,0 +1,96 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/link" +) + +func TestASecretValueIsNeverAcceptedThroughAVerb(t *testing.T) { + t.Setenv(verbVar, "mesh-controller.command") + for _, args := range [][]string{ + {"accept", "anchor", "telegram", "telegram-token", "--from", "/dev/null"}, + {"accept", "anchor", "app", "db", "--from", "/dev/null", "--provider", "store"}, + } { + err := secretCommand(context.Background(), args) + if err == nil || !strings.Contains(err.Error(), "never through a verb") { + t.Errorf("%v: %v", args, err) + } + } +} + +func TestATrustedPartysSecretGivenAtTheTerminalIsAnnouncedBeforeItIsKept(t *testing.T) { + var order []string + announce := func(fail bool) func() error { + return func() error { + order = append(order, "announce") + if fail { + return errors.New("no channel") + } + return nil + } + } + keep := func() (bool, error) { order = append(order, "keep"); return false, nil } + + order = nil + if _, unannounced, err := keepGiven(true, announce(false), keep); err != nil || unannounced != nil || + strings.Join(order, ",") != "announce,keep" { + t.Errorf("trusted: %v %v, order %v; want announced, then kept", unannounced, err, order) + } + order = nil + if _, _, err := keepGiven(true, announce(true), keep); err == nil || strings.Join(order, ",") != "announce" { + t.Errorf("trusted, announcement failed: %v, order %v; want refused and nothing kept", err, order) + } + order = nil + if _, unannounced, err := keepGiven(false, announce(true), keep); err != nil || unannounced == nil || + strings.Join(order, ",") != "keep,announce" { + t.Errorf("not trusted: %v %v, order %v; want kept, then the failed announcement said", unannounced, err, order) + } + order = nil + failing := func() (bool, error) { order = append(order, "keep"); return false, errors.New("store away") } + if _, _, err := keepGiven(false, announce(false), failing); err == nil || strings.Join(order, ",") != "keep" { + t.Errorf("not trusted, keep failed: %v, order %v; want refused and nothing announced", err, order) + } +} + +func TestASecretAskIsNeverASecretRead(t *testing.T) { + t.Setenv(verbVar, "mesh-controller.secret-ask") + for _, args := range [][]string{ + {"ask", "anchor", "telegram", "telegram-token", "the-value"}, + {"ask", "anchor", "telegram"}, + {"ask", "anchor", "telegram", "telegram-token", "--from", "/dev/null"}, + } { + if err := secretCommand(context.Background(), args); err == nil { + t.Errorf("secret %v was taken", args) + } + } + for _, args := range [][]string{{"recover", "anchor", "telegram", "telegram-token"}, {"export"}} { + if err := terminalOnly(append([]string{"secret"}, args...)); err == nil { + t.Errorf("secret %v passed the terminal rule", args) + } + } +} + +func TestATrustedPartysSecretAtTheTerminalIsNotKeptWhenTellAndTheConditionBothFail(t *testing.T) { + ctx := context.Background() + askOperatorChannelFake(t, func(map[string]any) (json.RawMessage, error) { return nil, link.ErrNothingServes }) + // conditionsFrom is left nil and there is no bus to dial here, so the fallback condition cannot be + // raised either: both of ADR 0380's ways to tell the operator fail. + kept := false + if _, _, err := keepGiven(true, + func() error { + return announceSecretGiven(ctx, "anchor", "telegram", "telegram-token", "at the controller's terminal", "me") + }, + func() (bool, error) { kept = true; return false, nil }); err == nil { + t.Error("a trusted party's secret was kept though telling the operator and raising the condition both failed") + } + if kept { + t.Error("the value was kept despite the failure") + } +} diff --git a/cmd/mesh-controller/desk_secret_test.go b/cmd/mesh-controller/desk_secret_test.go index 85bbb40e..363dba59 100644 --- a/cmd/mesh-controller/desk_secret_test.go +++ b/cmd/mesh-controller/desk_secret_test.go @@ -263,18 +263,6 @@ func TestOnlyTheControllerMayAskTheDesksPrompt(t *testing.T) { // A value for a secret comes from the terminal or the desk, never through a verb (the review of 2026-10-09, // M4): `secret accept` with a value, run for a verb, is refused before anything is read. -func TestASecretValueIsNeverAcceptedThroughAVerb(t *testing.T) { - t.Setenv(verbVar, "mesh-controller.command") - for _, args := range [][]string{ - {"accept", "anchor", "telegram", "telegram-token", "--from", "/dev/null"}, - {"accept", "anchor", "app", "db", "--from", "/dev/null", "--provider", "store"}, - } { - err := secretCommand(context.Background(), args) - if err == nil || !strings.Contains(err.Error(), "never through a verb") { - t.Errorf("%v: %v", args, err) - } - } -} // The `give` and `secret-ask` verbs' own line passes the terminal-only rule of ADR 0266, and no `secret accept` // does: a value, a file, a provider or an extra word is still the terminal's alone (novox/hq ADR 0277). @@ -355,39 +343,6 @@ func TestOnlyTheDeskMachinesLauncherMayAnswerItsPrompt(t *testing.T) { // N1-give at the controller's terminal (the confirmation review of 2026-10-09): a trusted party's secret is // announced before it is kept, and not kept when the announcement fails; another module's is kept first and // a failed announcement is said, not undone. -func TestATrustedPartysSecretGivenAtTheTerminalIsAnnouncedBeforeItIsKept(t *testing.T) { - var order []string - announce := func(fail bool) func() error { - return func() error { - order = append(order, "announce") - if fail { - return errors.New("no channel") - } - return nil - } - } - keep := func() (bool, error) { order = append(order, "keep"); return false, nil } - - order = nil - if _, unannounced, err := keepGiven(true, announce(false), keep); err != nil || unannounced != nil || - strings.Join(order, ",") != "announce,keep" { - t.Errorf("trusted: %v %v, order %v; want announced, then kept", unannounced, err, order) - } - order = nil - if _, _, err := keepGiven(true, announce(true), keep); err == nil || strings.Join(order, ",") != "announce" { - t.Errorf("trusted, announcement failed: %v, order %v; want refused and nothing kept", err, order) - } - order = nil - if _, unannounced, err := keepGiven(false, announce(true), keep); err != nil || unannounced == nil || - strings.Join(order, ",") != "keep,announce" { - t.Errorf("not trusted: %v %v, order %v; want kept, then the failed announcement said", unannounced, err, order) - } - order = nil - failing := func() (bool, error) { order = append(order, "keep"); return false, errors.New("store away") } - if _, _, err := keepGiven(false, announce(false), failing); err == nil || strings.Join(order, ",") != "keep" { - t.Errorf("not trusted, keep failed: %v, order %v; want refused and nothing announced", err, order) - } -} // A machine the mesh does not know, as the secret's or as the desk, is refused before anybody is asked to type. func TestAGiveNamingAMachineTheMeshDoesNotKnowAsksNobody(t *testing.T) { @@ -539,23 +494,6 @@ func TestASecretAskIsBoundedAndItsEndRecorded(t *testing.T) { // A secret ask cannot be turned into a secret read: the line carries no value, the answer carries none, and every // other `secret` line is the terminal's. -func TestASecretAskIsNeverASecretRead(t *testing.T) { - t.Setenv(verbVar, "mesh-controller.secret-ask") - for _, args := range [][]string{ - {"ask", "anchor", "telegram", "telegram-token", "the-value"}, - {"ask", "anchor", "telegram"}, - {"ask", "anchor", "telegram", "telegram-token", "--from", "/dev/null"}, - } { - if err := secretCommand(context.Background(), args); err == nil { - t.Errorf("secret %v was taken", args) - } - } - for _, args := range [][]string{{"recover", "anchor", "telegram", "telegram-token"}, {"export"}} { - if err := terminalOnly(append([]string{"secret"}, args...)); err == nil { - t.Errorf("secret %v passed the terminal rule", args) - } - } -} // askOperatorChannelFake replaces askOperatorChannel for the test and records every call it was given. func askOperatorChannelFake(t *testing.T, answer func(args map[string]any) (json.RawMessage, error)) *[]map[string]any { @@ -664,23 +602,6 @@ func TestASuccessfulTellRetiresEveryOpenSecretGivenCondition(t *testing.T) { // novox/hq ADR 0380 clause 7: a trusted party's secret given at the controller's terminal is announced // before it is kept, and not kept when the announcement fails. It still fails when telling the operator // AND raising the fallback condition both fail — nothing silently keeps a trusted party's value unheard. -func TestATrustedPartysSecretAtTheTerminalIsNotKeptWhenTellAndTheConditionBothFail(t *testing.T) { - ctx := context.Background() - askOperatorChannelFake(t, func(map[string]any) (json.RawMessage, error) { return nil, link.ErrNothingServes }) - // conditionsFrom is left nil and there is no bus to dial here, so the fallback condition cannot be - // raised either: both of ADR 0380's ways to tell the operator fail. - kept := false - if _, _, err := keepGiven(true, - func() error { - return announceSecretGiven(ctx, "anchor", "telegram", "telegram-token", "at the controller's terminal", "me") - }, - func() (bool, error) { kept = true; return false, nil }); err == nil { - t.Error("a trusted party's secret was kept though telling the operator and raising the condition both failed") - } - if kept { - t.Error("the value was kept despite the failure") - } -} // Review round 2 of mesh-controller (feat/mail-seats, head 5f24a47d): ADR 0380 §9's fallback — raising the // secret-given condition — answers only link.ErrNothingServes and link.ErrGrantRefused. Any other failure of diff --git a/cmd/mesh-controller/doctor_integration_test.go b/cmd/mesh-controller/doctor_integration_test.go new file mode 100644 index 00000000..33a1a2e6 --- /dev/null +++ b/cmd/mesh-controller/doctor_integration_test.go @@ -0,0 +1,246 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "errors" + "slices" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestARunKeepsWhatEachProbeFoundAndSaysItsHeartbeat(t *testing.T) { + failing := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Token: "refused", + Severity: conditions.Urgent, Summary: "anchor's node-engine would refuse its declaration"} + var broken atomic.Bool + broken.Store(true) + withProbes(t, + probe{ID: "P1", Asserts: "passes", Kind: "never", Phase: 1, + run: func(context.Context, *doctor) ([]conditions.Observation, error) { return nil, nil }}, + probe{ID: "P2", Asserts: "finds a fault", Kind: "declaration-refused", Phase: 1, + run: func(context.Context, *doctor) ([]conditions.Observation, error) { + if broken.Load() { + return []conditions.Observation{failing}, nil + } + return nil, nil + }}, + probe{ID: "P3", Asserts: "cannot run", Kind: "x", Phase: 1, + run: func(context.Context, *doctor) ([]conditions.Observation, error) { + if broken.Load() { + return nil, errors.New("the store is away") + } + return nil, nil + }}, + probe{ID: "P4", Asserts: "hangs", Kind: "x", Phase: 1, + run: func(ctx context.Context, _ *doctor) ([]conditions.Observation, error) { + if broken.Load() { + <-ctx.Done() + time.Sleep(50 * time.Millisecond) + } + return nil, nil + }}, + probe{ID: "P5", Asserts: "later", Kind: "x", Phase: 2, Deferred: "not yet"}, + ) + before := probeWithin + probeWithin = 200 * time.Millisecond + t.Cleanup(func() { probeWithin = before }) + + store := conditions.NewInMemory() + told := &conditions.Told{} + k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store}) + defer k.Close(context.Background()) + d := &doctor{keeper: k, teller: told, host: "anchor"} + // A probe that cannot run is said in the verdict at once, and as a condition when the next run cannot + // run it either (novox/hq issue 277): one timed-out question is not a probe gone blind. + first := d.runOnce(t.Context(), "a test") + if first.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) { + t.Fatalf("counted %+v", first.Counts) + } + if open, _ := k.Open(t.Context()); len(open) != 1 || open[0].Key != "machine.anchor.refused" { + t.Fatalf("after one run, open: %+v", open) + } + run := d.runOnce(t.Context(), "a test") + if run.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) { + t.Fatalf("counted %+v", run.Counts) + } + open, err := k.Open(t.Context()) + if err != nil { + t.Fatal(err) + } + var keys []string + for _, c := range open { + keys = append(keys, c.Key+"="+c.Kind) + } + for _, want := range []string{"machine.anchor.refused=declaration-refused", "probe.P3.failed=probe-failed", + "probe.P4.failed=probe-failed"} { + if !slices.Contains(keys, want) { + t.Errorf("%s is not open: %v", want, keys) + } + } + // The heartbeat, in the shape mesh-watcher reads (the contract with the operator's channel). + if len(told.Names) != 2 || told.Names[1] != conditions.HeartbeatEvent { + t.Fatalf("said %v", told.Names) + } + if d.lastRunEnded().IsZero() || d.lastRun().Run != run.Run { + t.Fatal("the run is not the last verdict") + } + body, _ := json.Marshal(run) + var shape map[string]any + _ = json.Unmarshal(body, &shape) + for _, field := range []string{"run", "at", "interval-seconds", "counts", "probes", "controller"} { + if _, ok := shape[field]; !ok { + t.Errorf("the heartbeat carries no %q: %s", field, body) + } + } + + // Mended: the next run clears every one of them. + broken.Store(false) + d.runOnce(t.Context(), "a test") + if open, _ := k.Open(t.Context()); len(open) != 0 { + t.Fatalf("a passing run left open %+v", open) + } +} + +func TestEveryMachineOfAHealthyMeshComposesAndValidates(t *testing.T) { + open := aMesh(t) + got, err := probeDeclarations(t.Context(), &doctor{open: open}) + if err != nil { + t.Fatal(err) + } + if len(got) != 0 { + t.Fatalf("a healthy mesh failed D1: %+v", got) + } +} + +func TestAMachineWhoseDeclarationDoesNotComposeIsSaid(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + one, two := rivals() + register(t, open, one) + register(t, open, two) + for _, m := range []string{"rival-one", "rival-two"} { + if _, err := assign(ctx, open, "laptop", m); err != nil && m == "rival-one" { + t.Fatal(err) + } + } + got, err := probeDeclarations(ctx, &doctor{open: open}) + linted(got) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "the-seat") { + t.Fatalf("%+v", got) + } +} + +func TestNatsTheBusIsWhatTheControllerDefines(t *testing.T) { + url := testbus.URL(t) + open := aMesh(t) + js, err := broker.Dial(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} { + _ = js.Context().DeleteStream(s) + } + if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil { + t.Fatal(err) + } + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + d := &doctor{open: open, js: js} + for _, p := range []func(context.Context, *doctor) ([]conditions.Observation, error){probeConsumers, probeStreams} { + got, err := p(t.Context(), d) + if err != nil || len(got) != 0 { + t.Fatalf("a bus just raised fails: %+v %v", got, err) + } + } + if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil { + t.Fatal(err) + } + info, err := js.Context().StreamInfo("EVENTS") + if err != nil { + t.Fatal(err) + } + cfg := info.Config + cfg.MaxMsgsPerSubject = 3 + if _, err := js.Context().UpdateStream(&cfg); err != nil { + t.Fatal(err) + } + consumers, err := probeConsumers(t.Context(), d) + if err != nil || len(consumers) != 1 || consumers[0].Key() != "bus.NODES.laptop.missing" { + t.Fatalf("the deleted consumer: %+v %v", consumers, err) + } + streams, err := probeStreams(t.Context(), d) + if err != nil || len(streams) != 1 || !strings.Contains(streams[0].Summary, "per subject") { + t.Fatalf("the redefined stream: %+v %v", streams, err) + } +} + +func TestNatsTheBusSaysAConsumerGaveUpAndOneWasDeleted(t *testing.T) { + url := testbus.URL(t) + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + heard := make(chan *nats.Msg, 16) + for _, subject := range broker.BusAdvisories { + if _, err := conn.ChanSubscribe(subject, heard); err != nil { + t.Fatal(err) + } + } + api, _ := jetstream.New(conn) + _ = api.DeleteStream(t.Context(), "SEAT_ADVISED") + stream, err := api.CreateStream(t.Context(), jetstream.StreamConfig{Name: "SEAT_ADVISED", Subjects: []string{"advised.>"}}) + if err != nil { + t.Fatal(err) + } + defer func() { _ = api.DeleteStream(context.Background(), "SEAT_ADVISED") }() + consumer, err := stream.CreateConsumer(t.Context(), jetstream.ConsumerConfig{Durable: "SEAT_ADVISED_worker", + AckPolicy: jetstream.AckExplicitPolicy, MaxDeliver: 1, AckWait: 100 * time.Millisecond}) + if err != nil { + t.Fatal(err) + } + if _, err := api.Publish(t.Context(), "advised.x", []byte("x")); err != nil { + t.Fatal(err) + } + if _, err := consumer.Fetch(1, jetstream.FetchMaxWait(time.Second)); err != nil { + t.Fatal(err) + } + // A seat's worker, so the deletion is of a consumer the mesh names (link.MeshNamed). + // Not acknowledged: after its one delivery the consumer gives up on it — on the next fetch. + time.Sleep(300 * time.Millisecond) + _, _ = consumer.Fetch(1, jetstream.FetchMaxWait(300*time.Millisecond)) + if err := stream.DeleteConsumer(t.Context(), "SEAT_ADVISED_worker"); err != nil { + t.Fatal(err) + } + kinds := map[string]string{} + deadline := time.After(5 * time.Second) + for len(kinds) < 2 { + select { + case m := <-heard: + if a, ok := link.ReadAdvisory(m.Subject, m.Data); ok { + kinds[a.Kind] = a.Said + } + case <-deadline: + t.Fatalf("the bus said only %v", kinds) + } + } + if !strings.Contains(kinds["max-deliveries"], "gave up") || !strings.Contains(kinds["consumer-lost"], "was deleted") { + t.Fatalf("%v", kinds) + } +} diff --git a/cmd/mesh-controller/doctor_test.go b/cmd/mesh-controller/doctor_test.go index f62990d3..8f5b0506 100644 --- a/cmd/mesh-controller/doctor_test.go +++ b/cmd/mesh-controller/doctor_test.go @@ -2,25 +2,18 @@ package main import ( "context" - "encoding/json" - "errors" "net" "slices" "strings" - "sync/atomic" "testing" "time" - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" "golang.org/x/net/dns/dnsmessage" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/link" - - "github.com/novox/mesh-controller/internal/testbus" ) // The self-check (novox/hq to-be 45 §4): a probe that fails raises its condition, one that cannot run @@ -34,98 +27,6 @@ func withProbes(t *testing.T, probes ...probe) { t.Cleanup(func() { probeRegistry = before }) } -func TestARunKeepsWhatEachProbeFoundAndSaysItsHeartbeat(t *testing.T) { - failing := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Token: "refused", - Severity: conditions.Urgent, Summary: "anchor's node-engine would refuse its declaration"} - var broken atomic.Bool - broken.Store(true) - withProbes(t, - probe{ID: "P1", Asserts: "passes", Kind: "never", Phase: 1, - run: func(context.Context, *doctor) ([]conditions.Observation, error) { return nil, nil }}, - probe{ID: "P2", Asserts: "finds a fault", Kind: "declaration-refused", Phase: 1, - run: func(context.Context, *doctor) ([]conditions.Observation, error) { - if broken.Load() { - return []conditions.Observation{failing}, nil - } - return nil, nil - }}, - probe{ID: "P3", Asserts: "cannot run", Kind: "x", Phase: 1, - run: func(context.Context, *doctor) ([]conditions.Observation, error) { - if broken.Load() { - return nil, errors.New("the store is away") - } - return nil, nil - }}, - probe{ID: "P4", Asserts: "hangs", Kind: "x", Phase: 1, - run: func(ctx context.Context, _ *doctor) ([]conditions.Observation, error) { - if broken.Load() { - <-ctx.Done() - time.Sleep(50 * time.Millisecond) - } - return nil, nil - }}, - probe{ID: "P5", Asserts: "later", Kind: "x", Phase: 2, Deferred: "not yet"}, - ) - before := probeWithin - probeWithin = 200 * time.Millisecond - t.Cleanup(func() { probeWithin = before }) - - store := conditions.NewInMemory() - told := &conditions.Told{} - k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store}) - defer k.Close(context.Background()) - d := &doctor{keeper: k, teller: told, host: "anchor"} - // A probe that cannot run is said in the verdict at once, and as a condition when the next run cannot - // run it either (novox/hq issue 277): one timed-out question is not a probe gone blind. - first := d.runOnce(t.Context(), "a test") - if first.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) { - t.Fatalf("counted %+v", first.Counts) - } - if open, _ := k.Open(t.Context()); len(open) != 1 || open[0].Key != "machine.anchor.refused" { - t.Fatalf("after one run, open: %+v", open) - } - run := d.runOnce(t.Context(), "a test") - if run.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) { - t.Fatalf("counted %+v", run.Counts) - } - open, err := k.Open(t.Context()) - if err != nil { - t.Fatal(err) - } - var keys []string - for _, c := range open { - keys = append(keys, c.Key+"="+c.Kind) - } - for _, want := range []string{"machine.anchor.refused=declaration-refused", "probe.P3.failed=probe-failed", - "probe.P4.failed=probe-failed"} { - if !slices.Contains(keys, want) { - t.Errorf("%s is not open: %v", want, keys) - } - } - // The heartbeat, in the shape mesh-watcher reads (the contract with the operator's channel). - if len(told.Names) != 2 || told.Names[1] != conditions.HeartbeatEvent { - t.Fatalf("said %v", told.Names) - } - if d.lastRunEnded().IsZero() || d.lastRun().Run != run.Run { - t.Fatal("the run is not the last verdict") - } - body, _ := json.Marshal(run) - var shape map[string]any - _ = json.Unmarshal(body, &shape) - for _, field := range []string{"run", "at", "interval-seconds", "counts", "probes", "controller"} { - if _, ok := shape[field]; !ok { - t.Errorf("the heartbeat carries no %q: %s", field, body) - } - } - - // Mended: the next run clears every one of them. - broken.Store(false) - d.runOnce(t.Context(), "a test") - if open, _ := k.Open(t.Context()); len(open) != 0 { - t.Fatalf("a passing run left open %+v", open) - } -} - // **The registry says what each probe asserts**, and a probe not built says why and when. func TestTheRegistryIsTheDesignsLiveForm(t *testing.T) { seen := map[string]bool{} @@ -164,38 +65,8 @@ func TestWatchdogsThatStoppedAreSaid(t *testing.T) { } // **D1 composes every machine of a healthy mesh and the host's own validator takes each.** -func TestEveryMachineOfAHealthyMeshComposesAndValidates(t *testing.T) { - open := aMesh(t) - got, err := probeDeclarations(t.Context(), &doctor{open: open}) - if err != nil { - t.Fatal(err) - } - if len(got) != 0 { - t.Fatalf("a healthy mesh failed D1: %+v", got) - } -} // **D1 names a machine nothing can be sent to**, and the network that cannot be computed for it. -func TestAMachineWhoseDeclarationDoesNotComposeIsSaid(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - one, two := rivals() - register(t, open, one) - register(t, open, two) - for _, m := range []string{"rival-one", "rival-two"} { - if _, err := assign(ctx, open, "laptop", m); err != nil && m == "rival-one" { - t.Fatal(err) - } - } - got, err := probeDeclarations(ctx, &doctor{open: open}) - linted(got) - if err != nil { - t.Fatal(err) - } - if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "the-seat") { - t.Fatalf("%+v", got) - } -} // **D2: a resolver answering NXDOMAIN for IPv6 is wrong** — musl takes it as no such name (issue 262). func TestAResolverAnsweringNoSuchNameForIPv6IsWrong(t *testing.T) { @@ -250,107 +121,8 @@ func TestAResolverAnsweringNoSuchNameForIPv6IsWrong(t *testing.T) { // **D6, D7: what the controller defines is what it finds**, and a consumer deleted or a stream // redefined is said — against a real bus, raised by the same derivation the controller starts with. -func TestNatsTheBusIsWhatTheControllerDefines(t *testing.T) { - url := testbus.URL(t) - open := aMesh(t) - js, err := broker.Dial(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} { - _ = js.Context().DeleteStream(s) - } - if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil { - t.Fatal(err) - } - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - d := &doctor{open: open, js: js} - for _, p := range []func(context.Context, *doctor) ([]conditions.Observation, error){probeConsumers, probeStreams} { - got, err := p(t.Context(), d) - if err != nil || len(got) != 0 { - t.Fatalf("a bus just raised fails: %+v %v", got, err) - } - } - if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil { - t.Fatal(err) - } - info, err := js.Context().StreamInfo("EVENTS") - if err != nil { - t.Fatal(err) - } - cfg := info.Config - cfg.MaxMsgsPerSubject = 3 - if _, err := js.Context().UpdateStream(&cfg); err != nil { - t.Fatal(err) - } - consumers, err := probeConsumers(t.Context(), d) - if err != nil || len(consumers) != 1 || consumers[0].Key() != "bus.NODES.laptop.missing" { - t.Fatalf("the deleted consumer: %+v %v", consumers, err) - } - streams, err := probeStreams(t.Context(), d) - if err != nil || len(streams) != 1 || !strings.Contains(streams[0].Summary, "per subject") { - t.Fatalf("the redefined stream: %+v %v", streams, err) - } -} // **S9 hears the bus**: a consumer that gives up on a message, and one deleted, as the server says. -func TestNatsTheBusSaysAConsumerGaveUpAndOneWasDeleted(t *testing.T) { - url := testbus.URL(t) - conn, err := nats.Connect(url) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - heard := make(chan *nats.Msg, 16) - for _, subject := range broker.BusAdvisories { - if _, err := conn.ChanSubscribe(subject, heard); err != nil { - t.Fatal(err) - } - } - api, _ := jetstream.New(conn) - _ = api.DeleteStream(t.Context(), "SEAT_ADVISED") - stream, err := api.CreateStream(t.Context(), jetstream.StreamConfig{Name: "SEAT_ADVISED", Subjects: []string{"advised.>"}}) - if err != nil { - t.Fatal(err) - } - defer func() { _ = api.DeleteStream(context.Background(), "SEAT_ADVISED") }() - consumer, err := stream.CreateConsumer(t.Context(), jetstream.ConsumerConfig{Durable: "SEAT_ADVISED_worker", - AckPolicy: jetstream.AckExplicitPolicy, MaxDeliver: 1, AckWait: 100 * time.Millisecond}) - if err != nil { - t.Fatal(err) - } - if _, err := api.Publish(t.Context(), "advised.x", []byte("x")); err != nil { - t.Fatal(err) - } - if _, err := consumer.Fetch(1, jetstream.FetchMaxWait(time.Second)); err != nil { - t.Fatal(err) - } - // A seat's worker, so the deletion is of a consumer the mesh names (link.MeshNamed). - // Not acknowledged: after its one delivery the consumer gives up on it — on the next fetch. - time.Sleep(300 * time.Millisecond) - _, _ = consumer.Fetch(1, jetstream.FetchMaxWait(300*time.Millisecond)) - if err := stream.DeleteConsumer(t.Context(), "SEAT_ADVISED_worker"); err != nil { - t.Fatal(err) - } - kinds := map[string]string{} - deadline := time.After(5 * time.Second) - for len(kinds) < 2 { - select { - case m := <-heard: - if a, ok := link.ReadAdvisory(m.Subject, m.Data); ok { - kinds[a.Kind] = a.Said - } - case <-deadline: - t.Fatalf("the bus said only %v", kinds) - } - } - if !strings.Contains(kinds["max-deliveries"], "gave up") || !strings.Contains(kinds["consumer-lost"], "was deleted") { - t.Fatalf("%v", kinds) - } -} // **D10 compares a node-engine with what it is delivered as, not with its commit** (2026-10-06: every // machine read as behind right after a push sent it the current build — it says the digest-named diff --git a/cmd/mesh-controller/dry_run_integration_test.go b/cmd/mesh-controller/dry_run_integration_test.go new file mode 100644 index 00000000..a8cab99e --- /dev/null +++ b/cmd/mesh-controller/dry_run_integration_test.go @@ -0,0 +1,20 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "testing" + + "github.com/novox/mesh-controller/internal/link" +) + +func TestADryRunsOutcomeIsTakenInByNothing(t *testing.T) { + err := builds{}.Built(t.Context(), link.BuildResult{ + ID: "build-1", Repository: "ssh://forge/app.git", Ref: "unreviewed", Module: "app", DryRun: true, + Manifest: json.RawMessage(`{"module":"app","version":"1"}`), + }) + if err != nil { + t.Fatalf("a dry run's outcome was not simply set aside: %v", err) + } +} diff --git a/cmd/mesh-controller/dry_run_test.go b/cmd/mesh-controller/dry_run_test.go index 0f087346..53767a0a 100644 --- a/cmd/mesh-controller/dry_run_test.go +++ b/cmd/mesh-controller/dry_run_test.go @@ -9,15 +9,6 @@ import ( // A dry run's outcome is looked at, never taken in (novox/hq issue 240). The daemon here holds no // store at all, so anything that tried to record or register would fail rather than pass quietly. -func TestADryRunsOutcomeIsTakenInByNothing(t *testing.T) { - err := builds{}.Built(t.Context(), link.BuildResult{ - ID: "build-1", Repository: "ssh://forge/app.git", Ref: "unreviewed", Module: "app", DryRun: true, - Manifest: json.RawMessage(`{"module":"app","version":"1"}`), - }) - if err != nil { - t.Fatalf("a dry run's outcome was not simply set aside: %v", err) - } -} // The mark survives the wire both ways: asked as a dry run, answered as one. func TestTheDryRunMarkTravelsWithTheBuild(t *testing.T) { diff --git a/cmd/mesh-controller/epoch_send_test.go b/cmd/mesh-controller/epoch_send_test.go index ae6adbdc..9e138c3c 100644 --- a/cmd/mesh-controller/epoch_send_test.go +++ b/cmd/mesh-controller/epoch_send_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/facts_test.go b/cmd/mesh-controller/facts_test.go index d5b3ac63..7f3c9544 100644 --- a/cmd/mesh-controller/facts_test.go +++ b/cmd/mesh-controller/facts_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/failed_wait_provider_test.go b/cmd/mesh-controller/failed_wait_provider_test.go index 64a7a960..2790e340 100644 --- a/cmd/mesh-controller/failed_wait_provider_test.go +++ b/cmd/mesh-controller/failed_wait_provider_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/found_wait_integration_test.go b/cmd/mesh-controller/found_wait_integration_test.go new file mode 100644 index 00000000..1a1f193c --- /dev/null +++ b/cmd/mesh-controller/found_wait_integration_test.go @@ -0,0 +1,127 @@ +//go:build integration + +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + for _, c := range []struct { + name string + found time.Duration // when the directory was found, against now + passes bool + }{ + {"found a day before the send", -24 * time.Hour, true}, + {"found by this send", time.Hour, false}, + } { + t.Run(c.name, func(t *testing.T) { + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + releaseHeard = func(context.Context, *stores) (map[string]bool, error) { + return map[string]bool{"anchor": true, "laptop": true}, nil + } + backlogFacts := gatherGateFacts + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + f, err := backlogFacts(ctx, open, component) + // The used-as-found condition, raised after the send (its second statement): its own kind, never a + // fault the gate reads as the build's. + f.judged, f.openErr = true, nil + f.open = append(f.open, conditions.Condition{Key: usedAsFoundKey("app", "anchor"), Kind: kindUsedAsFound, + Subject: conditions.Subject{Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"}, + Raised: time.Now()}) + f.health = map[string]inventory.NodeHealth{} + for _, n := range []string{"anchor", "laptop"} { + f.health[n] = inventory.NodeHealth{Node: n, HeardAt: time.Now(), Resources: []inventory.ResourceHealth{ + {Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy}, + foundDirectory("app", time.Now().Add(c.found)), + {Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateHealthy}}} + } + return f, err + } + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) + t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) + gateWithin, gateEvery, gateBound = time.Hour, 0, 300*time.Millisecond + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + advancePlans(ctx, b.open) + if p := b.release(t); p.State != inventory.PlanRolling { + break + } + time.Sleep(20 * time.Millisecond) + } + p := b.release(t) + v, found, err := inv.GateOf(ctx, "build-app-c2") + if c.passes { + if p.State != inventory.PlanDone || err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("the walk is %s (%s); app's verdict %+v: want the fix through", p.State, p.Note, v) + } + if !strings.Contains(v.Why+p.Note, "hand it over") { + t.Errorf("the wait is not carried: verdict %q, walk %q", v.Why, p.Note) + } + return + } + if p.State == inventory.PlanDone { + t.Fatalf("a directory this send found let the walk through: %s", p.Note) + } + }) + } +} + +func TestADirectoryUsedAsFoundIsItsOwnCondition(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + k, _ := withConditionsInMemory(t) + ctx := t.Context() + rs := map[string][]inventory.ResourceHealth{"notes": {foundDirectory("notes", time.Now().Add(-time.Hour))}} + for i := 0; i < 2; i++ { + if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": i + 1}, time.Now()); err != nil { + t.Fatal(err) + } + } + open, _ := k.Open(ctx) + var got *conditions.Condition + for i, c := range open { + if c.Key == usedAsFoundKey("notes", "laptop") { + got = &open[i] + } + if c.Kind == kindModuleUnhealthy { + t.Fatalf("raised as a fault: %+v", c) + } + } + if got == nil || got.Resolver != conditions.ResolverOperator || got.Severity == conditions.Urgent || + !strings.Contains(got.Summary, "notes.data") { + t.Fatalf("the condition: %+v", got) + } + // Long open is still not urgent: only a person can hand it over, and nothing is broken by the wait. + if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": 3}, time.Now().Add(48*time.Hour)); err != nil { + t.Fatal(err) + } + open, _ = k.Open(ctx) + for _, c := range open { + if c.Key == usedAsFoundKey("notes", "laptop") && c.Severity == conditions.Urgent { + t.Fatal("a directory used as found became urgent") + } + } + if err := judgeModuleHealth(ctx, nil, k, "laptop", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil { + t.Fatal(err) + } + open, _ = k.Open(ctx) + for _, c := range open { + if c.Key == usedAsFoundKey("notes", "laptop") { + t.Fatal("not cleared once handed over") + } + } +} diff --git a/cmd/mesh-controller/found_wait_test.go b/cmd/mesh-controller/found_wait_test.go index caa75ac7..54621738 100644 --- a/cmd/mesh-controller/found_wait_test.go +++ b/cmd/mesh-controller/found_wait_test.go @@ -1,12 +1,10 @@ package main import ( - "context" "strings" "testing" "time" - "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -48,116 +46,5 @@ func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) { // The whole walk: a module whose directory was used as found long before still gets its fix to every machine, // its pass kept and the wait said; a directory this very send found holds it and puts it back. -func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - for _, c := range []struct { - name string - found time.Duration // when the directory was found, against now - passes bool - }{ - {"found a day before the send", -24 * time.Hour, true}, - {"found by this send", time.Hour, false}, - } { - t.Run(c.name, func(t *testing.T) { - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - releaseHeard = func(context.Context, *stores) (map[string]bool, error) { - return map[string]bool{"anchor": true, "laptop": true}, nil - } - backlogFacts := gatherGateFacts - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f, err := backlogFacts(ctx, open, component) - // The used-as-found condition, raised after the send (its second statement): its own kind, never a - // fault the gate reads as the build's. - f.judged, f.openErr = true, nil - f.open = append(f.open, conditions.Condition{Key: usedAsFoundKey("app", "anchor"), Kind: kindUsedAsFound, - Subject: conditions.Subject{Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"}, - Raised: time.Now()}) - f.health = map[string]inventory.NodeHealth{} - for _, n := range []string{"anchor", "laptop"} { - f.health[n] = inventory.NodeHealth{Node: n, HeardAt: time.Now(), Resources: []inventory.ResourceHealth{ - {Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy}, - foundDirectory("app", time.Now().Add(c.found)), - {Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateHealthy}}} - } - return f, err - } - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) - gateWithin, gateEvery, gateBound = time.Hour, 0, 300*time.Millisecond - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - advancePlans(ctx, b.open) - if p := b.release(t); p.State != inventory.PlanRolling { - break - } - time.Sleep(20 * time.Millisecond) - } - p := b.release(t) - v, found, err := inv.GateOf(ctx, "build-app-c2") - if c.passes { - if p.State != inventory.PlanDone || err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("the walk is %s (%s); app's verdict %+v: want the fix through", p.State, p.Note, v) - } - if !strings.Contains(v.Why+p.Note, "hand it over") { - t.Errorf("the wait is not carried: verdict %q, walk %q", v.Why, p.Note) - } - return - } - if p.State == inventory.PlanDone { - t.Fatalf("a directory this send found let the walk through: %s", p.Note) - } - }) - } -} // The condition says the wait in its own kind: the operator's, never urgent, and cleared once handed over. -func TestADirectoryUsedAsFoundIsItsOwnCondition(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - k, _ := withConditionsInMemory(t) - ctx := t.Context() - rs := map[string][]inventory.ResourceHealth{"notes": {foundDirectory("notes", time.Now().Add(-time.Hour))}} - for i := 0; i < 2; i++ { - if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": i + 1}, time.Now()); err != nil { - t.Fatal(err) - } - } - open, _ := k.Open(ctx) - var got *conditions.Condition - for i, c := range open { - if c.Key == usedAsFoundKey("notes", "laptop") { - got = &open[i] - } - if c.Kind == kindModuleUnhealthy { - t.Fatalf("raised as a fault: %+v", c) - } - } - if got == nil || got.Resolver != conditions.ResolverOperator || got.Severity == conditions.Urgent || - !strings.Contains(got.Summary, "notes.data") { - t.Fatalf("the condition: %+v", got) - } - // Long open is still not urgent: only a person can hand it over, and nothing is broken by the wait. - if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": 3}, time.Now().Add(48*time.Hour)); err != nil { - t.Fatal(err) - } - open, _ = k.Open(ctx) - for _, c := range open { - if c.Key == usedAsFoundKey("notes", "laptop") && c.Severity == conditions.Urgent { - t.Fatal("a directory used as found became urgent") - } - } - if err := judgeModuleHealth(ctx, nil, k, "laptop", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil { - t.Fatal(err) - } - open, _ = k.Open(ctx) - for _, c := range open { - if c.Key == usedAsFoundKey("notes", "laptop") { - t.Fatal("not cleared once handed over") - } - } -} diff --git a/cmd/mesh-controller/gate_followup_test.go b/cmd/mesh-controller/gate_followup_test.go index a78963dd..9ba07bf3 100644 --- a/cmd/mesh-controller/gate_followup_test.go +++ b/cmd/mesh-controller/gate_followup_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/gate_integration_test.go b/cmd/mesh-controller/gate_integration_test.go new file mode 100644 index 00000000..fff52fa6 --- /dev/null +++ b/cmd/mesh-controller/gate_integration_test.go @@ -0,0 +1,482 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" + "github.com/novox/mesh-controller/internal/link" +) + +func aGateMesh(t *testing.T) *gateMesh { + t.Helper() + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + g := &gateMesh{open: open, health: map[string]health{}} + // The gate's own tests judge a first machine, as walks a controller before novox/hq ADR 0360 started still are. + wasGated := gateFirstMachine + gateFirstMachine = true + t.Cleanup(func() { gateFirstMachine = wasGated }) + g.keeper, _ = withConditionsInMemory(t) + g.told = &conditions.Told{} + was := doctorFrom + doctorFrom = &doctor{open: open, keeper: g.keeper, teller: g.told} + t.Cleanup(func() { doctorFrom = was }) + + for _, b := range []inventory.Build{ + {ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app", + Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)}, + {ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app", + Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)}, + } { + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit}) + b.Manifest = manifest + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + } + register := func(commit string, asked time.Time) { + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit, + Head: commit, Asked: asked}); err != nil { + t.Fatal(err) + } + } + register("c1", time.Now().Add(-2*time.Hour)) + for _, n := range []string{"anchor", "laptop"} { + if _, err := inv.Assign(ctx, n, "app"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil { + t.Fatal(err) + } + } + register("c2", time.Now().Add(-time.Minute)) + + // Every send: recorded with the build the module is at, applied and reported by the machine. + n := 0 + wasSend := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + g.sent = append(g.sent, append([]string(nil), names...)) + current, err := open.inventory.CurrentBuilds(ctx) + if err != nil { + return nil, err + } + for _, node := range names { + n++ + digest := fmt.Sprintf("d-%s-%d", node, n) + if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil { + return nil, err + } + if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, + Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { + return nil, err + } + } + return names, nil + } + t.Cleanup(func() { sendRollout = wasSend }) + + // What a judging reads: the store's reports, and a health the test says per machine. + wasGather := gatherGateFacts + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, + rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} + reports, err := open.inventory.LastReports(ctx) + if err != nil { + return f, err + } + for _, r := range reports { + if g.health[r.Node] == healthBroken { + r.Outcome = inventory.OutcomeFailed + } + f.reports[r.Node] = r + } + for node, h := range g.health { + if h == healthNotYet { + // Not reported on the send: neither the send made last, nor the gate's own (issue 352). + f.rolledBack[node] = nil + r := f.reports[node] + r.Current, r.Declared, r.ReportedSequence = false, "", 0 + f.reports[node] = r + } + } + return f, nil + } + t.Cleanup(func() { gatherGateFacts = wasGather }) + + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) + // One judging per advance until a test says otherwise: a pass waits gateEvery for the next. + gateWithin, gateEvery = time.Hour, time.Hour + t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) + + built := time.Now().UTC() + plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", + Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, + Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", + Build: "build-2"}}} + if err := inv.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + return g +} + +func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) { + judgeLeavesAsTheCore(t) + g := aGateMesh(t) + ctx := t.Context() + inv := g.open.inventory + + advancePlans(ctx, g.open) // the first machine is sent the new build + if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}}) { + t.Fatalf("sent %v, not the first machine alone", g.sent) + } + if p := g.plan(t); p.Modules["app"].Previous != "c1" { + t.Fatalf("the build the first machine ran before was not kept: %+v", p.Modules["app"]) + } + + g.health["anchor"] = healthBroken // the new build breaks its first machine, after one good judging + gateEvery = 0 + advancePlans(ctx, g.open) + + p := g.plan(t) + gate := p.Modules["app"].Gate + if p.State != inventory.PlanFailed || gate == nil || gate.Verdict != inventory.GateFailed || + gate.Rollback != inventory.RolledBack { + t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate) + } + if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"anchor"}}) { + t.Fatalf("sent %v: the rollback goes to the first machine, and nothing reaches laptop", g.sent) + } + if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" { + t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit) + } + if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c1" { + t.Fatalf("anchor was last sent %v, not the previous build", sent) + } + if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["app"] != "c1" { + t.Fatalf("laptop was sent the failed build: %v", sent) + } + if failed, err := inv.GateFailed(ctx, "build-2"); err != nil || !failed { + t.Fatalf("the build is not marked failed at its gate: %v %v", failed, err) + } + + // Said: a condition for the operator, and the event. + open, err := g.keeper.Open(ctx) + if err != nil { + t.Fatal(err) + } + var key string + for _, c := range open { + if c.Kind == kindRolledBack { + key = c.Key + } + } + if key != "build.app.anchor.rolled-back" { + t.Fatalf("no rolled-back condition for app on anchor: %+v", open) + } + saidIt := false + for _, e := range g.told.Said() { + saidIt = saidIt || e.Event == link.KeyRolledBack + } + if !saidIt { + t.Fatalf("the rollback was not said as an event: %+v", g.told.Said()) + } + + // Never again: more passes send nothing, a second judging rolls nothing back, and the failed build + // heard again is not registered. + advancePlans(ctx, g.open) + state := p.Modules["app"] + gateFailed(ctx, g.open, &p, "app", state, []string{"anchor"}, "again") + if len(g.sent) != 2 { + t.Fatalf("sent again after the rollback: %v", g.sent) + } + _, _, err = takeIn(ctx, inv, link.BuildResult{ID: "build-2", Repository: "novox/mesh-catalog", Path: "modules/app", + Commit: "c2", Manifest: mustJSON(t, catalogue.Manifest{Module: "app", Version: "c2"}), + Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}) + if err == nil || !strings.Contains(err.Error(), "failed its gate") { + t.Fatalf("the failed build was registered again: %v", err) + } + if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" { + t.Fatalf("the module moved to %s", current["app"].Commit) + } + if _, err := retryPlan(ctx, g.open, "plan-gate"); err == nil || !strings.Contains(err.Error(), "failed its gate") { + t.Fatalf("a plan stopped at a failed gate was retried: %v", err) + } + + // The probe keeps the condition while the newest verdict is the failure. + obs, err := probeGates(ctx, doctorFrom) + if err != nil || len(obs) != 1 || obs[0].Key() != key { + t.Fatalf("the gate's probe found %+v %v", obs, err) + } +} + +func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) { + g := aGateMesh(t) + ctx := t.Context() + gateEvery = 0 + for i := 0; i < 6; i++ { + advancePlans(ctx, g.open) + } + p := g.plan(t) + if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"laptop"}}) { + t.Fatalf("sent %v", g.sent) + } + gate := p.Modules["app"].Gate + if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gateWanted(gate) { + t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate) + } + if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("the verdict was not kept: %+v %v %v", v, found, err) + } + if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 { + t.Fatalf("a passing build left a condition: %+v %v", obs, err) + } +} + +func TestABuildRollsToEveryMachineAtOnceWithNoGate(t *testing.T) { + g := aGateMesh(t) + gateFirstMachine = false + ctx := t.Context() + judged := false + wasGather := gatherGateFacts + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + judged = true + return wasGather(ctx, open, component) + } + t.Cleanup(func() { gatherGateFacts = wasGather }) + for i := 0; i < 6; i++ { + advancePlans(ctx, g.open) + } + p := g.plan(t) + if !reflect.DeepEqual(g.sent, [][]string{{"anchor", "laptop"}}) { + t.Fatalf("sent %v, not both machines in one send", g.sent) + } + if s := p.Modules["app"]; p.State != inventory.PlanDone || s.Gate != nil || s.FirstAt != nil || judged { + t.Fatalf("the plan is %s (%s), app %+v, judged %v", p.State, p.Note, s, judged) + } + if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 { + t.Fatalf("a rolled build left a condition: %+v %v", obs, err) + } +} + +func TestALeafModulePassesOnOneHealthyReading(t *testing.T) { + g := aGateMesh(t) + ctx := t.Context() + gateWithin = time.Hour // a core module could not pass on fewer readings for an hour + advancePlans(ctx, g.open) // the first machine is sent the new build + advancePlans(ctx, g.open) // one healthy judging + p := g.plan(t) + gate := p.Modules["app"].Gate + if gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes != 1 || gate.Wants != 1 { + t.Fatalf("a leaf module's gate after one healthy judging: %+v (plan %s: %s)", gate, p.State, p.Note) + } + if !strings.Contains(gate.Why, "healthy 1 times") { + t.Errorf("the verdict says %q", gate.Why) + } +} + +func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) { + judgeLeavesAsTheCore(t) + g := aGateMesh(t) + ctx := t.Context() + advancePlans(ctx, g.open) + g.health["anchor"] = healthNotYet + gateEvery = 0 + advancePlans(ctx, g.open) + if p := g.plan(t); !p.Open() || len(g.sent) != 1 { + t.Fatalf("judged before the bound: %s %v", p.State, g.sent) + } + gateBound = -time.Second + advancePlans(ctx, g.open) + p := g.plan(t) + if gate := p.Modules["app"].Gate; p.State != inventory.PlanFailed || gate.Verdict != inventory.GateFailed || + !strings.Contains(gate.Why, "not healthy within") || gate.Rollback != inventory.RolledBack { + t.Fatalf("the plan is %s, its gate %+v", p.State, gate) + } +} + +func TestTheBusIsNeverRolledOutAutomatically(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + bus := catalogue.Manifest{Module: "nats", Version: "2", Provides: []catalogue.Offer{{Name: "mesh-bus"}}, + Upgrade: &catalogue.UpgradePolicy{Policy: catalogue.PolicyRoll}} + if err := inv.RegisterModule(ctx, bus, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", + Path: "modules/nats", BuiltFrom: "n2", Head: "n2"}); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, "anchor", "nats"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"nats": "n1"}); err != nil { + t.Fatal(err) + } + u, err := inv.UpgradeOf(ctx, "nats") + if err != nil || u.RollOut || u.From != catalogue.FromBus { + t.Fatalf("the bus's policy is %+v %v", u, err) + } + if err := inv.SetUpgradeOf(ctx, "nats", inventory.Upgrade{RollOut: true}); !errors.Is(err, inventory.ErrBusIsPlanned) { + t.Fatalf("a person rolled the bus out: %v", err) + } + var sent [][]string + was := sendRollout + sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) { + sent = append(sent, names) + return names, nil + } + t.Cleanup(func() { sendRollout = was }) + now := time.Now().UTC() + plan := inventory.Plan{ID: "plan-bus", Repository: "novox/mesh-catalog", Commit: "n2", Created: now, + State: inventory.PlanBuilding, Tiers: [][]string{{"nats"}}, + Modules: map[string]*inventory.PlanModule{"nats": {State: "built", BuiltAt: &now, Commit: "n2", Build: "b"}}} + if err := inv.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + advancePlans(ctx, open) + if p, _ := inv.PlanByID(ctx, "plan-bus"); p.State != inventory.PlanDone || len(sent) != 0 { + t.Fatalf("a plan sent the bus: %s %v", p.State, sent) + } + held, err := busHeld(ctx, inv, []string{"anchor", "laptop"}) + if err != nil || !strings.Contains(held["anchor"], "planned step") || held["laptop"] != "" { + t.Fatalf("a push may send the bus's machine: %v %v", held, err) + } + // Nor may any other send — a plan's for another module on that machine carried the bus with it. + if _, err := sendToEach(ctx, open, []string{"laptop", "anchor"}); !errors.Is(err, errBusWaits) { + t.Fatalf("a send to the bus's machine was not refused: %v", err) + } + // A rebuild that made the same artifacts is no move. + for _, b := range []inventory.Build{{ID: "nb1", Module: "nats", Commit: "n1"}, {ID: "nb2", Module: "nats", Commit: "n2"}} { + b.Made = []inventory.Artifact{{Name: "server", Kind: "image", Reference: "registry/nats@sha256:same"}} + b.Asked, b.At = time.Now(), time.Now() + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + } + if held, err := busHeld(ctx, inv, []string{"anchor"}); err != nil || len(held) != 0 { + t.Fatalf("a rebuild that changes nothing held the bus's machine: %v %v", held, err) + } + if err := inv.RecordBuild(ctx, inventory.Build{ID: "nb3", Module: "nats", Commit: "n2", Asked: time.Now().Add(time.Second), + At: time.Now().Add(time.Second), Made: []inventory.Artifact{{Name: "server", Kind: "image", + Reference: "registry/nats@sha256:new"}}}); err != nil { + t.Fatal(err) + } + // The planned step refuses to start without its word on reversibility, and without a snapshot taken + // first by the bus machine's backup holder. + if err := busCommand(ctx, []string{"upgrade", "--why", "2.11"}); err == nil || !strings.Contains(err.Error(), "reversible") { + t.Fatalf("a bus upgrade started without saying whether it can be reverted: %v", err) + } + wasSnapshot := takeBusSnapshot + t.Cleanup(func() { takeBusSnapshot = wasSnapshot }) + takeBusSnapshot = func(context.Context, string, string) (string, error) { return "", errors.New("no holder answers") } + if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err == nil || + !strings.Contains(err.Error(), "snapshotted") || len(sent) != 0 { + t.Fatalf("a bus upgrade started without its snapshot: %v, sent %v", err, sent) + } + takeBusSnapshot = func(_ context.Context, module, node string) (string, error) { + return node + "'s restore point of " + module, nil + } + if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(sent, [][]string{{"anchor"}}) { + t.Fatalf("the step sent %v, not the bus's machine", sent) + } + s, found, err := inv.LatestBusStep(ctx) + if err != nil || !found || s.Snapshot != "anchor's restore point of nats" || s.Ended != nil || + !reflect.DeepEqual(s.Machines, []string{"anchor"}) { + t.Fatalf("the step is %+v %v %v", s, found, err) + } +} + +func TestAMergeThatDeletesAModulePlansNothingToBuildForIt(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + asked := asksRecorded(t) + for _, name := range []string{"gone", "kept"} { + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: name, Version: "1"}, inventory.Source{ + Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + name, BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + } + m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1deadbeef", + Paths: []string{"modules/gone/module.json", "modules/gone/index.ts"}, + Removed: []string{"modules/gone/module.json", "modules/gone/index.ts"}} + if err := (following{open: open}).SourceMoved(ctx, m); err != nil { + t.Fatal(err) + } + if len(*asked) != 0 { + t.Fatalf("a deleted module was asked to build: %v", *asked) + } + if plans, _ := inv.OpenPlans(ctx); len(plans) != 0 { + t.Fatalf("a merge that only deleted a module made a plan: %+v", plans) + } + shelf, err := inv.Catalogue(ctx) + if err != nil { + t.Fatal(err) + } + if _, still := shelf["gone"]; still { + t.Fatal("a deleted module nothing holds was not forgotten") + } + + // Without the announcer saying what went: the build finds no manifest, and the plan goes on. + asked0 := time.Now().UTC().Add(-time.Minute) + plan := inventory.Plan{ID: "plan-deleted", Repository: "novox/mesh-catalog", Commit: "c2", Created: asked0, + State: inventory.PlanBuilding, Tiers: [][]string{{"kept"}}, + Modules: map[string]*inventory.PlanModule{"kept": {State: "asked", AskedAt: &asked0, Build: "build-k"}}} + if err := inv.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + planBuilt(ctx, open, "kept", "", "http://forge/novox/mesh-catalog.git has no module.json at modules/kept, so "+ + "there is nothing saying what it is: open …/module.json: no such file or directory", asked0, "build-k") + p, _ := inv.PlanByID(ctx, "plan-deleted") + if p.State == inventory.PlanFailed || p.Modules["kept"].State != planDeleted { + t.Fatalf("a module deleted at its source failed the plan: %s %+v", p.State, p.Modules["kept"]) + } +} + +func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) { + open := aMesh(t) + d := &doctor{open: open} + at := time.Now().UTC() + witnessed.heard("control", []lease.Rollback{ + {Component: lease.ComponentController, From: "sha256:new", To: "sha256:old", Outcome: lease.OutcomeRolledBack, + Why: "the new controller did not take the lease within 60s", At: at}, + {Component: lease.ComponentNodeTools, From: "sha256:t2", Outcome: lease.OutcomeUnwitnessed, Why: "no grant", At: at}, + }, at) + t.Cleanup(func() { witnessed.heard("control", nil, time.Now()) }) + obs, err := probeGates(t.Context(), d) + if err != nil { + t.Fatal(err) + } + got := map[string]conditions.Severity{} + for _, o := range obs { + got[o.Key()] = o.Severity + } + want := map[string]conditions.Severity{"core.controller.control.rolled-back": conditions.Urgent, + "core.node-tools.control.unwitnessed": conditions.Warning} + if !reflect.DeepEqual(got, want) { + t.Fatalf("the witness's verdicts are %v", got) + } + witnessed.heard("control", nil, time.Now()) + if obs, err := probeGates(t.Context(), d); err != nil || len(obs) != 0 { + t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err) + } +} diff --git a/cmd/mesh-controller/gate_test.go b/cmd/mesh-controller/gate_test.go index 651c90ec..589c4aa4 100644 --- a/cmd/mesh-controller/gate_test.go +++ b/cmd/mesh-controller/gate_test.go @@ -4,8 +4,6 @@ import ( "context" "encoding/json" "errors" - "fmt" - "reflect" "strings" "testing" "time" @@ -30,124 +28,6 @@ type gateMesh struct { told *conditions.Told } -func aGateMesh(t *testing.T) *gateMesh { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - g := &gateMesh{open: open, health: map[string]health{}} - // The gate's own tests judge a first machine, as walks a controller before novox/hq ADR 0360 started still are. - wasGated := gateFirstMachine - gateFirstMachine = true - t.Cleanup(func() { gateFirstMachine = wasGated }) - g.keeper, _ = withConditionsInMemory(t) - g.told = &conditions.Told{} - was := doctorFrom - doctorFrom = &doctor{open: open, keeper: g.keeper, teller: g.told} - t.Cleanup(func() { doctorFrom = was }) - - for _, b := range []inventory.Build{ - {ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app", - Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)}, - {ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app", - Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)}, - } { - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit}) - b.Manifest = manifest - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - register := func(commit string, asked time.Time) { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit, - Head: commit, Asked: asked}); err != nil { - t.Fatal(err) - } - } - register("c1", time.Now().Add(-2*time.Hour)) - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, "app"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil { - t.Fatal(err) - } - } - register("c2", time.Now().Add(-time.Minute)) - - // Every send: recorded with the build the module is at, applied and reported by the machine. - n := 0 - wasSend := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - g.sent = append(g.sent, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - for _, node := range names { - n++ - digest := fmt.Sprintf("d-%s-%d", node, n) - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil { - return nil, err - } - if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, - Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { - return nil, err - } - } - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - - // What a judging reads: the store's reports, and a health the test says per machine. - wasGather := gatherGateFacts - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, - rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} - reports, err := open.inventory.LastReports(ctx) - if err != nil { - return f, err - } - for _, r := range reports { - if g.health[r.Node] == healthBroken { - r.Outcome = inventory.OutcomeFailed - } - f.reports[r.Node] = r - } - for node, h := range g.health { - if h == healthNotYet { - // Not reported on the send: neither the send made last, nor the gate's own (issue 352). - f.rolledBack[node] = nil - r := f.reports[node] - r.Current, r.Declared, r.ReportedSequence = false, "", 0 - f.reports[node] = r - } - } - return f, nil - } - t.Cleanup(func() { gatherGateFacts = wasGather }) - - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - // One judging per advance until a test says otherwise: a pass waits gateEvery for the next. - gateWithin, gateEvery = time.Hour, time.Hour - t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) - - built := time.Now().UTC() - plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", - Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", - Build: "build-2"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - return g -} - func (g *gateMesh) plan(t *testing.T) inventory.Plan { t.Helper() p, err := g.open.inventory.PlanByID(t.Context(), "plan-gate") @@ -160,187 +40,17 @@ func (g *gateMesh) plan(t *testing.T) inventory.Plan { // A module's build that fails its gate on the first machine is put back there — the previous build // registered and sent to it again — and never reaches the second machine; it is marked, said as a // condition and an event, never registered or rolled back again. -func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) { - judgeLeavesAsTheCore(t) - g := aGateMesh(t) - ctx := t.Context() - inv := g.open.inventory - - advancePlans(ctx, g.open) // the first machine is sent the new build - if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v, not the first machine alone", g.sent) - } - if p := g.plan(t); p.Modules["app"].Previous != "c1" { - t.Fatalf("the build the first machine ran before was not kept: %+v", p.Modules["app"]) - } - - g.health["anchor"] = healthBroken // the new build breaks its first machine, after one good judging - gateEvery = 0 - advancePlans(ctx, g.open) - - p := g.plan(t) - gate := p.Modules["app"].Gate - if p.State != inventory.PlanFailed || gate == nil || gate.Verdict != inventory.GateFailed || - gate.Rollback != inventory.RolledBack { - t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate) - } - if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"anchor"}}) { - t.Fatalf("sent %v: the rollback goes to the first machine, and nothing reaches laptop", g.sent) - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" { - t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit) - } - if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c1" { - t.Fatalf("anchor was last sent %v, not the previous build", sent) - } - if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["app"] != "c1" { - t.Fatalf("laptop was sent the failed build: %v", sent) - } - if failed, err := inv.GateFailed(ctx, "build-2"); err != nil || !failed { - t.Fatalf("the build is not marked failed at its gate: %v %v", failed, err) - } - - // Said: a condition for the operator, and the event. - open, err := g.keeper.Open(ctx) - if err != nil { - t.Fatal(err) - } - var key string - for _, c := range open { - if c.Kind == kindRolledBack { - key = c.Key - } - } - if key != "build.app.anchor.rolled-back" { - t.Fatalf("no rolled-back condition for app on anchor: %+v", open) - } - saidIt := false - for _, e := range g.told.Said() { - saidIt = saidIt || e.Event == link.KeyRolledBack - } - if !saidIt { - t.Fatalf("the rollback was not said as an event: %+v", g.told.Said()) - } - - // Never again: more passes send nothing, a second judging rolls nothing back, and the failed build - // heard again is not registered. - advancePlans(ctx, g.open) - state := p.Modules["app"] - gateFailed(ctx, g.open, &p, "app", state, []string{"anchor"}, "again") - if len(g.sent) != 2 { - t.Fatalf("sent again after the rollback: %v", g.sent) - } - _, _, err = takeIn(ctx, inv, link.BuildResult{ID: "build-2", Repository: "novox/mesh-catalog", Path: "modules/app", - Commit: "c2", Manifest: mustJSON(t, catalogue.Manifest{Module: "app", Version: "c2"}), - Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}) - if err == nil || !strings.Contains(err.Error(), "failed its gate") { - t.Fatalf("the failed build was registered again: %v", err) - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" { - t.Fatalf("the module moved to %s", current["app"].Commit) - } - if _, err := retryPlan(ctx, g.open, "plan-gate"); err == nil || !strings.Contains(err.Error(), "failed its gate") { - t.Fatalf("a plan stopped at a failed gate was retried: %v", err) - } - - // The probe keeps the condition while the newest verdict is the failure. - obs, err := probeGates(ctx, doctorFrom) - if err != nil || len(obs) != 1 || obs[0].Key() != key { - t.Fatalf("the gate's probe found %+v %v", obs, err) - } -} // A passing build rolls everywhere with no hand: judged healthy on its first machine three times, then // the rest are sent, the verdict kept, and the plan done. -func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) { - g := aGateMesh(t) - ctx := t.Context() - gateEvery = 0 - for i := 0; i < 6; i++ { - advancePlans(ctx, g.open) - } - p := g.plan(t) - if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"laptop"}}) { - t.Fatalf("sent %v", g.sent) - } - gate := p.Modules["app"].Gate - if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gateWanted(gate) { - t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate) - } - if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("the verdict was not kept: %+v %v %v", v, found, err) - } - if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 { - t.Fatalf("a passing build left a condition: %+v %v", obs, err) - } -} // A build rolls everywhere with no hand and no gate (novox/hq ADR 0360 §1): both machines in one send, no health // judged, and the walk done once both reported on the send. Before, anchor was judged first, then laptop sent. -func TestABuildRollsToEveryMachineAtOnceWithNoGate(t *testing.T) { - g := aGateMesh(t) - gateFirstMachine = false - ctx := t.Context() - judged := false - wasGather := gatherGateFacts - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - judged = true - return wasGather(ctx, open, component) - } - t.Cleanup(func() { gatherGateFacts = wasGather }) - for i := 0; i < 6; i++ { - advancePlans(ctx, g.open) - } - p := g.plan(t) - if !reflect.DeepEqual(g.sent, [][]string{{"anchor", "laptop"}}) { - t.Fatalf("sent %v, not both machines in one send", g.sent) - } - if s := p.Modules["app"]; p.State != inventory.PlanDone || s.Gate != nil || s.FirstAt != nil || judged { - t.Fatalf("the plan is %s (%s), app %+v, judged %v", p.State, p.Note, s, judged) - } - if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 { - t.Fatalf("a rolled build left a condition: %+v %v", obs, err) - } -} // A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the // core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524). -func TestALeafModulePassesOnOneHealthyReading(t *testing.T) { - g := aGateMesh(t) - ctx := t.Context() - gateWithin = time.Hour // a core module could not pass on fewer readings for an hour - advancePlans(ctx, g.open) // the first machine is sent the new build - advancePlans(ctx, g.open) // one healthy judging - p := g.plan(t) - gate := p.Modules["app"].Gate - if gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes != 1 || gate.Wants != 1 { - t.Fatalf("a leaf module's gate after one healthy judging: %+v (plan %s: %s)", gate, p.State, p.Note) - } - if !strings.Contains(gate.Why, "healthy 1 times") { - t.Errorf("the verdict says %q", gate.Why) - } -} // A first machine that never becomes healthy fails its gate at the bound, and is put back. -func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) { - judgeLeavesAsTheCore(t) - g := aGateMesh(t) - ctx := t.Context() - advancePlans(ctx, g.open) - g.health["anchor"] = healthNotYet - gateEvery = 0 - advancePlans(ctx, g.open) - if p := g.plan(t); !p.Open() || len(g.sent) != 1 { - t.Fatalf("judged before the bound: %s %v", p.State, g.sent) - } - gateBound = -time.Second - advancePlans(ctx, g.open) - p := g.plan(t) - if gate := p.Modules["app"].Gate; p.State != inventory.PlanFailed || gate.Verdict != inventory.GateFailed || - !strings.Contains(gate.Why, "not healthy within") || gate.Rollback != inventory.RolledBack { - t.Fatalf("the plan is %s, its gate %+v", p.State, gate) - } -} // The health a judging finds, per core component and for a module. func TestTheHealthDefinitions(t *testing.T) { @@ -458,147 +168,9 @@ func TestTheHealthDefinitions(t *testing.T) { // The bus is never rolled out: its policy records whatever is said, a person's roll-out is refused, // a plan builds it and sends nothing, and a push naming its machine is refused while a new build waits. -func TestTheBusIsNeverRolledOutAutomatically(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - bus := catalogue.Manifest{Module: "nats", Version: "2", Provides: []catalogue.Offer{{Name: "mesh-bus"}}, - Upgrade: &catalogue.UpgradePolicy{Policy: catalogue.PolicyRoll}} - if err := inv.RegisterModule(ctx, bus, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", - Path: "modules/nats", BuiltFrom: "n2", Head: "n2"}); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, "anchor", "nats"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"nats": "n1"}); err != nil { - t.Fatal(err) - } - u, err := inv.UpgradeOf(ctx, "nats") - if err != nil || u.RollOut || u.From != catalogue.FromBus { - t.Fatalf("the bus's policy is %+v %v", u, err) - } - if err := inv.SetUpgradeOf(ctx, "nats", inventory.Upgrade{RollOut: true}); !errors.Is(err, inventory.ErrBusIsPlanned) { - t.Fatalf("a person rolled the bus out: %v", err) - } - var sent [][]string - was := sendRollout - sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) { - sent = append(sent, names) - return names, nil - } - t.Cleanup(func() { sendRollout = was }) - now := time.Now().UTC() - plan := inventory.Plan{ID: "plan-bus", Repository: "novox/mesh-catalog", Commit: "n2", Created: now, - State: inventory.PlanBuilding, Tiers: [][]string{{"nats"}}, - Modules: map[string]*inventory.PlanModule{"nats": {State: "built", BuiltAt: &now, Commit: "n2", Build: "b"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - advancePlans(ctx, open) - if p, _ := inv.PlanByID(ctx, "plan-bus"); p.State != inventory.PlanDone || len(sent) != 0 { - t.Fatalf("a plan sent the bus: %s %v", p.State, sent) - } - held, err := busHeld(ctx, inv, []string{"anchor", "laptop"}) - if err != nil || !strings.Contains(held["anchor"], "planned step") || held["laptop"] != "" { - t.Fatalf("a push may send the bus's machine: %v %v", held, err) - } - // Nor may any other send — a plan's for another module on that machine carried the bus with it. - if _, err := sendToEach(ctx, open, []string{"laptop", "anchor"}); !errors.Is(err, errBusWaits) { - t.Fatalf("a send to the bus's machine was not refused: %v", err) - } - // A rebuild that made the same artifacts is no move. - for _, b := range []inventory.Build{{ID: "nb1", Module: "nats", Commit: "n1"}, {ID: "nb2", Module: "nats", Commit: "n2"}} { - b.Made = []inventory.Artifact{{Name: "server", Kind: "image", Reference: "registry/nats@sha256:same"}} - b.Asked, b.At = time.Now(), time.Now() - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - if held, err := busHeld(ctx, inv, []string{"anchor"}); err != nil || len(held) != 0 { - t.Fatalf("a rebuild that changes nothing held the bus's machine: %v %v", held, err) - } - if err := inv.RecordBuild(ctx, inventory.Build{ID: "nb3", Module: "nats", Commit: "n2", Asked: time.Now().Add(time.Second), - At: time.Now().Add(time.Second), Made: []inventory.Artifact{{Name: "server", Kind: "image", - Reference: "registry/nats@sha256:new"}}}); err != nil { - t.Fatal(err) - } - // The planned step refuses to start without its word on reversibility, and without a snapshot taken - // first by the bus machine's backup holder. - if err := busCommand(ctx, []string{"upgrade", "--why", "2.11"}); err == nil || !strings.Contains(err.Error(), "reversible") { - t.Fatalf("a bus upgrade started without saying whether it can be reverted: %v", err) - } - wasSnapshot := takeBusSnapshot - t.Cleanup(func() { takeBusSnapshot = wasSnapshot }) - takeBusSnapshot = func(context.Context, string, string) (string, error) { return "", errors.New("no holder answers") } - if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err == nil || - !strings.Contains(err.Error(), "snapshotted") || len(sent) != 0 { - t.Fatalf("a bus upgrade started without its snapshot: %v, sent %v", err, sent) - } - takeBusSnapshot = func(_ context.Context, module, node string) (string, error) { - return node + "'s restore point of " + module, nil - } - if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(sent, [][]string{{"anchor"}}) { - t.Fatalf("the step sent %v, not the bus's machine", sent) - } - s, found, err := inv.LatestBusStep(ctx) - if err != nil || !found || s.Snapshot != "anchor's restore point of nats" || s.Ended != nil || - !reflect.DeepEqual(s.Machines, []string{"anchor"}) { - t.Fatalf("the step is %+v %v %v", s, found, err) - } -} // A merge that deletes a module's directory builds nothing for it: the module is forgotten where // nothing holds it, and a plan whose build finds no manifest goes on instead of failing. -func TestAMergeThatDeletesAModulePlansNothingToBuildForIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - asked := asksRecorded(t) - for _, name := range []string{"gone", "kept"} { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: name, Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + name, BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1deadbeef", - Paths: []string{"modules/gone/module.json", "modules/gone/index.ts"}, - Removed: []string{"modules/gone/module.json", "modules/gone/index.ts"}} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - if len(*asked) != 0 { - t.Fatalf("a deleted module was asked to build: %v", *asked) - } - if plans, _ := inv.OpenPlans(ctx); len(plans) != 0 { - t.Fatalf("a merge that only deleted a module made a plan: %+v", plans) - } - shelf, err := inv.Catalogue(ctx) - if err != nil { - t.Fatal(err) - } - if _, still := shelf["gone"]; still { - t.Fatal("a deleted module nothing holds was not forgotten") - } - - // Without the announcer saying what went: the build finds no manifest, and the plan goes on. - asked0 := time.Now().UTC().Add(-time.Minute) - plan := inventory.Plan{ID: "plan-deleted", Repository: "novox/mesh-catalog", Commit: "c2", Created: asked0, - State: inventory.PlanBuilding, Tiers: [][]string{{"kept"}}, - Modules: map[string]*inventory.PlanModule{"kept": {State: "asked", AskedAt: &asked0, Build: "build-k"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - planBuilt(ctx, open, "kept", "", "http://forge/novox/mesh-catalog.git has no module.json at modules/kept, so "+ - "there is nothing saying what it is: open …/module.json: no such file or directory", asked0, "build-k") - p, _ := inv.PlanByID(ctx, "plan-deleted") - if p.State == inventory.PlanFailed || p.Modules["kept"].State != planDeleted { - t.Fatalf("a module deleted at its source failed the plan: %s %+v", p.State, p.Modules["kept"]) - } -} func mustJSON(t *testing.T, v any) []byte { t.Helper() @@ -621,34 +193,6 @@ func nodeID(t *testing.T, open *stores, name string) string { // What a machine's witness says in its reports is a condition while it says it, by the host's words: // `core...`, urgent for a rollback, a warning when it could not judge — and // gone with the first report that no longer carries it. -func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) { - open := aMesh(t) - d := &doctor{open: open} - at := time.Now().UTC() - witnessed.heard("control", []lease.Rollback{ - {Component: lease.ComponentController, From: "sha256:new", To: "sha256:old", Outcome: lease.OutcomeRolledBack, - Why: "the new controller did not take the lease within 60s", At: at}, - {Component: lease.ComponentNodeTools, From: "sha256:t2", Outcome: lease.OutcomeUnwitnessed, Why: "no grant", At: at}, - }, at) - t.Cleanup(func() { witnessed.heard("control", nil, time.Now()) }) - obs, err := probeGates(t.Context(), d) - if err != nil { - t.Fatal(err) - } - got := map[string]conditions.Severity{} - for _, o := range obs { - got[o.Key()] = o.Severity - } - want := map[string]conditions.Severity{"core.controller.control.rolled-back": conditions.Urgent, - "core.node-tools.control.unwitnessed": conditions.Warning} - if !reflect.DeepEqual(got, want) { - t.Fatalf("the witness's verdicts are %v", got) - } - witnessed.heard("control", nil, time.Now()) - if obs, err := probeGates(t.Context(), d); err != nil || len(obs) != 0 { - t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err) - } -} // judgeLeavesAsTheCore makes a test's leaf modules judged as a core module is, three readings after the settle // time, for a test of the gate's other rules (a fault, the bound, a supersession) that needs a judging still diff --git a/cmd/mesh-controller/grant_follows_binding_integration_test.go b/cmd/mesh-controller/grant_follows_binding_integration_test.go new file mode 100644 index 00000000..74622b3d --- /dev/null +++ b/cmd/mesh-controller/grant_follows_binding_integration_test.go @@ -0,0 +1,157 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestAConsumerPinnedBackIsNoLongerGrantedByTheProviderItLeft(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + for _, m := range storeManifests() { + register(t, open, m) + } + if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "anchor", "store"); err != nil { + t.Fatal(err) + } + if err := inv.HoldSeat(ctx, "mesh-store", catalogue.ScopeMesh, "anchor", "store"); err != nil { + t.Fatal(err) + } + for _, a := range [][2]string{{"laptop", "store"}, {"laptop", "board"}} { + if _, err := assign(ctx, open, a[0], a[1]); err != nil { + t.Fatalf("assign %s %s: %v", a[0], a[1], err) + } + } + + // Bound to the anchor's store, and sent so: the credential from the anchor is made and recorded. + if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil { + t.Fatal(err) + } + plan, _, err := planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" { + t.Fatalf("pinned to the anchor and bound to %s", n.From) + } + if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil { + t.Fatal(err) + } + grants, _, unbound, err := grantsFor(ctx, open, "anchor") + if err != nil { + t.Fatal(err) + } + if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); !ok || g.From != "board" || len(unbound) != 0 { + t.Fatalf("a consumer bound to the anchor is not granted there: %+v, unbound %+v", grants, unbound) + } + + // Pinned back beside its data, as the operator did. + if err := inv.PinProvision(ctx, "laptop", "postgres-database", "laptop", "store"); err != nil { + t.Fatal(err) + } + plan, _, err = planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" { + t.Fatalf("pinned back to the laptop and bound to %s", n.From) + } + if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil { + t.Fatal(err) + } + holders, err := inv.HoldersOf(ctx, "postgres-database", "laptop") + if err != nil { + t.Fatal(err) + } + if len(holders) != 2 { + t.Fatalf("today's state is two credentials on record, one from each store: %+v", holders) + } + + // The store it left: no longer granted, and said. + grants, _, unbound, err = grantsFor(ctx, open, "anchor") + if err != nil { + t.Fatal(err) + } + if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); ok && g.From != "" { + t.Fatalf("the anchor's store is still granted a consumer bound to the laptop's: %+v", g) + } + if len(unbound) != 1 || unbound[0].Module != "board" || unbound[0].Provider != "anchor" || + strings.Join(unbound[0].BoundTo, ",") != "laptop" { + t.Fatalf("the consumer that moved is not the one said: %+v", unbound) + } + planned, settings, err := planFor(ctx, open, "anchor") + if err != nil { + t.Fatal(err) + } + declared, err := declarationFor(ctx, open, "anchor", planned, settings) + if err != nil { + t.Fatal(err) + } + if got := declared.Received["store"]["postgres-database"]; len(got) != 0 { + t.Fatalf("the anchor's store is still told about %+v", got) + } + said := printed(t, func() error { reportLeftOut("anchor", declared); return nil }) + if !strings.Contains(said, "board on laptop is bound to laptop for postgres-database, not to anchor") || + !strings.Contains(said, "cleanup delete") { + t.Fatalf("the push does not say whom the anchor no longer grants:\n%s", said) + } + + // The store it is bound to: granted. + grants, _, unbound, err = grantsFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); !ok || g.From != "board" || len(unbound) != 0 { + t.Fatalf("the consumer is not granted by the store it is bound to: %+v, unbound %+v", grants, unbound) + } + + // And the credential from the store it left is kept: the key to the login and data held there. + if holders, err = inv.HoldersOf(ctx, "postgres-database", "laptop"); err != nil || len(holders) != 2 { + t.Fatalf("a credential was forgotten while its provider still holds the login: %+v, %v", holders, err) + } +} + +func TestAConsumerWhoseResolutionCannotBeReadIsNotWithdrawn(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + for _, m := range storeManifests() { + register(t, open, m) + } + for _, a := range [][2]string{{"anchor", "store"}, {"laptop", "board"}} { + if _, err := assign(ctx, open, a[0], a[1]); err != nil { + t.Fatalf("assign %s %s: %v", a[0], a[1], err) + } + } + if _, _, err := planFor(ctx, open, "laptop"); err != nil { + t.Fatal(err) + } + + // The laptop's key changes to one nothing can seal to: its resolution cannot be completed, and + // that is not its set failing to compose. + laptop, err := inv.NodeByName(ctx, "laptop") + if err != nil { + t.Fatal(err) + } + if err := inv.RecordSealingKey(ctx, laptop.ID, "not a key"); err != nil { + t.Fatal(err) + } + if _, _, err := planFor(ctx, open, "laptop"); err == nil || unresolvable(err) { + t.Fatalf("the seam this test relies on moved: %v", err) + } + grants, _, unbound, err := grantsFor(ctx, open, "anchor") + if err == nil { + t.Fatalf("an unreadable consumer was answered: grants %+v, unbound %+v", grants, unbound) + } + if !strings.Contains(err.Error(), "what laptop asked of postgres-database cannot be read") { + t.Fatalf("the error does not say whose resolution could not be read: %v", err) + } +} diff --git a/cmd/mesh-controller/grant_follows_binding_test.go b/cmd/mesh-controller/grant_follows_binding_test.go index 306de2ba..9b9c6cd4 100644 --- a/cmd/mesh-controller/grant_follows_binding_test.go +++ b/cmd/mesh-controller/grant_follows_binding_test.go @@ -26,154 +26,9 @@ func grantOf(grants []catalogue.Grant, provision, consumer, module string) (cata // credentials are on record. The store it left is no longer granted it — so it retires it and keeps // its data (ADR 0230) — and says so; the store it is bound to keeps its grant; and the credential from // the store it left stays on record. -func TestAConsumerPinnedBackIsNoLongerGrantedByTheProviderItLeft(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - for _, m := range storeManifests() { - register(t, open, m) - } - if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "anchor", "store"); err != nil { - t.Fatal(err) - } - if err := inv.HoldSeat(ctx, "mesh-store", catalogue.ScopeMesh, "anchor", "store"); err != nil { - t.Fatal(err) - } - for _, a := range [][2]string{{"laptop", "store"}, {"laptop", "board"}} { - if _, err := assign(ctx, open, a[0], a[1]); err != nil { - t.Fatalf("assign %s %s: %v", a[0], a[1], err) - } - } - - // Bound to the anchor's store, and sent so: the credential from the anchor is made and recorded. - if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil { - t.Fatal(err) - } - plan, _, err := planFor(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" { - t.Fatalf("pinned to the anchor and bound to %s", n.From) - } - if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil { - t.Fatal(err) - } - grants, _, unbound, err := grantsFor(ctx, open, "anchor") - if err != nil { - t.Fatal(err) - } - if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); !ok || g.From != "board" || len(unbound) != 0 { - t.Fatalf("a consumer bound to the anchor is not granted there: %+v, unbound %+v", grants, unbound) - } - - // Pinned back beside its data, as the operator did. - if err := inv.PinProvision(ctx, "laptop", "postgres-database", "laptop", "store"); err != nil { - t.Fatal(err) - } - plan, _, err = planFor(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" { - t.Fatalf("pinned back to the laptop and bound to %s", n.From) - } - if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil { - t.Fatal(err) - } - holders, err := inv.HoldersOf(ctx, "postgres-database", "laptop") - if err != nil { - t.Fatal(err) - } - if len(holders) != 2 { - t.Fatalf("today's state is two credentials on record, one from each store: %+v", holders) - } - - // The store it left: no longer granted, and said. - grants, _, unbound, err = grantsFor(ctx, open, "anchor") - if err != nil { - t.Fatal(err) - } - if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); ok && g.From != "" { - t.Fatalf("the anchor's store is still granted a consumer bound to the laptop's: %+v", g) - } - if len(unbound) != 1 || unbound[0].Module != "board" || unbound[0].Provider != "anchor" || - strings.Join(unbound[0].BoundTo, ",") != "laptop" { - t.Fatalf("the consumer that moved is not the one said: %+v", unbound) - } - planned, settings, err := planFor(ctx, open, "anchor") - if err != nil { - t.Fatal(err) - } - declared, err := declarationFor(ctx, open, "anchor", planned, settings) - if err != nil { - t.Fatal(err) - } - if got := declared.Received["store"]["postgres-database"]; len(got) != 0 { - t.Fatalf("the anchor's store is still told about %+v", got) - } - said := printed(t, func() error { reportLeftOut("anchor", declared); return nil }) - if !strings.Contains(said, "board on laptop is bound to laptop for postgres-database, not to anchor") || - !strings.Contains(said, "cleanup delete") { - t.Fatalf("the push does not say whom the anchor no longer grants:\n%s", said) - } - - // The store it is bound to: granted. - grants, _, unbound, err = grantsFor(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); !ok || g.From != "board" || len(unbound) != 0 { - t.Fatalf("the consumer is not granted by the store it is bound to: %+v, unbound %+v", grants, unbound) - } - - // And the credential from the store it left is kept: the key to the login and data held there. - if holders, err = inv.HoldersOf(ctx, "postgres-database", "laptop"); err != nil || len(holders) != 2 { - t.Fatalf("a credential was forgotten while its provider still holds the login: %+v, %v", holders, err) - } -} // A consumer whose resolution cannot be read is an error, never a consumer bound nowhere — withdrawing // a grant on that reading would take its access away (issue 152). -func TestAConsumerWhoseResolutionCannotBeReadIsNotWithdrawn(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - for _, m := range storeManifests() { - register(t, open, m) - } - for _, a := range [][2]string{{"anchor", "store"}, {"laptop", "board"}} { - if _, err := assign(ctx, open, a[0], a[1]); err != nil { - t.Fatalf("assign %s %s: %v", a[0], a[1], err) - } - } - if _, _, err := planFor(ctx, open, "laptop"); err != nil { - t.Fatal(err) - } - - // The laptop's key changes to one nothing can seal to: its resolution cannot be completed, and - // that is not its set failing to compose. - laptop, err := inv.NodeByName(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if err := inv.RecordSealingKey(ctx, laptop.ID, "not a key"); err != nil { - t.Fatal(err) - } - if _, _, err := planFor(ctx, open, "laptop"); err == nil || unresolvable(err) { - t.Fatalf("the seam this test relies on moved: %v", err) - } - grants, _, unbound, err := grantsFor(ctx, open, "anchor") - if err == nil { - t.Fatalf("an unreadable consumer was answered: grants %+v, unbound %+v", grants, unbound) - } - if !strings.Contains(err.Error(), "what laptop asked of postgres-database cannot be read") { - t.Fatalf("the error does not say whose resolution could not be read: %v", err) - } -} // The rule itself, on a resolution: bound is the provider a need for exactly that credential is // answered by, never one answered by a record, and a different local name is a different credential. diff --git a/cmd/mesh-controller/grants_step_integration_test.go b/cmd/mesh-controller/grants_step_integration_test.go new file mode 100644 index 00000000..c40c1bd3 --- /dev/null +++ b/cmd/mesh-controller/grants_step_integration_test.go @@ -0,0 +1,27 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// The store test's manifests are manifests the controller takes in: checked here, where no store is needed, +// so a fixture the module rules refuse fails before the store test is run (repo-check of #230 at 325575b2). +func TestTheGrantsStepFixturesAreManifests(t *testing.T) { + for name, m := range map[string]map[string]any{"nats": natsFixture(), "restic-c1": resticFixture(false), + "restic-c2": resticFixture(true), "wallpaper": wallpaperFixture()} { + m["module"], m["version"] = strings.SplitN(name, "-", 2)[0], "1" + raw, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + if _, err := catalogue.ParseManifest(raw); err != nil { + t.Errorf("%s: %v", name, err) + } + } +} diff --git a/cmd/mesh-controller/grants_step_store_test.go b/cmd/mesh-controller/grants_step_store_test.go index 095a84f1..705dc8fb 100644 --- a/cmd/mesh-controller/grants_step_store_test.go +++ b/cmd/mesh-controller/grants_step_store_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/grants_step_test.go b/cmd/mesh-controller/grants_step_test.go index 70e275b2..48b34bb5 100644 --- a/cmd/mesh-controller/grants_step_test.go +++ b/cmd/mesh-controller/grants_step_test.go @@ -177,22 +177,6 @@ func TestTheGrantsStepKeepsOneMachinesBuilds(t *testing.T) { } } -// The store test's manifests are manifests the controller takes in: checked here, where no store is needed, -// so a fixture the module rules refuse fails before the store test is run (repo-check of #230 at 325575b2). -func TestTheGrantsStepFixturesAreManifests(t *testing.T) { - for name, m := range map[string]map[string]any{"nats": natsFixture(), "restic-c1": resticFixture(false), - "restic-c2": resticFixture(true), "wallpaper": wallpaperFixture()} { - m["module"], m["version"] = strings.SplitN(name, "-", 2)[0], "1" - raw, err := json.Marshal(m) - if err != nil { - t.Fatal(err) - } - if _, err := catalogue.ParseManifest(raw); err != nil { - t.Errorf("%s: %v", name, err) - } - } -} - // The delivery plan says the step before the merge. func TestAChangePlanSaysTheGrantsStep(t *testing.T) { const catalogue_ = "http://forge.internal:20000/novox/mesh-catalog.git" diff --git a/cmd/mesh-controller/healers_integration_test.go b/cmd/mesh-controller/healers_integration_test.go new file mode 100644 index 00000000..3a55294d --- /dev/null +++ b/cmd/mesh-controller/healers_integration_test.go @@ -0,0 +1,449 @@ +//go:build integration + +package main + +import ( + "context" + "errors" + "slices" + "strconv" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestH1AsksAMachineToReportAndSendsItAgain(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + h, told, clock := healingOn(t, open) + record, err := open.inventory.NodeByName(ctx, "laptop") + if err != nil { + t.Fatal(err) + } + if err := open.inventory.RecordSent(ctx, record.ID, "d2", nil); err != nil { + t.Fatal(err) + } + key := sentNotReported(t, "laptop") + + // First: the report had not reached the mesh, and asking for it brings it. + asked := 0 + h.askReport = func(ctx context.Context, node string) error { + asked++ + if node != "laptop" { + t.Errorf("asked %s", node) + } + _, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeApplied, + At: time.Now().Add(time.Second), Declared: "d2"}) + return err + } + h.tick(ctx) + c, found, err := conditionsFrom.Get(ctx, key) + if err != nil || !found { + t.Fatalf("the condition is gone after the act — a healer cleared it, not an observation: %v", err) + } + if asked != 1 || len(c.Tried) != 1 || c.Tried[0].By != "healer H1" || !strings.Contains(c.Tried[0].Outcome, "acted") || + c.Resolver != "healer:H1" { + t.Fatalf("after the first act: asked %d, %+v", asked, c) + } + if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H1" || acts[0].Outcome != inventory.HealActed || + acts[0].Condition != key || acts[0].By != "healer H1" || acts[0].Epoch != 57 { + t.Fatalf("the act was not said as healer-acted: %+v", acts) + } + + // Within its settle, nothing more: the observation has its turn. + clock.pass(time.Minute) + h.tick(ctx) + if asked != 1 { + t.Fatalf("acted again inside the settle: asked %d", asked) + } + + // Second: the machine says nothing, so it is sent again. + clock.pass(3 * time.Minute) + sent := 0 + h.askReport = func(context.Context, string) error { asked++; return nil } + h.sendAgain = func(_ context.Context, node string) error { sent++; return nil } + h.tick(ctx) + if asked != 2 || sent != 1 { + t.Fatalf("the second act: asked %d, sent %d", asked, sent) + } + c, _, _ = conditionsFrom.Get(ctx, key) + if len(c.Tried) != 2 || !strings.Contains(c.Tried[1].Outcome, "sent again") || !strings.Contains(c.Tried[1].Outcome, "act 2 of 2") { + t.Fatalf("tried %+v", c.Tried) + } + + // The budget is spent: the operator's, urgent, said; and no healer touches it again. + clock.pass(4 * time.Minute) + h.tick(ctx) + c, _, _ = conditionsFrom.Get(ctx, key) + if !c.Escalated() || c.Severity != conditions.Urgent || len(c.Tried) != 3 || + !strings.Contains(c.Tried[2].Outcome, "budget of 2") { + t.Fatalf("not handed to the operator: %+v", c) + } + if acts := told.said(); len(acts) != 3 || acts[2].Outcome != inventory.HealEscalated { + t.Fatalf("the escalation was not said: %+v", acts) + } + clock.pass(time.Hour) + h.tick(ctx) + if asked != 2 || sent != 1 || len(told.said()) != 3 { + t.Fatalf("a healer acted on a condition the operator holds: asked %d sent %d said %d", asked, sent, len(told.said())) + } + heals, err := open.inventory.HealsSince(ctx, time.Now().Add(-time.Hour)) + if err != nil || len(heals) != 3 || heals[0].Outcome != inventory.HealActed || heals[1].Outcome != inventory.HealActed || + heals[2].Outcome != inventory.HealEscalated || heals[0].Epoch != 57 { + t.Fatalf("the heals kept: %+v %v", heals, err) + } + // And a heal is not a hand act: what S15 counts never sees it. + for _, x := range heals { + if x.Healer == "" || strings.HasPrefix(x.Act, "hand-act") { + t.Errorf("a heal reads as a hand act: %+v", x) + } + } +} + +func TestNoHealerActsWithoutTheLease(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + h, told, _ := healingOn(t, open) + sentNotReported(t, "laptop") + h.epoch = func(context.Context) (uint64, error) { return 0, nil } + h.tick(ctx) + h.epoch = func(context.Context) (uint64, error) { return 0, errors.New("the lease is not held") } + h.tick(ctx) + h.epoch = func(context.Context) (uint64, error) { return 57, nil } + h.acting = func() bool { return false } + h.tick(ctx) + if len(told.said()) != 0 { + t.Fatalf("a healer acted without the lease: %+v", told.said()) + } +} + +func TestTheBrakeStopsEveryHealerAndSaysSo(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + h, told, clock := healingOn(t, open) + for i := 0; i < healBrakeLimit; i++ { + if _, err := open.inventory.BeginHeal(ctx, inventory.Heal{Healer: "H3", ConditionKey: "seat.x.anchor.silent", + Kind: "holder-silent", Act: "asserted", Outcome: inventory.HealActed, + At: clock.now().Add(-time.Duration(healBrakeLimit-i) * time.Minute)}); err != nil { + t.Fatal(err) + } + } + sentNotReported(t, "laptop") + h.tick(ctx) // the fakes fail the test if anything acts + braked, found, err := conditionsFrom.Get(ctx, "mesh.healers.braked") + if err != nil || !found || braked.Severity != conditions.Urgent || !strings.Contains(braked.Evidence[0].Said, "H3 on seat.x.anchor.silent ×12") { + t.Fatalf("the brake was not said: %+v %v", braked, err) + } + if len(told.said()) != 0 { + t.Fatalf("a healer acted under the brake: %+v", told.said()) + } + // Past the hour of the first act, still held: it lets go an hour after the last. + clock.pass(30 * time.Minute) + h.tick(ctx) + if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); !held { + t.Fatal("the brake let go before an hour had passed since the last act") + } + clock.pass(31 * time.Minute) + asked := 0 + h.askReport = func(context.Context, string) error { asked++; return nil } + h.sendAgain = func(context.Context, string) error { return nil } + h.tick(ctx) + if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); held { + t.Fatal("the brake held an hour after the last act") + } + if asked != 1 { + t.Fatalf("the healers did not act again after the brake let go: asked %d", asked) + } +} + +func TestH2ClosesAPlanAnotherHasTakenOver(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + h, told, _ := healingOn(t, open) + created := time.Now().UTC().Add(-time.Hour) + older := inventory.Plan{ID: "plan-old", Repository: "novox/app", Branch: "main", Commit: "0ld0ld0", Created: created, + State: inventory.PlanBuilding, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "asked"}}} + waiting := inventory.Plan{ID: "plan-other", Repository: "novox/other", Branch: "main", Commit: "07he707", Created: created, + State: inventory.PlanBuilding, Tiers: [][]string{{"b"}}, Modules: map[string]*inventory.PlanModule{"b": {State: "asked"}}} + newer := inventory.Plan{ID: "plan-new", Repository: "novox/app", Branch: "main", Commit: "new0new", Created: created.Add(time.Minute), + State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}} + for _, p := range []*inventory.Plan{&older, &waiting, &newer} { + if err := open.inventory.SavePlan(ctx, p); err != nil { + t.Fatal(err) + } + } + for _, id := range []string{"plan-old", "plan-other"} { + if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopePlan, ID: id, Kind: "stalled", + Severity: conditions.Warning, Summary: id + " is stalled", Source: "S3"}); err != nil { + t.Fatal(err) + } + } + h.tick(ctx) + closed, err := open.inventory.PlanByID(ctx, "plan-old") + if err != nil || closed.State != inventory.PlanSuperseded || !strings.Contains(closed.Note, "closed by healer H2") || + !strings.Contains(closed.Note, "plan-new") { + t.Fatalf("the superseded plan: %+v %v", closed, err) + } + if other, _ := open.inventory.PlanByID(ctx, "plan-other"); other.State != inventory.PlanBuilding { + t.Fatalf("a plan still waiting was closed: %+v", other) + } + if c, _, _ := conditionsFrom.Get(ctx, "plan.plan-other.stalled"); len(c.Tried) != 0 || c.Resolver != conditions.ResolverSelf { + t.Fatalf("a plan H2 does not repair was touched: %+v", c) + } + if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H2" || acts[0].Condition != "plan.plan-old.stalled" { + t.Fatalf("said %+v", acts) + } + // Finished: every module built, none rolled out unsent — closed as done. + done := inventory.Plan{ID: "plan-done", Repository: "novox/third", Branch: "main", Commit: "d0ned0n", Created: created, + State: inventory.PlanRolling, Tier: 0, Tiers: [][]string{{"c"}}, Modules: map[string]*inventory.PlanModule{"c": {State: "built"}}} + if err := open.inventory.SavePlan(ctx, &done); err != nil { + t.Fatal(err) + } + if state, why, err := planStale(ctx, open.inventory, done); err != nil || state != inventory.PlanDone || !strings.Contains(why, "finished") { + t.Fatalf("a finished plan reads %q %q %v", state, why, err) + } +} + +func TestH4ResetsOnlyWhatTheTableMarksResettable(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + h, _, _ := healingOn(t, open) + marked := 0 + for _, c := range broker.MeshConsumers() { + if c.Resettable != "" { + marked++ + if c.Stream != broker.EventsStream || c.Name != broker.ControllerName { + t.Errorf("%s on %s is marked resettable: only the controller's own events consumer is", c.Name, c.Stream) + } + } + } + if marked != 1 { + t.Fatalf("%d consumers are marked resettable, want the controller's events consumer alone", marked) + } + for _, who := range []string{"EVENTS.anchor_shop", "CONTROL.controller"} { + if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "behind", + Kind: kindConsumerBehind, Severity: conditions.Warning, Summary: who + " is far behind", Source: "D6"}); err != nil { + t.Fatal(err) + } + } + h.tick(ctx) // the fake reset fails the test if it is called + reset := "" + h.resetConsumer = func(stream, name string) (string, error) { + reset = stream + "." + name + return "it was 1500 behind", nil + } + if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopeBus, ID: "EVENTS.controller", + Token: "behind", Kind: kindConsumerBehind, Severity: conditions.Warning, Summary: "far behind", Source: "D6"}); err != nil { + t.Fatal(err) + } + h.tick(ctx) + if reset != "EVENTS.controller" { + t.Fatalf("reset %q", reset) + } +} + +func TestNatsH3AssertsAMissingConsumerAgainAndBrakesAfterItsBudget(t *testing.T) { + url := testbus.URL(t) + open := aMesh(t) + ctx := t.Context() + js, err := broker.Dial(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} { + _ = js.Context().DeleteStream(s) + } + if _, err := assertBusObjects(ctx, open.inventory, js); err != nil { + t.Fatal(err) + } + h, told, clock := healingOn(t, open) + h.js = js + h.assertObjects = func(ctx context.Context) error { return assertOnSend(ctx, open.inventory, js, " ") } + d := &doctor{open: open, js: js} + probe := func() { + t.Helper() + found, err := probeConsumers(ctx, d) + if err != nil { + t.Fatal(err) + } + if err := conditionsFrom.Reconcile(ctx, "D6", kindedAs(found, "consumer-wrong")); err != nil { + t.Fatal(err) + } + } + const key = "bus.NODES.laptop.missing" + + if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil { + t.Fatal(err) + } + probe() + if c, raised, _ := conditionsFrom.Get(ctx, key); !raised || c.Kind != "consumer-lost" { + t.Fatalf("the deleted consumer was not raised: %+v", c) + } + h.tick(ctx) + if _, err := js.Context().ConsumerInfo("NODES", "laptop"); err != nil { + t.Fatalf("H3 did not make the consumer again: %v", err) + } + c, still, _ := conditionsFrom.Get(ctx, key) + if !still || len(c.Tried) != 1 || c.Tried[0].By != "healer H3" || c.Resolver != "healer:H3" { + t.Fatalf("the act is not in the condition, or the healer cleared it: %+v", c) + } + probe() + if _, still, _ := conditionsFrom.Get(ctx, key); still { + t.Fatal("the probe's next run did not clear what H3 repaired") + } + // Kept in the history as the keeper says it, a moment later. + var cleared *conditions.Event + for wait := time.Now().Add(5 * time.Second); cleared == nil && time.Now().Before(wait); time.Sleep(20 * time.Millisecond) { + history, err := conditionsFrom.HistorySince(ctx, time.Now().Add(-time.Minute)) + if err != nil { + t.Fatal(err) + } + for i := range history { + if history[i].Key == key && history[i].Change == conditions.ChangeCleared { + cleared = &history[i] + } + } + } + if cleared == nil || !strings.Contains(cleared.Why, "D6 no longer observes it") || len(cleared.Tried) != 1 { + t.Fatalf("the clearing is not the probe's, or forgets what was tried: %+v", cleared) + } + + // Again within the hour: the budget is one, so after its settle the operator is told, and H3 stops. + clock.pass(10 * time.Minute) + if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil { + t.Fatal(err) + } + probe() + h.assertObjects = func(context.Context) error { t.Error("H3 acted past its budget"); return nil } + h.tick(ctx) + c, _, _ = conditionsFrom.Get(ctx, key) + if !c.Escalated() || c.Severity != conditions.Urgent || c.Count != 2 { + t.Fatalf("the spent budget was not handed to the operator: %+v", c) + } + acts := told.said() + if len(acts) != 2 || acts[0].Outcome != inventory.HealActed || acts[1].Outcome != inventory.HealEscalated { + t.Fatalf("said %+v", acts) + } + clock.pass(2 * time.Hour) + h.tick(ctx) + if len(told.said()) != 2 { + t.Fatal("a healer acted on what the operator holds") + } +} + +func TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers(t *testing.T) { + url := testbus.URL(t) + open := aMesh(t) + ctx := t.Context() + js, err := broker.Dial(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} { + _ = js.Context().DeleteStream(s) + } + if _, err := assertBusObjects(ctx, open.inventory, js); err != nil { + t.Fatal(err) + } + // Bound as the controller binds it (link/receive_nats.go), taking one and acknowledging none. + events := make(chan *nats.Msg, 64) + sub, err := js.Context().ChanSubscribe("", events, nats.Bind(broker.EventsStream, broker.ControllerName)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + followed := broker.ControllerFollows[0] + for i := 0; i < consumerFarBehind+200; i++ { + if _, err := js.Context().Publish(followed, []byte(`{"n":`+strconv.Itoa(i)+`}`)); err != nil { + t.Fatal(err) + } + } + d := &doctor{open: open, js: js} + probe := func() []conditions.Observation { + t.Helper() + found, err := probeConsumers(ctx, d) + if err != nil { + t.Fatal(err) + } + if err := conditionsFrom.Reconcile(ctx, "D6", kindedAs(found, "consumer-wrong")); err != nil { + t.Fatal(err) + } + return found + } + probe() + const key = "bus.EVENTS.controller.behind" + if c, raised, _ := conditionsFrom.Get(ctx, key); !raised || c.Kind != kindConsumerBehind { + t.Fatalf("a consumer %d behind was not raised: %+v", consumerFarBehind+200, c) + } + h, told, _ := healingOn(t, open) + h.resetConsumer = func(stream, name string) (string, error) { + before, after, err := js.ResetConsumer(stream, name) + if err != nil { + return "", err + } + return "it was " + strconv.FormatUint(before.Pending, 10) + " behind; " + strconv.FormatUint(after.Pending, 10) + + " pending now", nil + } + h.tick(ctx) + if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H4" || acts[0].Outcome != inventory.HealActed { + t.Fatalf("said %+v", acts) + } + if found := probe(); len(found) != 0 { + t.Fatalf("after the reset the probe still finds %+v", found) + } + if _, still, _ := conditionsFrom.Get(ctx, key); still { + t.Fatal("the probe's next run did not clear what H4 repaired") + } + // What comes next still reaches the controller. + for len(events) > 0 { + <-events + } + if _, err := js.Context().Publish(followed, []byte(`{"after":"the reset"}`)); err != nil { + t.Fatal(err) + } + deadline := time.After(10 * time.Second) + for { + select { + case m := <-events: + if strings.Contains(string(m.Data), "after") { + return + } + _ = m.Ack() + case <-deadline: + t.Fatal("the controller's subscription heard nothing after its consumer was reset: it would be deaf") + } + } +} + +func TestNatsAskToReportReachesTheMachine(t *testing.T) { + url := testbus.URL(t) + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(conn.Close) + asked, err := conn.SubscribeSync(broker.AskReportSubject("laptop")) + if err != nil { + t.Fatal(err) + } + if err := askToReport(t.Context(), conn, "laptop"); err != nil { + t.Fatal(err) + } + msg, err := asked.NextMsg(5 * time.Second) + if err != nil || !strings.Contains(string(msg.Data), "healer H1") { + t.Fatalf("the machine heard %v, %v", msg, err) + } + perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindNode, Node: "laptop", PasswordHash: "x"}) + if err != nil || !slices.Contains(perms.Subscribe, "mesh.node.laptop.ask.report") || + slices.Contains(perms.Subscribe, "mesh.node.anchor.ask.report") { + t.Fatalf("a machine's grant for the question: %v %v", perms.Subscribe, err) + } +} diff --git a/cmd/mesh-controller/healers_test.go b/cmd/mesh-controller/healers_test.go index dc2d0754..11829431 100644 --- a/cmd/mesh-controller/healers_test.go +++ b/cmd/mesh-controller/healers_test.go @@ -6,20 +6,14 @@ import ( "errors" "os" "slices" - "strconv" "strings" "sync" "testing" "time" - "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" - - "github.com/novox/mesh-controller/internal/testbus" ) // The test generated from the healer registry (novox/hq to-be 45 §7, ADR 0227 rule 7 "how it is @@ -56,9 +50,14 @@ var inducedFailures = map[string]string{ func TestEveryHealerAnswersAKindTheMeshRaisesWithABudgetABrakeAndItsEvent(t *testing.T) { kinds := raisedKinds() - source, err := os.ReadFile("healers_test.go") - if err != nil { - t.Fatal(err) + // The induced failures are tests of the integration level (a bus), in the file beside this one (issue 895). + var source []byte + for _, name := range []string{"healers_test.go", "healers_integration_test.go"} { + more, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + source = append(source, more...) } seen := map[string]bool{} answered := map[string]string{} @@ -195,446 +194,24 @@ func sentNotReported(t *testing.T, node string) string { // times): the machine is asked to report; a report that names what it was sent is all it takes, and one // that does not — or none — is a send of its current declaration again. Each act kept in `tried` as // `healer H1`, said as healer-acted, counted; twice, then the operator's, urgent; then nothing more. -func TestH1AsksAMachineToReportAndSendsItAgain(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - h, told, clock := healingOn(t, open) - record, err := open.inventory.NodeByName(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if err := open.inventory.RecordSent(ctx, record.ID, "d2", nil); err != nil { - t.Fatal(err) - } - key := sentNotReported(t, "laptop") - - // First: the report had not reached the mesh, and asking for it brings it. - asked := 0 - h.askReport = func(ctx context.Context, node string) error { - asked++ - if node != "laptop" { - t.Errorf("asked %s", node) - } - _, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeApplied, - At: time.Now().Add(time.Second), Declared: "d2"}) - return err - } - h.tick(ctx) - c, found, err := conditionsFrom.Get(ctx, key) - if err != nil || !found { - t.Fatalf("the condition is gone after the act — a healer cleared it, not an observation: %v", err) - } - if asked != 1 || len(c.Tried) != 1 || c.Tried[0].By != "healer H1" || !strings.Contains(c.Tried[0].Outcome, "acted") || - c.Resolver != "healer:H1" { - t.Fatalf("after the first act: asked %d, %+v", asked, c) - } - if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H1" || acts[0].Outcome != inventory.HealActed || - acts[0].Condition != key || acts[0].By != "healer H1" || acts[0].Epoch != 57 { - t.Fatalf("the act was not said as healer-acted: %+v", acts) - } - - // Within its settle, nothing more: the observation has its turn. - clock.pass(time.Minute) - h.tick(ctx) - if asked != 1 { - t.Fatalf("acted again inside the settle: asked %d", asked) - } - - // Second: the machine says nothing, so it is sent again. - clock.pass(3 * time.Minute) - sent := 0 - h.askReport = func(context.Context, string) error { asked++; return nil } - h.sendAgain = func(_ context.Context, node string) error { sent++; return nil } - h.tick(ctx) - if asked != 2 || sent != 1 { - t.Fatalf("the second act: asked %d, sent %d", asked, sent) - } - c, _, _ = conditionsFrom.Get(ctx, key) - if len(c.Tried) != 2 || !strings.Contains(c.Tried[1].Outcome, "sent again") || !strings.Contains(c.Tried[1].Outcome, "act 2 of 2") { - t.Fatalf("tried %+v", c.Tried) - } - - // The budget is spent: the operator's, urgent, said; and no healer touches it again. - clock.pass(4 * time.Minute) - h.tick(ctx) - c, _, _ = conditionsFrom.Get(ctx, key) - if !c.Escalated() || c.Severity != conditions.Urgent || len(c.Tried) != 3 || - !strings.Contains(c.Tried[2].Outcome, "budget of 2") { - t.Fatalf("not handed to the operator: %+v", c) - } - if acts := told.said(); len(acts) != 3 || acts[2].Outcome != inventory.HealEscalated { - t.Fatalf("the escalation was not said: %+v", acts) - } - clock.pass(time.Hour) - h.tick(ctx) - if asked != 2 || sent != 1 || len(told.said()) != 3 { - t.Fatalf("a healer acted on a condition the operator holds: asked %d sent %d said %d", asked, sent, len(told.said())) - } - heals, err := open.inventory.HealsSince(ctx, time.Now().Add(-time.Hour)) - if err != nil || len(heals) != 3 || heals[0].Outcome != inventory.HealActed || heals[1].Outcome != inventory.HealActed || - heals[2].Outcome != inventory.HealEscalated || heals[0].Epoch != 57 { - t.Fatalf("the heals kept: %+v %v", heals, err) - } - // And a heal is not a hand act: what S15 counts never sees it. - for _, x := range heals { - if x.Healer == "" || strings.HasPrefix(x.Act, "hand-act") { - t.Errorf("a heal reads as a hand act: %+v", x) - } - } -} // **Only the controller holding the lease heals** (to-be 45 §6): unleased, or standing by, nothing. -func TestNoHealerActsWithoutTheLease(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - h, told, _ := healingOn(t, open) - sentNotReported(t, "laptop") - h.epoch = func(context.Context) (uint64, error) { return 0, nil } - h.tick(ctx) - h.epoch = func(context.Context) (uint64, error) { return 0, errors.New("the lease is not held") } - h.tick(ctx) - h.epoch = func(context.Context) (uint64, error) { return 57, nil } - h.acting = func() bool { return false } - h.tick(ctx) - if len(told.said()) != 0 { - t.Fatalf("a healer acted without the lease: %+v", told.said()) - } -} // **The mesh-wide brake**: a dozen acts in an hour and every healer stops, said urgently, until an hour // after the last. -func TestTheBrakeStopsEveryHealerAndSaysSo(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - h, told, clock := healingOn(t, open) - for i := 0; i < healBrakeLimit; i++ { - if _, err := open.inventory.BeginHeal(ctx, inventory.Heal{Healer: "H3", ConditionKey: "seat.x.anchor.silent", - Kind: "holder-silent", Act: "asserted", Outcome: inventory.HealActed, - At: clock.now().Add(-time.Duration(healBrakeLimit-i) * time.Minute)}); err != nil { - t.Fatal(err) - } - } - sentNotReported(t, "laptop") - h.tick(ctx) // the fakes fail the test if anything acts - braked, found, err := conditionsFrom.Get(ctx, "mesh.healers.braked") - if err != nil || !found || braked.Severity != conditions.Urgent || !strings.Contains(braked.Evidence[0].Said, "H3 on seat.x.anchor.silent ×12") { - t.Fatalf("the brake was not said: %+v %v", braked, err) - } - if len(told.said()) != 0 { - t.Fatalf("a healer acted under the brake: %+v", told.said()) - } - // Past the hour of the first act, still held: it lets go an hour after the last. - clock.pass(30 * time.Minute) - h.tick(ctx) - if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); !held { - t.Fatal("the brake let go before an hour had passed since the last act") - } - clock.pass(31 * time.Minute) - asked := 0 - h.askReport = func(context.Context, string) error { asked++; return nil } - h.sendAgain = func(context.Context, string) error { return nil } - h.tick(ctx) - if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); held { - t.Fatal("the brake held an hour after the last act") - } - if asked != 1 { - t.Fatalf("the healers did not act again after the brake let go: asked %d", asked) - } -} // **H2 closes a plan another has taken over**, with its note — and leaves a plan alone whose wait is // still to come: that one is its own signal's, not a healer's. -func TestH2ClosesAPlanAnotherHasTakenOver(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - h, told, _ := healingOn(t, open) - created := time.Now().UTC().Add(-time.Hour) - older := inventory.Plan{ID: "plan-old", Repository: "novox/app", Branch: "main", Commit: "0ld0ld0", Created: created, - State: inventory.PlanBuilding, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "asked"}}} - waiting := inventory.Plan{ID: "plan-other", Repository: "novox/other", Branch: "main", Commit: "07he707", Created: created, - State: inventory.PlanBuilding, Tiers: [][]string{{"b"}}, Modules: map[string]*inventory.PlanModule{"b": {State: "asked"}}} - newer := inventory.Plan{ID: "plan-new", Repository: "novox/app", Branch: "main", Commit: "new0new", Created: created.Add(time.Minute), - State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}} - for _, p := range []*inventory.Plan{&older, &waiting, &newer} { - if err := open.inventory.SavePlan(ctx, p); err != nil { - t.Fatal(err) - } - } - for _, id := range []string{"plan-old", "plan-other"} { - if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopePlan, ID: id, Kind: "stalled", - Severity: conditions.Warning, Summary: id + " is stalled", Source: "S3"}); err != nil { - t.Fatal(err) - } - } - h.tick(ctx) - closed, err := open.inventory.PlanByID(ctx, "plan-old") - if err != nil || closed.State != inventory.PlanSuperseded || !strings.Contains(closed.Note, "closed by healer H2") || - !strings.Contains(closed.Note, "plan-new") { - t.Fatalf("the superseded plan: %+v %v", closed, err) - } - if other, _ := open.inventory.PlanByID(ctx, "plan-other"); other.State != inventory.PlanBuilding { - t.Fatalf("a plan still waiting was closed: %+v", other) - } - if c, _, _ := conditionsFrom.Get(ctx, "plan.plan-other.stalled"); len(c.Tried) != 0 || c.Resolver != conditions.ResolverSelf { - t.Fatalf("a plan H2 does not repair was touched: %+v", c) - } - if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H2" || acts[0].Condition != "plan.plan-old.stalled" { - t.Fatalf("said %+v", acts) - } - // Finished: every module built, none rolled out unsent — closed as done. - done := inventory.Plan{ID: "plan-done", Repository: "novox/third", Branch: "main", Commit: "d0ned0n", Created: created, - State: inventory.PlanRolling, Tier: 0, Tiers: [][]string{{"c"}}, Modules: map[string]*inventory.PlanModule{"c": {State: "built"}}} - if err := open.inventory.SavePlan(ctx, &done); err != nil { - t.Fatal(err) - } - if state, why, err := planStale(ctx, open.inventory, done); err != nil || state != inventory.PlanDone || !strings.Contains(why, "finished") { - t.Fatalf("a finished plan reads %q %q %v", state, why, err) - } -} // **H4 only for a consumer the stream table marks resettable**: a module's consumer far behind is said // and left — what a reset drops, nothing would catch up for it. -func TestH4ResetsOnlyWhatTheTableMarksResettable(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - h, _, _ := healingOn(t, open) - marked := 0 - for _, c := range broker.MeshConsumers() { - if c.Resettable != "" { - marked++ - if c.Stream != broker.EventsStream || c.Name != broker.ControllerName { - t.Errorf("%s on %s is marked resettable: only the controller's own events consumer is", c.Name, c.Stream) - } - } - } - if marked != 1 { - t.Fatalf("%d consumers are marked resettable, want the controller's events consumer alone", marked) - } - for _, who := range []string{"EVENTS.anchor_shop", "CONTROL.controller"} { - if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "behind", - Kind: kindConsumerBehind, Severity: conditions.Warning, Summary: who + " is far behind", Source: "D6"}); err != nil { - t.Fatal(err) - } - } - h.tick(ctx) // the fake reset fails the test if it is called - reset := "" - h.resetConsumer = func(stream, name string) (string, error) { - reset = stream + "." + name - return "it was 1500 behind", nil - } - if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopeBus, ID: "EVENTS.controller", - Token: "behind", Kind: kindConsumerBehind, Severity: conditions.Warning, Summary: "far behind", Source: "D6"}); err != nil { - t.Fatal(err) - } - h.tick(ctx) - if reset != "EVENTS.controller" { - t.Fatalf("reset %q", reset) - } -} // **H3 against a real bus** (issue 208's note): a consumer the mesh expects, deleted; D6 says so; H3 // asserts the bus's objects the way a send does, and D6's next run clears it — the healer never does. // Deleted again within the hour, the budget is spent: the operator's, urgent, and H3 stops. -func TestNatsH3AssertsAMissingConsumerAgainAndBrakesAfterItsBudget(t *testing.T) { - url := testbus.URL(t) - open := aMesh(t) - ctx := t.Context() - js, err := broker.Dial(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} { - _ = js.Context().DeleteStream(s) - } - if _, err := assertBusObjects(ctx, open.inventory, js); err != nil { - t.Fatal(err) - } - h, told, clock := healingOn(t, open) - h.js = js - h.assertObjects = func(ctx context.Context) error { return assertOnSend(ctx, open.inventory, js, " ") } - d := &doctor{open: open, js: js} - probe := func() { - t.Helper() - found, err := probeConsumers(ctx, d) - if err != nil { - t.Fatal(err) - } - if err := conditionsFrom.Reconcile(ctx, "D6", kindedAs(found, "consumer-wrong")); err != nil { - t.Fatal(err) - } - } - const key = "bus.NODES.laptop.missing" - - if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil { - t.Fatal(err) - } - probe() - if c, raised, _ := conditionsFrom.Get(ctx, key); !raised || c.Kind != "consumer-lost" { - t.Fatalf("the deleted consumer was not raised: %+v", c) - } - h.tick(ctx) - if _, err := js.Context().ConsumerInfo("NODES", "laptop"); err != nil { - t.Fatalf("H3 did not make the consumer again: %v", err) - } - c, still, _ := conditionsFrom.Get(ctx, key) - if !still || len(c.Tried) != 1 || c.Tried[0].By != "healer H3" || c.Resolver != "healer:H3" { - t.Fatalf("the act is not in the condition, or the healer cleared it: %+v", c) - } - probe() - if _, still, _ := conditionsFrom.Get(ctx, key); still { - t.Fatal("the probe's next run did not clear what H3 repaired") - } - // Kept in the history as the keeper says it, a moment later. - var cleared *conditions.Event - for wait := time.Now().Add(5 * time.Second); cleared == nil && time.Now().Before(wait); time.Sleep(20 * time.Millisecond) { - history, err := conditionsFrom.HistorySince(ctx, time.Now().Add(-time.Minute)) - if err != nil { - t.Fatal(err) - } - for i := range history { - if history[i].Key == key && history[i].Change == conditions.ChangeCleared { - cleared = &history[i] - } - } - } - if cleared == nil || !strings.Contains(cleared.Why, "D6 no longer observes it") || len(cleared.Tried) != 1 { - t.Fatalf("the clearing is not the probe's, or forgets what was tried: %+v", cleared) - } - - // Again within the hour: the budget is one, so after its settle the operator is told, and H3 stops. - clock.pass(10 * time.Minute) - if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil { - t.Fatal(err) - } - probe() - h.assertObjects = func(context.Context) error { t.Error("H3 acted past its budget"); return nil } - h.tick(ctx) - c, _, _ = conditionsFrom.Get(ctx, key) - if !c.Escalated() || c.Severity != conditions.Urgent || c.Count != 2 { - t.Fatalf("the spent budget was not handed to the operator: %+v", c) - } - acts := told.said() - if len(acts) != 2 || acts[0].Outcome != inventory.HealActed || acts[1].Outcome != inventory.HealEscalated { - t.Fatalf("said %+v", acts) - } - clock.pass(2 * time.Hour) - h.tick(ctx) - if len(told.said()) != 2 { - t.Fatal("a healer acted on what the operator holds") - } -} // **H4 against a real bus** (issue 248): the controller's events consumer a long way behind — the week it // once replayed — is re-made from now by the mesh itself, and the controller's bound subscription still // receives what comes next: a reset that left the controller deaf would be the incident. -func TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers(t *testing.T) { - url := testbus.URL(t) - open := aMesh(t) - ctx := t.Context() - js, err := broker.Dial(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} { - _ = js.Context().DeleteStream(s) - } - if _, err := assertBusObjects(ctx, open.inventory, js); err != nil { - t.Fatal(err) - } - // Bound as the controller binds it (link/receive_nats.go), taking one and acknowledging none. - events := make(chan *nats.Msg, 64) - sub, err := js.Context().ChanSubscribe("", events, nats.Bind(broker.EventsStream, broker.ControllerName)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = sub.Unsubscribe() }) - followed := broker.ControllerFollows[0] - for i := 0; i < consumerFarBehind+200; i++ { - if _, err := js.Context().Publish(followed, []byte(`{"n":`+strconv.Itoa(i)+`}`)); err != nil { - t.Fatal(err) - } - } - d := &doctor{open: open, js: js} - probe := func() []conditions.Observation { - t.Helper() - found, err := probeConsumers(ctx, d) - if err != nil { - t.Fatal(err) - } - if err := conditionsFrom.Reconcile(ctx, "D6", kindedAs(found, "consumer-wrong")); err != nil { - t.Fatal(err) - } - return found - } - probe() - const key = "bus.EVENTS.controller.behind" - if c, raised, _ := conditionsFrom.Get(ctx, key); !raised || c.Kind != kindConsumerBehind { - t.Fatalf("a consumer %d behind was not raised: %+v", consumerFarBehind+200, c) - } - h, told, _ := healingOn(t, open) - h.resetConsumer = func(stream, name string) (string, error) { - before, after, err := js.ResetConsumer(stream, name) - if err != nil { - return "", err - } - return "it was " + strconv.FormatUint(before.Pending, 10) + " behind; " + strconv.FormatUint(after.Pending, 10) + - " pending now", nil - } - h.tick(ctx) - if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H4" || acts[0].Outcome != inventory.HealActed { - t.Fatalf("said %+v", acts) - } - if found := probe(); len(found) != 0 { - t.Fatalf("after the reset the probe still finds %+v", found) - } - if _, still, _ := conditionsFrom.Get(ctx, key); still { - t.Fatal("the probe's next run did not clear what H4 repaired") - } - // What comes next still reaches the controller. - for len(events) > 0 { - <-events - } - if _, err := js.Context().Publish(followed, []byte(`{"after":"the reset"}`)); err != nil { - t.Fatal(err) - } - deadline := time.After(10 * time.Second) - for { - select { - case m := <-events: - if strings.Contains(string(m.Data), "after") { - return - } - _ = m.Ack() - case <-deadline: - t.Fatal("the controller's subscription heard nothing after its consumer was reset: it would be deaf") - } - } -} // **H1's question reaches the machine**, on the subject its grant lets it hear and nothing else. -func TestNatsAskToReportReachesTheMachine(t *testing.T) { - url := testbus.URL(t) - conn, err := nats.Connect(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - asked, err := conn.SubscribeSync(broker.AskReportSubject("laptop")) - if err != nil { - t.Fatal(err) - } - if err := askToReport(t.Context(), conn, "laptop"); err != nil { - t.Fatal(err) - } - msg, err := asked.NextMsg(5 * time.Second) - if err != nil || !strings.Contains(string(msg.Data), "healer H1") { - t.Fatalf("the machine heard %v, %v", msg, err) - } - perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindNode, Node: "laptop", PasswordHash: "x"}) - if err != nil || !slices.Contains(perms.Subscribe, "mesh.node.laptop.ask.report") || - slices.Contains(perms.Subscribe, "mesh.node.anchor.ask.report") { - t.Fatalf("a machine's grant for the question: %v %v", perms.Subscribe, err) - } -} diff --git a/cmd/mesh-controller/held_back_integration_test.go b/cmd/mesh-controller/held_back_integration_test.go new file mode 100644 index 00000000..e8e2eb5d --- /dev/null +++ b/cmd/mesh-controller/held_back_integration_test.go @@ -0,0 +1,182 @@ +//go:build integration + +package main + +import ( + "bytes" + "reflect" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + asked := time.Now().Add(-time.Hour) + aResolver(t, open, "c1c1c1c1c1", asked) + for _, node := range []string{"anchor", "laptop"} { + if _, err := inv.Assign(ctx, node, "resolver"); err != nil { + t.Fatal(err) + } + } + // Recorded by a person's choice: the default rolls out since novox/hq ADR 0236. + if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{Why: "each machine checked by hand"}); err != nil { + t.Fatal(err) + } + gens, err := generators(ctx, open) + if err != nil { + t.Fatal(err) + } + compose := composeForPush(open, gens) + d := &recordedDelivery{inv: inv} + if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil { + t.Fatal(err) + } + if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" { + t.Fatalf("the send did not record the build it carried: %v %v %v", builds, known, err) + } + digestOfLaptop := func() string { + sent, err := inv.Outstanding(ctx, "laptop") + if err != nil { + t.Fatal(err) + } + return sent + } + before := digestOfLaptop() + + // The change merges; the policy is record. `push anchor` sends the anchor... + aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute)) + d.declared = nil + if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, "", nil); err != nil { + t.Fatal(err) + } + // ...and its cascade leaves the laptop, saying so. + var said bytes.Buffer + d.declared = nil + refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", nil, &said) + if err != nil || len(refused) != 0 { + t.Fatalf("the cascade failed: %v %v", refused, err) + } + if len(d.declared) != 0 { + t.Fatalf("the cascade sent %v a build its policy records", d.declared) + } + if digestOfLaptop() != before { + t.Fatal("the laptop's last send moved: it was sent the held build") + } + for _, want := range []string{"laptop is behind and was not sent", "resolver would move from c1c1c1c1 to c2c2c2c2", + "upgrade policy records", "`push laptop` sends it"} { + if !strings.Contains(said.String(), want) { + t.Errorf("the push did not say %q:\n%s", want, said.String()) + } + } + + // Held and owed something else at once — a peer joined: still not sent, and both said: why it + // is held, and that what else it is owed waits with it. + aThirdMachine(t, open) + d.declared = nil + if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil { + t.Fatal(err) + } + d.declared = nil + said.Reset() + if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, + compose, d, "", nil, &said); err != nil { + t.Fatal(err) + } + if len(d.declared) != 0 || digestOfLaptop() != before { + t.Fatalf("a held machine owed a consequence was sent: %v", d.declared) + } + if !strings.Contains(said.String(), "resolver would move") || !strings.Contains(said.String(), "anything else it is owed") { + t.Fatalf("the push did not say both:\n%s", said.String()) + } + + // A policy that rolls out, and a build no gate has seen: still held — a cascade does not judge it + // (novox/hq ADR 0236). + if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil { + t.Fatal(err) + } + said.Reset() + if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, + compose, d, "", nil, &said); err != nil { + t.Fatal(err) + } + if len(d.declared) != 0 || !strings.Contains(said.String(), "has passed no gate yet") { + t.Fatalf("a cascade carried a build no gate has seen: %v\n%s", d.declared, said.String()) + } + // Once it passed a gate on some machine, the laptop is a consequence like any other, and sent. + if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: "build-c2", Module: "resolver", Commit: "c2c2c2c2c2", + Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil { + t.Fatal(err) + } + said.Reset() + if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, + compose, d, "", nil, &said); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before { + t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String()) + } + if builds, _, _ := inv.SentBuilds(ctx, "laptop"); builds["resolver"] != "c2c2c2c2c2" { + t.Fatalf("the new send did not record the new build: %v", builds) + } +} + +func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + aResolver(t, open, "c1c1c1c1c1", time.Now().Add(-time.Hour)) + if _, err := inv.Assign(ctx, "laptop", "resolver"); err != nil { + t.Fatal(err) + } + gens, err := generators(ctx, open) + if err != nil { + t.Fatal(err) + } + compose := composeForPush(open, gens) + d := &recordedDelivery{inv: inv} + if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil { + t.Fatal(err) + } + + // `push spare`, the machine just placed: the others' peers change with it. + aThirdMachine(t, open) + if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil { + t.Fatal(err) + } + d.declared = nil + var said bytes.Buffer + if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) { + t.Fatalf("a consequence nothing holds was not sent: %v\n%s", d.declared, said.String()) + } + if strings.Contains(said.String(), "was not sent") { + t.Fatalf("a machine nothing holds was said to be held:\n%s", said.String()) + } + + // A machine whose last send was not recorded — a declaration sent by hand — is held until named. + record, err := inv.NodeByName(ctx, "laptop") + if err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, record.ID, "sent-by-hand", nil); err != nil { + t.Fatal(err) + } + d.declared = nil + said.Reset() + if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil { + t.Fatal(err) + } + // The anchor, the hub, may still be settling from the machine placed above; the laptop is the + // question. + if slices.Contains(d.declared, "laptop") || !strings.Contains(said.String(), "laptop is behind and was not sent") { + t.Fatalf("a machine whose last send is not known was sent: %v\n%s", d.declared, said.String()) + } +} diff --git a/cmd/mesh-controller/held_back_test.go b/cmd/mesh-controller/held_back_test.go index f9d2ba06..2d5eb506 100644 --- a/cmd/mesh-controller/held_back_test.go +++ b/cmd/mesh-controller/held_back_test.go @@ -1,11 +1,9 @@ package main import ( - "bytes" "context" "encoding/json" "reflect" - "slices" "strings" "testing" "time" @@ -182,172 +180,6 @@ func aThirdMachine(t *testing.T, open *stores) { // The issue as it happened, against the real stores: a change merged with the policy `record`, a push // naming the anchor, and the laptop — running the same module — left with what it had, by name. -func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - asked := time.Now().Add(-time.Hour) - aResolver(t, open, "c1c1c1c1c1", asked) - for _, node := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, node, "resolver"); err != nil { - t.Fatal(err) - } - } - // Recorded by a person's choice: the default rolls out since novox/hq ADR 0236. - if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{Why: "each machine checked by hand"}); err != nil { - t.Fatal(err) - } - gens, err := generators(ctx, open) - if err != nil { - t.Fatal(err) - } - compose := composeForPush(open, gens) - d := &recordedDelivery{inv: inv} - if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil { - t.Fatal(err) - } - if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" { - t.Fatalf("the send did not record the build it carried: %v %v %v", builds, known, err) - } - digestOfLaptop := func() string { - sent, err := inv.Outstanding(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - return sent - } - before := digestOfLaptop() - - // The change merges; the policy is record. `push anchor` sends the anchor... - aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute)) - d.declared = nil - if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, "", nil); err != nil { - t.Fatal(err) - } - // ...and its cascade leaves the laptop, saying so. - var said bytes.Buffer - d.declared = nil - refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", nil, &said) - if err != nil || len(refused) != 0 { - t.Fatalf("the cascade failed: %v %v", refused, err) - } - if len(d.declared) != 0 { - t.Fatalf("the cascade sent %v a build its policy records", d.declared) - } - if digestOfLaptop() != before { - t.Fatal("the laptop's last send moved: it was sent the held build") - } - for _, want := range []string{"laptop is behind and was not sent", "resolver would move from c1c1c1c1 to c2c2c2c2", - "upgrade policy records", "`push laptop` sends it"} { - if !strings.Contains(said.String(), want) { - t.Errorf("the push did not say %q:\n%s", want, said.String()) - } - } - - // Held and owed something else at once — a peer joined: still not sent, and both said: why it - // is held, and that what else it is owed waits with it. - aThirdMachine(t, open) - d.declared = nil - if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil { - t.Fatal(err) - } - d.declared = nil - said.Reset() - if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, - compose, d, "", nil, &said); err != nil { - t.Fatal(err) - } - if len(d.declared) != 0 || digestOfLaptop() != before { - t.Fatalf("a held machine owed a consequence was sent: %v", d.declared) - } - if !strings.Contains(said.String(), "resolver would move") || !strings.Contains(said.String(), "anything else it is owed") { - t.Fatalf("the push did not say both:\n%s", said.String()) - } - - // A policy that rolls out, and a build no gate has seen: still held — a cascade does not judge it - // (novox/hq ADR 0236). - if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil { - t.Fatal(err) - } - said.Reset() - if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, - compose, d, "", nil, &said); err != nil { - t.Fatal(err) - } - if len(d.declared) != 0 || !strings.Contains(said.String(), "has passed no gate yet") { - t.Fatalf("a cascade carried a build no gate has seen: %v\n%s", d.declared, said.String()) - } - // Once it passed a gate on some machine, the laptop is a consequence like any other, and sent. - if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: "build-c2", Module: "resolver", Commit: "c2c2c2c2c2", - Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil { - t.Fatal(err) - } - said.Reset() - if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, - compose, d, "", nil, &said); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before { - t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String()) - } - if builds, _, _ := inv.SentBuilds(ctx, "laptop"); builds["resolver"] != "c2c2c2c2c2" { - t.Fatalf("the new send did not record the new build: %v", builds) - } -} // Issue 057's case is unchanged: a machine whose builds are all current and whose declaration moved // for another reason is sent by a push that names someone else. -func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - aResolver(t, open, "c1c1c1c1c1", time.Now().Add(-time.Hour)) - if _, err := inv.Assign(ctx, "laptop", "resolver"); err != nil { - t.Fatal(err) - } - gens, err := generators(ctx, open) - if err != nil { - t.Fatal(err) - } - compose := composeForPush(open, gens) - d := &recordedDelivery{inv: inv} - if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil { - t.Fatal(err) - } - - // `push spare`, the machine just placed: the others' peers change with it. - aThirdMachine(t, open) - if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil { - t.Fatal(err) - } - d.declared = nil - var said bytes.Buffer - if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) { - t.Fatalf("a consequence nothing holds was not sent: %v\n%s", d.declared, said.String()) - } - if strings.Contains(said.String(), "was not sent") { - t.Fatalf("a machine nothing holds was said to be held:\n%s", said.String()) - } - - // A machine whose last send was not recorded — a declaration sent by hand — is held until named. - record, err := inv.NodeByName(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, record.ID, "sent-by-hand", nil); err != nil { - t.Fatal(err) - } - d.declared = nil - said.Reset() - if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil { - t.Fatal(err) - } - // The anchor, the hub, may still be settling from the machine placed above; the laptop is the - // question. - if slices.Contains(d.declared, "laptop") || !strings.Contains(said.String(), "laptop is behind and was not sent") { - t.Fatalf("a machine whose last send is not known was sent: %v\n%s", d.declared, said.String()) - } -} diff --git a/cmd/mesh-controller/hold_test.go b/cmd/mesh-controller/hold_test.go index 8301b42c..818247f7 100644 --- a/cmd/mesh-controller/hold_test.go +++ b/cmd/mesh-controller/hold_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/holdings_test.go b/cmd/mesh-controller/holdings_test.go index 83267096..efab46cd 100644 --- a/cmd/mesh-controller/holdings_test.go +++ b/cmd/mesh-controller/holdings_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/hosts_behind_integration_test.go b/cmd/mesh-controller/hosts_behind_integration_test.go new file mode 100644 index 00000000..4bced031 --- /dev/null +++ b/cmd/mesh-controller/hosts_behind_integration_test.go @@ -0,0 +1,42 @@ +//go:build integration + +package main + +import "testing" + +func TestAReportedHostVersionIsKeptAndReadBack(t *testing.T) { + // The machine has sent this since ADR 0141 and the controller's own copy of the report did not + // have the field, so it was unmarshalled into nothing. End to end through the store, because the + // fault was a field that existed on one side of the wire only. + open := aMesh(t) + record, err := open.inventory.NodeByName(t.Context(), "anchor") + if err != nil { + t.Fatal(err) + } + if record.HostVersion != "" { + t.Fatalf("a machine that never reported one has host version %q", record.HostVersion) + } + if err := open.inventory.RecordHostVersion(t.Context(), record.ID, "ced54d4"); err != nil { + t.Fatal(err) + } + again, err := open.inventory.NodeByName(t.Context(), "anchor") + if err != nil { + t.Fatal(err) + } + if again.HostVersion != "ced54d4" { + t.Fatalf("the reported host version read back as %q", again.HostVersion) + } + // An empty report never clears what a machine last said: a bare word that the node is there says + // nothing about its host. + if err := open.inventory.RecordHostVersion(t.Context(), record.ID, " "); err != nil { + t.Fatal(err) + } + kept, err := open.inventory.NodeByName(t.Context(), "anchor") + if err != nil { + t.Fatal(err) + } + if kept.HostVersion != "ced54d4" { + t.Fatalf("a report carrying no host version cleared what the machine had said: %q", + kept.HostVersion) + } +} diff --git a/cmd/mesh-controller/hosts_behind_test.go b/cmd/mesh-controller/hosts_behind_test.go index 75634937..5fb0f822 100644 --- a/cmd/mesh-controller/hosts_behind_test.go +++ b/cmd/mesh-controller/hosts_behind_test.go @@ -72,40 +72,3 @@ func TestAMeshWhereNothingReportedAHostStatesNoDisagreement(t *testing.T) { t.Fatalf("a mesh told no host version reported a split: %v", split) } } - -func TestAReportedHostVersionIsKeptAndReadBack(t *testing.T) { - // The machine has sent this since ADR 0141 and the controller's own copy of the report did not - // have the field, so it was unmarshalled into nothing. End to end through the store, because the - // fault was a field that existed on one side of the wire only. - open := aMesh(t) - record, err := open.inventory.NodeByName(t.Context(), "anchor") - if err != nil { - t.Fatal(err) - } - if record.HostVersion != "" { - t.Fatalf("a machine that never reported one has host version %q", record.HostVersion) - } - if err := open.inventory.RecordHostVersion(t.Context(), record.ID, "ced54d4"); err != nil { - t.Fatal(err) - } - again, err := open.inventory.NodeByName(t.Context(), "anchor") - if err != nil { - t.Fatal(err) - } - if again.HostVersion != "ced54d4" { - t.Fatalf("the reported host version read back as %q", again.HostVersion) - } - // An empty report never clears what a machine last said: a bare word that the node is there says - // nothing about its host. - if err := open.inventory.RecordHostVersion(t.Context(), record.ID, " "); err != nil { - t.Fatal(err) - } - kept, err := open.inventory.NodeByName(t.Context(), "anchor") - if err != nil { - t.Fatal(err) - } - if kept.HostVersion != "ced54d4" { - t.Fatalf("a report carrying no host version cleared what the machine had said: %q", - kept.HostVersion) - } -} diff --git a/cmd/mesh-controller/identity_bound_integration_test.go b/cmd/mesh-controller/identity_bound_integration_test.go new file mode 100644 index 00000000..8195ab04 --- /dev/null +++ b/cmd/mesh-controller/identity_bound_integration_test.go @@ -0,0 +1,107 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestAnOverflowingConsumerNeverRefusesItsProvidersMachine(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "objects", Version: "1", + Provides: []catalogue.Offer{{Name: "s3-bucket", Scope: catalogue.ScopeMesh, + Identity: &catalogue.OfferIdentity{Max: 20, In: "an S3 access key"}}}, + Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}}) + register(t, open, catalogue.Manifest{Module: "resolver", Version: "1", + Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}}) + register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1", + Requires: []string{"wildcard-resolution"}}) + register(t, open, catalogue.Manifest{Module: "photoalbum", Version: "1", Requires: []string{"s3-bucket"}}) + register(t, open, catalogue.Manifest{Module: "files", Version: "1", Requires: []string{"s3-bucket"}}) + for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"}, + {"laptop", "networkmanager"}, {"laptop", "photoalbum"}, {"laptop", "files"}} { + if _, err := assign(ctx, open, a[0], a[1]); err != nil { + t.Fatalf("assign %s %s: %v", a[0], a[1], err) + } + } + + // The consumer's machine resolves, and says which of its modules no provider will grant. + consumer, _, err := planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + over := consumer.Overflowing() + if len(over) != 1 || over[0].Module != "photoalbum" || over[0].Provision != "s3-bucket" { + t.Fatalf("the consumer's side does not name exactly photoalbum: %+v", over) + } + + // The provider's machine composes. Under ADR 0049's one bound this was a refusal naming + // networkmanager, and no push to the provider could go through. + plan, settings, err := planFor(ctx, open, "anchor") + if err != nil { + t.Fatal(err) + } + declared, err := declarationFor(ctx, open, "anchor", plan, settings) + if err != nil { + t.Fatalf("one consumer's identity refused its provider's whole machine: %v", err) + } + if len(declared.withheld) != 1 || declared.withheld[0].Identity != "mesh_laptop_photoalbum" { + t.Fatalf("the overflowing consumer is not the one withheld: %+v", declared.withheld) + } + grants, _, _, err := grantsFor(ctx, open, "anchor") + if err != nil { + t.Fatal(err) + } + var keyless bool + for _, g := range grants { + keyless = keyless || g.Provision == "wildcard-resolution" && g.From == "networkmanager" + } + if !keyless { + t.Fatalf("networkmanager, 26 characters, is not granted the keyless resolver provision: %+v", grants) + } + var granted []string + for _, c := range declared.Received["objects"]["s3-bucket"] { + granted = append(granted, c.From) + } + if strings.Join(granted, ",") != "files" { + t.Fatalf("the object store grants %v; files and only files fit", granted) + } + said := printed(t, func() error { reportLeftOut("anchor", declared); return nil }) + if !strings.Contains(said, `photoalbum on laptop requires s3-bucket from anchor`) || + !strings.Contains(said, "left out of anchor's grants") { + t.Fatalf("the push does not say whom it leaves out:\n%s", said) + } + + // And `status` names it, and does not call the mesh well while it stands. + asked, err := theThreeQuestions(ctx, open) + if err != nil { + t.Fatal(err) + } + if len(asked.overflowing) != 1 || asked.overflowing[0].Module != "photoalbum" { + t.Fatalf("status does not carry the overflow: %+v", asked.overflowing) + } + if asked.well() { + t.Fatal("a mesh with a consumer left out of its grants reads as well") + } + shown := printed(t, func() error { return printStatus(asked) }) + if !strings.Contains(shown, "identified too long for a provision they require") || + !strings.Contains(shown, "mesh_laptop_photoalbum") { + t.Fatalf("status does not say it:\n%s", shown) + } + body, err := statusAsJSON(asked) + if err != nil { + t.Fatal(err) + } + var doc struct { + Overflowing []catalogue.Overflow `json:"overflowing"` + } + if err := json.Unmarshal(body, &doc); err != nil || len(doc.Overflowing) != 1 || + doc.Overflowing[0].Bound.Max != 20 { + t.Fatalf("the document does not carry it: %v\n%s", err, body) + } +} diff --git a/cmd/mesh-controller/identity_bound_test.go b/cmd/mesh-controller/identity_bound_test.go index 25745587..a3160af9 100644 --- a/cmd/mesh-controller/identity_bound_test.go +++ b/cmd/mesh-controller/identity_bound_test.go @@ -2,13 +2,10 @@ package main import ( "bytes" - "encoding/json" "os" "path/filepath" "strings" "testing" - - "github.com/novox/mesh-controller/internal/catalogue" ) // novox/hq ADR 0225, issue 263: a consumer's identity is bounded by the provision it requires, an @@ -53,98 +50,3 @@ func TestModuleCheckRefusesAnIdentityThatOverflowsWhatItRequires(t *testing.T) { // resolver provision, and a second consumer there overflows an object store's access key. The // provider's machine still composes; the overflowing consumer is left out of its grants and named, // by push and by `status`, and the keyless consumer is granted with its long name. -func TestAnOverflowingConsumerNeverRefusesItsProvidersMachine(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "objects", Version: "1", - Provides: []catalogue.Offer{{Name: "s3-bucket", Scope: catalogue.ScopeMesh, - Identity: &catalogue.OfferIdentity{Max: 20, In: "an S3 access key"}}}, - Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}}) - register(t, open, catalogue.Manifest{Module: "resolver", Version: "1", - Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}}) - register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1", - Requires: []string{"wildcard-resolution"}}) - register(t, open, catalogue.Manifest{Module: "photoalbum", Version: "1", Requires: []string{"s3-bucket"}}) - register(t, open, catalogue.Manifest{Module: "files", Version: "1", Requires: []string{"s3-bucket"}}) - for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"}, - {"laptop", "networkmanager"}, {"laptop", "photoalbum"}, {"laptop", "files"}} { - if _, err := assign(ctx, open, a[0], a[1]); err != nil { - t.Fatalf("assign %s %s: %v", a[0], a[1], err) - } - } - - // The consumer's machine resolves, and says which of its modules no provider will grant. - consumer, _, err := planFor(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - over := consumer.Overflowing() - if len(over) != 1 || over[0].Module != "photoalbum" || over[0].Provision != "s3-bucket" { - t.Fatalf("the consumer's side does not name exactly photoalbum: %+v", over) - } - - // The provider's machine composes. Under ADR 0049's one bound this was a refusal naming - // networkmanager, and no push to the provider could go through. - plan, settings, err := planFor(ctx, open, "anchor") - if err != nil { - t.Fatal(err) - } - declared, err := declarationFor(ctx, open, "anchor", plan, settings) - if err != nil { - t.Fatalf("one consumer's identity refused its provider's whole machine: %v", err) - } - if len(declared.withheld) != 1 || declared.withheld[0].Identity != "mesh_laptop_photoalbum" { - t.Fatalf("the overflowing consumer is not the one withheld: %+v", declared.withheld) - } - grants, _, _, err := grantsFor(ctx, open, "anchor") - if err != nil { - t.Fatal(err) - } - var keyless bool - for _, g := range grants { - keyless = keyless || g.Provision == "wildcard-resolution" && g.From == "networkmanager" - } - if !keyless { - t.Fatalf("networkmanager, 26 characters, is not granted the keyless resolver provision: %+v", grants) - } - var granted []string - for _, c := range declared.Received["objects"]["s3-bucket"] { - granted = append(granted, c.From) - } - if strings.Join(granted, ",") != "files" { - t.Fatalf("the object store grants %v; files and only files fit", granted) - } - said := printed(t, func() error { reportLeftOut("anchor", declared); return nil }) - if !strings.Contains(said, `photoalbum on laptop requires s3-bucket from anchor`) || - !strings.Contains(said, "left out of anchor's grants") { - t.Fatalf("the push does not say whom it leaves out:\n%s", said) - } - - // And `status` names it, and does not call the mesh well while it stands. - asked, err := theThreeQuestions(ctx, open) - if err != nil { - t.Fatal(err) - } - if len(asked.overflowing) != 1 || asked.overflowing[0].Module != "photoalbum" { - t.Fatalf("status does not carry the overflow: %+v", asked.overflowing) - } - if asked.well() { - t.Fatal("a mesh with a consumer left out of its grants reads as well") - } - shown := printed(t, func() error { return printStatus(asked) }) - if !strings.Contains(shown, "identified too long for a provision they require") || - !strings.Contains(shown, "mesh_laptop_photoalbum") { - t.Fatalf("status does not say it:\n%s", shown) - } - body, err := statusAsJSON(asked) - if err != nil { - t.Fatal(err) - } - var doc struct { - Overflowing []catalogue.Overflow `json:"overflowing"` - } - if err := json.Unmarshal(body, &doc); err != nil || len(doc.Overflowing) != 1 || - doc.Overflowing[0].Bound.Max != 20 { - t.Fatalf("the document does not carry it: %v\n%s", err, body) - } -} diff --git a/cmd/mesh-controller/issue201_integration_test.go b/cmd/mesh-controller/issue201_integration_test.go new file mode 100644 index 00000000..413b2f1c --- /dev/null +++ b/cmd/mesh-controller/issue201_integration_test.go @@ -0,0 +1,181 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" +) + +func core201(t *testing.T) *stores { + t.Helper() + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute) + record := func(commit string, asked time.Time) inventory.Build { + manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"}) + b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit, + Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}} + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + return b + } + before := record("c1aaaaaaaaaa", old) + if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: before.Commit, + Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil { + t.Fatal(err) + } + record("c2bbbbbbbbbb", now) + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{ + Repository: "novox/mesh-controller", BuiltFrom: "c2bbbbbbbbbb", Head: "c2bbbbbbbbbb", Asked: now}); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}); err != nil { + t.Fatal(err) + } + if err := inv.RestoreModule(ctx, before); err != nil { + t.Fatal(err) + } + wasMoves := machineMoves + machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { + modules, err := open.inventory.Assigned(ctx, node) + if err != nil { + return nil, err + } + sent, known, err := open.inventory.SentBuilds(ctx, node) + if err != nil { + return nil, err + } + return f.moves(node, modules, sent, known, all), nil + } + t.Cleanup(func() { machineMoves = wasMoves }) + return open +} + +func TestReplay201(t *testing.T) { + open := core201(t) + ctx := t.Context() + k, _ := withConditionsInMemory(t) + + judged := withScope(ctx, sendScope{judged: map[string]bool{"anchor": true}}) + if kept, err := ungatedIn(judged, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 { + t.Fatalf("a plan's send to the machine is made, the rest of it with it: %v %v", kept, err) + } + keeps, err := recordedKept(keepingRecorded(judged), open, "anchor") + if err != nil || keeps["mesh-controller"] != "c2bbbbbbbbbb" { + t.Fatalf("the send keeps the newer controller there: %v %v", keeps, err) + } + shelf := map[string]catalogue.Manifest{"mesh-controller": {Module: "mesh-controller", Version: "old"}} + if _, err := keepRecorded(keepingRecorded(judged), open, "anchor", shelf); err != nil || shelf["mesh-controller"].Version != "1" { + t.Fatalf("the send composes the kept build's manifest: %+v %v", shelf, err) + } + key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key() + if c, found, err := k.Get(ctx, key); err != nil || !found || !strings.Contains(c.Summary, "c2bbbbbb") { + t.Fatalf("the condition %s is raised, saying the build kept: %+v %v %v", key, c, found, err) + } + + putBack := keepingRecorded(withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true})) + if keeps, err := recordedKept(putBack, open, "anchor"); err != nil || len(keeps) != 0 { + t.Fatalf("the gate's own put-back of the controller carries it back: %v %v", keeps, err) + } + + settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}) + if _, found, _ := k.Get(ctx, key); !found { + t.Fatal("a send that kept the newer controller cleared the condition") + } + settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c1aaaaaaaaaa"}) + if _, found, _ := k.Get(ctx, key); found { + t.Fatal("a send carrying the mesh's record — a person's push — leaves the condition open") + } +} + +func TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend(t *testing.T) { + open := core201(t) + ctx := t.Context() + k, _ := withConditionsInMemory(t) + if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: "build-mesh-controller-c2bbbbbbbbbb", + Module: "mesh-controller", Commit: "c2bbbbbbbbbb", Previous: "c1aaaaaaaaaa", Machines: []string{"anchor"}, + Verdict: inventory.GateFailed, Rollback: inventory.NotRolledBack}); err != nil { + t.Fatal(err) + } + healer := withScope(ctx, sendScope{}) + if kept, err := ungatedIn(healer, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 { + t.Fatalf("the healer's send is made: %v %v", kept, err) + } + if keeps, err := recordedKept(keepingRecorded(healer), open, "anchor"); err != nil || len(keeps) != 0 { + t.Fatalf("the failed controller was kept over the build put back: %v %v", keeps, err) + } + if open, _ := k.Open(ctx); len(open) != 0 { + t.Fatalf("a completed put-back raised a condition: %+v", open) + } +} + +func TestAGateJudgesTheOtherMovesBesideAKeptCore(t *testing.T) { + open := core201(t) + ctx := t.Context() + withConditionsInMemory(t) + inv := open.inventory + old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute) + for _, b := range []struct { + commit string + at time.Time + }{{"a1aaaaaaaaaa", old}, {"a2bbbbbbbbbb", now}} { + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-" + b.commit, Module: "app", Commit: b.commit, + Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: b.at, At: b.at, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-" + b.commit}}}); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ + Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: b.commit, Head: b.commit, + Asked: b.at}); err != nil { + t.Fatal(err) + } + } + if _, err := inv.Assign(ctx, "anchor", "app"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-2", + map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a1aaaaaaaaaa"}); err != nil { + t.Fatal(err) + } + // The send as it is made: the controller kept at c2, the app moved to the record. + wasSend := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + return names, open.inventory.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-3", + map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a2bbbbbbbbbb"}) + } + t.Cleanup(func() { sendRollout = wasSend }) + + moves, sent, _, err := gatedSend(ctx, open, "anchor", nil) + if err != nil || len(sent) != 1 { + t.Fatalf("the gated send is made: %v %v", sent, err) + } + if len(moves) != 1 || moves[0].Module != "app" || moves[0].To != "a2bbbbbbbbbb" { + t.Fatalf("the gate judges the app's move and not the kept controller: %+v", moves) + } + builds, _, err := inv.SentBuilds(ctx, "anchor") + if err != nil { + t.Fatal(err) + } + g := &inventory.PlanGate{Carried: moves} + f := gateFacts{commits: judgedCommits(g, nil), sentBuilds: map[string]map[string]string{"anchor": builds}} + for _, m := range []string{"app", "mesh-controller"} { + if why, superseded := f.supersededOn(m, "anchor"); superseded { + t.Fatalf("the gate is superseded by its own send: %s", why) + } + } +} diff --git a/cmd/mesh-controller/issue201_test.go b/cmd/mesh-controller/issue201_test.go index 2e9c5c91..7bd7085c 100644 --- a/cmd/mesh-controller/issue201_test.go +++ b/cmd/mesh-controller/issue201_test.go @@ -1,14 +1,10 @@ package main import ( - "context" - "encoding/json" "strings" "testing" "time" - "github.com/novox/mesh-controller/internal/catalogue" - "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/lease" ) @@ -82,178 +78,16 @@ func TestACoreModuleIsKeptOverAnOlderRecord(t *testing.T) { // core201 is a mesh whose control node, anchor, was sent the newer controller c2 while the mesh's record of the // controller is the build before it, c1, which passed its gate when it rolled out. -func core201(t *testing.T) *stores { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute) - record := func(commit string, asked time.Time) inventory.Build { - manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"}) - b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit, - Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}} - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - return b - } - before := record("c1aaaaaaaaaa", old) - if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: before.Commit, - Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil { - t.Fatal(err) - } - record("c2bbbbbbbbbb", now) - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-controller", BuiltFrom: "c2bbbbbbbbbb", Head: "c2bbbbbbbbbb", Asked: now}); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}); err != nil { - t.Fatal(err) - } - if err := inv.RestoreModule(ctx, before); err != nil { - t.Fatal(err) - } - wasMoves := machineMoves - machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { - modules, err := open.inventory.Assigned(ctx, node) - if err != nil { - return nil, err - } - sent, known, err := open.inventory.SentBuilds(ctx, node) - if err != nil { - return nil, err - } - return f.moves(node, modules, sent, known, all), nil - } - t.Cleanup(func() { machineMoves = wasMoves }) - return open -} // TestReplay201 replays the race on a store: the control node runs the newer controller and the mesh's record is // the build before it. A plan's send there — the node-engine's plan of 2026-10-02 — goes, keeps the newer // controller, and says so as a condition; the gate's own put-back carries it back; a send that carries the record, // as a person's push does, clears the condition. -func TestReplay201(t *testing.T) { - open := core201(t) - ctx := t.Context() - k, _ := withConditionsInMemory(t) - - judged := withScope(ctx, sendScope{judged: map[string]bool{"anchor": true}}) - if kept, err := ungatedIn(judged, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 { - t.Fatalf("a plan's send to the machine is made, the rest of it with it: %v %v", kept, err) - } - keeps, err := recordedKept(keepingRecorded(judged), open, "anchor") - if err != nil || keeps["mesh-controller"] != "c2bbbbbbbbbb" { - t.Fatalf("the send keeps the newer controller there: %v %v", keeps, err) - } - shelf := map[string]catalogue.Manifest{"mesh-controller": {Module: "mesh-controller", Version: "old"}} - if _, err := keepRecorded(keepingRecorded(judged), open, "anchor", shelf); err != nil || shelf["mesh-controller"].Version != "1" { - t.Fatalf("the send composes the kept build's manifest: %+v %v", shelf, err) - } - key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key() - if c, found, err := k.Get(ctx, key); err != nil || !found || !strings.Contains(c.Summary, "c2bbbbbb") { - t.Fatalf("the condition %s is raised, saying the build kept: %+v %v %v", key, c, found, err) - } - - putBack := keepingRecorded(withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true})) - if keeps, err := recordedKept(putBack, open, "anchor"); err != nil || len(keeps) != 0 { - t.Fatalf("the gate's own put-back of the controller carries it back: %v %v", keeps, err) - } - - settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}) - if _, found, _ := k.Get(ctx, key); !found { - t.Fatal("a send that kept the newer controller cleared the condition") - } - settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c1aaaaaaaaaa"}) - if _, found, _ := k.Get(ctx, key); found { - t.Fatal("a send carrying the mesh's record — a person's push — leaves the condition open") - } -} // The review of #273: a gate failed the newer controller, put the record back, and the put-back's own send was // refused. The next automatic send — a healer's, which judges nothing — completes the put-back: a build that // failed its gate is never kept. -func TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend(t *testing.T) { - open := core201(t) - ctx := t.Context() - k, _ := withConditionsInMemory(t) - if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: "build-mesh-controller-c2bbbbbbbbbb", - Module: "mesh-controller", Commit: "c2bbbbbbbbbb", Previous: "c1aaaaaaaaaa", Machines: []string{"anchor"}, - Verdict: inventory.GateFailed, Rollback: inventory.NotRolledBack}); err != nil { - t.Fatal(err) - } - healer := withScope(ctx, sendScope{}) - if kept, err := ungatedIn(healer, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 { - t.Fatalf("the healer's send is made: %v %v", kept, err) - } - if keeps, err := recordedKept(keepingRecorded(healer), open, "anchor"); err != nil || len(keeps) != 0 { - t.Fatalf("the failed controller was kept over the build put back: %v %v", keeps, err) - } - if open, _ := k.Open(ctx); len(open) != 0 { - t.Fatalf("a completed put-back raised a condition: %+v", open) - } -} // Review 91 of #273: a gated send to a machine whose controller is kept over an older record leaves the kept // controller out of what its gate judges. Judged as moving back to the record, the gate would read the send's own // kept controller as another build sent there after it (issue 352) and be superseded, judging nothing. -func TestAGateJudgesTheOtherMovesBesideAKeptCore(t *testing.T) { - open := core201(t) - ctx := t.Context() - withConditionsInMemory(t) - inv := open.inventory - old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute) - for _, b := range []struct { - commit string - at time.Time - }{{"a1aaaaaaaaaa", old}, {"a2bbbbbbbbbb", now}} { - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-" + b.commit, Module: "app", Commit: b.commit, - Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: b.at, At: b.at, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-" + b.commit}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: b.commit, Head: b.commit, - Asked: b.at}); err != nil { - t.Fatal(err) - } - } - if _, err := inv.Assign(ctx, "anchor", "app"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-2", - map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a1aaaaaaaaaa"}); err != nil { - t.Fatal(err) - } - // The send as it is made: the controller kept at c2, the app moved to the record. - wasSend := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - return names, open.inventory.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-3", - map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a2bbbbbbbbbb"}) - } - t.Cleanup(func() { sendRollout = wasSend }) - - moves, sent, _, err := gatedSend(ctx, open, "anchor", nil) - if err != nil || len(sent) != 1 { - t.Fatalf("the gated send is made: %v %v", sent, err) - } - if len(moves) != 1 || moves[0].Module != "app" || moves[0].To != "a2bbbbbbbbbb" { - t.Fatalf("the gate judges the app's move and not the kept controller: %+v", moves) - } - builds, _, err := inv.SentBuilds(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - g := &inventory.PlanGate{Carried: moves} - f := gateFacts{commits: judgedCommits(g, nil), sentBuilds: map[string]map[string]string{"anchor": builds}} - for _, m := range []string{"app", "mesh-controller"} { - if why, superseded := f.supersededOn(m, "anchor"); superseded { - t.Fatalf("the gate is superseded by its own send: %s", why) - } - } -} diff --git a/cmd/mesh-controller/issue348_integration_test.go b/cmd/mesh-controller/issue348_integration_test.go new file mode 100644 index 00000000..b2c5f29c --- /dev/null +++ b/cmd/mesh-controller/issue348_integration_test.go @@ -0,0 +1,199 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" + "github.com/novox/mesh-controller/internal/link" +) + +func TestReplay348(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv, k := open.inventory, conditionsFrom + say := func(at time.Time, n *link.NetworkHealth) { + t.Helper() + if err := stateHealth(ctx, inv, k, "anchor", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil { + t.Fatal(err) + } + } + network := func() (conditions.Condition, bool) { + t.Helper() + list, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + for _, c := range list { + if c.Key == "machine.anchor.network" { + return c, true + } + } + return conditions.Condition{}, false + } + + // 10:58:45 — the second failing look: raised. + say(h0, networkSaying(namesRefused(link.StateUnhealthy, 2))) + raised, ok := network() + if !ok { + t.Fatal("the resolver refusing on the control node raised nothing") + } + time.Sleep(5 * time.Millisecond) + sent := time.Now().UTC() + time.Sleep(5 * time.Millisecond) + + // 10:59:42 — the restarted engine's first statement: one look failed, a second decides. + say(h0.Add(time.Minute), networkSaying(namesRefused(link.StateUnknown, 1))) + if _, ok := network(); !ok { + t.Fatal("a statement that judged nothing yet cleared the condition: the restarted engine's first look " + + "said the fault was gone while it still refused") + } + // 11:00:23 — its second look: unhealthy again, the same raising. + say(h0.Add(2*time.Minute), networkSaying(namesRefused(link.StateUnhealthy, 2))) + again, ok := network() + if !ok || !again.Raised.Equal(raised.Raised) || again.Count != 1 { + t.Fatalf("the same raising was not kept: raised %s (first %s), count %d", again.Raised, raised.Raised, again.Count) + } + + // The gate on the control node, for a build sent after the fault began. + open2, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + f := gateFacts{judged: true, open: open2} + if w := aboutTheMachine("anchor", []string{"mesh-host"}, sent, f); w.whole != "" || len(w.on) != 0 { + t.Fatalf("a fault from before the send held the build: %+v", w) + } + + // Decided healthy: cleared. + say(h0.Add(3*time.Minute), networkSaying(link.NetworkPart{Part: link.PartNames, State: link.StateHealthy, Since: h0})) + if c, ok := network(); ok { + t.Fatalf("a statement that decides the names healthy left %s open", c.Key) + } +} + +// namesRefused is the control node's names part as its node-engine said it in the outage: its own +// resolver, at its own address, refusing. +func namesRefused(state string, streak int) link.NetworkPart { + p := link.NetworkPart{Part: link.PartNames, State: state, Since: h0, Streak: streak} + if state == link.StateUnhealthy { + p.Reason = "1 of its 2 resolvers do not answer as the mesh's do" + p.Said = "10.77.0.1 — anchor.internal (IPv4): read udp 10.77.0.1:35244->10.77.0.1:53: read: connection refused" + p.Toward = []string{"10.77.0.1"} + } + return p +} + +func networkSaying(parts ...link.NetworkPart) *link.NetworkHealth { + state := link.StateHealthy + for _, p := range parts { + switch { + case p.State == link.StateUnhealthy: + state = link.StateUnhealthy + case p.State == link.StateUnknown && state == link.StateHealthy: + state = link.StateUnknown + } + } + return &link.NetworkHealth{State: state, Since: h0, Parts: parts} +} + +// A fault there at the send, cleared and reopened after it, is not raised since the send; one that cleared +// before the send and came back after it is — a send that breaks a recovered machine fails its gate (review +// of mesh-controller PR 179, A2). Read through OpenAt, by both of the gate's readings. +func TestAFaultThatFlappedAfterTheSendIsNotTheSendsAndOneThatRecoveredBeforeItIs(t *testing.T) { + since := h0 + network := func(first time.Time, gaps ...conditions.Gap) conditions.Condition { + raised := since.Add(time.Minute) + if len(gaps) > 0 { + raised = gaps[len(gaps)-1].Reopened + } + return conditions.Condition{Key: "machine.anchor.network", Kind: kindMachineNetwork, + Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor"}, + Summary: "anchor's network is not healthy", Source: sourceNetwork, First: first, Gaps: gaps, Raised: raised} + } + held := func(c conditions.Condition) bool { + return aboutTheMachine("anchor", []string{"mesh-controller"}, since, gateFacts{judged: true, + open: []conditions.Condition{c}}).whole != "" + } + // The day's case: raised before the send, cleared 8 s after it, reopened 49 s after it. + flapped := network(since.Add(-49*time.Second), + conditions.Gap{Cleared: since.Add(8 * time.Second), Reopened: since.Add(49 * time.Second)}) + if held(flapped) { + t.Fatal("a fault there at the send, flapping after it, held the machine") + } + // Recovered before the send, broken again after it: the send's. + recovered := network(since.Add(-time.Hour), + conditions.Gap{Cleared: since.Add(-30 * time.Second), Reopened: since.Add(20 * time.Second)}) + if !held(recovered) { + t.Fatal("a machine recovered at the send and broken after it passed the gate") + } + // An older gap, before the send, and the fault there at the send: not the send's. + twice := network(since.Add(-time.Hour), + conditions.Gap{Cleared: since.Add(-50 * time.Minute), Reopened: since.Add(-45 * time.Minute)}, + conditions.Gap{Cleared: since.Add(10 * time.Second), Reopened: since.Add(30 * time.Second)}) + if held(twice) { + t.Fatal("a fault there at the send, with an older gap, held the machine") + } + // Raised after the send, never cleared: the send's. + if !held(network(time.Time{})) { + t.Fatal("a fault raised after the send held nothing") + } + // And a module's own, through judgeHealth. + at := since.Add(2 * time.Minute) + g := gateFacts{judged: true, now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor", + Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{}, + served: map[string]served{}, rolledBack: map[string][]lease.Rollback{}, + open: []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{ + Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Summary: "failing", + First: since.Add(-time.Hour), Raised: since.Add(time.Minute), + Gaps: []conditions.Gap{{Cleared: since.Add(5 * time.Second), Reopened: since.Add(time.Minute)}}}}} + if _, why := judgeHealth("app", "", catalogue.Manifest{Module: "app"}, "anchor", since, g); strings.HasPrefix(why, "raised since it was sent") { + t.Fatalf("a module's own fault there at the send: %s", why) + } + g.open[0].Gaps[0].Cleared = since.Add(-5 * time.Second) + if _, why := judgeHealth("app", "", catalogue.Manifest{Module: "app"}, "anchor", since, g); !strings.HasPrefix(why, "raised since it was sent") { + t.Fatalf("a module's own fault, recovered at the send and back after it, was not counted: %s", why) + } +} + +// Only an undecided part holds a condition that names it; a condition about another part clears, and a +// statement unknown as a whole holds every part (review of PR 179, A4). Pure. +func TestAnUndecidedPartHoldsOnlyWhatNamesIt(t *testing.T) { + f := netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateUnknown, inventory.NetworkPart{Part: link.PartNames, State: link.StateUnknown, Streak: 1}, + inventory.NetworkPart{Part: link.PartRoute, State: link.StateHealthy}), + "laptop": aNetwork(link.StateHealthy, inventory.NetworkPart{Part: link.PartNames, State: link.StateHealthy}), + "spare": aNetwork(link.StateStarting, inventory.NetworkPart{Part: link.PartTunnel, State: link.StateHealthy}), + "other": aNetwork(link.StateStarting, inventory.NetworkPart{Part: link.PartTunnel, State: link.StateStarting}), + }) + u := undecidedParts(f) + if !u["anchor"][link.PartNames] || u["anchor"][link.PartRoute] || u["laptop"] != nil || !u["spare"]["*"] || + !u["other"][link.PartTunnel] || u["other"]["*"] { + t.Fatalf("undecided: %v", u) + } + about := func(machine, said string, also ...string) conditions.Condition { + return conditions.Condition{Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: machine, + Machine: machine, Also: also}, Evidence: []conditions.Evidence{{Said: said}}} + } + for _, c := range []struct { + c conditions.Condition + held bool + }{ + {about("anchor", "names since 2026-10-09 10:58:45 UTC: 10.77.0.1 — refused"), true}, + {about("anchor", "route since 2026-10-09 10:58:45 UTC: no default route"), false}, + {about("laptop", "names since 2026-10-09 10:58:45 UTC: refused"), false}, + {about("spare", "route since …: no default route"), true}, + {about("hub", "anchor: names: refused", "anchor"), true}, + {about("hub", "anchor: tunnel: no handshake", "anchor"), false}, + } { + if got := heldUndecided(c.c, u); got != c.held { + t.Errorf("%s %q held %v, want %v", c.c.Subject.Machine, c.c.Evidence[0].Said, got, c.held) + } + } +} diff --git a/cmd/mesh-controller/issue348_test.go b/cmd/mesh-controller/issue348_test.go index 7c0bd4a7..1f8544c4 100644 --- a/cmd/mesh-controller/issue348_test.go +++ b/cmd/mesh-controller/issue348_test.go @@ -1,15 +1,9 @@ package main import ( - "strings" "testing" - "time" - "github.com/novox/mesh-controller/internal/catalogue" - "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" - "github.com/novox/mesh-controller/internal/link" ) // novox/hq issue 348: on 2026-10-09 the control node's resolver stopped answering on its private address @@ -20,192 +14,9 @@ import ( // builds failed their gate at 11:10 with "raised since it was sent" and were put back, for a fault that // began before they were sent. -// namesRefused is the control node's names part as its node-engine said it in the outage: its own -// resolver, at its own address, refusing. -func namesRefused(state string, streak int) link.NetworkPart { - p := link.NetworkPart{Part: link.PartNames, State: state, Since: h0, Streak: streak} - if state == link.StateUnhealthy { - p.Reason = "1 of its 2 resolvers do not answer as the mesh's do" - p.Said = "10.77.0.1 — anchor.internal (IPv4): read udp 10.77.0.1:35244->10.77.0.1:53: read: connection refused" - p.Toward = []string{"10.77.0.1"} - } - return p -} - -func networkSaying(parts ...link.NetworkPart) *link.NetworkHealth { - state := link.StateHealthy - for _, p := range parts { - switch { - case p.State == link.StateUnhealthy: - state = link.StateUnhealthy - case p.State == link.StateUnknown && state == link.StateHealthy: - state = link.StateUnknown - } - } - return &link.NetworkHealth{State: state, Since: h0, Parts: parts} -} - // TestReplay348 replays the statements of the outage: the condition raised before the send is not // cleared by the restarted engine's first, undecided statement, and the gate does not count it against // the builds sent after it began. -func TestReplay348(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv, k := open.inventory, conditionsFrom - say := func(at time.Time, n *link.NetworkHealth) { - t.Helper() - if err := stateHealth(ctx, inv, k, "anchor", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil { - t.Fatal(err) - } - } - network := func() (conditions.Condition, bool) { - t.Helper() - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - for _, c := range list { - if c.Key == "machine.anchor.network" { - return c, true - } - } - return conditions.Condition{}, false - } - - // 10:58:45 — the second failing look: raised. - say(h0, networkSaying(namesRefused(link.StateUnhealthy, 2))) - raised, ok := network() - if !ok { - t.Fatal("the resolver refusing on the control node raised nothing") - } - time.Sleep(5 * time.Millisecond) - sent := time.Now().UTC() - time.Sleep(5 * time.Millisecond) - - // 10:59:42 — the restarted engine's first statement: one look failed, a second decides. - say(h0.Add(time.Minute), networkSaying(namesRefused(link.StateUnknown, 1))) - if _, ok := network(); !ok { - t.Fatal("a statement that judged nothing yet cleared the condition: the restarted engine's first look " + - "said the fault was gone while it still refused") - } - // 11:00:23 — its second look: unhealthy again, the same raising. - say(h0.Add(2*time.Minute), networkSaying(namesRefused(link.StateUnhealthy, 2))) - again, ok := network() - if !ok || !again.Raised.Equal(raised.Raised) || again.Count != 1 { - t.Fatalf("the same raising was not kept: raised %s (first %s), count %d", again.Raised, raised.Raised, again.Count) - } - - // The gate on the control node, for a build sent after the fault began. - open2, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - f := gateFacts{judged: true, open: open2} - if w := aboutTheMachine("anchor", []string{"mesh-host"}, sent, f); w.whole != "" || len(w.on) != 0 { - t.Fatalf("a fault from before the send held the build: %+v", w) - } - - // Decided healthy: cleared. - say(h0.Add(3*time.Minute), networkSaying(link.NetworkPart{Part: link.PartNames, State: link.StateHealthy, Since: h0})) - if c, ok := network(); ok { - t.Fatalf("a statement that decides the names healthy left %s open", c.Key) - } -} - -// A fault there at the send, cleared and reopened after it, is not raised since the send; one that cleared -// before the send and came back after it is — a send that breaks a recovered machine fails its gate (review -// of mesh-controller PR 179, A2). Read through OpenAt, by both of the gate's readings. -func TestAFaultThatFlappedAfterTheSendIsNotTheSendsAndOneThatRecoveredBeforeItIs(t *testing.T) { - since := h0 - network := func(first time.Time, gaps ...conditions.Gap) conditions.Condition { - raised := since.Add(time.Minute) - if len(gaps) > 0 { - raised = gaps[len(gaps)-1].Reopened - } - return conditions.Condition{Key: "machine.anchor.network", Kind: kindMachineNetwork, - Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor"}, - Summary: "anchor's network is not healthy", Source: sourceNetwork, First: first, Gaps: gaps, Raised: raised} - } - held := func(c conditions.Condition) bool { - return aboutTheMachine("anchor", []string{"mesh-controller"}, since, gateFacts{judged: true, - open: []conditions.Condition{c}}).whole != "" - } - // The day's case: raised before the send, cleared 8 s after it, reopened 49 s after it. - flapped := network(since.Add(-49*time.Second), - conditions.Gap{Cleared: since.Add(8 * time.Second), Reopened: since.Add(49 * time.Second)}) - if held(flapped) { - t.Fatal("a fault there at the send, flapping after it, held the machine") - } - // Recovered before the send, broken again after it: the send's. - recovered := network(since.Add(-time.Hour), - conditions.Gap{Cleared: since.Add(-30 * time.Second), Reopened: since.Add(20 * time.Second)}) - if !held(recovered) { - t.Fatal("a machine recovered at the send and broken after it passed the gate") - } - // An older gap, before the send, and the fault there at the send: not the send's. - twice := network(since.Add(-time.Hour), - conditions.Gap{Cleared: since.Add(-50 * time.Minute), Reopened: since.Add(-45 * time.Minute)}, - conditions.Gap{Cleared: since.Add(10 * time.Second), Reopened: since.Add(30 * time.Second)}) - if held(twice) { - t.Fatal("a fault there at the send, with an older gap, held the machine") - } - // Raised after the send, never cleared: the send's. - if !held(network(time.Time{})) { - t.Fatal("a fault raised after the send held nothing") - } - // And a module's own, through judgeHealth. - at := since.Add(2 * time.Minute) - g := gateFacts{judged: true, now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor", - Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{}, - served: map[string]served{}, rolledBack: map[string][]lease.Rollback{}, - open: []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{ - Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Summary: "failing", - First: since.Add(-time.Hour), Raised: since.Add(time.Minute), - Gaps: []conditions.Gap{{Cleared: since.Add(5 * time.Second), Reopened: since.Add(time.Minute)}}}}} - if _, why := judgeHealth("app", "", catalogue.Manifest{Module: "app"}, "anchor", since, g); strings.HasPrefix(why, "raised since it was sent") { - t.Fatalf("a module's own fault there at the send: %s", why) - } - g.open[0].Gaps[0].Cleared = since.Add(-5 * time.Second) - if _, why := judgeHealth("app", "", catalogue.Manifest{Module: "app"}, "anchor", since, g); !strings.HasPrefix(why, "raised since it was sent") { - t.Fatalf("a module's own fault, recovered at the send and back after it, was not counted: %s", why) - } -} - -// Only an undecided part holds a condition that names it; a condition about another part clears, and a -// statement unknown as a whole holds every part (review of PR 179, A4). Pure. -func TestAnUndecidedPartHoldsOnlyWhatNamesIt(t *testing.T) { - f := netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateUnknown, inventory.NetworkPart{Part: link.PartNames, State: link.StateUnknown, Streak: 1}, - inventory.NetworkPart{Part: link.PartRoute, State: link.StateHealthy}), - "laptop": aNetwork(link.StateHealthy, inventory.NetworkPart{Part: link.PartNames, State: link.StateHealthy}), - "spare": aNetwork(link.StateStarting, inventory.NetworkPart{Part: link.PartTunnel, State: link.StateHealthy}), - "other": aNetwork(link.StateStarting, inventory.NetworkPart{Part: link.PartTunnel, State: link.StateStarting}), - }) - u := undecidedParts(f) - if !u["anchor"][link.PartNames] || u["anchor"][link.PartRoute] || u["laptop"] != nil || !u["spare"]["*"] || - !u["other"][link.PartTunnel] || u["other"]["*"] { - t.Fatalf("undecided: %v", u) - } - about := func(machine, said string, also ...string) conditions.Condition { - return conditions.Condition{Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: machine, - Machine: machine, Also: also}, Evidence: []conditions.Evidence{{Said: said}}} - } - for _, c := range []struct { - c conditions.Condition - held bool - }{ - {about("anchor", "names since 2026-10-09 10:58:45 UTC: 10.77.0.1 — refused"), true}, - {about("anchor", "route since 2026-10-09 10:58:45 UTC: no default route"), false}, - {about("laptop", "names since 2026-10-09 10:58:45 UTC: refused"), false}, - {about("spare", "route since …: no default route"), true}, - {about("hub", "anchor: names: refused", "anchor"), true}, - {about("hub", "anchor: tunnel: no handshake", "anchor"), false}, - } { - if got := heldUndecided(c.c, u); got != c.held { - t.Errorf("%s %q held %v, want %v", c.c.Subject.Machine, c.c.Evidence[0].Said, got, c.held) - } - } -} // A release walks its modules without a record per module: D10 counts what its tier names as rolling, // so the node-engine a release walks is not "behind, and no plan is rolling it out" on its first machine. diff --git a/cmd/mesh-controller/issue352_integration_test.go b/cmd/mesh-controller/issue352_integration_test.go new file mode 100644 index 00000000..7f604afe --- /dev/null +++ b/cmd/mesh-controller/issue352_integration_test.go @@ -0,0 +1,227 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestReplay352(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + judgeLeavesAsTheCore(t) + t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn) + t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging) +} + +func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) { + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + advancePlans(ctx, b.open) // anchor is sent, and the fixture reports it applied + // 16:31:37 — another plan sends anchor a newer declaration, which it has not reported on. + carried, _, _ := inv.SentBuilds(ctx, "anchor") + if err := inv.RecordSent(ctx, nodeID(t, b.open, "anchor"), "d-anchor-newer", carried); err != nil { + t.Fatal(err) + } + reports, _ := inv.LastReports(ctx) + for _, r := range reports { + if r.Node == "anchor" && r.Current { + t.Fatal("the fixture's newer send reads as reported") + } + } + gateEvery, gateBound = 0, 0 // past the bound at once: before the fix, "has not reported" fails it here + for i := 0; i < 4; i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + if p.State == inventory.PlanFailed || strings.Contains(p.Note, "has not reported") { + t.Fatalf("the release failed on the newer send: %s %s", p.State, p.Note) + } + if g := p.Release.Gate; g != nil && (g.Sent == nil || g.Sent["anchor"].Digest == "") { + t.Fatalf("the gate does not keep what it sent: %+v", g) + } + if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("app's pass on anchor was not kept: %+v %v %v", v, found, err) + } + if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { + t.Fatalf("app was put back to %s", current["app"].Commit) + } +} + +func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + advancePlans(ctx, b.open) // the release judges app c2 on anchor + _, _, _, err := gatedSend(ctx, b.open, "anchor", []inventory.CarriedMove{{Module: "app", Node: "anchor", From: "c2", To: "c3", Build: "build-app-c3"}}) + if !errors.Is(err, errWalkedElsewhere) || !strings.Contains(err.Error(), "release-") { + t.Fatalf("a newer build of a judged module was sent under the release's gate: %v", err) + } + if len(b.sent) != 1 { + t.Fatalf("sent %v", b.sent) + } +} + +func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + judgeLeavesAsTheCore(t) + g := aGateMesh(t) + ctx := t.Context() + inv := g.open.inventory + advancePlans(ctx, g.open) // anchor is sent app c2 first + if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-c3", map[string]string{"app": "c3"}); err != nil { + t.Fatal(err) + } + gateEvery = 0 + advancePlans(ctx, g.open) + p := g.plan(t) + if p.State != inventory.PlanSuperseded || !strings.Contains(p.Note, "superseded") || !strings.Contains(p.Note, "c3") { + t.Fatalf("the plan is %s: %s", p.State, p.Note) + } + // Nothing put back: the registered build stands, the build is not marked, and the plan's gate made no + // rollback (a release may walk what the other send left waiting on anchor; that is not a put-back). + if r := p.Modules["app"].Gate.Rollback; r != "" { + t.Fatalf("a superseded judging made a rollback: %q", r) + } + if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { + t.Fatalf("app was put back to %s", current["app"].Commit) + } + if failed, _ := inv.GateFailed(ctx, "build-2"); failed { + t.Fatal("a superseded build was marked failed") + } +} + +func testASendThatMovesTheModuleSupersedesTheJudging(t *testing.T) { + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + advancePlans(ctx, b.open) + // Another send moves app on anchor to a build this gate does not judge. + carried, _, _ := inv.SentBuilds(ctx, "anchor") + carried["app"] = "c3" + if err := inv.RecordSent(ctx, nodeID(t, b.open, "anchor"), "d-anchor-c3", carried); err != nil { + t.Fatal(err) + } + gateEvery = 0 + advancePlans(ctx, b.open) + p := b.release(t) + if p.State != inventory.PlanSuperseded || !strings.Contains(p.Note, "superseded") || !strings.Contains(p.Note, "c3") { + t.Fatalf("the release is %s: %s", p.State, p.Note) + } + if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}}) { + t.Fatalf("sent %v: a superseded judging puts nothing back", b.sent) + } + if _, found, _ := inv.GateOf(ctx, "build-app-c2"); found { + t.Fatal("a superseded judging kept a verdict") + } + if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { + t.Fatalf("app was put back to %s", current["app"].Commit) + } +} + +func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + keeper, _ := withConditionsInMemory(t) + told := &conditions.Told{} + was := doctorFrom + doctorFrom = &doctor{open: open, keeper: keeper, teller: told} + t.Cleanup(func() { doctorFrom = was }) + wasSend := sendRollout + var sent [][]string + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + sent = append(sent, names) + return names, nil + } + t.Cleanup(func() { sendRollout = wasSend }) + + build := func(id, commit, digest string, asked time.Time) inventory.Build { + manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: commit}) + b := inventory.Build{ID: id, Module: "mesh-controller", Commit: commit, Repository: "novox/mesh-controller", Path: ".", + Manifest: manifest, Asked: asked, At: asked, Made: []inventory.Artifact{{Name: "controller", Kind: catalogue.ArtifactBundle, + Reference: "mesh-artifact://mesh-controller/controller/blobs/sha256:" + digest}}} + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: commit}, + inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Path: ".", BuiltFrom: commit, Head: commit, Asked: asked}); err != nil { + t.Fatal(err) + } + return b + } + previous := build("build-old", "ef26d4cb", strings.Repeat("1", 64), time.Now().Add(-2*time.Hour)) + failed := build("build-new", "e6b00e2e", strings.Repeat("2", 64), time.Now().Add(-time.Minute)) + if versionOfBuild(previous) != strings.Repeat("1", 12) { + t.Fatalf("the build's version is %q", versionOfBuild(previous)) + } + applied, err := inv.SchemaApplied(ctx) + if err != nil || applied < 87 { + t.Fatalf("the store's schema reaches %d (%v)", applied, err) + } + // The build before never recorded what it reads: not proved, refused. + if why, ok := controllerSchemaAllows(ctx, inv, previous); ok || !strings.Contains(why, "never recorded") { + t.Fatalf("an unknown reach: %v %q", ok, why) + } + // It reads less than the store has: refused, naming both. + if err := inv.RecordSchemaReach(ctx, versionOfBuild(previous), applied-1); err != nil { + t.Fatal(err) + } + if why, ok := controllerSchemaAllows(ctx, inv, previous); ok || !strings.Contains(why, "is at") { + t.Fatalf("a reach behind the store: %v %q", ok, why) + } + // Through the gate: the failed build is marked, nothing is put back, nothing is sent, the condition is urgent. + at := time.Now().Add(-5 * time.Minute) + state := &inventory.PlanModule{Build: failed.ID, Commit: failed.Commit, Previous: previous.Commit, First: []string{"anchor"}, FirstAt: &at} + p := inventory.Plan{ID: "plan-352", Repository: "novox/mesh-controller", Branch: "main", Commit: failed.Commit, Created: at, + State: inventory.PlanRolling, Tiers: [][]string{{"mesh-controller"}}, Modules: map[string]*inventory.PlanModule{"mesh-controller": state}} + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + gateFailed(ctx, open, &p, "mesh-controller", state, []string{"anchor"}, "not healthy within 10m0s of its apply") + if state.Gate.Rollback != inventory.NotRolledBack || !strings.Contains(p.Note, "NOT put back") || !strings.Contains(p.Note, "the current build is kept") { + t.Fatalf("rollback %q: %s", state.Gate.Rollback, p.Note) + } + if len(sent) != 0 { + t.Fatalf("sent %v: nothing is put back", sent) + } + if current, _ := inv.CurrentBuilds(ctx); current["mesh-controller"].Commit != failed.Commit { + t.Fatalf("the module was put back to %s", current["mesh-controller"].Commit) + } + if marked, _ := inv.GateFailed(ctx, failed.ID); !marked { + t.Fatal("the failed build is not marked failed at its gate") + } + open2, _ := keeper.Open(ctx) + var found bool + for _, c := range open2 { + if c.Kind == kindRollbackFailed && c.Severity == conditions.Urgent && strings.Contains(c.Summary, "current build is kept") { + found = true + } + } + if !found { + t.Fatalf("no urgent rollback-failed condition saying the current build is kept: %+v", open2) + } + // Reaching the store: allowed. + if err := inv.RecordSchemaReach(ctx, versionOfBuild(previous), applied); err != nil { + t.Fatal(err) + } + if why, ok := controllerSchemaAllows(ctx, inv, previous); !ok { + t.Fatalf("a build that reads the whole schema was refused: %q", why) + } + // A build with no bundle to know it by: refused. + if why, ok := controllerSchemaAllows(ctx, inv, inventory.Build{Commit: "x"}); ok || !strings.Contains(why, "names no bundle") { + t.Fatalf("a build without a bundle: %v %q", ok, why) + } +} diff --git a/cmd/mesh-controller/issue352_test.go b/cmd/mesh-controller/issue352_test.go index e3a6f7e1..3baa9b57 100644 --- a/cmd/mesh-controller/issue352_test.go +++ b/cmd/mesh-controller/issue352_test.go @@ -1,16 +1,11 @@ package main import ( - "context" - "encoding/json" - "errors" - "reflect" "strings" "testing" "time" "github.com/novox/mesh-controller/internal/catalogue" - "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/lease" ) @@ -24,121 +19,10 @@ import ( // TestReplay352 replays the walk on the backlog fixture: the release sends anchor and anchor reports; // another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module // to another build supersedes the judging: no verdict, nothing put back. -func TestReplay352(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn) - t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging) -} - -func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) { - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - advancePlans(ctx, b.open) // anchor is sent, and the fixture reports it applied - // 16:31:37 — another plan sends anchor a newer declaration, which it has not reported on. - carried, _, _ := inv.SentBuilds(ctx, "anchor") - if err := inv.RecordSent(ctx, nodeID(t, b.open, "anchor"), "d-anchor-newer", carried); err != nil { - t.Fatal(err) - } - reports, _ := inv.LastReports(ctx) - for _, r := range reports { - if r.Node == "anchor" && r.Current { - t.Fatal("the fixture's newer send reads as reported") - } - } - gateEvery, gateBound = 0, 0 // past the bound at once: before the fix, "has not reported" fails it here - for i := 0; i < 4; i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - if p.State == inventory.PlanFailed || strings.Contains(p.Note, "has not reported") { - t.Fatalf("the release failed on the newer send: %s %s", p.State, p.Note) - } - if g := p.Release.Gate; g != nil && (g.Sent == nil || g.Sent["anchor"].Digest == "") { - t.Fatalf("the gate does not keep what it sent: %+v", g) - } - if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("app's pass on anchor was not kept: %+v %v %v", v, found, err) - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { - t.Fatalf("app was put back to %s", current["app"].Commit) - } -} // A plan's own first send waits while a release judges the same module on that machine with another build. -func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - advancePlans(ctx, b.open) // the release judges app c2 on anchor - _, _, _, err := gatedSend(ctx, b.open, "anchor", []inventory.CarriedMove{{Module: "app", Node: "anchor", From: "c2", To: "c3", Build: "build-app-c3"}}) - if !errors.Is(err, errWalkedElsewhere) || !strings.Contains(err.Error(), "release-") { - t.Fatalf("a newer build of a judged module was sent under the release's gate: %v", err) - } - if len(b.sent) != 1 { - t.Fatalf("sent %v", b.sent) - } -} // A merge plan's judging is superseded the same way: another send moved its module on the first machine. -func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - g := aGateMesh(t) - ctx := t.Context() - inv := g.open.inventory - advancePlans(ctx, g.open) // anchor is sent app c2 first - if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-c3", map[string]string{"app": "c3"}); err != nil { - t.Fatal(err) - } - gateEvery = 0 - advancePlans(ctx, g.open) - p := g.plan(t) - if p.State != inventory.PlanSuperseded || !strings.Contains(p.Note, "superseded") || !strings.Contains(p.Note, "c3") { - t.Fatalf("the plan is %s: %s", p.State, p.Note) - } - // Nothing put back: the registered build stands, the build is not marked, and the plan's gate made no - // rollback (a release may walk what the other send left waiting on anchor; that is not a put-back). - if r := p.Modules["app"].Gate.Rollback; r != "" { - t.Fatalf("a superseded judging made a rollback: %q", r) - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { - t.Fatalf("app was put back to %s", current["app"].Commit) - } - if failed, _ := inv.GateFailed(ctx, "build-2"); failed { - t.Fatal("a superseded build was marked failed") - } -} - -func testASendThatMovesTheModuleSupersedesTheJudging(t *testing.T) { - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - advancePlans(ctx, b.open) - // Another send moves app on anchor to a build this gate does not judge. - carried, _, _ := inv.SentBuilds(ctx, "anchor") - carried["app"] = "c3" - if err := inv.RecordSent(ctx, nodeID(t, b.open, "anchor"), "d-anchor-c3", carried); err != nil { - t.Fatal(err) - } - gateEvery = 0 - advancePlans(ctx, b.open) - p := b.release(t) - if p.State != inventory.PlanSuperseded || !strings.Contains(p.Note, "superseded") || !strings.Contains(p.Note, "c3") { - t.Fatalf("the release is %s: %s", p.State, p.Note) - } - if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: a superseded judging puts nothing back", b.sent) - } - if _, found, _ := inv.GateOf(ctx, "build-app-c2"); found { - t.Fatal("a superseded judging kept a verdict") - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { - t.Fatalf("app was put back to %s", current["app"].Commit) - } -} // A report is on the gate's own send: the declaration itself, or one sequenced after it; a gate kept // without its send reads the report against the send made last, as before. Pure. @@ -254,101 +138,6 @@ func TestAWalkWaitsForAJudgingOfTheSameModuleOnThatMachine(t *testing.T) { // The controller is never put back to a build that reaches less of the store's schema than the store // has, or to one that never said what it reaches: the current build is kept, and the condition says so. -func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - keeper, _ := withConditionsInMemory(t) - told := &conditions.Told{} - was := doctorFrom - doctorFrom = &doctor{open: open, keeper: keeper, teller: told} - t.Cleanup(func() { doctorFrom = was }) - wasSend := sendRollout - var sent [][]string - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - sent = append(sent, names) - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - - build := func(id, commit, digest string, asked time.Time) inventory.Build { - manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: commit}) - b := inventory.Build{ID: id, Module: "mesh-controller", Commit: commit, Repository: "novox/mesh-controller", Path: ".", - Manifest: manifest, Asked: asked, At: asked, Made: []inventory.Artifact{{Name: "controller", Kind: catalogue.ArtifactBundle, - Reference: "mesh-artifact://mesh-controller/controller/blobs/sha256:" + digest}}} - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: commit}, - inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Path: ".", BuiltFrom: commit, Head: commit, Asked: asked}); err != nil { - t.Fatal(err) - } - return b - } - previous := build("build-old", "ef26d4cb", strings.Repeat("1", 64), time.Now().Add(-2*time.Hour)) - failed := build("build-new", "e6b00e2e", strings.Repeat("2", 64), time.Now().Add(-time.Minute)) - if versionOfBuild(previous) != strings.Repeat("1", 12) { - t.Fatalf("the build's version is %q", versionOfBuild(previous)) - } - applied, err := inv.SchemaApplied(ctx) - if err != nil || applied < 87 { - t.Fatalf("the store's schema reaches %d (%v)", applied, err) - } - // The build before never recorded what it reads: not proved, refused. - if why, ok := controllerSchemaAllows(ctx, inv, previous); ok || !strings.Contains(why, "never recorded") { - t.Fatalf("an unknown reach: %v %q", ok, why) - } - // It reads less than the store has: refused, naming both. - if err := inv.RecordSchemaReach(ctx, versionOfBuild(previous), applied-1); err != nil { - t.Fatal(err) - } - if why, ok := controllerSchemaAllows(ctx, inv, previous); ok || !strings.Contains(why, "is at") { - t.Fatalf("a reach behind the store: %v %q", ok, why) - } - // Through the gate: the failed build is marked, nothing is put back, nothing is sent, the condition is urgent. - at := time.Now().Add(-5 * time.Minute) - state := &inventory.PlanModule{Build: failed.ID, Commit: failed.Commit, Previous: previous.Commit, First: []string{"anchor"}, FirstAt: &at} - p := inventory.Plan{ID: "plan-352", Repository: "novox/mesh-controller", Branch: "main", Commit: failed.Commit, Created: at, - State: inventory.PlanRolling, Tiers: [][]string{{"mesh-controller"}}, Modules: map[string]*inventory.PlanModule{"mesh-controller": state}} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - gateFailed(ctx, open, &p, "mesh-controller", state, []string{"anchor"}, "not healthy within 10m0s of its apply") - if state.Gate.Rollback != inventory.NotRolledBack || !strings.Contains(p.Note, "NOT put back") || !strings.Contains(p.Note, "the current build is kept") { - t.Fatalf("rollback %q: %s", state.Gate.Rollback, p.Note) - } - if len(sent) != 0 { - t.Fatalf("sent %v: nothing is put back", sent) - } - if current, _ := inv.CurrentBuilds(ctx); current["mesh-controller"].Commit != failed.Commit { - t.Fatalf("the module was put back to %s", current["mesh-controller"].Commit) - } - if marked, _ := inv.GateFailed(ctx, failed.ID); !marked { - t.Fatal("the failed build is not marked failed at its gate") - } - open2, _ := keeper.Open(ctx) - var found bool - for _, c := range open2 { - if c.Kind == kindRollbackFailed && c.Severity == conditions.Urgent && strings.Contains(c.Summary, "current build is kept") { - found = true - } - } - if !found { - t.Fatalf("no urgent rollback-failed condition saying the current build is kept: %+v", open2) - } - // Reaching the store: allowed. - if err := inv.RecordSchemaReach(ctx, versionOfBuild(previous), applied); err != nil { - t.Fatal(err) - } - if why, ok := controllerSchemaAllows(ctx, inv, previous); !ok { - t.Fatalf("a build that reads the whole schema was refused: %q", why) - } - // A build with no bundle to know it by: refused. - if why, ok := controllerSchemaAllows(ctx, inv, inventory.Build{Commit: "x"}); ok || !strings.Contains(why, "names no bundle") { - t.Fatalf("a build without a bundle: %v %q", ok, why) - } -} // The lease's holder names the build the declaration told it it is, and the version stamp only without one. func TestTheHolderNamesTheBuildTheDeclarationToldIt(t *testing.T) { diff --git a/cmd/mesh-controller/issue459_integration_test.go b/cmd/mesh-controller/issue459_integration_test.go new file mode 100644 index 00000000..14c9acc0 --- /dev/null +++ b/cmd/mesh-controller/issue459_integration_test.go @@ -0,0 +1,293 @@ +//go:build integration + +package main + +import ( + "errors" + "reflect" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestAStopWithoutAMergeWalksTheOthersAgain(t *testing.T) { + open := windowed(t) + groupsWaiting(t, "pr/cat1aaaaaaaa") + endsItsLine(t) + asksRecorded(t) + ctx := t.Context() + cat := catalogueMerge("cat1aaaaaaaa", "app", t0) + one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) + hear(t, open, cat, t0.Add(time.Second)) + hear(t, open, one, t0.Add(6*time.Second)) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + if len(ws) != 1 || !ws[0].Open() { + t.Fatalf("no open walk to stop: %+v", ws) + } + stopped := ws[0] + p, said, err := stopWalkWithout(ctx, open.inventory, stopped.ID, "mesh-delivery for jochen", "its group stopped", + []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute)) + if err != nil { + t.Fatal(err) + } + if p.State != inventory.PlanFailed || p.Delivery.Stopped == "" || p.Delivery.WalkedAgainBy == "" || + !reflect.DeepEqual(p.Delivery.LeftOut, []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: cat.Commit}}) { + t.Fatalf("the stopped walk reads %s %+v", p.State, p.Delivery) + } + if said.WalkedAgainBy != p.Delivery.WalkedAgainBy || !reflect.DeepEqual(said.WalkedAgain, []string{"novox/one@" + one.Commit}) { + t.Fatalf("the stop answered %+v", said) + } + again, err := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy) + if err != nil || len(again) != 1 || again[0].Commit != one.Commit { + t.Fatalf("the fresh batch answers %+v (%v); want the other repository's merge alone", again, err) + } + if left, _ := open.inventory.MergesOf(ctx, stopped.ID); len(left) != 1 || left[0].Commit != cat.Commit { + t.Fatalf("the stopped walk still answers %+v; want the left-out merge alone", left) + } + cutAt(t, open, t0.Add(4*time.Minute)) + fresh, err := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy) + if err != nil || fresh.Batch() || !fresh.Open() { + t.Fatalf("the fresh batch was not cut into a walk: %s %v", fresh.State, err) + } + if _, in := fresh.Modules["one"]; !in { + t.Fatalf("the fresh walk does not build one: %v", fresh.Modules) + } + if _, in := fresh.Modules["app"]; in { + t.Fatalf("the fresh walk builds app, whose merge was left out: %v", fresh.Modules) + } + if len(fresh.Delivery.Merges) != 1 || fresh.Delivery.Merges[0].Commit != one.Commit { + t.Fatalf("the fresh walk answers %+v", fresh.Delivery.Merges) + } + // A stopped walk starts no search of its own (ADR 0276 decision 3 is for a failed one). + cutAt(t, open, t0.Add(5*time.Minute)) + if alone, _ := open.inventory.AloneMerges(ctx); len(alone) != 0 { + t.Fatalf("a merge of the stopped walk waits to be walked alone: %+v", alone) + } +} + +func TestAStopWithoutAMergeEndsTheMergesThatContainIt(t *testing.T) { + open := windowed(t) + groupsWaiting(t, "pr/cat1aaaaaaaa") + endsItsLine(t) + asksRecorded(t) + ctx := t.Context() + first := catalogueMerge("cat1aaaaaaaa", "app", t0) + later := catalogueMerge("cat2cccccccc", "notes", t0.Add(10*time.Second)) + one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), first.Head) + hear(t, open, first, t0.Add(time.Second)) + hear(t, open, one, t0.Add(6*time.Second)) + hear(t, open, later, t0.Add(11*time.Second)) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + walk := ws[0] + + if _, _, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "x", + []string{"novox/mesh-catalog@ffffffffffff"}, t0.Add(3*time.Minute)); err == nil { + t.Fatal("a stop leaving out a merge the walk does not answer was taken") + } + if p, _ := open.inventory.PlanByID(ctx, walk.ID); !p.Open() { + t.Fatalf("a refused stop ended the walk: %s", p.State) + } + + p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", + []string{"novox/mesh-catalog@" + first.Commit}, t0.Add(3*time.Minute)) + if err != nil { + t.Fatal(err) + } + want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: first.Commit}, + {Repository: "novox/mesh-catalog", Commit: later.Commit, Contains: first.Commit}} + if !reflect.DeepEqual(p.Delivery.LeftOut, want) || !reflect.DeepEqual(said.LeftOut, want) { + t.Fatalf("left out %+v, said %+v; want %+v", p.Delivery.LeftOut, said.LeftOut, want) + } + if !strings.Contains(p.Note, later.Commit[:8]) || !strings.Contains(p.Note, p.Delivery.WalkedAgainBy) { + t.Fatalf("the stopped walk's note does not name the merge ended with it and the fresh batch: %q", p.Note) + } + again, _ := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy) + if len(again) != 1 || again[0].Commit != one.Commit { + t.Fatalf("walked again: %+v; want the other repository's merge alone", again) + } + + // Nothing left once the other repository's merge is left out too: the fresh walk is stopped, and nothing more. + cutAt(t, open, t0.Add(4*time.Minute)) + fresh, _ := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy) + q, said, err := stopWalkWithout(ctx, open.inventory, fresh.ID, "mesh-delivery for jochen", "that one too", + []string{"novox/one@" + one.Commit}, t0.Add(5*time.Minute)) + if err != nil || q.State != inventory.PlanFailed || q.Delivery.WalkedAgainBy != "" || said.WalkedAgainBy != "" { + t.Fatalf("a stop leaving nothing to walk: %s %+v %+v %v", q.State, q.Delivery, said, err) + } + if _, bs := walks(t, open); len(bs) != 0 { + t.Fatalf("a batch of nothing was made: %+v", bs) + } +} + +func TestAStopWalksTheOthersAgainInTheOpenBatch(t *testing.T) { + open := windowed(t) + groupsWaiting(t, "pr/cat1aaaaaaaa") + endsItsLine(t) + asksRecorded(t) + ctx := t.Context() + cat := catalogueMerge("cat1aaaaaaaa", "app", t0) + one1 := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) + hear(t, open, cat, t0.Add(time.Second)) + hear(t, open, one1, t0.Add(6*time.Second)) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + walk := ws[0] + one2 := repoMerge("one", "one2cccccccc", t0.Add(3*time.Minute)) + hear(t, open, one2, t0.Add(3*time.Minute)) + _, bs := walks(t, open) + if len(bs) != 1 { + t.Fatalf("the merge heard while the walk ran is in %d batches", len(bs)) + } + waiting := bs[0] + p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", + []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second)) + if err != nil { + t.Fatal(err) + } + if p.Delivery.WalkedAgainBy != waiting.ID || said.WalkedAgainBy != waiting.ID { + t.Fatalf("walked again by %q (said %q); want the open batch %s", p.Delivery.WalkedAgainBy, said.WalkedAgainBy, waiting.ID) + } + if _, bs := walks(t, open); len(bs) != 1 { + t.Fatalf("%d batches; want the open one alone", len(bs)) + } + b, _ := open.inventory.PlanByID(ctx, waiting.ID) + if b.Delivery.Batch == nil || !b.Delivery.Batch.Closed || !slices.Contains(b.Delivery.Batch.Withheld, "app") { + t.Fatalf("the open batch reads %+v; want its window closed and app withheld", b.Delivery.Batch) + } + // Its window closed by the stop, not by the clock: cut at once. + cutAt(t, open, t0.Add(3*time.Minute+11*time.Second)) + fresh, _ := open.inventory.PlanByID(ctx, waiting.ID) + if fresh.Batch() || fresh.CommitOf("novox/one") != one2.Commit { + t.Fatalf("the batch is %s at %q; want it cut, walking the newer commit", fresh.State, fresh.CommitOf("novox/one")) + } + var answered []string + for _, m := range fresh.Delivery.Merges { + answered = append(answered, m.Commit+">"+m.Carried) + } + if !slices.Contains(answered, one1.Commit+">"+one2.Commit) || !slices.Contains(answered, one2.Commit+">") { + t.Fatalf("the walk answers %v; want the earlier merge carried by the later", answered) + } +} + +func TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt(t *testing.T) { + open := windowed(t) + groupsWaiting(t, "pr/cat1aaaaaaaa") + endsItsLine(t) + asksRecorded(t) + ctx := t.Context() + cat := catalogueMerge("cat1aaaaaaaa", "app", t0) + one1 := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) + hear(t, open, cat, t0.Add(time.Second)) + hear(t, open, one1, t0.Add(6*time.Second)) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + walk := ws[0] + sent := t0.Add(150 * time.Second) + for name, m := range walk.Modules { + if name == "app" || name == "one" { + m.First, m.FirstAt = []string{"anchor"}, &sent + } + } + if err := open.inventory.SavePlan(ctx, &walk); err != nil { + t.Fatal(err) + } + cat2 := catalogueMerge("cat2dddddddd", "notes", t0.Add(3*time.Minute)) + hear(t, open, cat2, t0.Add(3*time.Minute)) + _, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", + []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second)) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(said.Kept, []inventory.PlanKept{{Module: "app", Machines: []string{"anchor"}}}) { + t.Fatalf("kept %+v; want app on anchor alone (one is walked again)", said.Kept) + } + p, _ := open.inventory.PlanByID(ctx, walk.ID) + if !reflect.DeepEqual(p.Delivery.Kept, said.Kept) || !strings.Contains(p.Note, "app on anchor") { + t.Fatalf("the stopped walk's record does not name what stays: %+v %q", p.Delivery.Kept, p.Note) + } + if len(said.StillCarriedBy) != 1 || !strings.Contains(said.StillCarriedBy[0], cat2.Commit) { + t.Fatalf("still carried by %v; want the later catalogue merge waiting in the batch", said.StillCarriedBy) + } + if _, err := retryPlan(ctx, open, walk.ID); err == nil || !strings.Contains(err.Error(), "was stopped without") { + t.Fatalf("a walk stopped without some merges was retried, or refused for another reason: %v", err) + } +} + +func TestAStopWithoutAMergeLeavesItOutOfABatch(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + endsItsLine(t) + asksRecorded(t) + ctx := t.Context() + cat := catalogueMerge("cat1aaaaaaaa", "app", t0) + one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) + hear(t, open, cat, t0.Add(time.Second)) + hear(t, open, one, t0.Add(6*time.Second)) + _, bs := walks(t, open) + b := bs[0] + r, said, err := stopWalkWithout(ctx, open.inventory, b.ID, "mesh-delivery for jochen", "its group stopped", + []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(10*time.Second)) + if err != nil { + t.Fatal(err) + } + if said.WalkedAgainBy != b.ID || said.Stopped == b.ID { + t.Fatalf("said %+v; want a stopped record of its own, and the batch walking the others", said) + } + if left, _ := open.inventory.MergesOf(ctx, said.Stopped); len(left) != 1 || left[0].Commit != cat.Commit { + t.Fatalf("the stopped record holds %+v", left) + } + if stopped, _ := open.inventory.PlanByID(ctx, said.Stopped); stopped.State != inventory.PlanFailed || + stopped.Delivery.Stopped == "" || stopped.Delivery.WalkedAgainBy != b.ID { + t.Fatalf("the stopped record reads %s %+v", stopped.State, stopped.Delivery) + } + _ = r + cutAt(t, open, t0.Add(3*time.Minute)) + w, _ := open.inventory.PlanByID(ctx, b.ID) + if _, in := w.Modules["app"]; in || w.Batch() { + t.Fatalf("the batch's walk is %s and builds %v; want it cut without app", w.State, w.Modules) + } + // A batch whose every merge is left out is stopped itself. + hear(t, open, catalogueMerge("cat3eeeeeeee", "notes", t0.Add(4*time.Minute)), t0.Add(4*time.Minute)) + _, bs = walks(t, open) + last := bs[0] + if _, said, err := stopWalkWithout(ctx, open.inventory, last.ID, "mesh-delivery for jochen", "x", + []string{"novox/mesh-catalog@cat3eeeeeeee"}, t0.Add(4*time.Minute+time.Second)); err != nil || said.Stopped != last.ID { + t.Fatalf("a batch left with nothing: %+v %v", said, err) + } + if p, _ := open.inventory.PlanByID(ctx, last.ID); p.State != inventory.PlanFailed || p.Batch() { + t.Fatalf("the emptied batch is %s", p.State) + } +} + +func TestSavingPlansAndMovingMergesIsOneAct(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + endsItsLine(t) + ctx := t.Context() + hear(t, open, catalogueMerge("cat1aaaaaaaa", "app", t0), t0.Add(time.Second)) + _, bs := walks(t, open) + stale := bs[0] + moved := stale + moved.Note = "written by somebody else" + if err := open.inventory.SavePlan(ctx, &moved); err != nil { + t.Fatal(err) + } + fresh := inventory.Plan{ID: "plan-fresh", Repository: "novox/mesh-catalog", Branch: "main", Commit: "x", Created: t0, + State: inventory.PlanAssembling, Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{}} + err := open.inventory.SavePlansMoving(ctx, []*inventory.Plan{&fresh, &stale}, + []inventory.MergeMove{{Repository: "novox/mesh-catalog", Commit: "cat1aaaaaaaa", Plan: "plan-fresh"}}) + if !errors.Is(err, inventory.ErrPlanMoved) { + t.Fatalf("a stale plan in the act: %v", err) + } + if _, err := open.inventory.PlanByID(ctx, "plan-fresh"); err == nil { + t.Fatal("the new plan was written though the act was refused") + } + if m, _, _ := open.inventory.MergeOf(ctx, "novox/mesh-catalog", "cat1aaaaaaaa"); m.Plan != stale.ID { + t.Fatalf("the merge moved to %q though the act was refused", m.Plan) + } +} diff --git a/cmd/mesh-controller/issue459_test.go b/cmd/mesh-controller/issue459_test.go index 7fec24d4..9c79df8c 100644 --- a/cmd/mesh-controller/issue459_test.go +++ b/cmd/mesh-controller/issue459_test.go @@ -2,10 +2,8 @@ package main import ( "encoding/json" - "errors" "fmt" "reflect" - "slices" "strings" "testing" "time" @@ -121,297 +119,24 @@ func TestDeliveryStopTakesTheMergesToLeaveOut(t *testing.T) { // A walk of the catalogue's merge and another repository's is stopped without the catalogue's: it ends stopped, // names the batch that walks the other merge again and the merge it left out, and that batch is cut into a walk // that builds the other repository's module alone. -func TestAStopWithoutAMergeWalksTheOthersAgain(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/cat1aaaaaaaa") - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - cat := catalogueMerge("cat1aaaaaaaa", "app", t0) - one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) - hear(t, open, cat, t0.Add(time.Second)) - hear(t, open, one, t0.Add(6*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - if len(ws) != 1 || !ws[0].Open() { - t.Fatalf("no open walk to stop: %+v", ws) - } - stopped := ws[0] - p, said, err := stopWalkWithout(ctx, open.inventory, stopped.ID, "mesh-delivery for jochen", "its group stopped", - []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute)) - if err != nil { - t.Fatal(err) - } - if p.State != inventory.PlanFailed || p.Delivery.Stopped == "" || p.Delivery.WalkedAgainBy == "" || - !reflect.DeepEqual(p.Delivery.LeftOut, []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: cat.Commit}}) { - t.Fatalf("the stopped walk reads %s %+v", p.State, p.Delivery) - } - if said.WalkedAgainBy != p.Delivery.WalkedAgainBy || !reflect.DeepEqual(said.WalkedAgain, []string{"novox/one@" + one.Commit}) { - t.Fatalf("the stop answered %+v", said) - } - again, err := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy) - if err != nil || len(again) != 1 || again[0].Commit != one.Commit { - t.Fatalf("the fresh batch answers %+v (%v); want the other repository's merge alone", again, err) - } - if left, _ := open.inventory.MergesOf(ctx, stopped.ID); len(left) != 1 || left[0].Commit != cat.Commit { - t.Fatalf("the stopped walk still answers %+v; want the left-out merge alone", left) - } - cutAt(t, open, t0.Add(4*time.Minute)) - fresh, err := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy) - if err != nil || fresh.Batch() || !fresh.Open() { - t.Fatalf("the fresh batch was not cut into a walk: %s %v", fresh.State, err) - } - if _, in := fresh.Modules["one"]; !in { - t.Fatalf("the fresh walk does not build one: %v", fresh.Modules) - } - if _, in := fresh.Modules["app"]; in { - t.Fatalf("the fresh walk builds app, whose merge was left out: %v", fresh.Modules) - } - if len(fresh.Delivery.Merges) != 1 || fresh.Delivery.Merges[0].Commit != one.Commit { - t.Fatalf("the fresh walk answers %+v", fresh.Delivery.Merges) - } - // A stopped walk starts no search of its own (ADR 0276 decision 3 is for a failed one). - cutAt(t, open, t0.Add(5*time.Minute)) - if alone, _ := open.inventory.AloneMerges(ctx); len(alone) != 0 { - t.Fatalf("a merge of the stopped walk waits to be walked alone: %+v", alone) - } -} // A later merge of the stopped merge's repository contains it: it is left out too, named with the merge it // contains, and only the other repository's merge is walked again. With nothing left to walk, the walk is only // ended. A merge the walk does not answer is refused, and nothing ends. -func TestAStopWithoutAMergeEndsTheMergesThatContainIt(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/cat1aaaaaaaa") - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - first := catalogueMerge("cat1aaaaaaaa", "app", t0) - later := catalogueMerge("cat2cccccccc", "notes", t0.Add(10*time.Second)) - one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), first.Head) - hear(t, open, first, t0.Add(time.Second)) - hear(t, open, one, t0.Add(6*time.Second)) - hear(t, open, later, t0.Add(11*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - walk := ws[0] - - if _, _, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "x", - []string{"novox/mesh-catalog@ffffffffffff"}, t0.Add(3*time.Minute)); err == nil { - t.Fatal("a stop leaving out a merge the walk does not answer was taken") - } - if p, _ := open.inventory.PlanByID(ctx, walk.ID); !p.Open() { - t.Fatalf("a refused stop ended the walk: %s", p.State) - } - - p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", - []string{"novox/mesh-catalog@" + first.Commit}, t0.Add(3*time.Minute)) - if err != nil { - t.Fatal(err) - } - want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: first.Commit}, - {Repository: "novox/mesh-catalog", Commit: later.Commit, Contains: first.Commit}} - if !reflect.DeepEqual(p.Delivery.LeftOut, want) || !reflect.DeepEqual(said.LeftOut, want) { - t.Fatalf("left out %+v, said %+v; want %+v", p.Delivery.LeftOut, said.LeftOut, want) - } - if !strings.Contains(p.Note, later.Commit[:8]) || !strings.Contains(p.Note, p.Delivery.WalkedAgainBy) { - t.Fatalf("the stopped walk's note does not name the merge ended with it and the fresh batch: %q", p.Note) - } - again, _ := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy) - if len(again) != 1 || again[0].Commit != one.Commit { - t.Fatalf("walked again: %+v; want the other repository's merge alone", again) - } - - // Nothing left once the other repository's merge is left out too: the fresh walk is stopped, and nothing more. - cutAt(t, open, t0.Add(4*time.Minute)) - fresh, _ := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy) - q, said, err := stopWalkWithout(ctx, open.inventory, fresh.ID, "mesh-delivery for jochen", "that one too", - []string{"novox/one@" + one.Commit}, t0.Add(5*time.Minute)) - if err != nil || q.State != inventory.PlanFailed || q.Delivery.WalkedAgainBy != "" || said.WalkedAgainBy != "" { - t.Fatalf("a stop leaving nothing to walk: %s %+v %+v %v", q.State, q.Delivery, said, err) - } - if _, bs := walks(t, open); len(bs) != 0 { - t.Fatalf("a batch of nothing was made: %+v", bs) - } -} // A merge heard while the walk ran waits in the open batch, at a later commit of the same repository. The merge // walked again joins that batch, never a batch of its own that would walk the older commit after the newer one // (review of #232): its window closes, and its walk is at the newer commit, answering both. -func TestAStopWalksTheOthersAgainInTheOpenBatch(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/cat1aaaaaaaa") - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - cat := catalogueMerge("cat1aaaaaaaa", "app", t0) - one1 := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) - hear(t, open, cat, t0.Add(time.Second)) - hear(t, open, one1, t0.Add(6*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - walk := ws[0] - one2 := repoMerge("one", "one2cccccccc", t0.Add(3*time.Minute)) - hear(t, open, one2, t0.Add(3*time.Minute)) - _, bs := walks(t, open) - if len(bs) != 1 { - t.Fatalf("the merge heard while the walk ran is in %d batches", len(bs)) - } - waiting := bs[0] - p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", - []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second)) - if err != nil { - t.Fatal(err) - } - if p.Delivery.WalkedAgainBy != waiting.ID || said.WalkedAgainBy != waiting.ID { - t.Fatalf("walked again by %q (said %q); want the open batch %s", p.Delivery.WalkedAgainBy, said.WalkedAgainBy, waiting.ID) - } - if _, bs := walks(t, open); len(bs) != 1 { - t.Fatalf("%d batches; want the open one alone", len(bs)) - } - b, _ := open.inventory.PlanByID(ctx, waiting.ID) - if b.Delivery.Batch == nil || !b.Delivery.Batch.Closed || !slices.Contains(b.Delivery.Batch.Withheld, "app") { - t.Fatalf("the open batch reads %+v; want its window closed and app withheld", b.Delivery.Batch) - } - // Its window closed by the stop, not by the clock: cut at once. - cutAt(t, open, t0.Add(3*time.Minute+11*time.Second)) - fresh, _ := open.inventory.PlanByID(ctx, waiting.ID) - if fresh.Batch() || fresh.CommitOf("novox/one") != one2.Commit { - t.Fatalf("the batch is %s at %q; want it cut, walking the newer commit", fresh.State, fresh.CommitOf("novox/one")) - } - var answered []string - for _, m := range fresh.Delivery.Merges { - answered = append(answered, m.Commit+">"+m.Carried) - } - if !slices.Contains(answered, one1.Commit+">"+one2.Commit) || !slices.Contains(answered, one2.Commit+">") { - t.Fatalf("the walk answers %v; want the earlier merge carried by the later", answered) - } -} // A stop names what the stopped walk already sent and nothing walks again, on which machines; and a later merge of // the stopped merge's repository that waits in a batch, which will deliver the stopped change. A stopped walk // that walked its merges again is refused a retry, saying why. -func TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/cat1aaaaaaaa") - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - cat := catalogueMerge("cat1aaaaaaaa", "app", t0) - one1 := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) - hear(t, open, cat, t0.Add(time.Second)) - hear(t, open, one1, t0.Add(6*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - walk := ws[0] - sent := t0.Add(150 * time.Second) - for name, m := range walk.Modules { - if name == "app" || name == "one" { - m.First, m.FirstAt = []string{"anchor"}, &sent - } - } - if err := open.inventory.SavePlan(ctx, &walk); err != nil { - t.Fatal(err) - } - cat2 := catalogueMerge("cat2dddddddd", "notes", t0.Add(3*time.Minute)) - hear(t, open, cat2, t0.Add(3*time.Minute)) - _, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", - []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second)) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(said.Kept, []inventory.PlanKept{{Module: "app", Machines: []string{"anchor"}}}) { - t.Fatalf("kept %+v; want app on anchor alone (one is walked again)", said.Kept) - } - p, _ := open.inventory.PlanByID(ctx, walk.ID) - if !reflect.DeepEqual(p.Delivery.Kept, said.Kept) || !strings.Contains(p.Note, "app on anchor") { - t.Fatalf("the stopped walk's record does not name what stays: %+v %q", p.Delivery.Kept, p.Note) - } - if len(said.StillCarriedBy) != 1 || !strings.Contains(said.StillCarriedBy[0], cat2.Commit) { - t.Fatalf("still carried by %v; want the later catalogue merge waiting in the batch", said.StillCarriedBy) - } - if _, err := retryPlan(ctx, open, walk.ID); err == nil || !strings.Contains(err.Error(), "was stopped without") { - t.Fatalf("a walk stopped without some merges was retried, or refused for another reason: %v", err) - } -} // A batch not yet cut is stopped without a merge: the merge goes to a stopped record of its own, the batch goes on // with the others and withholds the stopped merge's modules; with every merge left out, the batch is stopped. -func TestAStopWithoutAMergeLeavesItOutOfABatch(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - cat := catalogueMerge("cat1aaaaaaaa", "app", t0) - one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) - hear(t, open, cat, t0.Add(time.Second)) - hear(t, open, one, t0.Add(6*time.Second)) - _, bs := walks(t, open) - b := bs[0] - r, said, err := stopWalkWithout(ctx, open.inventory, b.ID, "mesh-delivery for jochen", "its group stopped", - []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(10*time.Second)) - if err != nil { - t.Fatal(err) - } - if said.WalkedAgainBy != b.ID || said.Stopped == b.ID { - t.Fatalf("said %+v; want a stopped record of its own, and the batch walking the others", said) - } - if left, _ := open.inventory.MergesOf(ctx, said.Stopped); len(left) != 1 || left[0].Commit != cat.Commit { - t.Fatalf("the stopped record holds %+v", left) - } - if stopped, _ := open.inventory.PlanByID(ctx, said.Stopped); stopped.State != inventory.PlanFailed || - stopped.Delivery.Stopped == "" || stopped.Delivery.WalkedAgainBy != b.ID { - t.Fatalf("the stopped record reads %s %+v", stopped.State, stopped.Delivery) - } - _ = r - cutAt(t, open, t0.Add(3*time.Minute)) - w, _ := open.inventory.PlanByID(ctx, b.ID) - if _, in := w.Modules["app"]; in || w.Batch() { - t.Fatalf("the batch's walk is %s and builds %v; want it cut without app", w.State, w.Modules) - } - // A batch whose every merge is left out is stopped itself. - hear(t, open, catalogueMerge("cat3eeeeeeee", "notes", t0.Add(4*time.Minute)), t0.Add(4*time.Minute)) - _, bs = walks(t, open) - last := bs[0] - if _, said, err := stopWalkWithout(ctx, open.inventory, last.ID, "mesh-delivery for jochen", "x", - []string{"novox/mesh-catalog@cat3eeeeeeee"}, t0.Add(4*time.Minute+time.Second)); err != nil || said.Stopped != last.ID { - t.Fatalf("a batch left with nothing: %+v %v", said, err) - } - if p, _ := open.inventory.PlanByID(ctx, last.ID); p.State != inventory.PlanFailed || p.Batch() { - t.Fatalf("the emptied batch is %s", p.State) - } -} // The stopped walk, the batch and the merges moved are one act (ADR 0299 decision 7): a plan written by somebody // else since it was read refuses all of it, and no merge moves, no plan is written. -func TestSavingPlansAndMovingMergesIsOneAct(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - endsItsLine(t) - ctx := t.Context() - hear(t, open, catalogueMerge("cat1aaaaaaaa", "app", t0), t0.Add(time.Second)) - _, bs := walks(t, open) - stale := bs[0] - moved := stale - moved.Note = "written by somebody else" - if err := open.inventory.SavePlan(ctx, &moved); err != nil { - t.Fatal(err) - } - fresh := inventory.Plan{ID: "plan-fresh", Repository: "novox/mesh-catalog", Branch: "main", Commit: "x", Created: t0, - State: inventory.PlanAssembling, Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{}} - err := open.inventory.SavePlansMoving(ctx, []*inventory.Plan{&fresh, &stale}, - []inventory.MergeMove{{Repository: "novox/mesh-catalog", Commit: "cat1aaaaaaaa", Plan: "plan-fresh"}}) - if !errors.Is(err, inventory.ErrPlanMoved) { - t.Fatalf("a stale plan in the act: %v", err) - } - if _, err := open.inventory.PlanByID(ctx, "plan-fresh"); err == nil { - t.Fatal("the new plan was written though the act was refused") - } - if m, _, _ := open.inventory.MergeOf(ctx, "novox/mesh-catalog", "cat1aaaaaaaa"); m.Plan != stale.ID { - t.Fatalf("the merge moved to %q though the act was refused", m.Plan) - } -} // What a stopped walk leaves on the machines names every machine its sends reached (review of #232): a module sent // with no first machine reached every machine running it; one sent first reached those, and the rest it was sent diff --git a/cmd/mesh-controller/issue611_integration_test.go b/cmd/mesh-controller/issue611_integration_test.go new file mode 100644 index 00000000..31aa2acb --- /dev/null +++ b/cmd/mesh-controller/issue611_integration_test.go @@ -0,0 +1,54 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestAWalkThatMovedNothingSaysWhichModulesMatchedWhatRuns(t *testing.T) { + open := windowed(t) + endsItsLine(t) + asksRecorded(t) + ctx := t.Context() + inv := open.inventory + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-one", Repository: "novox/one", Ref: "main", + Module: "one", Commit: "c0", On: "builder", Asked: t0.Add(-time.Hour), + Sources: []inventory.BuildSource{{Paths: []string{"main.go", "go.mod"}}}}); err != nil { + t.Fatal(err) + } + readme := repoMerge("one", "one1aaaaaaaa", t0) + readme.Paths = []string{"README.md"} + hear(t, open, readme, t0.Add(time.Second)) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + if len(ws) != 1 { + t.Fatalf("walks: %+v", ws) + } + w := ws[0] + if w.State != inventory.PlanDone || len(w.Tiers) != 0 || len(w.Modules) != 0 { + t.Fatalf("the walk reads %s, tiers %v, modules %v; want done with nothing to walk", w.State, w.Tiers, w.Modules) + } + if w.Delivery == nil || !strings.Contains(w.Delivery.Unchanged["one"], "holds none of the 1 changed file(s)") { + t.Fatalf("the walk does not say one matched what runs: %+v", w.Delivery) + } + if !strings.Contains(w.Note, "unchanged") || !strings.Contains(w.Note, "one") { + t.Errorf("the walk's note does not name it: %q", w.Note) + } + + // A merge that changes the build source is walked, and says nothing unchanged. + code := repoMerge("one", "one2bbbbbbbb", t0.Add(3*time.Minute)) + hear(t, open, code, t0.Add(3*time.Minute+time.Second)) + cutAt(t, open, t0.Add(6*time.Minute)) + ws, _ = walks(t, open) + if len(ws) != 2 { + t.Fatalf("walks: %+v", ws) + } + if _, built := ws[0].Modules["one"]; !built || ws[0].Delivery == nil || len(ws[0].Delivery.Unchanged) != 0 { + t.Fatalf("the walk of a change to main.go reads modules %v, delivery %+v", ws[0].Modules, ws[0].Delivery) + } +} diff --git a/cmd/mesh-controller/issue611_test.go b/cmd/mesh-controller/issue611_test.go index 2b2a8e7b..71235868 100644 --- a/cmd/mesh-controller/issue611_test.go +++ b/cmd/mesh-controller/issue611_test.go @@ -5,7 +5,6 @@ import ( "sort" "strings" "testing" - "time" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" @@ -95,48 +94,6 @@ func TestAWalkSaysUnchangedOnlyWhatItDoesNotBuild(t *testing.T) { // Against the store, the path a real merge takes: a merge whose module recorded a build source that holds none // of its files is cut into a walk done with nothing to walk, whose record names the module unchanged and why. -func TestAWalkThatMovedNothingSaysWhichModulesMatchedWhatRuns(t *testing.T) { - open := windowed(t) - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - inv := open.inventory - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-one", Repository: "novox/one", Ref: "main", - Module: "one", Commit: "c0", On: "builder", Asked: t0.Add(-time.Hour), - Sources: []inventory.BuildSource{{Paths: []string{"main.go", "go.mod"}}}}); err != nil { - t.Fatal(err) - } - readme := repoMerge("one", "one1aaaaaaaa", t0) - readme.Paths = []string{"README.md"} - hear(t, open, readme, t0.Add(time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - if len(ws) != 1 { - t.Fatalf("walks: %+v", ws) - } - w := ws[0] - if w.State != inventory.PlanDone || len(w.Tiers) != 0 || len(w.Modules) != 0 { - t.Fatalf("the walk reads %s, tiers %v, modules %v; want done with nothing to walk", w.State, w.Tiers, w.Modules) - } - if w.Delivery == nil || !strings.Contains(w.Delivery.Unchanged["one"], "holds none of the 1 changed file(s)") { - t.Fatalf("the walk does not say one matched what runs: %+v", w.Delivery) - } - if !strings.Contains(w.Note, "unchanged") || !strings.Contains(w.Note, "one") { - t.Errorf("the walk's note does not name it: %q", w.Note) - } - - // A merge that changes the build source is walked, and says nothing unchanged. - code := repoMerge("one", "one2bbbbbbbb", t0.Add(3*time.Minute)) - hear(t, open, code, t0.Add(3*time.Minute+time.Second)) - cutAt(t, open, t0.Add(6*time.Minute)) - ws, _ = walks(t, open) - if len(ws) != 2 { - t.Fatalf("walks: %+v", ws) - } - if _, built := ws[0].Modules["one"]; !built || ws[0].Delivery == nil || len(ws[0].Delivery.Unchanged) != 0 { - t.Fatalf("the walk of a change to main.go reads modules %v, delivery %+v", ws[0].Modules, ws[0].Delivery) - } -} func sortedOf(names []string) []string { out := append([]string{}, names...) diff --git a/cmd/mesh-controller/issue648_test.go b/cmd/mesh-controller/issue648_test.go index 0557e774..89fc9bb5 100644 --- a/cmd/mesh-controller/issue648_test.go +++ b/cmd/mesh-controller/issue648_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/issue_204_integration_test.go b/cmd/mesh-controller/issue_204_integration_test.go new file mode 100644 index 00000000..666519c8 --- /dev/null +++ b/cmd/mesh-controller/issue_204_integration_test.go @@ -0,0 +1,33 @@ +//go:build integration + +package main + +import ( + "context" + "testing" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) { + inv := inventory.ForTest(t) + ctx, cancel := context.WithCancel(t.Context()) + if _, err := inv.AddNode(ctx, "anchor"); err != nil { + t.Fatal(err) + } + cancel() // the sender is going away: its context is cancelled between the send and the record + body := []byte(`{"declaration":1,"resources":[]}`) + digest, err := recordSent(ctx, inv, "anchor", body, nil, 0, sentRecord{sender: "a test"}) + if err != nil { + // NodeByName on the cancelled context may itself refuse; the record must still be possible + // through the detached context, so look the node up again on a live one. + t.Fatalf("recording a send after cancellation failed: %v", err) + } + outstanding, err := inv.Outstanding(t.Context(), "anchor") + if err != nil { + t.Fatal(err) + } + if outstanding != digest || digest != digestOf(body) { + t.Fatalf("the send was not recorded: outstanding %q, sent %q", outstanding, digest) + } +} diff --git a/cmd/mesh-controller/issue_204_test.go b/cmd/mesh-controller/issue_204_test.go index 1779292a..950fd9cb 100644 --- a/cmd/mesh-controller/issue_204_test.go +++ b/cmd/mesh-controller/issue_204_test.go @@ -3,8 +3,6 @@ package main import ( "context" "testing" - - "github.com/novox/mesh-controller/internal/inventory" ) // A declaration composed earlier is numbered lower than one composed later, whatever order the two @@ -69,25 +67,3 @@ func TestTheNumberIsTakenBeforeComposingAndItsFailureIsARefusal(t *testing.T) { // What was sent is written down even when the sender's context is already cancelled (issue 204): a // controller replaced mid-send had told the machine and never recorded it, so status read "applied, // current" over a machine that had just been sent something else. -func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) { - inv := inventory.ForTest(t) - ctx, cancel := context.WithCancel(t.Context()) - if _, err := inv.AddNode(ctx, "anchor"); err != nil { - t.Fatal(err) - } - cancel() // the sender is going away: its context is cancelled between the send and the record - body := []byte(`{"declaration":1,"resources":[]}`) - digest, err := recordSent(ctx, inv, "anchor", body, nil, 0, sentRecord{sender: "a test"}) - if err != nil { - // NodeByName on the cancelled context may itself refuse; the record must still be possible - // through the detached context, so look the node up again on a live one. - t.Fatalf("recording a send after cancellation failed: %v", err) - } - outstanding, err := inv.Outstanding(t.Context(), "anchor") - if err != nil { - t.Fatal(err) - } - if outstanding != digest || digest != digestOf(body) { - t.Fatalf("the send was not recorded: outstanding %q, sent %q", outstanding, digest) - } -} diff --git a/cmd/mesh-controller/issued_integration_test.go b/cmd/mesh-controller/issued_integration_test.go new file mode 100644 index 00000000..023be0c0 --- /dev/null +++ b/cmd/mesh-controller/issued_integration_test.go @@ -0,0 +1,96 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/secrets" +) + +func TestAnIssuedValueIsTakenSealedFromTheProviderAndSent(t *testing.T) { + var r issuedRecorder + words, err := r.taker(t, issuedSealed(t, "the-servers-own-token")).take("plex-api") + if err != nil { + t.Fatal(err) + } + if len(r.kept) != 1 || r.kept[0] != "the-servers-own-token" { + t.Fatalf("the provider's value was not kept: %d kept", len(r.kept)) + } + if len(r.sent) != 1 || strings.Join(r.sent[0], ",") != "ace" { + t.Fatalf("the holders' machines were not sent: %v", r.sent) + } + if len(r.acts) != 1 || r.acts[0].Verb != handActIssued { + t.Fatalf("the change was not recorded in the hand-act log: %+v", r.acts) + } + for _, said := range []string{words, r.acts[0].Why, strings.Join(r.acts[0].Args, " ")} { + if strings.Contains(said, "the-servers-own-token") || strings.Contains(said, inventory.IssuedStamp("the-servers-own-token")) { + t.Fatalf("the value or its fingerprint was said: %q", said) + } + } + if !strings.Contains(words, "kometa on ace") || !strings.Contains(words, "tautulli on ace") { + t.Fatalf("the answer does not name the holders: %q", words) + } +} + +func TestAnIssuedValueEveryHolderHoldsSendsNobody(t *testing.T) { + r := issuedRecorder{again: true} + words, err := r.taker(t, issuedSealed(t, "the-servers-own-token")).take("plex-api") + if err != nil { + t.Fatal(err) + } + if len(r.sent) != 0 || len(r.acts) != 0 { + t.Fatalf("an unchanged value was sent or recorded: %v %v", r.sent, r.acts) + } + if !strings.Contains(words, "nothing was sent") { + t.Fatalf("the answer does not say nothing was sent: %q", words) + } +} + +func TestOnlyAValueSealedToTheCallIsTaken(t *testing.T) { + other, _, _ := secrets.Keypair() + elsewhere, _ := secrets.Seal(other, []byte("somebody's value")) + raw, _ := json.Marshal(map[string]string{"sealed": elsewhere}) + plain, _ := json.Marshal(map[string]string{"value": "the-servers-own-token"}) + for name, ask := range map[string]func(issuedProvider, map[string]any) (link.Answer, error){ + "sealed to another key": func(issuedProvider, map[string]any) (link.Answer, error) { return link.Answer{Result: raw}, nil }, + "answered in the clear": func(issuedProvider, map[string]any) (link.Answer, error) { return link.Answer{Result: plain}, nil }, + "refused": func(issuedProvider, map[string]any) (link.Answer, error) { + return link.Answer{Error: "the controller alone"}, nil + }, + "not reached": func(issuedProvider, map[string]any) (link.Answer, error) { + return link.Answer{}, errors.New("503 no responders") + }, + "sealed empty": issuedSealed(t, " "), + } { + var r issuedRecorder + if _, err := r.taker(t, ask).take("plex-api"); err == nil { + t.Errorf("%s: taken", name) + } + if len(r.kept) != 0 || len(r.sent) != 0 { + t.Errorf("%s: something was kept or sent", name) + } + } +} + +func TestAProvisionNoOfferSaysIsIssuedIsRefused(t *testing.T) { + asked := false + taker := issuedTaker{ + providers: func(string) ([]issuedProvider, bool, error) { return nil, false, nil }, + ask: func(issuedProvider, map[string]any) (link.Answer, error) { + asked = true + return link.Answer{}, nil + }, + } + if _, err := taker.take("postgres-database"); err == nil || !strings.Contains(err.Error(), "ADR 0305") { + t.Fatalf("expected a refusal naming the record, got %v", err) + } + if asked { + t.Fatal("a provider was asked for a provision no offer says its software issues") + } +} diff --git a/cmd/mesh-controller/issued_test.go b/cmd/mesh-controller/issued_test.go index f2b9da84..f2819a76 100644 --- a/cmd/mesh-controller/issued_test.go +++ b/cmd/mesh-controller/issued_test.go @@ -2,7 +2,6 @@ package main import ( "encoding/json" - "errors" "strings" "testing" @@ -100,90 +99,12 @@ func (r *issuedRecorder) taker(t *testing.T, ask func(issuedProvider, map[string } // The value the provider's tool sealed to the call is kept, recorded and sent, and the answer never says it. -func TestAnIssuedValueIsTakenSealedFromTheProviderAndSent(t *testing.T) { - var r issuedRecorder - words, err := r.taker(t, issuedSealed(t, "the-servers-own-token")).take("plex-api") - if err != nil { - t.Fatal(err) - } - if len(r.kept) != 1 || r.kept[0] != "the-servers-own-token" { - t.Fatalf("the provider's value was not kept: %d kept", len(r.kept)) - } - if len(r.sent) != 1 || strings.Join(r.sent[0], ",") != "ace" { - t.Fatalf("the holders' machines were not sent: %v", r.sent) - } - if len(r.acts) != 1 || r.acts[0].Verb != handActIssued { - t.Fatalf("the change was not recorded in the hand-act log: %+v", r.acts) - } - for _, said := range []string{words, r.acts[0].Why, strings.Join(r.acts[0].Args, " ")} { - if strings.Contains(said, "the-servers-own-token") || strings.Contains(said, inventory.IssuedStamp("the-servers-own-token")) { - t.Fatalf("the value or its fingerprint was said: %q", said) - } - } - if !strings.Contains(words, "kometa on ace") || !strings.Contains(words, "tautulli on ace") { - t.Fatalf("the answer does not name the holders: %q", words) - } -} // The same value again writes nothing, records nothing and sends nobody (ADR 0305 §5). -func TestAnIssuedValueEveryHolderHoldsSendsNobody(t *testing.T) { - r := issuedRecorder{again: true} - words, err := r.taker(t, issuedSealed(t, "the-servers-own-token")).take("plex-api") - if err != nil { - t.Fatal(err) - } - if len(r.sent) != 0 || len(r.acts) != 0 { - t.Fatalf("an unchanged value was sent or recorded: %v %v", r.sent, r.acts) - } - if !strings.Contains(words, "nothing was sent") { - t.Fatalf("the answer does not say nothing was sent: %q", words) - } -} // A value not sealed to the call's key is not taken, nor is a refusal, nor an empty answer (ADR 0305 §2). -func TestOnlyAValueSealedToTheCallIsTaken(t *testing.T) { - other, _, _ := secrets.Keypair() - elsewhere, _ := secrets.Seal(other, []byte("somebody's value")) - raw, _ := json.Marshal(map[string]string{"sealed": elsewhere}) - plain, _ := json.Marshal(map[string]string{"value": "the-servers-own-token"}) - for name, ask := range map[string]func(issuedProvider, map[string]any) (link.Answer, error){ - "sealed to another key": func(issuedProvider, map[string]any) (link.Answer, error) { return link.Answer{Result: raw}, nil }, - "answered in the clear": func(issuedProvider, map[string]any) (link.Answer, error) { return link.Answer{Result: plain}, nil }, - "refused": func(issuedProvider, map[string]any) (link.Answer, error) { - return link.Answer{Error: "the controller alone"}, nil - }, - "not reached": func(issuedProvider, map[string]any) (link.Answer, error) { - return link.Answer{}, errors.New("503 no responders") - }, - "sealed empty": issuedSealed(t, " "), - } { - var r issuedRecorder - if _, err := r.taker(t, ask).take("plex-api"); err == nil { - t.Errorf("%s: taken", name) - } - if len(r.kept) != 0 || len(r.sent) != 0 { - t.Errorf("%s: something was kept or sent", name) - } - } -} // A provision no offer says its software issues is refused before anything is asked. -func TestAProvisionNoOfferSaysIsIssuedIsRefused(t *testing.T) { - asked := false - taker := issuedTaker{ - providers: func(string) ([]issuedProvider, bool, error) { return nil, false, nil }, - ask: func(issuedProvider, map[string]any) (link.Answer, error) { - asked = true - return link.Answer{}, nil - }, - } - if _, err := taker.take("postgres-database"); err == nil || !strings.Contains(err.Error(), "ADR 0305") { - t.Fatalf("expected a refusal naming the record, got %v", err) - } - if asked { - t.Fatal("a provider was asked for a provision no offer says its software issues") - } -} // Which assignments are asked: the modules on the providing machine whose offer says so, with their tool. func TestTheProvidersAskedAreTheOffersThatSaySo(t *testing.T) { diff --git a/cmd/mesh-controller/issued_to_vault_test.go b/cmd/mesh-controller/issued_to_vault_test.go index d2525d65..6f518918 100644 --- a/cmd/mesh-controller/issued_to_vault_test.go +++ b/cmd/mesh-controller/issued_to_vault_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/join_through_the_tunnel_test.go b/cmd/mesh-controller/join_through_the_tunnel_test.go index 5782644b..19cc37b2 100644 --- a/cmd/mesh-controller/join_through_the_tunnel_test.go +++ b/cmd/mesh-controller/join_through_the_tunnel_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/last_send_integration_test.go b/cmd/mesh-controller/last_send_integration_test.go new file mode 100644 index 00000000..4cb33fb7 --- /dev/null +++ b/cmd/mesh-controller/last_send_integration_test.go @@ -0,0 +1,51 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestNodeShowAndStatusReadTheLastSendFromTheStore(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + record, err := open.inventory.NodeByName(ctx, "laptop") + if err != nil { + t.Fatal(err) + } + if err := open.inventory.RecordSentWith(ctx, record.ID, "d12", nil, 57, 40, inventory.Send{Sequence: 12, + Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12", + Modules: []string{"docker", "sudo"}}); err != nil { + t.Fatal(err) + } + if _, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeRefused, + Declared: "d12", Refused: "unknown field \"generation\""}); err != nil { + t.Fatal(err) + } + shown := printed(t, func() error { return showNode(ctx, open, "laptop") }) + for _, want := range []string{"a one-shot push by jochen at a shell on anchor", "lease epoch 57", "generation 40", + "docker, sudo", "unknown field \"generation\""} { + if !strings.Contains(shown, want) { + t.Errorf("node show laptop does not say %q:\n%s", want, shown) + } + } + body := printed(t, func() error { return statusFor(ctx, open, true) }) + var got struct { + LastSent []struct { + Node string `json:"node"` + Sender string `json:"sender"` + Answer string `json:"answer"` + } `json:"lastSent"` + } + if err := json.Unmarshal([]byte(body), &got); err != nil { + t.Fatalf("%v:\n%s", err, body) + } + if len(got.LastSent) != 1 || got.LastSent[0].Node != "laptop" || got.LastSent[0].Answer != "refused" || + got.LastSent[0].Sender != "a one-shot push by jochen at a shell on anchor" { + t.Fatalf("status --json's last sends read %+v", got.LastSent) + } +} diff --git a/cmd/mesh-controller/last_send_test.go b/cmd/mesh-controller/last_send_test.go index 0fecd717..33c329f7 100644 --- a/cmd/mesh-controller/last_send_test.go +++ b/cmd/mesh-controller/last_send_test.go @@ -119,42 +119,3 @@ func TestStatusJSONCarriesEachMachinesLastSend(t *testing.T) { // And through the store: a send recorded and refused by the machine is what `node show` and `status --json` // read back. -func TestNodeShowAndStatusReadTheLastSendFromTheStore(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - record, err := open.inventory.NodeByName(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if err := open.inventory.RecordSentWith(ctx, record.ID, "d12", nil, 57, 40, inventory.Send{Sequence: 12, - Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12", - Modules: []string{"docker", "sudo"}}); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeRefused, - Declared: "d12", Refused: "unknown field \"generation\""}); err != nil { - t.Fatal(err) - } - shown := printed(t, func() error { return showNode(ctx, open, "laptop") }) - for _, want := range []string{"a one-shot push by jochen at a shell on anchor", "lease epoch 57", "generation 40", - "docker, sudo", "unknown field \"generation\""} { - if !strings.Contains(shown, want) { - t.Errorf("node show laptop does not say %q:\n%s", want, shown) - } - } - body := printed(t, func() error { return statusFor(ctx, open, true) }) - var got struct { - LastSent []struct { - Node string `json:"node"` - Sender string `json:"sender"` - Answer string `json:"answer"` - } `json:"lastSent"` - } - if err := json.Unmarshal([]byte(body), &got); err != nil { - t.Fatalf("%v:\n%s", err, body) - } - if len(got.LastSent) != 1 || got.LastSent[0].Node != "laptop" || got.LastSent[0].Answer != "refused" || - got.LastSent[0].Sender != "a one-shot push by jochen at a shell on anchor" { - t.Fatalf("status --json's last sends read %+v", got.LastSent) - } -} diff --git a/cmd/mesh-controller/lean_walk_integration_test.go b/cmd/mesh-controller/lean_walk_integration_test.go new file mode 100644 index 00000000..bc5ebd47 --- /dev/null +++ b/cmd/mesh-controller/lean_walk_integration_test.go @@ -0,0 +1,141 @@ +//go:build integration + +package main + +import ( + "context" + "slices" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestAReleaseSendsEveryMachineAtOnce(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklogOnThree(t) + ctx := t.Context() + said := events(t) + for i := 0; i < 12; i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + if p.State != inventory.PlanDone || len(p.Release.Done) != 3 { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + for _, n := range []string{"desk", "laptop", "anchor"} { + if !slices.Contains(*said, "send "+n) { + t.Fatalf("%s was not sent: %v", n, *said) + } + } + if slices.Contains(*said, "judge") { + t.Fatalf("a machine was judged by its health: %v", *said) + } + if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) + } +} + +func TestAMachineThatFailsItsApplyIsNamedAndNothingIsPutBack(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklogOnThree(t) + ctx := t.Context() + inv := b.open.inventory + b.broken["desk"] = true // desk fails what it is sent + advancePlans(ctx, b.open) + for i := 0; i < 12 && b.release(t).Open(); i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + if !slices.Contains(p.Release.Done, "laptop") || !slices.Contains(p.Release.Done, "anchor") { + t.Fatalf("the others have no verdict of their own: %+v", p.Release) + } + if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) + } + if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c2" { + t.Fatalf("desk was put back: %v", sent) + } +} + +func TestAReleaseSendKeptAndMadeIsNotMadeAgainAtOnce(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + p, err := releaseBacklog(ctx, b.open, "") + if err != nil || p == nil { + t.Fatalf("no release opened: %v", err) + } + if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gates["anchor"] == nil || !p.Release.Gates["anchor"].Pending { + t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err) + } + if err := inv.SavePlan(ctx, p); err != nil { + t.Fatal(err) + } + // The send is made, and the controller is replaced before it keeps it. + if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil { + t.Fatal(err) + } + advancePlans(ctx, b.open) + if n := len(slices.DeleteFunc(slices.Clone(b.sent), func(s []string) bool { return !slices.Equal(s, []string{"anchor"}) })); n != 1 { + t.Fatalf("anchor was sent again: %v", b.sent) + } + g := b.release(t).Release.Gates["anchor"] + if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 { + t.Fatalf("anchor is not judged on the send made: %+v", g) + } +} + +func TestAMachineSilentPastItsBoundIsLeftAndSaidAtOnce(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + reporting := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing + b.sent = append(b.sent, names) + current, _ := open.inventory.CurrentBuilds(ctx) + return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep", + map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit}) + } + return reporting(ctx, open, names) + } + t.Cleanup(func() { sendRollout = reporting }) + for i := 0; i < 8; i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + g := p.Release.Gates["laptop"] + if !p.Open() || g == nil || g.Verdict != "" || g.Silent { + t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g) + } + since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) + g.Since = &since + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + advancePlans(ctx, b.open) + p = b.release(t) + // anchor applied in the same step, so the release is done, laptop named silent (novox/hq ADR 0360 §7). + if p.State != inventory.PlanDone || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed { + t.Fatal("a silent machine's build was marked failed") + } + if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" { + t.Fatalf("laptop was put back: %v", sent) + } + obs, err := silentObservation(ctx, inv) + if err != nil { + t.Fatal(err) + } + if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) { + t.Fatalf("the silence is not said: %+v", obs) + } +} diff --git a/cmd/mesh-controller/lean_walk_test.go b/cmd/mesh-controller/lean_walk_test.go index eced96cf..5f6ad330 100644 --- a/cmd/mesh-controller/lean_walk_test.go +++ b/cmd/mesh-controller/lean_walk_test.go @@ -1,7 +1,6 @@ package main import ( - "context" "slices" "strings" "testing" @@ -53,140 +52,15 @@ func withTheGate(t *testing.T) { // Every machine heard from is sent at once and passed on its applied report, with no judging (novox/hq ADR 0360 §7); // before, the first machine was judged before the rest were sent (ADR 0353 §1). -func TestAReleaseSendsEveryMachineAtOnce(t *testing.T) { - judgeLeavesAsTheCore(t) - b := aBacklogOnThree(t) - ctx := t.Context() - said := events(t) - for i := 0; i < 12; i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - if p.State != inventory.PlanDone || len(p.Release.Done) != 3 { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - for _, n := range []string{"desk", "laptop", "anchor"} { - if !slices.Contains(*said, "send "+n) { - t.Fatalf("%s was not sent: %v", n, *said) - } - } - if slices.Contains(*said, "judge") { - t.Fatalf("a machine was judged by its health: %v", *said) - } - if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) - } -} // A machine that reports its apply failed is named and fails the release, with nothing put back, and every other // machine goes on to its own report (novox/hq ADR 0360 §5, §7); before, it was put back (ADR 0353 §3). -func TestAMachineThatFailsItsApplyIsNamedAndNothingIsPutBack(t *testing.T) { - judgeLeavesAsTheCore(t) - b := aBacklogOnThree(t) - ctx := t.Context() - inv := b.open.inventory - b.broken["desk"] = true // desk fails what it is sent - advancePlans(ctx, b.open) - for i := 0; i < 12 && b.release(t).Open(); i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - if !slices.Contains(p.Release.Done, "laptop") || !slices.Contains(p.Release.Done, "anchor") { - t.Fatalf("the others have no verdict of their own: %+v", p.Release) - } - if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) - } - if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c2" { - t.Fatalf("desk was put back: %v", sent) - } -} // A release's send kept before it was made, and made, is judged by the controller that takes over without being // made again (ADR 0353 §8), with every machine sent at once (ADR 0360 §7). -func TestAReleaseSendKeptAndMadeIsNotMadeAgainAtOnce(t *testing.T) { - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - p, err := releaseBacklog(ctx, b.open, "") - if err != nil || p == nil { - t.Fatalf("no release opened: %v", err) - } - if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gates["anchor"] == nil || !p.Release.Gates["anchor"].Pending { - t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err) - } - if err := inv.SavePlan(ctx, p); err != nil { - t.Fatal(err) - } - // The send is made, and the controller is replaced before it keeps it. - if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, b.open) - if n := len(slices.DeleteFunc(slices.Clone(b.sent), func(s []string) bool { return !slices.Equal(s, []string{"anchor"}) })); n != 1 { - t.Fatalf("anchor was sent again: %v", b.sent) - } - g := b.release(t).Release.Gates["anchor"] - if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 { - t.Fatalf("anchor is not judged on the send made: %+v", g) - } -} // A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put // back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4); its step also sent anchor, so the release is done, not failed (ADR 0360 §7). -func TestAMachineSilentPastItsBoundIsLeftAndSaidAtOnce(t *testing.T) { - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - reporting := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing - b.sent = append(b.sent, names) - current, _ := open.inventory.CurrentBuilds(ctx) - return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep", - map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit}) - } - return reporting(ctx, open, names) - } - t.Cleanup(func() { sendRollout = reporting }) - for i := 0; i < 8; i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - g := p.Release.Gates["laptop"] - if !p.Open() || g == nil || g.Verdict != "" || g.Silent { - t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g) - } - since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) - g.Since = &since - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - advancePlans(ctx, b.open) - p = b.release(t) - // anchor applied in the same step, so the release is done, laptop named silent (novox/hq ADR 0360 §7). - if p.State != inventory.PlanDone || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed { - t.Fatal("a silent machine's build was marked failed") - } - if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" { - t.Fatalf("laptop was put back: %v", sent) - } - obs, err := silentObservation(ctx, inv) - if err != nil { - t.Fatal(err) - } - if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) { - t.Fatalf("the silence is not said: %+v", obs) - } -} // Issue 840 (novox/hq ADR 0362 §1-5): a walk's end counts a machine's send of a provisioned module applied only once // the engine and the machine's provisioner client both said so. diff --git a/cmd/mesh-controller/lease_command_test.go b/cmd/mesh-controller/lease_command_test.go index af20840e..f8c0c811 100644 --- a/cmd/mesh-controller/lease_command_test.go +++ b/cmd/mesh-controller/lease_command_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/lease_integration_test.go b/cmd/mesh-controller/lease_integration_test.go new file mode 100644 index 00000000..4c7e980f --- /dev/null +++ b/cmd/mesh-controller/lease_integration_test.go @@ -0,0 +1,154 @@ +//go:build integration + +package main + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" + "github.com/novox/mesh-controller/internal/testbus" +) + +func aBusForTheLease(t *testing.T) (string, jetstream.KeyValue) { + t.Helper() + url := testbus.URL(t) + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(conn.Close) + js, err := jetstream.New(conn) + if err != nil { + t.Fatal(err) + } + _ = js.DeleteKeyValue(t.Context(), broker.LeaseBucket) + if err := broker.EnsureLeaseBucket(t.Context(), js); err != nil { + t.Fatal(err) + } + kv, err := js.KeyValue(t.Context(), broker.LeaseBucket) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = js.DeleteKeyValue(context.Background(), broker.LeaseBucket) }) + return url, kv +} + +func TestTwoControllersOneActs(t *testing.T) { + url, kv := aBusForTheLease(t) + inv := inventory.ForTest(t) + ctx := t.Context() + + // Each renews five times and looks again ten times as often as a serving controller does, so the + // handover and the loss below are seen in a second, not in a renewal of five (novox/hq issue 591): what + // is checked is the order of events, which does not change with the interval. Each renewal still has a + // serving controller's five seconds to answer (acting.go), so a slow bus does not lose A the lease. + const renewEvery, poll = lease.RenewEvery / 5, lease.Poll / 10 + + // Controller A takes the lease. + a := &actor{renewEvery: renewEvery, poll: poll} + aCtx, stopA := context.WithCancel(ctx) + defer stopA() + lostA, err := a.serveUnderTheLease(aCtx, inv, url) + if err != nil { + t.Fatal(err) + } + epochA, err := a.epoch(ctx) + if err != nil || epochA == 0 { + t.Fatalf("A holds no epoch: %d, %v", epochA, err) + } + + // Controller B starts while A holds it, and waits — acting on nothing meanwhile. + b := &actor{renewEvery: renewEvery, poll: poll} + bCtx, stopB := context.WithCancel(ctx) + defer stopB() + tookB := make(chan (<-chan struct{}), 1) + go func() { + lost, err := b.serveUnderTheLease(bCtx, inv, url) + if err != nil { + t.Errorf("B: %v", err) + close(tookB) + return + } + tookB <- lost + }() + // B looks fifteen times, across one of A's renewals at least. + select { + case <-tookB: + t.Fatal("B took the lease while A held it") + case <-time.After(15 * poll): + } + if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) { + t.Fatalf("B, waiting, may act: %v", err) + } + + // A hands over, as a controller being replaced does: B takes the lease at once, at a higher epoch. + stopA() + a.release() + var lostB <-chan struct{} + select { + case lostB = <-tookB: + case <-time.After(10 * time.Second): + t.Fatal("B did not take the lease A gave back") + } + select { + case <-lostA: + default: + t.Fatal("A, having given the lease back, is not told it no longer holds it") + } + epochB, err := b.epoch(ctx) + if err != nil || epochB <= epochA { + t.Fatalf("B acts as epoch %d after A's %d (%v): an epoch only grows", epochB, epochA, err) + } + if _, err := a.epoch(ctx); err == nil { + t.Fatal("A acts after giving the lease back") + } + ea, _, _ := inv.EpochOf(ctx, epochA) + eb, _, _ := inv.EpochOf(ctx, epochB) + if ea.How != inventory.EpochReleased || eb.Ended != nil || eb.Instance != instance { + t.Fatalf("the record of the handover reads %+v then %+v", ea, eb) + } + + // Something else writes the lease's key — a third controller on a clock that read it as expired: + // B's next renewal is refused, and B stops acting at once. + if _, err := kv.Put(ctx, lease.Key, []byte(`{"instance":"a third controller","epoch":1}`)); err != nil { + t.Fatal(err) + } + select { + case <-lostB: + case <-time.After(2 * renewEvery): + t.Fatal("B was not told it lost the lease") + } + if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) { + t.Fatalf("B acts after losing the lease: %v", err) + } + // Nothing B does is written: a declaration's number is not taken, a plan is not saved, a condition + // is not raised. + inv.ActsUnder(b.epoch) + if _, err := inv.AddNode(ctx, "anchor"); err != nil { + t.Fatal(err) + } + saved := inventory.Plan{ID: "plan-after-loss", Repository: "novox/app", Commit: "c0ffee00", Created: time.Now(), + State: inventory.PlanBuilding} + if err := inv.SavePlan(ctx, &saved); err == nil { + t.Fatal("B wrote a plan after losing the lease") + } + store := conditions.NewInMemory() + keeper := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store, + Epoch: func() (uint64, error) { return b.epoch(ctx) }}) + defer keeper.Close(context.Background()) + if _, err := keeper.Observe(ctx, conditions.Observation{Scope: conditions.ScopeCore, ID: "x", Kind: "x", + Severity: conditions.Warning, Summary: "x", Source: "test"}); err == nil { + t.Fatal("B raised a condition after losing the lease") + } + if ended, _, _ := inv.EpochOf(ctx, epochB); ended.How != inventory.EpochLost { + t.Fatalf("B's epoch does not say it was lost: %+v", ended) + } +} diff --git a/cmd/mesh-controller/lease_test.go b/cmd/mesh-controller/lease_test.go index 5356a116..2aa9114d 100644 --- a/cmd/mesh-controller/lease_test.go +++ b/cmd/mesh-controller/lease_test.go @@ -4,17 +4,8 @@ import ( "context" "errors" "testing" - "time" - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/lease" - - "github.com/novox/mesh-controller/internal/testbus" ) // Two controllers at once (novox/hq to-be 45 §6, issue 204; the half of replay R1 that lives here): two @@ -26,141 +17,6 @@ import ( // MESH_TEST_POSTGRES=… go test ./cmd/mesh-controller/ -run Controllers (each test on a bus of its own: internal/testbus) // aBusForTheLease is the test bus with the controller's lease bucket new. -func aBusForTheLease(t *testing.T) (string, jetstream.KeyValue) { - t.Helper() - url := testbus.URL(t) - conn, err := nats.Connect(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - js, err := jetstream.New(conn) - if err != nil { - t.Fatal(err) - } - _ = js.DeleteKeyValue(t.Context(), broker.LeaseBucket) - if err := broker.EnsureLeaseBucket(t.Context(), js); err != nil { - t.Fatal(err) - } - kv, err := js.KeyValue(t.Context(), broker.LeaseBucket) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = js.DeleteKeyValue(context.Background(), broker.LeaseBucket) }) - return url, kv -} - -func TestTwoControllersOneActs(t *testing.T) { - url, kv := aBusForTheLease(t) - inv := inventory.ForTest(t) - ctx := t.Context() - - // Each renews five times and looks again ten times as often as a serving controller does, so the - // handover and the loss below are seen in a second, not in a renewal of five (novox/hq issue 591): what - // is checked is the order of events, which does not change with the interval. Each renewal still has a - // serving controller's five seconds to answer (acting.go), so a slow bus does not lose A the lease. - const renewEvery, poll = lease.RenewEvery / 5, lease.Poll / 10 - - // Controller A takes the lease. - a := &actor{renewEvery: renewEvery, poll: poll} - aCtx, stopA := context.WithCancel(ctx) - defer stopA() - lostA, err := a.serveUnderTheLease(aCtx, inv, url) - if err != nil { - t.Fatal(err) - } - epochA, err := a.epoch(ctx) - if err != nil || epochA == 0 { - t.Fatalf("A holds no epoch: %d, %v", epochA, err) - } - - // Controller B starts while A holds it, and waits — acting on nothing meanwhile. - b := &actor{renewEvery: renewEvery, poll: poll} - bCtx, stopB := context.WithCancel(ctx) - defer stopB() - tookB := make(chan (<-chan struct{}), 1) - go func() { - lost, err := b.serveUnderTheLease(bCtx, inv, url) - if err != nil { - t.Errorf("B: %v", err) - close(tookB) - return - } - tookB <- lost - }() - // B looks fifteen times, across one of A's renewals at least. - select { - case <-tookB: - t.Fatal("B took the lease while A held it") - case <-time.After(15 * poll): - } - if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) { - t.Fatalf("B, waiting, may act: %v", err) - } - - // A hands over, as a controller being replaced does: B takes the lease at once, at a higher epoch. - stopA() - a.release() - var lostB <-chan struct{} - select { - case lostB = <-tookB: - case <-time.After(10 * time.Second): - t.Fatal("B did not take the lease A gave back") - } - select { - case <-lostA: - default: - t.Fatal("A, having given the lease back, is not told it no longer holds it") - } - epochB, err := b.epoch(ctx) - if err != nil || epochB <= epochA { - t.Fatalf("B acts as epoch %d after A's %d (%v): an epoch only grows", epochB, epochA, err) - } - if _, err := a.epoch(ctx); err == nil { - t.Fatal("A acts after giving the lease back") - } - ea, _, _ := inv.EpochOf(ctx, epochA) - eb, _, _ := inv.EpochOf(ctx, epochB) - if ea.How != inventory.EpochReleased || eb.Ended != nil || eb.Instance != instance { - t.Fatalf("the record of the handover reads %+v then %+v", ea, eb) - } - - // Something else writes the lease's key — a third controller on a clock that read it as expired: - // B's next renewal is refused, and B stops acting at once. - if _, err := kv.Put(ctx, lease.Key, []byte(`{"instance":"a third controller","epoch":1}`)); err != nil { - t.Fatal(err) - } - select { - case <-lostB: - case <-time.After(2 * renewEvery): - t.Fatal("B was not told it lost the lease") - } - if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) { - t.Fatalf("B acts after losing the lease: %v", err) - } - // Nothing B does is written: a declaration's number is not taken, a plan is not saved, a condition - // is not raised. - inv.ActsUnder(b.epoch) - if _, err := inv.AddNode(ctx, "anchor"); err != nil { - t.Fatal(err) - } - saved := inventory.Plan{ID: "plan-after-loss", Repository: "novox/app", Commit: "c0ffee00", Created: time.Now(), - State: inventory.PlanBuilding} - if err := inv.SavePlan(ctx, &saved); err == nil { - t.Fatal("B wrote a plan after losing the lease") - } - store := conditions.NewInMemory() - keeper := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store, - Epoch: func() (uint64, error) { return b.epoch(ctx) }}) - defer keeper.Close(context.Background()) - if _, err := keeper.Observe(ctx, conditions.Observation{Scope: conditions.ScopeCore, ID: "x", Kind: "x", - Severity: conditions.Warning, Summary: "x", Source: "test"}); err == nil { - t.Fatal("B raised a condition after losing the lease") - } - if ended, _, _ := inv.EpochOf(ctx, epochB); ended.How != inventory.EpochLost { - t.Fatalf("B's epoch does not say it was lost: %+v", ended) - } -} // A controller whose bus refuses it the lease's key, with nobody holding it, serves without the lease: // it acts with no epoch — refused by no node-engine — says so, and takes the lease once it can. diff --git a/cmd/mesh-controller/left_out_integration_test.go b/cmd/mesh-controller/left_out_integration_test.go new file mode 100644 index 00000000..feae280f --- /dev/null +++ b/cmd/mesh-controller/left_out_integration_test.go @@ -0,0 +1,127 @@ +//go:build integration + +package main + +import ( + "context" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestALeftOutModulesConditionClearsWhenItComposesAgainOrIsUnassigned(t *testing.T) { + plan, settings := leftOutNFS(t), catalogue.SettingsBy(nil) + withProbes(t, probe{ID: probeLeftOutID, Asserts: "the test's", Kind: kindLeftOut, Phase: 1, + run: func(context.Context, *doctor) ([]conditions.Observation, error) { + return leftOutObservations("anchor", leftOutOf(plan, settings, false)), nil + }}) + store := conditions.NewInMemory() + k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store}) + defer k.Close(context.Background()) + d := &doctor{keeper: k, teller: &conditions.Told{}, host: "anchor"} + key := "module.nfs-server.anchor.left-out" + openOnes := func() map[string]conditions.Condition { + t.Helper() + open, err := k.Open(t.Context()) + if err != nil { + t.Fatal(err) + } + out := map[string]conditions.Condition{} + for _, c := range open { + out[c.Key] = c + } + return out + } + + d.runOnce(t.Context(), "a test") + c, raised := openOnes()[key] + if !raised || c.Kind != kindNeedsOperator || c.Severity != conditions.Warning { + t.Fatalf("a module left out raised %+v", openOnes()) + } + // A statement from the machine that says nothing of it — the module runs nothing there — leaves it open. + if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil, + time.Now().Add(72*time.Hour)); err != nil { + t.Fatal(err) + } + if _, still := openOnes()[key]; !still { + t.Fatal("a health statement that says nothing of the module cleared its left-out condition") + } + d.runOnce(t.Context(), "a test") + if c := openOnes()[key]; c.Severity != conditions.Warning { + t.Fatalf("after a health statement and days, it is %+v", openOnes()) + } + + // The setting given: it composes, and the condition clears. + settings = sharesGiven("library=/srv/library") + d.runOnce(t.Context(), "a test") + if _, still := openOnes()[key]; still { + t.Fatalf("composed again, still open: %+v", openOnes()) + } + + // Left out again, then unassigned: no longer in the machine's set, and it clears. + settings = nil + d.runOnce(t.Context(), "a test") + if _, raised := openOnes()[key]; !raised { + t.Fatal("left out again and not raised") + } + plan = catalogue.Resolution{} + d.runOnce(t.Context(), "a test") + if _, still := openOnes()[key]; still { + t.Fatalf("unassigned, still open: %+v", openOnes()) + } +} + +func TestAWaitsNeedsOperatorStillClearsWhenItsStatementStopsNamingItBesideALeftOutOne(t *testing.T) { + store := conditions.NewInMemory() + k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store}) + defer k.Close(context.Background()) + left := leftOutObservations("anchor", leftOutOf(leftOutNFS(t), nil, false)) + if err := k.Reconcile(t.Context(), probeLeftOutID, left); err != nil { + t.Fatal(err) + } + waiting := map[string][]inventory.ResourceHealth{"notes": {{Module: "notes", Resource: "server", State: link.StateWaiting, + Waits: []inventory.Wait{{Setting: "domain", What: "the domain it serves"}}}}} + if err := judgeModuleHealth(t.Context(), nil, k, "anchor", waiting, map[string]int{"notes": 2}, time.Now()); err != nil { + t.Fatal(err) + } + waitKey, leftKey := needsOperatorKey("notes", "anchor"), "module.nfs-server.anchor.left-out" + open := func() map[string]conditions.Condition { + t.Helper() + list, err := k.Open(t.Context()) + if err != nil { + t.Fatal(err) + } + out := map[string]conditions.Condition{} + for _, c := range list { + out[c.Key] = c + } + return out + } + if c, raised := open()[waitKey]; !raised || c.Kind != kindNeedsOperator { + t.Fatalf("the wait raised %+v", open()) + } + if c := open()[leftKey]; c.Kind != kindNeedsOperator { + t.Fatalf("the left-out condition is not open as needs-operator: %+v", open()) + } + // D1 runs again and still finds nfs-server left out: its reconcile clears only what D1 raised, never the wait, + // which the machine's statement raised. + if err := k.Reconcile(t.Context(), probeLeftOutID, left); err != nil { + t.Fatal(err) + } + if _, still := open()[waitKey]; !still { + t.Fatalf("D1's reconcile cleared the wait's needs-operator: %+v", open()) + } + if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil { + t.Fatal(err) + } + if _, still := open()[waitKey]; still { + t.Fatalf("the statement no longer names the wait, and its needs-operator is still open: %+v", open()) + } + if _, still := open()[leftKey]; !still { + t.Fatalf("the left-out condition was cleared by a health statement: %+v", open()) + } +} diff --git a/cmd/mesh-controller/left_out_test.go b/cmd/mesh-controller/left_out_test.go index 7c75d307..795bef55 100644 --- a/cmd/mesh-controller/left_out_test.go +++ b/cmd/mesh-controller/left_out_test.go @@ -1,18 +1,14 @@ package main import ( - "context" "encoding/json" "os" "slices" "strings" "testing" - "time" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" ) // novox/hq issue 380: nfs-server was assigned to the home server for days and every send left it out — "nfs-server @@ -95,67 +91,6 @@ func TestAModuleLeftOutForAnotherCauseIsAWarningWithTheReasonAsEvidence(t *testi // The self-check raises it, keeps it a warning however long it stands, and clears it when the module composes // again or is no longer assigned; a machine's health statement neither clears nor raises it. -func TestALeftOutModulesConditionClearsWhenItComposesAgainOrIsUnassigned(t *testing.T) { - plan, settings := leftOutNFS(t), catalogue.SettingsBy(nil) - withProbes(t, probe{ID: probeLeftOutID, Asserts: "the test's", Kind: kindLeftOut, Phase: 1, - run: func(context.Context, *doctor) ([]conditions.Observation, error) { - return leftOutObservations("anchor", leftOutOf(plan, settings, false)), nil - }}) - store := conditions.NewInMemory() - k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store}) - defer k.Close(context.Background()) - d := &doctor{keeper: k, teller: &conditions.Told{}, host: "anchor"} - key := "module.nfs-server.anchor.left-out" - openOnes := func() map[string]conditions.Condition { - t.Helper() - open, err := k.Open(t.Context()) - if err != nil { - t.Fatal(err) - } - out := map[string]conditions.Condition{} - for _, c := range open { - out[c.Key] = c - } - return out - } - - d.runOnce(t.Context(), "a test") - c, raised := openOnes()[key] - if !raised || c.Kind != kindNeedsOperator || c.Severity != conditions.Warning { - t.Fatalf("a module left out raised %+v", openOnes()) - } - // A statement from the machine that says nothing of it — the module runs nothing there — leaves it open. - if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil, - time.Now().Add(72*time.Hour)); err != nil { - t.Fatal(err) - } - if _, still := openOnes()[key]; !still { - t.Fatal("a health statement that says nothing of the module cleared its left-out condition") - } - d.runOnce(t.Context(), "a test") - if c := openOnes()[key]; c.Severity != conditions.Warning { - t.Fatalf("after a health statement and days, it is %+v", openOnes()) - } - - // The setting given: it composes, and the condition clears. - settings = sharesGiven("library=/srv/library") - d.runOnce(t.Context(), "a test") - if _, still := openOnes()[key]; still { - t.Fatalf("composed again, still open: %+v", openOnes()) - } - - // Left out again, then unassigned: no longer in the machine's set, and it clears. - settings = nil - d.runOnce(t.Context(), "a test") - if _, raised := openOnes()[key]; !raised { - t.Fatal("left out again and not raised") - } - plan = catalogue.Resolution{} - d.runOnce(t.Context(), "a test") - if _, still := openOnes()[key]; still { - t.Fatalf("unassigned, still open: %+v", openOnes()) - } -} func TestTheLeftOutProbeIsInTheRegistry(t *testing.T) { for _, p := range probeRegistry { @@ -207,53 +142,3 @@ func TestStatusAndNodeListAModuleAssignedAndNotApplied(t *testing.T) { // The skip is this probe's alone (review of #223): a part waiting for the operator (ADR 0283) keeps its own // needs-operator condition, `module...needs-operator`, which still clears on the first statement that no // longer names it — beside a left-out condition on the same machine, which stays. -func TestAWaitsNeedsOperatorStillClearsWhenItsStatementStopsNamingItBesideALeftOutOne(t *testing.T) { - store := conditions.NewInMemory() - k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store}) - defer k.Close(context.Background()) - left := leftOutObservations("anchor", leftOutOf(leftOutNFS(t), nil, false)) - if err := k.Reconcile(t.Context(), probeLeftOutID, left); err != nil { - t.Fatal(err) - } - waiting := map[string][]inventory.ResourceHealth{"notes": {{Module: "notes", Resource: "server", State: link.StateWaiting, - Waits: []inventory.Wait{{Setting: "domain", What: "the domain it serves"}}}}} - if err := judgeModuleHealth(t.Context(), nil, k, "anchor", waiting, map[string]int{"notes": 2}, time.Now()); err != nil { - t.Fatal(err) - } - waitKey, leftKey := needsOperatorKey("notes", "anchor"), "module.nfs-server.anchor.left-out" - open := func() map[string]conditions.Condition { - t.Helper() - list, err := k.Open(t.Context()) - if err != nil { - t.Fatal(err) - } - out := map[string]conditions.Condition{} - for _, c := range list { - out[c.Key] = c - } - return out - } - if c, raised := open()[waitKey]; !raised || c.Kind != kindNeedsOperator { - t.Fatalf("the wait raised %+v", open()) - } - if c := open()[leftKey]; c.Kind != kindNeedsOperator { - t.Fatalf("the left-out condition is not open as needs-operator: %+v", open()) - } - // D1 runs again and still finds nfs-server left out: its reconcile clears only what D1 raised, never the wait, - // which the machine's statement raised. - if err := k.Reconcile(t.Context(), probeLeftOutID, left); err != nil { - t.Fatal(err) - } - if _, still := open()[waitKey]; !still { - t.Fatalf("D1's reconcile cleared the wait's needs-operator: %+v", open()) - } - if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil { - t.Fatal(err) - } - if _, still := open()[waitKey]; still { - t.Fatalf("the statement no longer names the wait, and its needs-operator is still open: %+v", open()) - } - if _, still := open()[leftKey]; !still { - t.Fatalf("the left-out condition was cleared by a health statement: %+v", open()) - } -} diff --git a/cmd/mesh-controller/lone_merge_test.go b/cmd/mesh-controller/lone_merge_test.go index c2bb3df6..4c9eb8f6 100644 --- a/cmd/mesh-controller/lone_merge_test.go +++ b/cmd/mesh-controller/lone_merge_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/machine_network_integration_test.go b/cmd/mesh-controller/machine_network_integration_test.go new file mode 100644 index 00000000..0a18cf2d --- /dev/null +++ b/cmd/mesh-controller/machine_network_integration_test.go @@ -0,0 +1,266 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "os" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestARewrittenResolverFileIsRaisedFromTheStatementAndClearedWhenWrittenBack(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv, k := open.inventory, conditionsFrom + say := func(at time.Time, n *link.NetworkHealth) { + t.Helper() + if err := stateHealth(ctx, inv, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil { + t.Fatal(err) + } + } + openKeys := func() []string { + t.Helper() + list, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + var keys []string + for _, c := range list { + keys = append(keys, c.Key) + } + return keys + } + healthy := &link.NetworkHealth{State: link.StateHealthy, Since: h0, Parts: []link.NetworkPart{ + {Part: link.PartResolvConf, State: link.StateHealthy, Since: h0}}} + rewritten := &link.NetworkHealth{State: link.StateUnhealthy, Since: h0.Add(time.Minute), Parts: []link.NetworkPart{ + {Part: link.PartResolvConf, State: link.StateUnhealthy, Reason: rewrittenByVPN.Reason, Said: rewrittenByVPN.Said, + Writer: "FortiClient", Owner: "networkmanager", Since: h0.Add(time.Minute)}, + {Part: link.PartNames, State: link.StateUnhealthy, Reason: namesThroughVPN.Reason, Said: namesThroughVPN.Said, + Toward: namesThroughVPN.Toward, Since: h0.Add(time.Minute)}}} + + say(h0, healthy) + if keys := openKeys(); len(keys) != 0 { + t.Fatalf("a healthy network raised %v", keys) + } + say(h0.Add(time.Minute), rewritten) + if keys := openKeys(); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" { + t.Fatalf("the rewrite raised %v", keys) + } + kept, had, err := inv.HealthOf(ctx, "laptop") + if err != nil || !had || kept.Network == nil || kept.Network.Parts[0].Writer != "FortiClient" { + t.Fatalf("the statement's network was not kept: %+v %v", kept.Network, err) + } + if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "FortiClient") || + !strings.Contains(lines, "unhealthy resolv-conf") { + t.Fatalf("node show says:\n%s", lines) + } + say(h0.Add(2*time.Minute), healthy) + if keys := openKeys(); len(keys) != 0 { + t.Fatalf("written back, still open: %v", keys) + } + // An engine older than the judging says no network: nothing raised, and node show says it is not known. + say(h0.Add(3*time.Minute), nil) + kept, had, _ = inv.HealthOf(ctx, "laptop") + if keys := openKeys(); len(keys) != 0 || kept.Network != nil { + t.Fatalf("an older engine: %v %+v", keys, kept.Network) + } + if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "older than that judging") { + t.Fatalf("node show says:\n%s", lines) + } +} + +func TestTheDrillsStatementsRaiseAndClearTheRewrite(t *testing.T) { + raw, err := os.ReadFile("testdata/network-drill.json") + if err != nil { + t.Fatal(err) + } + var said []link.Health + if err := json.Unmarshal(raw, &said); err != nil { + t.Fatal(err) + } + open := aMesh(t) + ctx := t.Context() + k := conditionsFrom + var raisedAt []int + for i, h := range said { + if err := stateHealth(ctx, open.inventory, k, "laptop", h, h.At); err != nil { + t.Fatal(err) + } + list, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + for _, c := range list { + if c.Key != "machine.laptop.networkmanager.rewritten" || !strings.Contains(c.Summary, "(FortiClient)") { + t.Fatalf("statement %d raised %s: %s", i, c.Key, c.Summary) + } + raisedAt = append(raisedAt, i) + } + } + // Five statements: healthy, healthy, one failing look (still healthy), unhealthy, written back. + if len(raisedAt) != 1 || raisedAt[0] != 3 { + t.Fatalf("the rewrite was open after statements %v; want after the fourth alone", raisedAt) + } +} + +func aNetwork(state string, parts ...inventory.NetworkPart) *inventory.NetworkHealth { + for i := range parts { + if parts[i].State == "" { + parts[i].State = link.StateUnhealthy + } + parts[i].Since = h0 + } + return &inventory.NetworkHealth{State: state, Since: h0, Parts: parts} +} + +func TestAResolverFileRewrittenIsOneFindingNamingItsWriterAndWhatItCosts(t *testing.T) { + obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateHealthy), + "laptop": aNetwork(link.StateUnhealthy, rewrittenByVPN, namesThroughVPN), + })) + if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" { + t.Fatalf("want one finding, the rewrite, got %v", keys) + } + o := obs[0] + for _, want := range []string{"laptop", "rewritten by another program (FortiClient)", "mesh names do not resolve", + "next reconcile"} { + if !strings.Contains(o.Summary, want) { + t.Errorf("the summary does not say %q: %s", want, o.Summary) + } + } + if strings.Contains(o.Summary, "172.16.") || strings.Contains(o.Summary, "/etc/") { + t.Errorf("an address or a path reached the summary: %s", o.Summary) + } + if !strings.Contains(o.Said, "172.16.5.5") || o.Severity != conditions.Warning || o.Machine != "laptop" { + t.Errorf("the evidence or the severity is wrong: %+v", o) + } +} + +func TestOneMachineFailingTowardAHealthyHubIsItsOwn(t *testing.T) { + obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateHealthy), + "laptop": aNetwork(link.StateUnhealthy, tunnelDown), + "spare": aNetwork(link.StateHealthy), + })) + if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" { + t.Fatalf("want the laptop's own, got %v", keys) + } + if obs[0].Severity != conditions.Warning { + t.Fatalf("a laptop's own tunnel is a warning, got %s", obs[0].Severity) + } +} + +func TestTwoMachinesThatCannotReachTheHubAreSaidOnceAtTheHub(t *testing.T) { + obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateHealthy), + "laptop": aNetwork(link.StateUnhealthy, tunnelDown), + "spare": aNetwork(link.StateUnhealthy, tunnelDown), + })) + if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.unreachable" { + t.Fatalf("want one condition at the hub, got %v", keys) + } + o := obs[0] + if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop, spare") || len(o.Also) != 2 { + t.Fatalf("the hub's condition is %+v", o) + } +} + +func TestWhatPointsAtASilentHubIsHeldUnderItsSilence(t *testing.T) { + f := netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, tunnelDown)}) + f.silent["anchor"] = true + if obs := networkObservations(f); len(obs) != 0 { + t.Fatalf("the hub's silence says it; got %v", keysOf(obs)) + } +} + +func TestAHubWhoseOwnNetworkIsUnhealthyListsWhoCannotReachIt(t *testing.T) { + obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateUnhealthy, noRoute), + "laptop": aNetwork(link.StateUnhealthy, tunnelDown), + })) + if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.network" { + t.Fatalf("want the hub's own, holding the laptop's, got %v", keys) + } + if o := obs[0]; o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop cannot reach it") { + t.Fatalf("the hub's condition is %+v", o) + } +} + +func TestOneMachineFailingAHealthyMeshResolverIsItsOwnAndTwoAreTheResolvers(t *testing.T) { + silentResolver := inventory.NetworkPart{Part: link.PartNames, Reason: "1 of its 2 resolvers do not answer as the mesh's do", + Said: "10.77.0.3 — no answer within 1s", Toward: []string{"10.77.0.3"}} + obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateHealthy), "spare": aNetwork(link.StateHealthy), + "laptop": aNetwork(link.StateUnhealthy, silentResolver), + })) + if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" { + t.Fatalf("one machine alone: want its own, got %v", keys) + } + obs = networkObservations(netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateUnhealthy, silentResolver), "spare": aNetwork(link.StateHealthy), + "laptop": aNetwork(link.StateUnhealthy, silentResolver), + })) + if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.spare.unreachable" { + t.Fatalf("two machines: want it said once at the resolver's machine, got %v", keys) + } + if !strings.Contains(obs[0].Summary, "its resolver") { + t.Fatalf("the resolver's machine is said %s", obs[0].Summary) + } +} + +func TestTheControlNodeAndTheBusAreUrgent(t *testing.T) { + f := netFacts(map[string]*inventory.NetworkHealth{"anchor": aNetwork(link.StateUnhealthy, rewrittenByVPN)}) + if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent { + t.Fatalf("the control node's rewritten file: %+v", obs) + } + bus := inventory.NetworkPart{Part: link.PartBus, Reason: "the bus cannot be reached", Said: "no link"} + f = netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, bus, noRoute)}) + if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent { + t.Fatalf("the bus unreachable from the laptop: %+v", obs) + } +} + +func TestAnEngineThatSaysNothingOfItsNetworkRaisesNothing(t *testing.T) { + f := netFacts(map[string]*inventory.NetworkHealth{"laptop": nil, "anchor": aNetwork(link.StateHealthy)}) + if obs := networkObservations(f); len(obs) != 0 { + t.Fatalf("got %v", keysOf(obs)) + } +} + +// The gate: a resolver file another program rewrote waits the judging rather than failing it at the +// bound; the same, when the send moved the module whose file it is, is that module's; a machine's own +// network fault holds the machine as a whole, as before. +func TestTheGateWaitsOnARewriteItDidNotMakeAndHoldsTheOwnerOnOneItMoved(t *testing.T) { + since := h0 + rewrite := conditions.Condition{Key: "machine.laptop.networkmanager.rewritten", Kind: kindNetworkRewritten, + Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop.networkmanager", Machine: "laptop"}, + Summary: "the resolver file was rewritten", Raised: since.Add(time.Minute), Source: sourceNetwork} + f := gateFacts{judged: true, open: []conditions.Condition{rewrite}} + if w := aboutTheMachine("laptop", []string{"letta"}, since, f); w.waiting == "" || w.whole != "" || len(w.on) != 0 { + t.Fatalf("a rewrite the send did not make: %+v", w) + } + if w := aboutTheMachine("laptop", []string{"networkmanager", "letta"}, since, f); w.on["networkmanager"] == "" || + w.on["letta"] != "" || w.waiting != "" { + t.Fatalf("a rewrite of the file a moved module owns: %+v", w) + } + own := conditions.Condition{Key: "machine.laptop.network", Kind: kindMachineNetwork, + Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"}, + Summary: "laptop's network is not healthy", Raised: since.Add(time.Minute), Source: sourceNetwork} + if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true, + open: []conditions.Condition{own}}); w.whole == "" || w.waiting != "" { + t.Fatalf("the machine's own network: %+v", w) + } + unreachable := conditions.Condition{Key: "machine.anchor.unreachable", Kind: kindNetworkUnreachable, + Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor", Also: []string{"laptop", "spare"}}, + Summary: "anchor cannot be reached", Raised: since.Add(time.Minute), Source: sourceNetwork} + if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true, + open: []conditions.Condition{unreachable}}); w.waiting == "" || w.whole != "" { + t.Fatalf("a machine that cannot reach the hub: %+v", w) + } +} diff --git a/cmd/mesh-controller/machine_network_test.go b/cmd/mesh-controller/machine_network_test.go index 20bea9dc..355f384f 100644 --- a/cmd/mesh-controller/machine_network_test.go +++ b/cmd/mesh-controller/machine_network_test.go @@ -1,12 +1,6 @@ package main import ( - "encoding/json" - "os" - "strings" - "testing" - "time" - "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" @@ -19,16 +13,6 @@ import ( // urgent; an engine that says nothing of its network raises nothing; and the gate waits on what is shown to // be another's. -func aNetwork(state string, parts ...inventory.NetworkPart) *inventory.NetworkHealth { - for i := range parts { - if parts[i].State == "" { - parts[i].State = link.StateUnhealthy - } - parts[i].Since = h0 - } - return &inventory.NetworkHealth{State: state, Since: h0, Parts: parts} -} - func netFacts(healths map[string]*inventory.NetworkHealth) networkFacts { f := networkFacts{healths: map[string]inventory.NodeHealth{}, hub: "anchor", control: "anchor", byAddress: map[string]string{"10.77.0.1": "anchor", "10.77.0.2": "laptop", "10.77.0.3": "spare"}, @@ -57,250 +41,11 @@ func keysOf(obs []conditions.Observation) []string { return keys } -func TestAResolverFileRewrittenIsOneFindingNamingItsWriterAndWhatItCosts(t *testing.T) { - obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateHealthy), - "laptop": aNetwork(link.StateUnhealthy, rewrittenByVPN, namesThroughVPN), - })) - if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" { - t.Fatalf("want one finding, the rewrite, got %v", keys) - } - o := obs[0] - for _, want := range []string{"laptop", "rewritten by another program (FortiClient)", "mesh names do not resolve", - "next reconcile"} { - if !strings.Contains(o.Summary, want) { - t.Errorf("the summary does not say %q: %s", want, o.Summary) - } - } - if strings.Contains(o.Summary, "172.16.") || strings.Contains(o.Summary, "/etc/") { - t.Errorf("an address or a path reached the summary: %s", o.Summary) - } - if !strings.Contains(o.Said, "172.16.5.5") || o.Severity != conditions.Warning || o.Machine != "laptop" { - t.Errorf("the evidence or the severity is wrong: %+v", o) - } -} - -func TestOneMachineFailingTowardAHealthyHubIsItsOwn(t *testing.T) { - obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateHealthy), - "laptop": aNetwork(link.StateUnhealthy, tunnelDown), - "spare": aNetwork(link.StateHealthy), - })) - if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" { - t.Fatalf("want the laptop's own, got %v", keys) - } - if obs[0].Severity != conditions.Warning { - t.Fatalf("a laptop's own tunnel is a warning, got %s", obs[0].Severity) - } -} - -func TestTwoMachinesThatCannotReachTheHubAreSaidOnceAtTheHub(t *testing.T) { - obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateHealthy), - "laptop": aNetwork(link.StateUnhealthy, tunnelDown), - "spare": aNetwork(link.StateUnhealthy, tunnelDown), - })) - if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.unreachable" { - t.Fatalf("want one condition at the hub, got %v", keys) - } - o := obs[0] - if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop, spare") || len(o.Also) != 2 { - t.Fatalf("the hub's condition is %+v", o) - } -} - -func TestWhatPointsAtASilentHubIsHeldUnderItsSilence(t *testing.T) { - f := netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, tunnelDown)}) - f.silent["anchor"] = true - if obs := networkObservations(f); len(obs) != 0 { - t.Fatalf("the hub's silence says it; got %v", keysOf(obs)) - } -} - -func TestAHubWhoseOwnNetworkIsUnhealthyListsWhoCannotReachIt(t *testing.T) { - obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateUnhealthy, noRoute), - "laptop": aNetwork(link.StateUnhealthy, tunnelDown), - })) - if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.network" { - t.Fatalf("want the hub's own, holding the laptop's, got %v", keys) - } - if o := obs[0]; o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop cannot reach it") { - t.Fatalf("the hub's condition is %+v", o) - } -} - -func TestOneMachineFailingAHealthyMeshResolverIsItsOwnAndTwoAreTheResolvers(t *testing.T) { - silentResolver := inventory.NetworkPart{Part: link.PartNames, Reason: "1 of its 2 resolvers do not answer as the mesh's do", - Said: "10.77.0.3 — no answer within 1s", Toward: []string{"10.77.0.3"}} - obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateHealthy), "spare": aNetwork(link.StateHealthy), - "laptop": aNetwork(link.StateUnhealthy, silentResolver), - })) - if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" { - t.Fatalf("one machine alone: want its own, got %v", keys) - } - obs = networkObservations(netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateUnhealthy, silentResolver), "spare": aNetwork(link.StateHealthy), - "laptop": aNetwork(link.StateUnhealthy, silentResolver), - })) - if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.spare.unreachable" { - t.Fatalf("two machines: want it said once at the resolver's machine, got %v", keys) - } - if !strings.Contains(obs[0].Summary, "its resolver") { - t.Fatalf("the resolver's machine is said %s", obs[0].Summary) - } -} - -func TestTheControlNodeAndTheBusAreUrgent(t *testing.T) { - f := netFacts(map[string]*inventory.NetworkHealth{"anchor": aNetwork(link.StateUnhealthy, rewrittenByVPN)}) - if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent { - t.Fatalf("the control node's rewritten file: %+v", obs) - } - bus := inventory.NetworkPart{Part: link.PartBus, Reason: "the bus cannot be reached", Said: "no link"} - f = netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, bus, noRoute)}) - if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent { - t.Fatalf("the bus unreachable from the laptop: %+v", obs) - } -} - -func TestAnEngineThatSaysNothingOfItsNetworkRaisesNothing(t *testing.T) { - f := netFacts(map[string]*inventory.NetworkHealth{"laptop": nil, "anchor": aNetwork(link.StateHealthy)}) - if obs := networkObservations(f); len(obs) != 0 { - t.Fatalf("got %v", keysOf(obs)) - } -} - // Through the store and the keeper: the statement kept, the rewrite raised from it, cleared when the file // is written back, and node show saying it. -func TestARewrittenResolverFileIsRaisedFromTheStatementAndClearedWhenWrittenBack(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv, k := open.inventory, conditionsFrom - say := func(at time.Time, n *link.NetworkHealth) { - t.Helper() - if err := stateHealth(ctx, inv, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil { - t.Fatal(err) - } - } - openKeys := func() []string { - t.Helper() - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - var keys []string - for _, c := range list { - keys = append(keys, c.Key) - } - return keys - } - healthy := &link.NetworkHealth{State: link.StateHealthy, Since: h0, Parts: []link.NetworkPart{ - {Part: link.PartResolvConf, State: link.StateHealthy, Since: h0}}} - rewritten := &link.NetworkHealth{State: link.StateUnhealthy, Since: h0.Add(time.Minute), Parts: []link.NetworkPart{ - {Part: link.PartResolvConf, State: link.StateUnhealthy, Reason: rewrittenByVPN.Reason, Said: rewrittenByVPN.Said, - Writer: "FortiClient", Owner: "networkmanager", Since: h0.Add(time.Minute)}, - {Part: link.PartNames, State: link.StateUnhealthy, Reason: namesThroughVPN.Reason, Said: namesThroughVPN.Said, - Toward: namesThroughVPN.Toward, Since: h0.Add(time.Minute)}}} - - say(h0, healthy) - if keys := openKeys(); len(keys) != 0 { - t.Fatalf("a healthy network raised %v", keys) - } - say(h0.Add(time.Minute), rewritten) - if keys := openKeys(); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" { - t.Fatalf("the rewrite raised %v", keys) - } - kept, had, err := inv.HealthOf(ctx, "laptop") - if err != nil || !had || kept.Network == nil || kept.Network.Parts[0].Writer != "FortiClient" { - t.Fatalf("the statement's network was not kept: %+v %v", kept.Network, err) - } - if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "FortiClient") || - !strings.Contains(lines, "unhealthy resolv-conf") { - t.Fatalf("node show says:\n%s", lines) - } - say(h0.Add(2*time.Minute), healthy) - if keys := openKeys(); len(keys) != 0 { - t.Fatalf("written back, still open: %v", keys) - } - // An engine older than the judging says no network: nothing raised, and node show says it is not known. - say(h0.Add(3*time.Minute), nil) - kept, had, _ = inv.HealthOf(ctx, "laptop") - if keys := openKeys(); len(keys) != 0 || kept.Network != nil { - t.Fatalf("an older engine: %v %+v", keys, kept.Network) - } - if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "older than that judging") { - t.Fatalf("node show says:\n%s", lines) - } -} - -// The gate: a resolver file another program rewrote waits the judging rather than failing it at the -// bound; the same, when the send moved the module whose file it is, is that module's; a machine's own -// network fault holds the machine as a whole, as before. -func TestTheGateWaitsOnARewriteItDidNotMakeAndHoldsTheOwnerOnOneItMoved(t *testing.T) { - since := h0 - rewrite := conditions.Condition{Key: "machine.laptop.networkmanager.rewritten", Kind: kindNetworkRewritten, - Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop.networkmanager", Machine: "laptop"}, - Summary: "the resolver file was rewritten", Raised: since.Add(time.Minute), Source: sourceNetwork} - f := gateFacts{judged: true, open: []conditions.Condition{rewrite}} - if w := aboutTheMachine("laptop", []string{"letta"}, since, f); w.waiting == "" || w.whole != "" || len(w.on) != 0 { - t.Fatalf("a rewrite the send did not make: %+v", w) - } - if w := aboutTheMachine("laptop", []string{"networkmanager", "letta"}, since, f); w.on["networkmanager"] == "" || - w.on["letta"] != "" || w.waiting != "" { - t.Fatalf("a rewrite of the file a moved module owns: %+v", w) - } - own := conditions.Condition{Key: "machine.laptop.network", Kind: kindMachineNetwork, - Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"}, - Summary: "laptop's network is not healthy", Raised: since.Add(time.Minute), Source: sourceNetwork} - if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true, - open: []conditions.Condition{own}}); w.whole == "" || w.waiting != "" { - t.Fatalf("the machine's own network: %+v", w) - } - unreachable := conditions.Condition{Key: "machine.anchor.unreachable", Kind: kindNetworkUnreachable, - Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor", Also: []string{"laptop", "spare"}}, - Summary: "anchor cannot be reached", Raised: since.Add(time.Minute), Source: sourceNetwork} - if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true, - open: []conditions.Condition{unreachable}}); w.waiting == "" || w.whole != "" { - t.Fatalf("a machine that cannot reach the hub: %+v", w) - } -} // TestTheDrillsStatementsRaiseAndClearTheRewrite replays the drill of ADR 0241 (mesh-host // internal/network TestDrill…): what a node-engine said in a throwaway container while its resolver file // was declared, rewritten as a VPN client rewrites it, and written back — recorded, with the mesh's names // and addresses replaced by this test mesh's. Healthy raises nothing; the rewrite, on its second look, is // raised as one finding naming the writer; written back, it clears. -func TestTheDrillsStatementsRaiseAndClearTheRewrite(t *testing.T) { - raw, err := os.ReadFile("testdata/network-drill.json") - if err != nil { - t.Fatal(err) - } - var said []link.Health - if err := json.Unmarshal(raw, &said); err != nil { - t.Fatal(err) - } - open := aMesh(t) - ctx := t.Context() - k := conditionsFrom - var raisedAt []int - for i, h := range said { - if err := stateHealth(ctx, open.inventory, k, "laptop", h, h.At); err != nil { - t.Fatal(err) - } - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - for _, c := range list { - if c.Key != "machine.laptop.networkmanager.rewritten" || !strings.Contains(c.Summary, "(FortiClient)") { - t.Fatalf("statement %d raised %s: %s", i, c.Key, c.Summary) - } - raisedAt = append(raisedAt, i) - } - } - // Five statements: healthy, healthy, one failing look (still healthy), unhealthy, written back. - if len(raisedAt) != 1 || raisedAt[0] != 3 { - t.Fatalf("the rewrite was open after statements %v; want after the fourth alone", raisedAt) - } -} diff --git a/cmd/mesh-controller/machine_units_integration_test.go b/cmd/mesh-controller/machine_units_integration_test.go new file mode 100644 index 00000000..fd9017e0 --- /dev/null +++ b/cmd/mesh-controller/machine_units_integration_test.go @@ -0,0 +1,125 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/link" +) + +func TestAFailedUnitIsTheModulesConditionAndTheMachinesOwnAreOneFinding(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv, k := open.inventory, conditionsFrom + openConditions := func() map[string]conditions.Condition { + t.Helper() + list, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + out := map[string]conditions.Condition{} + for _, c := range list { + out[c.Key] = c + } + return out + } + say := func(at time.Time, h link.Health) { + t.Helper() + if err := stateHealth(ctx, inv, k, "laptop", h, at); err != nil { + t.Fatal(err) + } + } + + say(h0, desktop(h0)) + got := openConditions() + machine, raised := got["machine.laptop.units"] + if !raised || len(got) != 1 { + t.Fatalf("the machine's own failed units are one finding at once (the engine looked twice): %v", got) + } + for _, unit := range []string{"mnt-recalbox.mount", "storage-media.mount", "greenclip.service"} { + if !strings.Contains(machine.Summary, unit) { + t.Errorf("the finding names %s: %s", unit, machine.Summary) + } + } + if strings.Contains(machine.Summary, "openrazer") || machine.Severity != conditions.Warning { + t.Errorf("the module's unit is the module's, and the machine's a warning: %+v", machine) + } + + say(h0.Add(time.Minute), desktop(h0.Add(time.Minute))) + got = openConditions() + module, raised := got["module.openrazer.laptop.unhealthy"] + if !raised || !strings.Contains(module.Summary, "openrazer-daemon.service") || + !strings.Contains(module.Summary, "account's own service manager") { + t.Fatalf("the module's failed unit raises its condition on the second statement, naming the unit: %+v", got) + } + + kept, had, err := inv.HealthOf(ctx, "laptop") + if err != nil || !had || kept.Units == nil || len(kept.Units.Failed) != 3 { + t.Fatalf("the statement's units were not kept: %+v %v", kept.Units, err) + } + lines := strings.Join(unitsLines(kept, had, h0), "\n") + for _, want := range []string{"degraded", "storage-media.mount (system) — no module places it", + "openrazer-daemon.service — openrazer's"} { + if !strings.Contains(lines, want) { + t.Errorf("node show does not say %q:\n%s", want, lines) + } + } + + // An older engine says nothing of its units: nothing cleared on its word. + older := desktop(h0.Add(2 * time.Minute)) + older.Units = nil + say(h0.Add(2*time.Minute), older) + if _, still := openConditions()["machine.laptop.units"]; !still { + t.Fatal("a statement saying nothing of units cleared the machine's finding") + } + + // The operator mends the mounts and removes the leftover link; the module's unit still fails. + mended := desktop(h0.Add(3 * time.Minute)) + mended.Units.Failed = []link.FailedUnit{} + say(h0.Add(3*time.Minute), mended) + got = openConditions() + if _, still := got["machine.laptop.units"]; still { + t.Fatalf("no failed unit of the machine's own, still open: %v", got) + } + if _, still := got["module.openrazer.laptop.unhealthy"]; !still { + t.Fatalf("the module's unit still fails, and its condition stands: %v", got) + } + + // And the module's unit mended: running, nothing open. + well := link.Health{Contract: link.ReadinessContract, At: h0.Add(4 * time.Minute), + Units: &link.UnitsHealth{State: link.UnitsRunning, Failed: []link.FailedUnit{}}} + say(h0.Add(4*time.Minute), well) + if got = openConditions(); len(got) != 0 { + t.Fatalf("a running service manager leaves nothing open: %v", got) + } +} + +// desktop is the workstation as found on 2026-10-08, with the mesh's names: the Razer daemon's packaged +// user unit, the module's; two mounts of the machine's own table and a unit a removed package left +// behind, the machine's. +func desktop(at time.Time) link.Health { + return link.Health{Contract: link.ReadinessContract, At: at, + Resources: []link.ResourceHealth{{Module: "openrazer", Resource: "openrazer.daemon", Kind: link.KindUnit, + Target: "openrazer-daemon.service", State: link.StateUnhealthy, + Reason: "failed in the account's own service manager (exit-code)", Since: h0, Streak: 2}}, + Units: &link.UnitsHealth{State: link.UnitsDegraded, Failed: []link.FailedUnit{ + {Unit: "mnt-recalbox.mount", Scope: "system", Load: "loaded", Result: "exit-code", Since: h0}, + {Unit: "storage-media.mount", Scope: "system", Load: "loaded", Result: "timeout", Since: h0}, + {Unit: "greenclip.service", Scope: "user", Load: "not-found", Result: "start-limit-hit", Since: h0}, + }}} +} + +func TestTheGateNeverHoldsASendOnTheMachinesOwnUnits(t *testing.T) { + since := h0 + units := conditions.Condition{Key: "machine.laptop.units", Kind: kindMachineUnits, + Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"}, + Summary: "laptop's service manager is degraded", Raised: since.Add(time.Minute), Source: sourceUnits} + if w := aboutTheMachine("laptop", []string{"openrazer"}, since, gateFacts{judged: true, + open: []conditions.Condition{units}}); w.whole != "" || w.waiting != "" || len(w.on) != 0 { + t.Fatalf("the machine's own failed units held a send: %+v", w) + } +} diff --git a/cmd/mesh-controller/machine_units_test.go b/cmd/mesh-controller/machine_units_test.go index 162a7ffa..e6e8146f 100644 --- a/cmd/mesh-controller/machine_units_test.go +++ b/cmd/mesh-controller/machine_units_test.go @@ -1,128 +1,6 @@ package main -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/link" -) - // A failed unit is never silent (novox/hq issue 315, "how it is checked"): a module's failed unit raises // the module's condition on the second statement, naming the unit; the machine's own failed units raise // one finding for the machine, listing them, cleared when none is listed; an engine that says nothing of // its units raises and clears nothing; and the gate never holds a send on the machine's own. - -// desktop is the workstation as found on 2026-10-08, with the mesh's names: the Razer daemon's packaged -// user unit, the module's; two mounts of the machine's own table and a unit a removed package left -// behind, the machine's. -func desktop(at time.Time) link.Health { - return link.Health{Contract: link.ReadinessContract, At: at, - Resources: []link.ResourceHealth{{Module: "openrazer", Resource: "openrazer.daemon", Kind: link.KindUnit, - Target: "openrazer-daemon.service", State: link.StateUnhealthy, - Reason: "failed in the account's own service manager (exit-code)", Since: h0, Streak: 2}}, - Units: &link.UnitsHealth{State: link.UnitsDegraded, Failed: []link.FailedUnit{ - {Unit: "mnt-recalbox.mount", Scope: "system", Load: "loaded", Result: "exit-code", Since: h0}, - {Unit: "storage-media.mount", Scope: "system", Load: "loaded", Result: "timeout", Since: h0}, - {Unit: "greenclip.service", Scope: "user", Load: "not-found", Result: "start-limit-hit", Since: h0}, - }}} -} - -func TestAFailedUnitIsTheModulesConditionAndTheMachinesOwnAreOneFinding(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv, k := open.inventory, conditionsFrom - openConditions := func() map[string]conditions.Condition { - t.Helper() - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - out := map[string]conditions.Condition{} - for _, c := range list { - out[c.Key] = c - } - return out - } - say := func(at time.Time, h link.Health) { - t.Helper() - if err := stateHealth(ctx, inv, k, "laptop", h, at); err != nil { - t.Fatal(err) - } - } - - say(h0, desktop(h0)) - got := openConditions() - machine, raised := got["machine.laptop.units"] - if !raised || len(got) != 1 { - t.Fatalf("the machine's own failed units are one finding at once (the engine looked twice): %v", got) - } - for _, unit := range []string{"mnt-recalbox.mount", "storage-media.mount", "greenclip.service"} { - if !strings.Contains(machine.Summary, unit) { - t.Errorf("the finding names %s: %s", unit, machine.Summary) - } - } - if strings.Contains(machine.Summary, "openrazer") || machine.Severity != conditions.Warning { - t.Errorf("the module's unit is the module's, and the machine's a warning: %+v", machine) - } - - say(h0.Add(time.Minute), desktop(h0.Add(time.Minute))) - got = openConditions() - module, raised := got["module.openrazer.laptop.unhealthy"] - if !raised || !strings.Contains(module.Summary, "openrazer-daemon.service") || - !strings.Contains(module.Summary, "account's own service manager") { - t.Fatalf("the module's failed unit raises its condition on the second statement, naming the unit: %+v", got) - } - - kept, had, err := inv.HealthOf(ctx, "laptop") - if err != nil || !had || kept.Units == nil || len(kept.Units.Failed) != 3 { - t.Fatalf("the statement's units were not kept: %+v %v", kept.Units, err) - } - lines := strings.Join(unitsLines(kept, had, h0), "\n") - for _, want := range []string{"degraded", "storage-media.mount (system) — no module places it", - "openrazer-daemon.service — openrazer's"} { - if !strings.Contains(lines, want) { - t.Errorf("node show does not say %q:\n%s", want, lines) - } - } - - // An older engine says nothing of its units: nothing cleared on its word. - older := desktop(h0.Add(2 * time.Minute)) - older.Units = nil - say(h0.Add(2*time.Minute), older) - if _, still := openConditions()["machine.laptop.units"]; !still { - t.Fatal("a statement saying nothing of units cleared the machine's finding") - } - - // The operator mends the mounts and removes the leftover link; the module's unit still fails. - mended := desktop(h0.Add(3 * time.Minute)) - mended.Units.Failed = []link.FailedUnit{} - say(h0.Add(3*time.Minute), mended) - got = openConditions() - if _, still := got["machine.laptop.units"]; still { - t.Fatalf("no failed unit of the machine's own, still open: %v", got) - } - if _, still := got["module.openrazer.laptop.unhealthy"]; !still { - t.Fatalf("the module's unit still fails, and its condition stands: %v", got) - } - - // And the module's unit mended: running, nothing open. - well := link.Health{Contract: link.ReadinessContract, At: h0.Add(4 * time.Minute), - Units: &link.UnitsHealth{State: link.UnitsRunning, Failed: []link.FailedUnit{}}} - say(h0.Add(4*time.Minute), well) - if got = openConditions(); len(got) != 0 { - t.Fatalf("a running service manager leaves nothing open: %v", got) - } -} - -func TestTheGateNeverHoldsASendOnTheMachinesOwnUnits(t *testing.T) { - since := h0 - units := conditions.Condition{Key: "machine.laptop.units", Kind: kindMachineUnits, - Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"}, - Summary: "laptop's service manager is degraded", Raised: since.Add(time.Minute), Source: sourceUnits} - if w := aboutTheMachine("laptop", []string{"openrazer"}, since, gateFacts{judged: true, - open: []conditions.Condition{units}}); w.whole != "" || w.waiting != "" || len(w.on) != 0 { - t.Fatalf("the machine's own failed units held a send: %+v", w) - } -} diff --git a/cmd/mesh-controller/merge_check_nested_test.go b/cmd/mesh-controller/merge_check_nested_test.go index e8888954..7dd78da6 100644 --- a/cmd/mesh-controller/merge_check_nested_test.go +++ b/cmd/mesh-controller/merge_check_nested_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/merge_gate_handing_integration_test.go b/cmd/mesh-controller/merge_gate_handing_integration_test.go new file mode 100644 index 00000000..74720f4c --- /dev/null +++ b/cmd/mesh-controller/merge_gate_handing_integration_test.go @@ -0,0 +1,63 @@ +//go:build integration + +package main + +import ( + "os" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestIssue969TheGateRaisesEachMachineAsItsEngineStatedItself(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + said := time.Date(2026, 10, 16, 12, 0, 0, 0, time.UTC) + for node, contract := range map[string]int{"anchor": link.HandedContract, "laptop": link.HandedContract - 1} { + if _, err := open.inventory.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: contract, + SaidAt: said, HeardAt: said}); err != nil { + t.Fatal(err) + } + } + f, err := gatherFacts(ctx, open, "2.11.17") + if err != nil { + t.Fatal(err) + } + if len(f.Machines) != 2 { + t.Fatalf("the snapshot holds %d machines", len(f.Machines)) + } + want := map[string]bool{} + for _, m := range f.Machines { + if m.Contract == 0 { + t.Fatalf("%s: the snapshot carries no contract", m.Name) + } + want[m.Name] = m.Contract >= link.HandedContract + } + handing := 0 + for _, hands := range want { + if hands { + handing++ + } + } + if handing != 1 { + t.Fatalf("one machine takes handed files in the snapshot, got %v", want) + } + + raised, drop, err := throwawayStores(ctx, os.Getenv("MESH_TEST_POSTGRES")) + if err != nil { + t.Fatal(err) + } + defer drop() + if _, err := raiseFromFacts(ctx, raised, f, map[string]catalogue.Manifest{}, map[string]inventory.Source{}); err != nil { + t.Fatal(err) + } + for name, hands := range want { + got, err := engineTakesHanded(ctx, raised.inventory, name) + if err != nil || got != hands { + t.Errorf("%s: the gate raises it as taking handed files = %v (%v), the mesh says %v", name, got, err, hands) + } + } +} diff --git a/cmd/mesh-controller/merge_gate_handing_test.go b/cmd/mesh-controller/merge_gate_handing_test.go index a5840284..4dfa0c91 100644 --- a/cmd/mesh-controller/merge_gate_handing_test.go +++ b/cmd/mesh-controller/merge_gate_handing_test.go @@ -1,13 +1,10 @@ package main import ( - "os" "testing" "time" - "github.com/novox/mesh-controller/internal/catalogue" snapshot "github.com/novox/mesh-controller/internal/facts" - "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -34,52 +31,3 @@ func TestIssue969TheGateGivesAMachineItsContractSoItTakesHandedFiles(t *testing. // gate raises it as taking handed files, and one whose statement is below it is raised as not taking them. Run in // the merge check with the other tests that need a store; the composition itself, handed here and refused there, is // the catalogue's TestAHandedModuleNamingARunTimeBoundKeyComposesOnlyWhereItIsHanded. -func TestIssue969TheGateRaisesEachMachineAsItsEngineStatedItself(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - said := time.Date(2026, 10, 16, 12, 0, 0, 0, time.UTC) - for node, contract := range map[string]int{"anchor": link.HandedContract, "laptop": link.HandedContract - 1} { - if _, err := open.inventory.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: contract, - SaidAt: said, HeardAt: said}); err != nil { - t.Fatal(err) - } - } - f, err := gatherFacts(ctx, open, "2.11.17") - if err != nil { - t.Fatal(err) - } - if len(f.Machines) != 2 { - t.Fatalf("the snapshot holds %d machines", len(f.Machines)) - } - want := map[string]bool{} - for _, m := range f.Machines { - if m.Contract == 0 { - t.Fatalf("%s: the snapshot carries no contract", m.Name) - } - want[m.Name] = m.Contract >= link.HandedContract - } - handing := 0 - for _, hands := range want { - if hands { - handing++ - } - } - if handing != 1 { - t.Fatalf("one machine takes handed files in the snapshot, got %v", want) - } - - raised, drop, err := throwawayStores(ctx, os.Getenv("MESH_TEST_POSTGRES")) - if err != nil { - t.Fatal(err) - } - defer drop() - if _, err := raiseFromFacts(ctx, raised, f, map[string]catalogue.Manifest{}, map[string]inventory.Source{}); err != nil { - t.Fatal(err) - } - for name, hands := range want { - got, err := engineTakesHanded(ctx, raised.inventory, name) - if err != nil || got != hands { - t.Errorf("%s: the gate raises it as taking handed files = %v (%v), the mesh says %v", name, got, err, hands) - } - } -} diff --git a/cmd/mesh-controller/merge_gate_integration_test.go b/cmd/mesh-controller/merge_gate_integration_test.go new file mode 100644 index 00000000..57ade01d --- /dev/null +++ b/cmd/mesh-controller/merge_gate_integration_test.go @@ -0,0 +1,528 @@ +//go:build integration + +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + snapshot "github.com/novox/mesh-controller/internal/facts" + "github.com/novox/mesh-controller/internal/inventory" +) + +func catalogueMesh(t *testing.T) (snapshot.Facts, map[string]string) { + t.Helper() + open := aMesh(t) + ctx := t.Context() + manifests := map[string]string{ + "objects": `{"module":"objects","version":"1", + "provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}], + "receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`, + "resolver": `{"module":"resolver","version":"1", + "provides":[{"name":"wildcard-resolution","scope":"mesh","identity":false}]}`, + "networkmanager": `{"module":"networkmanager","version":"1","requires":["wildcard-resolution"]}`, + "album": `{"module":"album","version":"1","requires":["s3-bucket"]}`, + "lemurs": `{"module":"lemurs","version":"1","capabilities":["systemd"], + "resources":[{"id":"service","type":"service","unit":"lemurs.service","state":"running","boot":"enabled"}]}`, + } + for name, raw := range manifests { + m, err := catalogue.ParseManifest([]byte(raw)) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", + Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil { + t.Fatal(err) + } + } + for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"}, {"laptop", "networkmanager"}, + {"laptop", "album"}, {"laptop", "lemurs"}} { + if _, err := assign(ctx, open, a[0], a[1]); err != nil { + t.Fatalf("assign %s %s: %v", a[0], a[1], err) + } + } + f, err := gatherFacts(ctx, open, "2.11.17") + if err != nil { + t.Fatal(err) + } + return f, manifests +} + +func gateJudged(t *testing.T, f snapshot.Facts, tree string, changed ...string) mergeVerdict { + t.Helper() + admin := os.Getenv("MESH_TEST_POSTGRES") + in := mergeCheckInput{facts: f, admin: admin, changed: changed} + if tree != "" { + in.repository, in.tree = "novox/mesh-catalog", tree + } + v, err := judgeChange(t.Context(), in) + if err != nil { + t.Fatal(err) + } + return v +} + +func TestTheMeshAsItIsPassesTheGate(t *testing.T) { + f, manifests := catalogueMesh(t) + for _, m := range f.Machines { + if !m.Declaration.Composes { + t.Fatalf("the mesh itself does not compose: %+v", m.Declaration) + } + } + v := gateJudged(t, f, aTree(t, manifests)) + if v.Verdict != "pass" { + t.Fatalf("an unchanged catalogue does not pass:\n%s", v.Report()) + } + for _, m := range v.Machines { + if !m.Base.Composes || !m.Change.Composes { + t.Errorf("%s does not compose in the gate's store as it does on the mesh: %+v / %+v", m.Described, m.Base, m.Change) + } + } +} + +func TestIssue263AnIdentityARealMachineNameOverflowsFailsThePullRequest(t *testing.T) { + f, manifests := catalogueMesh(t) + tree := aTree(t, withEdit(manifests, "networkmanager", + `{"module":"networkmanager","version":"1","requires":["wildcard-resolution","s3-bucket"]}`)) + v := gateJudged(t, f, tree) + if v.Verdict != "fail" { + t.Fatalf("the overflow passed the gate:\n%s", v.Report()) + } + report := v.Report() + if !strings.Contains(report, "networkmanager") || !strings.Contains(report, "s3-bucket") { + t.Errorf("the refusal does not name the module and the provision:\n%s", report) + } +} + +func TestIssue236AManifestTheNodeEngineRefusesFailsThePullRequest(t *testing.T) { + f, manifests := catalogueMesh(t) + broken := `{"module":"lemurs","version":"1","capabilities":["systemd"], + "resources":[{"id":"service","type":"service","unit":"lemurs.service","boot":"enabled"}]}` + v := gateJudged(t, f, aTree(t, withEdit(manifests, "lemurs", broken))) + if v.Verdict != "fail" || !strings.Contains(v.Report(), "lemurs") { + t.Fatalf("a service the node-engine refuses passed the gate:\n%s", v.Report()) + } + laptop := snapshot.Pseudonym("machine", "laptop") + if !strings.Contains(v.Report(), laptop) { + t.Errorf("the refusal does not name the machine it would be refused on:\n%s", v.Report()) + } + + // And as a new module, which no machine runs: tried on one that could. + newcomer := strings.ReplaceAll(broken, `"lemurs"`, `"greeter"`) + newcomer = strings.ReplaceAll(newcomer, "lemurs.service", "greeter.service") + v = gateJudged(t, f, aTree(t, withEdit(manifests, "greeter", newcomer))) + if v.Verdict != "fail" || !strings.Contains(v.Report(), "greeter, assigned to") { + t.Fatalf("a new module the node-engine would refuse passed the gate:\n%s", v.Report()) + } +} + +func TestAManifestTheRunningControllerCannotReadFailsThePullRequest(t *testing.T) { + f, manifests := catalogueMesh(t) + v := gateJudged(t, f, aTree(t, withEdit(manifests, "album", + `{"module":"album","version":"1","requires":["s3-bucket"],"a-field-of-a-newer-controller":true}`))) + if v.Verdict != "fail" || !strings.Contains(v.Report(), "refuses it") { + t.Fatalf("a manifest this controller cannot read passed:\n%s", v.Report()) + } +} + +func TestAModuleAMachineRunsRemovedFromItsSourceFails(t *testing.T) { + f, manifests := catalogueMesh(t) + v := gateJudged(t, f, aTree(t, withEdit(manifests, "album", ""))) + if v.Verdict != "fail" || !strings.Contains(v.Report(), "album is removed") { + t.Fatalf("removing a module a machine runs passed:\n%s", v.Report()) + } +} + +func TestIssue278AWideRebuildIsSaidBeforeTheMerge(t *testing.T) { + f, manifests := catalogueMesh(t) + was := wideRebuild + wideRebuild = 2 + t.Cleanup(func() { wideRebuild = was }) + for _, file := range []string{"modules/showcase/index.ts", "merge-check.sh", "README.md"} { + v := gateJudged(t, f, aTree(t, manifests), file) + if v.Width == nil || len(v.Width.Modules) != 0 || len(v.Width.Unread) != 1 || v.Verdict != "pass" { + t.Fatalf("%s, read by no build, reads %+v, %s", file, v.Width, v.Verdict) + } + if !strings.Contains(v.Report(), "read by no module's build") { + t.Errorf("why %s rebuilds nothing is not said:\n%s", file, v.Report()) + } + } + v := gateJudged(t, f, aTree(t, manifests), "modules/album/x.ts", "modules/objects/x.go", "modules/resolver/x.go") + if v.Width == nil || len(v.Width.Modules) < 3 || v.Verdict != "warning" { + t.Fatalf("a rebuild wider than the bound is not warned of: %+v, %s", v.Width, v.Verdict) + } + // With the reference module's definition in the tree, its directory is a module, held or not. + withShowcase := withEdit(manifests, "showcase", `{"module":"showcase","version":"1"}`) + v = gateJudged(t, f, aTree(t, withShowcase), "modules/showcase/index.ts") + if v.Width == nil || len(v.Width.Modules) != 0 || len(v.Width.Unread) != 0 { + t.Fatalf("a file of a module nobody holds reads %+v", v.Width) + } + v = gateJudged(t, f, aTree(t, manifests), "modules/album/module.json") + if v.Width == nil || strings.Join(v.Width.Modules, ",") != "album" || v.Verdict != "pass" { + t.Fatalf("a change to one module's directory reads %+v, %s", v.Width, v.Verdict) + } +} + +func TestADeliveryGroupsHeadsAreComposedTogether(t *testing.T) { + f, manifests := catalogueMeshWithAnApp(t) + needsTheApp := withEdit(manifests, "album", `{"module":"album","version":"1","requires":["s3-bucket","app-api"]}`) + catTree := aTree(t, needsTheApp) + alone := gateJudged(t, f, catTree, "modules/album/module.json") + if alone.Verdict != "fail" { + t.Fatalf("a requirement nothing provides passed alone:\n%s", alone.Report()) + } + app := t.TempDir() + if err := os.WriteFile(filepath.Join(app, "module.json"), []byte(`{"module":"app","version":"1", + "provides":[{"name":"app-api","scope":"mesh","identity":false}]}`), 0o644); err != nil { + t.Fatal(err) + } + in := mergeCheckInput{facts: f, admin: os.Getenv("MESH_TEST_POSTGRES"), repository: "novox/mesh-catalog", + tree: catTree, changed: []string{"modules/album/module.json"}, + group: []groupChange{{Repository: "novox/app", Tree: app, Changed: []string{"module.json"}}}} + together, err := judgeChange(t.Context(), in) + if err != nil { + t.Fatal(err) + } + if together.Verdict == "fail" { + t.Fatalf("the group composed together fails:\n%s", together.Report()) + } + if together.Modules["app"] != "changed" || together.Modules["album"] != "changed" { + t.Fatalf("the group's heads were not both laid over the mesh: %v", together.Modules) + } +} + +func catalogueMeshWithAnApp(t *testing.T) (snapshot.Facts, map[string]string) { + t.Helper() + open := aMesh(t) + ctx := t.Context() + manifests := map[string]string{ + "objects": `{"module":"objects","version":"1", + "provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}], + "receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`, + "album": `{"module":"album","version":"1","requires":["s3-bucket"]}`, + } + for name, raw := range manifests { + m, err := catalogue.ParseManifest([]byte(raw)) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", + Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil { + t.Fatal(err) + } + } + if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, + inventory.Source{Repository: "novox/app", BuiltFrom: "c0ffee"}); err != nil { + t.Fatal(err) + } + for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "app"}, {"laptop", "album"}} { + if _, err := assign(ctx, open, a[0], a[1]); err != nil { + t.Fatalf("assign %s %s: %v", a[0], a[1], err) + } + } + f, err := gatherFacts(ctx, open, "2.11.17") + if err != nil { + t.Fatal(err) + } + return f, manifests +} + +func busMesh(t *testing.T) snapshot.Facts { + t.Helper() + open := aMesh(t) + ctx := t.Context() + m, err := catalogue.ParseManifest([]byte(`{"module":"speaker","version":"1", + "own-secrets":{"broker":"/var/lib/speaker/broker"}}`)) + if err != nil { + t.Fatal(err) + } + if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", + Path: "modules/speaker", BuiltFrom: "c0ffee"}); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "laptop", "speaker"); err != nil { + t.Fatal(err) + } + user := broker.Principal{Kind: broker.KindModule, Node: "laptop", Module: "speaker"}.Username() + password, err := open.inventory.MintBusPassword(ctx, inventory.BusUser{Username: user, Kind: inventory.BusModule, + Node: "laptop", Module: "speaker"}) + if err != nil { + t.Fatal(err) + } + if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "speaker", "broker", + `{"user":"`+user+`","password":"`+password+`"}`); err != nil { + t.Fatal(err) + } + f, err := gatherFacts(ctx, open, "2.11.17") + if err != nil { + t.Fatal(err) + } + for _, mc := range f.Machines { + if !mc.Declaration.Composes { + t.Fatalf("the mesh itself does not compose: %+v", mc.Declaration) + } + } + return f +} + +func TestIssue285AModuleOnTheBusComposesInTheGate(t *testing.T) { + f := busMesh(t) + v := gateJudged(t, f, "") + if v.Verdict != "pass" { + t.Fatalf("the mesh as it is does not pass:\n%s", v.Report()) + } + for _, m := range v.Machines { + if !m.Base.Composes { + t.Errorf("%s composes on the mesh and not in the gate: %v", m.Described, m.Base.Problems) + } + } + if !strings.Contains(v.Summary, "2 of 2 compose") { + t.Errorf("the summary does not say every machine composes: %s", v.Summary) + } +} + +func TestIssue285AMachineTheGateCannotRaiseIsAnErrorNeverAPass(t *testing.T) { + f := busMesh(t) + for i := range f.Machines { + // A fact the snapshot does not carry: the module's credential, gone from the laptop's. + var kept []snapshot.Accepted + for _, a := range f.Machines[i].Accepted { + if a.Name != "broker" { + kept = append(kept, a) + } + } + f.Machines[i].Accepted = kept + } + v := gateJudged(t, f, "") + if v.Verdict != "error" { + t.Fatalf("a gate whose mesh does not compose said %s:\n%s", v.Verdict, v.Report()) + } + if len(v.Errors) != 1 || !strings.Contains(v.Errors[0], "speaker") { + t.Errorf("the error does not name what could not be raised: %v", v.Errors) + } +} + +func TestAGroupsMoveOfAModuleIsNeitherTwoDefinitionsNorARemoval(t *testing.T) { + const catalogueRepo, controllerRepo = "novox/mesh-catalog", "novox/mesh-controller" + built := func(repo, path string) inventory.Source { + return inventory.Source{Repository: "http://forge.internal:20000/" + repo + ".git", Path: path, BuiltFrom: "c0ffee"} + } + sources := map[string]inventory.Source{"mover": built(catalogueRepo, "modules/mover")} + raw := `{"module":"mover","version":"1"}` + base := map[string]catalogue.Manifest{} + m, err := catalogue.ParseManifest([]byte(raw)) + if err != nil { + t.Fatal(err) + } + base["mover"] = m + facts := snapshot.Facts{Machines: []snapshot.Machine{{Name: "anchor", Assigned: []string{"mover"}}}} + + catalogueWithout := aTree(t, map[string]string{"other": `{"module":"other","version":"1"}`}) + controllerWith := func() string { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "modules", "mover"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "modules", "mover", "module.json"), []byte(raw), 0o644); err != nil { + t.Fatal(err) + } + return dir + }() + removes := groupChange{Repository: catalogueRepo, Tree: catalogueWithout, Changed: []string{"modules/mover/module.json"}} + defines := groupChange{Repository: controllerRepo, Tree: controllerWith, Changed: []string{"modules/mover/module.json"}} + + lay := func(heads ...groupChange) (map[string]catalogue.Manifest, []string, map[string]string) { + t.Helper() + moves, err := movesInGroup(heads, sources) + if err != nil { + t.Fatal(err) + } + at := sources + if len(moves) > 0 { + at = map[string]inventory.Source{} + for k, v := range sources { + at[k] = v + } + for name, mv := range moves { + o := at[name] + o.Repository, o.Path = mv.To, mv.Dir + at[name] = o + } + } + shelf, said := base, map[string]string{} + var failures []string + for _, h := range heads { + var f []string + if shelf, f, err = shelfWithChange(shelf, at, facts, h.Repository, h.Tree, said, nil); err != nil { + t.Fatal(err) + } + failures = append(failures, f...) + } + for name := range moves { + said[name] = "moved" + } + return shelf, failures, said + } + for _, order := range [][]groupChange{{removes, defines}, {defines, removes}} { + shelf, failures, said := lay(order...) + if len(failures) > 0 { + t.Fatalf("a move of a module machines run, laid %s first, was refused: %v", order[0].Repository, failures) + } + if said["mover"] != "moved" { + t.Errorf("the move reads %v, not moved", said) + } + if _, held := shelf["mover"]; !held { + t.Errorf("the moved module is not on the shelf after laying %s first", order[0].Repository) + } + } + // Alone, a head is still refused: a second definition (issue 239), and a removal of what runs (ADR 0236). + if _, failures, _ := lay(defines); len(failures) != 1 || !strings.Contains(failures[0], "issue 239") { + t.Errorf("a definition of a module another repository builds, alone: %v", failures) + } + if _, failures, _ := lay(removes); len(failures) != 1 || !strings.Contains(failures[0], "ADR 0236") { + t.Errorf("the removal of a module machines run, alone: %v", failures) + } +} + +func TestADeliveryGroupMovingAModuleMachinesRunPassesTheGate(t *testing.T) { + f, manifests := catalogueMeshWithAnApp(t) + catTree := aTree(t, withEdit(manifests, "album", "")) + app := t.TempDir() + // The app module is built from this repository's root, and stays there: only album moves. + if err := os.WriteFile(filepath.Join(app, "module.json"), []byte(`{"module":"app","version":"1"}`), 0o644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(app, "modules", "album"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(app, "modules", "album", "module.json"), []byte(manifests["album"]), 0o644); err != nil { + t.Fatal(err) + } + removes := groupChange{Repository: "novox/mesh-catalog", Tree: catTree, Changed: []string{"modules/album/module.json"}} + defines := groupChange{Repository: "novox/app", Tree: app, Changed: []string{"modules/album/module.json"}} + judge := func(first groupChange, rest ...groupChange) mergeVerdict { + v, err := judgeChange(t.Context(), mergeCheckInput{facts: f, admin: os.Getenv("MESH_TEST_POSTGRES"), + repository: first.Repository, tree: first.Tree, changed: first.Changed, group: rest}) + if err != nil { + t.Fatal(err) + } + return v + } + for _, v := range []mergeVerdict{judge(removes, defines), judge(defines, removes)} { + if v.Verdict == "fail" || v.Modules["album"] != "moved" { + t.Fatalf("a group moving a module machines run: %v\n%s", v.Modules, v.Report()) + } + } + if v := judge(defines); v.Verdict != "fail" { + t.Errorf("a definition of a module another repository builds passed alone:\n%s", v.Report()) + } + if v := judge(removes); v.Verdict != "fail" { + t.Errorf("the removal of a module machines run passed alone:\n%s", v.Report()) + } +} + +func TestTheGateRefusesAnArtifactContextNamingAnotherRepository(t *testing.T) { + withContext := func(repo string) string { + return `{"module":"proxy","version":"1","build":{"artifacts":[{"name":"server","kind":"image","from":"Dockerfile",` + + `"context":{"seat":"git","repository":"` + repo + `","ref":"main"}}]}}` + } + judge := func(repo string) []string { + t.Helper() + tree := aTree(t, map[string]string{"proxy": withContext(repo)}) + _, failures, err := shelfWithChange(map[string]catalogue.Manifest{}, map[string]inventory.Source{}, + snapshot.Facts{}, "novox/mesh-controller", tree, map[string]string{}, nil) + if err != nil { + t.Fatal(err) + } + return failures + } + const want = "proxy builds artifact server from novox/mesh-catalog: a module lives whole in one repository (novox/hq ADR 0420 §1)" + failures := judge("novox/mesh-catalog") + if len(failures) != 1 || !strings.Contains(failures[0], want) { + t.Errorf("a context naming another repository: %v, want it to say %q", failures, want) + } + for _, f := range judge("novox/mesh-controller") { + if strings.Contains(f, "ADR 0420 §1") { + t.Errorf("a context naming the module's own repository was refused: %s", f) + } + } +} + +func TestAMemberJudgedAloneSeesAMoveThroughTheClonesBeside(t *testing.T) { + const catalogueRepo, controllerRepo = "novox/mesh-catalog", "novox/mesh-controller" + raw := `{"module":"mover","version":"1"}` + sources := map[string]inventory.Source{"mover": {Repository: "http://forge.internal:20000/" + catalogueRepo + ".git", + Path: "modules/mover", BuiltFrom: "c0ffee"}} + // A tree is written into dir: defining mover, or holding only another module. + write := func(dir string, defines bool) string { + name, body := "other", `{"module":"other","version":"1"}` + if defines { + name, body = "mover", raw + } + if err := os.MkdirAll(filepath.Join(dir, "modules", name), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "modules", name, "module.json"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return dir + } + defining := func() string { return write(t.TempDir(), true) } + without := func() string { return write(t.TempDir(), false) } + // beside lays out a MESH_CHECK_BESIDE directory holding the named repositories' trees, each defining mover or not. + beside := func(trees map[string]bool) string { + root := t.TempDir() + for name, defines := range trees { + write(filepath.Join(root, name), defines) + } + return root + } + moves := func(repository, tree, besideRoot string) map[string]moveOfModule { + t.Helper() + got, err := movesOfChange(mergeCheckInput{repository: repository, tree: tree, beside: besideRoot}, sources) + if err != nil { + t.Fatal(err) + } + return got + } + // The controller's member: defines mover; the catalogue beside has removed it. + if got := moves(controllerRepo, defining(), beside(map[string]bool{"mesh-catalog": false})); got["mover"].To != controllerRepo { + t.Errorf("the controller's change defining mover, the catalogue beside without it: moves %v", got) + } + // The sources the gate then judges with: built from the controller, the definition is not a second one (issue 239); + // unmoved, it still is. + judge := func(at map[string]inventory.Source) []string { + t.Helper() + _, failures, err := shelfWithChange(map[string]catalogue.Manifest{}, at, snapshot.Facts{}, controllerRepo, defining(), + map[string]string{}, nil) + if err != nil { + t.Fatal(err) + } + return failures + } + mv := moves(controllerRepo, defining(), beside(map[string]bool{"mesh-catalog": false}))["mover"] + moved := map[string]inventory.Source{"mover": {Repository: mv.To, Path: mv.Dir, BuiltFrom: "c0ffee"}} + if f := judge(moved); len(f) != 0 { + t.Errorf("a definition of a module the catalogue beside removed was refused: %v", f) + } + if f := judge(sources); len(f) != 1 || !strings.Contains(f[0], "issue 239") { + t.Errorf("with the catalogue beside at main, the definition: %v, want the issue 239 refusal", f) + } + if got := moves(controllerRepo, defining(), beside(map[string]bool{"mesh-catalog": true})); len(got) != 0 { + t.Errorf("the catalogue beside at main still defines mover, yet moves %v", got) + } + // The catalogue's member: removes mover; the controller beside defines it. + if got := moves(catalogueRepo, without(), beside(map[string]bool{"mesh-controller": true})); got["mover"].To != controllerRepo { + t.Errorf("the catalogue's change removing mover, the controller beside defining it: moves %v", got) + } + if got := moves(catalogueRepo, without(), beside(map[string]bool{"mesh-controller": false})); len(got) != 0 { + t.Errorf("the controller beside at main does not define mover, yet moves %v", got) + } + if got := moves(catalogueRepo, without(), ""); len(got) != 0 { + t.Errorf("nothing beside, yet moves %v", got) + } +} diff --git a/cmd/mesh-controller/merge_gate_test.go b/cmd/mesh-controller/merge_gate_test.go index 11c304c7..7960ea21 100644 --- a/cmd/mesh-controller/merge_gate_test.go +++ b/cmd/mesh-controller/merge_gate_test.go @@ -5,11 +5,6 @@ import ( "path/filepath" "strings" "testing" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/catalogue" - snapshot "github.com/novox/mesh-controller/internal/facts" - "github.com/novox/mesh-controller/internal/inventory" ) // The merge gate (novox/hq to-be 45 §9): a change judged against every machine of the snapshot, by the @@ -19,43 +14,6 @@ import ( // catalogueMesh is two machines and a catalogue repository: a resolver and an object store on the // anchor, and on the laptop a network manager requiring the resolver, an album requiring the object // store, and a login manager. Every module is registered from novox/mesh-catalog, as the mesh's are. -func catalogueMesh(t *testing.T) (snapshot.Facts, map[string]string) { - t.Helper() - open := aMesh(t) - ctx := t.Context() - manifests := map[string]string{ - "objects": `{"module":"objects","version":"1", - "provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}], - "receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`, - "resolver": `{"module":"resolver","version":"1", - "provides":[{"name":"wildcard-resolution","scope":"mesh","identity":false}]}`, - "networkmanager": `{"module":"networkmanager","version":"1","requires":["wildcard-resolution"]}`, - "album": `{"module":"album","version":"1","requires":["s3-bucket"]}`, - "lemurs": `{"module":"lemurs","version":"1","capabilities":["systemd"], - "resources":[{"id":"service","type":"service","unit":"lemurs.service","state":"running","boot":"enabled"}]}`, - } - for name, raw := range manifests { - m, err := catalogue.ParseManifest([]byte(raw)) - if err != nil { - t.Fatalf("%s: %v", name, err) - } - if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", - Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil { - t.Fatal(err) - } - } - for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"}, {"laptop", "networkmanager"}, - {"laptop", "album"}, {"laptop", "lemurs"}} { - if _, err := assign(ctx, open, a[0], a[1]); err != nil { - t.Fatalf("assign %s %s: %v", a[0], a[1], err) - } - } - f, err := gatherFacts(ctx, open, "2.11.17") - if err != nil { - t.Fatal(err) - } - return f, manifests -} // aTree is a checkout of the catalogue repository holding these manifests. func aTree(t *testing.T, manifests map[string]string) string { @@ -73,20 +31,6 @@ func aTree(t *testing.T, manifests map[string]string) string { return dir } -func gateJudged(t *testing.T, f snapshot.Facts, tree string, changed ...string) mergeVerdict { - t.Helper() - admin := os.Getenv("MESH_TEST_POSTGRES") - in := mergeCheckInput{facts: f, admin: admin, changed: changed} - if tree != "" { - in.repository, in.tree = "novox/mesh-catalog", tree - } - v, err := judgeChange(t.Context(), in) - if err != nil { - t.Fatal(err) - } - return v -} - func withEdit(manifests map[string]string, name, raw string) map[string]string { out := map[string]string{} for k, v := range manifests { @@ -101,271 +45,42 @@ func withEdit(manifests map[string]string, name, raw string) map[string]string { } // The mesh as it is passes: every machine composes in the gate's store as the controller composed it. -func TestTheMeshAsItIsPassesTheGate(t *testing.T) { - f, manifests := catalogueMesh(t) - for _, m := range f.Machines { - if !m.Declaration.Composes { - t.Fatalf("the mesh itself does not compose: %+v", m.Declaration) - } - } - v := gateJudged(t, f, aTree(t, manifests)) - if v.Verdict != "pass" { - t.Fatalf("an unchanged catalogue does not pass:\n%s", v.Report()) - } - for _, m := range v.Machines { - if !m.Base.Composes || !m.Change.Composes { - t.Errorf("%s does not compose in the gate's store as it does on the mesh: %+v / %+v", m.Described, m.Base, m.Change) - } - } -} // **Issue 263**: a change makes the network manager require the object store's provision; on a machine // whose name is six characters its identity is 26 against a bound of 20. Refused in the pull request, // naming the module, and not on the anchor after it merged. -func TestIssue263AnIdentityARealMachineNameOverflowsFailsThePullRequest(t *testing.T) { - f, manifests := catalogueMesh(t) - tree := aTree(t, withEdit(manifests, "networkmanager", - `{"module":"networkmanager","version":"1","requires":["wildcard-resolution","s3-bucket"]}`)) - v := gateJudged(t, f, tree) - if v.Verdict != "fail" { - t.Fatalf("the overflow passed the gate:\n%s", v.Report()) - } - report := v.Report() - if !strings.Contains(report, "networkmanager") || !strings.Contains(report, "s3-bucket") { - t.Errorf("the refusal does not name the module and the provision:\n%s", report) - } -} // **Issue 236**: a login manager's service that omits its state passed the catalogue check and was // refused whole by the node-engine on the first machine. A change to a module a machine runs, and a // module nobody runs yet, are both refused before merge, naming the machine and the module. -func TestIssue236AManifestTheNodeEngineRefusesFailsThePullRequest(t *testing.T) { - f, manifests := catalogueMesh(t) - broken := `{"module":"lemurs","version":"1","capabilities":["systemd"], - "resources":[{"id":"service","type":"service","unit":"lemurs.service","boot":"enabled"}]}` - v := gateJudged(t, f, aTree(t, withEdit(manifests, "lemurs", broken))) - if v.Verdict != "fail" || !strings.Contains(v.Report(), "lemurs") { - t.Fatalf("a service the node-engine refuses passed the gate:\n%s", v.Report()) - } - laptop := snapshot.Pseudonym("machine", "laptop") - if !strings.Contains(v.Report(), laptop) { - t.Errorf("the refusal does not name the machine it would be refused on:\n%s", v.Report()) - } - - // And as a new module, which no machine runs: tried on one that could. - newcomer := strings.ReplaceAll(broken, `"lemurs"`, `"greeter"`) - newcomer = strings.ReplaceAll(newcomer, "lemurs.service", "greeter.service") - v = gateJudged(t, f, aTree(t, withEdit(manifests, "greeter", newcomer))) - if v.Verdict != "fail" || !strings.Contains(v.Report(), "greeter, assigned to") { - t.Fatalf("a new module the node-engine would refuse passed the gate:\n%s", v.Report()) - } -} // **Version skew**: a manifest the controller judging it cannot read — the one the mesh runs, for a // catalogue change — fails here, not at registration after the merge. -func TestAManifestTheRunningControllerCannotReadFailsThePullRequest(t *testing.T) { - f, manifests := catalogueMesh(t) - v := gateJudged(t, f, aTree(t, withEdit(manifests, "album", - `{"module":"album","version":"1","requires":["s3-bucket"],"a-field-of-a-newer-controller":true}`))) - if v.Verdict != "fail" || !strings.Contains(v.Report(), "refuses it") { - t.Fatalf("a manifest this controller cannot read passed:\n%s", v.Report()) - } -} // A module a machine runs, removed from its source, fails until it is unassigned (ADR 0236). -func TestAModuleAMachineRunsRemovedFromItsSourceFails(t *testing.T) { - f, manifests := catalogueMesh(t) - v := gateJudged(t, f, aTree(t, withEdit(manifests, "album", ""))) - if v.Verdict != "fail" || !strings.Contains(v.Report(), "album is removed") { - t.Fatalf("removing a module a machine runs passed:\n%s", v.Report()) - } -} // **Issues 278 and 280**: a file in no held module's directory read as shared code, and a merge rebuilt the // catalogue. A file is a module's only by being inside it — no build reads one outside — so it rebuilds // nothing, and the gate says why; a merge that does rebuild widely is warned of. -func TestIssue278AWideRebuildIsSaidBeforeTheMerge(t *testing.T) { - f, manifests := catalogueMesh(t) - was := wideRebuild - wideRebuild = 2 - t.Cleanup(func() { wideRebuild = was }) - for _, file := range []string{"modules/showcase/index.ts", "merge-check.sh", "README.md"} { - v := gateJudged(t, f, aTree(t, manifests), file) - if v.Width == nil || len(v.Width.Modules) != 0 || len(v.Width.Unread) != 1 || v.Verdict != "pass" { - t.Fatalf("%s, read by no build, reads %+v, %s", file, v.Width, v.Verdict) - } - if !strings.Contains(v.Report(), "read by no module's build") { - t.Errorf("why %s rebuilds nothing is not said:\n%s", file, v.Report()) - } - } - v := gateJudged(t, f, aTree(t, manifests), "modules/album/x.ts", "modules/objects/x.go", "modules/resolver/x.go") - if v.Width == nil || len(v.Width.Modules) < 3 || v.Verdict != "warning" { - t.Fatalf("a rebuild wider than the bound is not warned of: %+v, %s", v.Width, v.Verdict) - } - // With the reference module's definition in the tree, its directory is a module, held or not. - withShowcase := withEdit(manifests, "showcase", `{"module":"showcase","version":"1"}`) - v = gateJudged(t, f, aTree(t, withShowcase), "modules/showcase/index.ts") - if v.Width == nil || len(v.Width.Modules) != 0 || len(v.Width.Unread) != 0 { - t.Fatalf("a file of a module nobody holds reads %+v", v.Width) - } - v = gateJudged(t, f, aTree(t, manifests), "modules/album/module.json") - if v.Width == nil || strings.Join(v.Width.Modules, ",") != "album" || v.Verdict != "pass" { - t.Fatalf("a change to one module's directory reads %+v, %s", v.Width, v.Verdict) - } -} // **A delivery group is judged as one future state** (novox/hq ADR 0239): a catalogue change that needs a // provision only another repository's change adds fails alone and passes composed with it; and a group // whose other head breaks what the first needs fails, naming it. -func TestADeliveryGroupsHeadsAreComposedTogether(t *testing.T) { - f, manifests := catalogueMeshWithAnApp(t) - needsTheApp := withEdit(manifests, "album", `{"module":"album","version":"1","requires":["s3-bucket","app-api"]}`) - catTree := aTree(t, needsTheApp) - alone := gateJudged(t, f, catTree, "modules/album/module.json") - if alone.Verdict != "fail" { - t.Fatalf("a requirement nothing provides passed alone:\n%s", alone.Report()) - } - app := t.TempDir() - if err := os.WriteFile(filepath.Join(app, "module.json"), []byte(`{"module":"app","version":"1", - "provides":[{"name":"app-api","scope":"mesh","identity":false}]}`), 0o644); err != nil { - t.Fatal(err) - } - in := mergeCheckInput{facts: f, admin: os.Getenv("MESH_TEST_POSTGRES"), repository: "novox/mesh-catalog", - tree: catTree, changed: []string{"modules/album/module.json"}, - group: []groupChange{{Repository: "novox/app", Tree: app, Changed: []string{"module.json"}}}} - together, err := judgeChange(t.Context(), in) - if err != nil { - t.Fatal(err) - } - if together.Verdict == "fail" { - t.Fatalf("the group composed together fails:\n%s", together.Report()) - } - if together.Modules["app"] != "changed" || together.Modules["album"] != "changed" { - t.Fatalf("the group's heads were not both laid over the mesh: %v", together.Modules) - } -} // catalogueMeshWithAnApp is catalogueMesh with an application built from a repository of its own, at its // root, on the anchor. -func catalogueMeshWithAnApp(t *testing.T) (snapshot.Facts, map[string]string) { - t.Helper() - open := aMesh(t) - ctx := t.Context() - manifests := map[string]string{ - "objects": `{"module":"objects","version":"1", - "provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}], - "receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`, - "album": `{"module":"album","version":"1","requires":["s3-bucket"]}`, - } - for name, raw := range manifests { - m, err := catalogue.ParseManifest([]byte(raw)) - if err != nil { - t.Fatalf("%s: %v", name, err) - } - if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", - Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil { - t.Fatal(err) - } - } - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, - inventory.Source{Repository: "novox/app", BuiltFrom: "c0ffee"}); err != nil { - t.Fatal(err) - } - for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "app"}, {"laptop", "album"}} { - if _, err := assign(ctx, open, a[0], a[1]); err != nil { - t.Fatalf("assign %s %s: %v", a[0], a[1], err) - } - } - f, err := gatherFacts(ctx, open, "2.11.17") - if err != nil { - t.Fatal(err) - } - return f, manifests -} // busMesh is aMesh with a module on the bus on the laptop, its account issued as `module issue` issues // one: minted, and its credential held as the module's own secret named broker. -func busMesh(t *testing.T) snapshot.Facts { - t.Helper() - open := aMesh(t) - ctx := t.Context() - m, err := catalogue.ParseManifest([]byte(`{"module":"speaker","version":"1", - "own-secrets":{"broker":"/var/lib/speaker/broker"}}`)) - if err != nil { - t.Fatal(err) - } - if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", - Path: "modules/speaker", BuiltFrom: "c0ffee"}); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "laptop", "speaker"); err != nil { - t.Fatal(err) - } - user := broker.Principal{Kind: broker.KindModule, Node: "laptop", Module: "speaker"}.Username() - password, err := open.inventory.MintBusPassword(ctx, inventory.BusUser{Username: user, Kind: inventory.BusModule, - Node: "laptop", Module: "speaker"}) - if err != nil { - t.Fatal(err) - } - if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "speaker", "broker", - `{"user":"`+user+`","password":"`+password+`"}`); err != nil { - t.Fatal(err) - } - f, err := gatherFacts(ctx, open, "2.11.17") - if err != nil { - t.Fatal(err) - } - for _, mc := range f.Machines { - if !mc.Declaration.Composes { - t.Fatalf("the mesh itself does not compose: %+v", mc.Declaration) - } - } - return f -} // **Issue 285**: every machine running a module on the bus failed to compose in the gate's store, with // the change and without — the store held the module's credential and no account for it, which // composition refuses (issue 203) — and the gate passed every change, "0 of 4 compose". The account the // mesh issued is raised with its credential, so the machine composes in the gate as on the mesh. -func TestIssue285AModuleOnTheBusComposesInTheGate(t *testing.T) { - f := busMesh(t) - v := gateJudged(t, f, "") - if v.Verdict != "pass" { - t.Fatalf("the mesh as it is does not pass:\n%s", v.Report()) - } - for _, m := range v.Machines { - if !m.Base.Composes { - t.Errorf("%s composes on the mesh and not in the gate: %v", m.Described, m.Base.Problems) - } - } - if !strings.Contains(v.Summary, "2 of 2 compose") { - t.Errorf("the summary does not say every machine composes: %s", v.Summary) - } -} // **Issue 285**: a machine the mesh composes that the gate cannot raise as it is leaves the change judged // against a machine that is not the mesh's — broken against broken, which passes whatever the change does. // That is an error, never a pass. -func TestIssue285AMachineTheGateCannotRaiseIsAnErrorNeverAPass(t *testing.T) { - f := busMesh(t) - for i := range f.Machines { - // A fact the snapshot does not carry: the module's credential, gone from the laptop's. - var kept []snapshot.Accepted - for _, a := range f.Machines[i].Accepted { - if a.Name != "broker" { - kept = append(kept, a) - } - } - f.Machines[i].Accepted = kept - } - v := gateJudged(t, f, "") - if v.Verdict != "error" { - t.Fatalf("a gate whose mesh does not compose said %s:\n%s", v.Verdict, v.Report()) - } - if len(v.Errors) != 1 || !strings.Contains(v.Errors[0], "speaker") { - t.Errorf("the error does not name what could not be raised: %v", v.Errors) - } -} // A path the snapshot withheld is given a path of its own where an access or a place needs one: a bare // "withheld" is no absolute path, and a machine whose setting composes on the mesh would not in the gate. @@ -439,231 +154,14 @@ func sdkPinnedByGoMod(t *testing.T) string { // neither a second definition of one module name (issue 239) nor the removal of a module machines run // (ADR 0236). Alone, each head is still refused. This lays the heads over the shelf as judgeChange does, // which needs no database. -func TestAGroupsMoveOfAModuleIsNeitherTwoDefinitionsNorARemoval(t *testing.T) { - const catalogueRepo, controllerRepo = "novox/mesh-catalog", "novox/mesh-controller" - built := func(repo, path string) inventory.Source { - return inventory.Source{Repository: "http://forge.internal:20000/" + repo + ".git", Path: path, BuiltFrom: "c0ffee"} - } - sources := map[string]inventory.Source{"mover": built(catalogueRepo, "modules/mover")} - raw := `{"module":"mover","version":"1"}` - base := map[string]catalogue.Manifest{} - m, err := catalogue.ParseManifest([]byte(raw)) - if err != nil { - t.Fatal(err) - } - base["mover"] = m - facts := snapshot.Facts{Machines: []snapshot.Machine{{Name: "anchor", Assigned: []string{"mover"}}}} - - catalogueWithout := aTree(t, map[string]string{"other": `{"module":"other","version":"1"}`}) - controllerWith := func() string { - dir := t.TempDir() - if err := os.MkdirAll(filepath.Join(dir, "modules", "mover"), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "modules", "mover", "module.json"), []byte(raw), 0o644); err != nil { - t.Fatal(err) - } - return dir - }() - removes := groupChange{Repository: catalogueRepo, Tree: catalogueWithout, Changed: []string{"modules/mover/module.json"}} - defines := groupChange{Repository: controllerRepo, Tree: controllerWith, Changed: []string{"modules/mover/module.json"}} - - lay := func(heads ...groupChange) (map[string]catalogue.Manifest, []string, map[string]string) { - t.Helper() - moves, err := movesInGroup(heads, sources) - if err != nil { - t.Fatal(err) - } - at := sources - if len(moves) > 0 { - at = map[string]inventory.Source{} - for k, v := range sources { - at[k] = v - } - for name, mv := range moves { - o := at[name] - o.Repository, o.Path = mv.To, mv.Dir - at[name] = o - } - } - shelf, said := base, map[string]string{} - var failures []string - for _, h := range heads { - var f []string - if shelf, f, err = shelfWithChange(shelf, at, facts, h.Repository, h.Tree, said, nil); err != nil { - t.Fatal(err) - } - failures = append(failures, f...) - } - for name := range moves { - said[name] = "moved" - } - return shelf, failures, said - } - for _, order := range [][]groupChange{{removes, defines}, {defines, removes}} { - shelf, failures, said := lay(order...) - if len(failures) > 0 { - t.Fatalf("a move of a module machines run, laid %s first, was refused: %v", order[0].Repository, failures) - } - if said["mover"] != "moved" { - t.Errorf("the move reads %v, not moved", said) - } - if _, held := shelf["mover"]; !held { - t.Errorf("the moved module is not on the shelf after laying %s first", order[0].Repository) - } - } - // Alone, a head is still refused: a second definition (issue 239), and a removal of what runs (ADR 0236). - if _, failures, _ := lay(defines); len(failures) != 1 || !strings.Contains(failures[0], "issue 239") { - t.Errorf("a definition of a module another repository builds, alone: %v", failures) - } - if _, failures, _ := lay(removes); len(failures) != 1 || !strings.Contains(failures[0], "ADR 0236") { - t.Errorf("the removal of a module machines run, alone: %v", failures) - } -} // A group that moves a module machines run passes the gate whole, in either order of its heads, and says the // module moved (novox/hq ADR 0420 §6, issue 1025); a head alone is still refused. Needs the test store. -func TestADeliveryGroupMovingAModuleMachinesRunPassesTheGate(t *testing.T) { - f, manifests := catalogueMeshWithAnApp(t) - catTree := aTree(t, withEdit(manifests, "album", "")) - app := t.TempDir() - // The app module is built from this repository's root, and stays there: only album moves. - if err := os.WriteFile(filepath.Join(app, "module.json"), []byte(`{"module":"app","version":"1"}`), 0o644); err != nil { - t.Fatal(err) - } - if err := os.MkdirAll(filepath.Join(app, "modules", "album"), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(app, "modules", "album", "module.json"), []byte(manifests["album"]), 0o644); err != nil { - t.Fatal(err) - } - removes := groupChange{Repository: "novox/mesh-catalog", Tree: catTree, Changed: []string{"modules/album/module.json"}} - defines := groupChange{Repository: "novox/app", Tree: app, Changed: []string{"modules/album/module.json"}} - judge := func(first groupChange, rest ...groupChange) mergeVerdict { - v, err := judgeChange(t.Context(), mergeCheckInput{facts: f, admin: os.Getenv("MESH_TEST_POSTGRES"), - repository: first.Repository, tree: first.Tree, changed: first.Changed, group: rest}) - if err != nil { - t.Fatal(err) - } - return v - } - for _, v := range []mergeVerdict{judge(removes, defines), judge(defines, removes)} { - if v.Verdict == "fail" || v.Modules["album"] != "moved" { - t.Fatalf("a group moving a module machines run: %v\n%s", v.Modules, v.Report()) - } - } - if v := judge(defines); v.Verdict != "fail" { - t.Errorf("a definition of a module another repository builds passed alone:\n%s", v.Report()) - } - if v := judge(removes); v.Verdict != "fail" { - t.Errorf("the removal of a module machines run passed alone:\n%s", v.Report()) - } -} // **A module lives whole in one repository** (novox/hq ADR 0420 §1, issue 1019): an artifact whose build // context names another repository is refused by the gate, and one naming its own passes it. -func TestTheGateRefusesAnArtifactContextNamingAnotherRepository(t *testing.T) { - withContext := func(repo string) string { - return `{"module":"proxy","version":"1","build":{"artifacts":[{"name":"server","kind":"image","from":"Dockerfile",` + - `"context":{"seat":"git","repository":"` + repo + `","ref":"main"}}]}}` - } - judge := func(repo string) []string { - t.Helper() - tree := aTree(t, map[string]string{"proxy": withContext(repo)}) - _, failures, err := shelfWithChange(map[string]catalogue.Manifest{}, map[string]inventory.Source{}, - snapshot.Facts{}, "novox/mesh-controller", tree, map[string]string{}, nil) - if err != nil { - t.Fatal(err) - } - return failures - } - const want = "proxy builds artifact server from novox/mesh-catalog: a module lives whole in one repository (novox/hq ADR 0420 §1)" - failures := judge("novox/mesh-catalog") - if len(failures) != 1 || !strings.Contains(failures[0], want) { - t.Errorf("a context naming another repository: %v, want it to say %q", failures, want) - } - for _, f := range judge("novox/mesh-controller") { - if strings.Contains(f, "ADR 0420 §1") { - t.Errorf("a context naming the module's own repository was refused: %s", f) - } - } -} // **A member of a delivery group judged alone sees a move** through the core clones beside its check // (novox/hq ADR 0420 §6, issue 1043): the controller's own gate, with the catalogue beside having removed the // module, is not refused for a second definition (issue 239); the catalogue's own gate, with the controller // beside defining it, is not refused for a removal (ADR 0236). A clone beside at main finds no move. -func TestAMemberJudgedAloneSeesAMoveThroughTheClonesBeside(t *testing.T) { - const catalogueRepo, controllerRepo = "novox/mesh-catalog", "novox/mesh-controller" - raw := `{"module":"mover","version":"1"}` - sources := map[string]inventory.Source{"mover": {Repository: "http://forge.internal:20000/" + catalogueRepo + ".git", - Path: "modules/mover", BuiltFrom: "c0ffee"}} - // A tree is written into dir: defining mover, or holding only another module. - write := func(dir string, defines bool) string { - name, body := "other", `{"module":"other","version":"1"}` - if defines { - name, body = "mover", raw - } - if err := os.MkdirAll(filepath.Join(dir, "modules", name), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "modules", name, "module.json"), []byte(body), 0o644); err != nil { - t.Fatal(err) - } - return dir - } - defining := func() string { return write(t.TempDir(), true) } - without := func() string { return write(t.TempDir(), false) } - // beside lays out a MESH_CHECK_BESIDE directory holding the named repositories' trees, each defining mover or not. - beside := func(trees map[string]bool) string { - root := t.TempDir() - for name, defines := range trees { - write(filepath.Join(root, name), defines) - } - return root - } - moves := func(repository, tree, besideRoot string) map[string]moveOfModule { - t.Helper() - got, err := movesOfChange(mergeCheckInput{repository: repository, tree: tree, beside: besideRoot}, sources) - if err != nil { - t.Fatal(err) - } - return got - } - // The controller's member: defines mover; the catalogue beside has removed it. - if got := moves(controllerRepo, defining(), beside(map[string]bool{"mesh-catalog": false})); got["mover"].To != controllerRepo { - t.Errorf("the controller's change defining mover, the catalogue beside without it: moves %v", got) - } - // The sources the gate then judges with: built from the controller, the definition is not a second one (issue 239); - // unmoved, it still is. - judge := func(at map[string]inventory.Source) []string { - t.Helper() - _, failures, err := shelfWithChange(map[string]catalogue.Manifest{}, at, snapshot.Facts{}, controllerRepo, defining(), - map[string]string{}, nil) - if err != nil { - t.Fatal(err) - } - return failures - } - mv := moves(controllerRepo, defining(), beside(map[string]bool{"mesh-catalog": false}))["mover"] - moved := map[string]inventory.Source{"mover": {Repository: mv.To, Path: mv.Dir, BuiltFrom: "c0ffee"}} - if f := judge(moved); len(f) != 0 { - t.Errorf("a definition of a module the catalogue beside removed was refused: %v", f) - } - if f := judge(sources); len(f) != 1 || !strings.Contains(f[0], "issue 239") { - t.Errorf("with the catalogue beside at main, the definition: %v, want the issue 239 refusal", f) - } - if got := moves(controllerRepo, defining(), beside(map[string]bool{"mesh-catalog": true})); len(got) != 0 { - t.Errorf("the catalogue beside at main still defines mover, yet moves %v", got) - } - // The catalogue's member: removes mover; the controller beside defines it. - if got := moves(catalogueRepo, without(), beside(map[string]bool{"mesh-controller": true})); got["mover"].To != controllerRepo { - t.Errorf("the catalogue's change removing mover, the controller beside defining it: moves %v", got) - } - if got := moves(catalogueRepo, without(), beside(map[string]bool{"mesh-controller": false})); len(got) != 0 { - t.Errorf("the controller beside at main does not define mover, yet moves %v", got) - } - if got := moves(catalogueRepo, without(), ""); len(got) != 0 { - t.Errorf("nothing beside, yet moves %v", got) - } -} diff --git a/cmd/mesh-controller/merge_on_the_bus_test.go b/cmd/mesh-controller/merge_on_the_bus_test.go index cc84fc3f..df3825e2 100644 --- a/cmd/mesh-controller/merge_on_the_bus_test.go +++ b/cmd/mesh-controller/merge_on_the_bus_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/merge_window_bus_integration_test.go b/cmd/mesh-controller/merge_window_bus_integration_test.go new file mode 100644 index 00000000..5cdae1e8 --- /dev/null +++ b/cmd/mesh-controller/merge_window_bus_integration_test.go @@ -0,0 +1,90 @@ +//go:build integration + +package main + +import ( + "context" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/testbus" +) + +func windowOnABus(t *testing.T) (*busWindow, <-chan string) { + t.Helper() + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + conn, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(conn.Close) + api, err := jetstream.New(conn) + if err != nil { + t.Fatal(err) + } + heard := make(chan string, 16) + ctx, stop := context.WithCancel(context.Background()) + t.Cleanup(stop) + w, err := newBusWindow(ctx, api, func(batch string) { heard <- batch }) + if err != nil { + t.Fatal(err) + } + return w, heard +} + +func TestTheQuietKeyExpiresAndTheWatcherNamesTheBatch(t *testing.T) { + w, heard := windowOnABus(t) + if err := w.Arm(t.Context(), "plan-1", time.Second, time.Minute); err != nil { + t.Fatal(err) + } + if b, ok := hears(t, heard, 10*time.Second); !ok || b != "plan-1" { + t.Fatalf("heard %q (%v); want plan-1", b, ok) + } +} + +func TestAMergeRestartsTheQuietKeyAndTheMostKeyClosesAllTheSame(t *testing.T) { + w, heard := windowOnABus(t) + ctx := t.Context() + if err := w.Arm(ctx, "plan-2", 2*time.Second, 5*time.Second); err != nil { + t.Fatal(err) + } + time.Sleep(1200 * time.Millisecond) + if err := w.Arm(ctx, "plan-2", 2*time.Second, 5*time.Second); err != nil { // a merge: quiet again, most as it was + t.Fatal(err) + } + if b, ok := hears(t, heard, 1500*time.Millisecond); ok { + t.Fatalf("heard %q before the restarted window was up", b) + } + // Another merge keeps the window open; the most (5 s from the first) closes it at about 5 s. + start := time.Now() + for time.Since(start) < 2500*time.Millisecond { + if err := w.Arm(ctx, "plan-2", 2*time.Second, 5*time.Second); err != nil { + t.Fatal(err) + } + time.Sleep(700 * time.Millisecond) + } + if b, ok := hears(t, heard, 8*time.Second); !ok || b != "plan-2" { + t.Fatalf("the most did not close the batch: heard %q (%v)", b, ok) + } +} + +func TestForgettingABatchWakesNobody(t *testing.T) { + w, heard := windowOnABus(t) + if err := w.Arm(t.Context(), "plan-3", time.Second, time.Minute); err != nil { + t.Fatal(err) + } + w.Forget(t.Context(), "plan-3") + if b, ok := hears(t, heard, 3*time.Second); ok { + t.Fatalf("a forgotten batch was heard expire: %q", b) + } +} diff --git a/cmd/mesh-controller/merge_window_bus_test.go b/cmd/mesh-controller/merge_window_bus_test.go index a2c24dc0..760a360e 100644 --- a/cmd/mesh-controller/merge_window_bus_test.go +++ b/cmd/mesh-controller/merge_window_bus_test.go @@ -1,47 +1,13 @@ package main import ( - "context" "testing" "time" - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/testbus" ) // windowOnABus is the merge window's clock on a real bus, and the batches its watcher heard expire. -func windowOnABus(t *testing.T) (*busWindow, <-chan string) { - t.Helper() - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - api, err := jetstream.New(conn) - if err != nil { - t.Fatal(err) - } - heard := make(chan string, 16) - ctx, stop := context.WithCancel(context.Background()) - t.Cleanup(stop) - w, err := newBusWindow(ctx, api, func(batch string) { heard <- batch }) - if err != nil { - t.Fatal(err) - } - return w, heard -} func hears(t *testing.T, heard <-chan string, within time.Duration) (string, bool) { t.Helper() @@ -55,55 +21,11 @@ func hears(t *testing.T, heard <-chan string, within time.Duration) (string, boo // A batch's quiet key expires when no merge came for the window, and the watcher names the batch (issue 973, // novox/hq ADR 0407 decision 1). -func TestTheQuietKeyExpiresAndTheWatcherNamesTheBatch(t *testing.T) { - w, heard := windowOnABus(t) - if err := w.Arm(t.Context(), "plan-1", time.Second, time.Minute); err != nil { - t.Fatal(err) - } - if b, ok := hears(t, heard, 10*time.Second); !ok || b != "plan-1" { - t.Fatalf("heard %q (%v); want plan-1", b, ok) - } -} // A merge into the batch makes the quiet key again, so the window restarts: it does not expire at the first // deadline, and the batch's most closes it all the same. -func TestAMergeRestartsTheQuietKeyAndTheMostKeyClosesAllTheSame(t *testing.T) { - w, heard := windowOnABus(t) - ctx := t.Context() - if err := w.Arm(ctx, "plan-2", 2*time.Second, 5*time.Second); err != nil { - t.Fatal(err) - } - time.Sleep(1200 * time.Millisecond) - if err := w.Arm(ctx, "plan-2", 2*time.Second, 5*time.Second); err != nil { // a merge: quiet again, most as it was - t.Fatal(err) - } - if b, ok := hears(t, heard, 1500*time.Millisecond); ok { - t.Fatalf("heard %q before the restarted window was up", b) - } - // Another merge keeps the window open; the most (5 s from the first) closes it at about 5 s. - start := time.Now() - for time.Since(start) < 2500*time.Millisecond { - if err := w.Arm(ctx, "plan-2", 2*time.Second, 5*time.Second); err != nil { - t.Fatal(err) - } - time.Sleep(700 * time.Millisecond) - } - if b, ok := hears(t, heard, 8*time.Second); !ok || b != "plan-2" { - t.Fatalf("the most did not close the batch: heard %q (%v)", b, ok) - } -} // A batch cut is forgotten: its keys are taken away by a delete, which is no expiry and wakes nobody. -func TestForgettingABatchWakesNobody(t *testing.T) { - w, heard := windowOnABus(t) - if err := w.Arm(t.Context(), "plan-3", time.Second, time.Minute); err != nil { - t.Fatal(err) - } - w.Forget(t.Context(), "plan-3") - if b, ok := hears(t, heard, 3*time.Second); ok { - t.Fatalf("a forgotten batch was heard expire: %q", b) - } -} // The deadlines a batch's keys are made with: what is left of its window and of its most, at least a second; // none for a batch whose window is closed or that is no batch. diff --git a/cmd/mesh-controller/mesh_for_test.go b/cmd/mesh-controller/mesh_for_test.go index eea81d13..aad02830 100644 --- a/cmd/mesh-controller/mesh_for_test.go +++ b/cmd/mesh-controller/mesh_for_test.go @@ -8,9 +8,10 @@ import ( "encoding/base64" "encoding/json" "fmt" - "github.com/novox/mesh-controller/internal/conditions" "testing" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/licences" diff --git a/cmd/mesh-controller/meshcli_integration_test.go b/cmd/mesh-controller/meshcli_integration_test.go new file mode 100644 index 00000000..94980ef7 --- /dev/null +++ b/cmd/mesh-controller/meshcli_integration_test.go @@ -0,0 +1,193 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestTheTerminalRunsWithoutAVerbAndAnOrdinaryCallNamesMeshCLI(t *testing.T) { + t.Setenv(echoEnvironment, "1") + t.Setenv("MESH_VERB", "leaked-from-the-serving-process") + // The serving controller marks itself (ADR 0266); its terminal line for mesh-cli is still the terminal's. + t.Setenv(servedVar, "1") + ctx := context.Background() + + a := runForMeshCLI(ctx, "control", cliAsked("operator", 1000, "status"), cliVerdict{terminal: true, why: "the terminal"}) + if a.Exit != 0 || a.Refused != "" || !a.Terminal { + t.Fatalf("the terminal's line did not run: %+v", a) + } + if got := string(a.Stdout); !strings.Contains(got, `verb=""`) || !strings.Contains(got, "operator through mesh-cli on control") || + !strings.Contains(got, "terminal=true") { + t.Fatalf("the terminal's line ran with %s", got) + } + + a = runForMeshCLI(ctx, "laptop", cliAsked("operator", 1000, "status"), cliVerdict{why: "not the terminal"}) + if a.Exit != 0 || a.Terminal || a.Why != "not the terminal" { + t.Fatalf("an ordinary line did not run as one: %+v", a) + } + if got := string(a.Stdout); !strings.Contains(got, `verb="mesh-cli"`) || !strings.Contains(got, "terminal=false") { + t.Fatalf("an ordinary line ran with %s", got) + } +} + +func TestAnOrdinaryCallMeetsTheCommandVerbsRefusals(t *testing.T) { + t.Setenv(echoEnvironment, "1") + ctx := context.Background() + ordinary := cliVerdict{why: "not the terminal"} + a := runForMeshCLI(ctx, "laptop", cliAsked("operator", 1000, "cleanup", "delete", "x"), ordinary) + if a.Refused == "" || len(a.Stdout) != 0 || a.Exit != 1 || a.Why != "not the terminal" { + t.Fatalf("a repair without --why ran as an ordinary call: %+v", a) + } + a = runForMeshCLI(ctx, "laptop", cliAsked("operator", 1000, "settings", "set", "claude-code", "{}"), ordinary) + if a.Refused != "" || !strings.Contains(string(a.Stdout), `verb="mesh-cli"`) { + t.Fatalf("an ordinary settings set did not run through the settings verb's path with MESH_VERB set: %+v", a) + } + for _, server := range []string{"serve", "api", "board"} { + a := runForMeshCLI(ctx, "control", cliAsked("operator", 1000, server), cliVerdict{terminal: true}) + if a.Refused == "" || len(a.Stdout) != 0 { + t.Fatalf("%s was run for mesh-cli: %+v", server, a) + } + } + a = runForMeshCLI(ctx, "control", cliAsked("agent", 1001, "status"), cliVerdict{refused: "agent is not answered"}) + if a.Refused != "agent is not answered" || len(a.Stdout) != 0 { + t.Fatalf("a refused line ran: %+v", a) + } +} + +func TestTheControllersModuleIsMovedAtTheTerminalAlone(t *testing.T) { + t.Setenv("MESH_VERB", "assign") + ctx := context.Background() + if _, err := assignWith(ctx, nil, "laptop", assignOptions{}, "zsh", "mesh-controller"); err == nil || + !strings.Contains(err.Error(), "terminal") { + t.Fatalf("assigning the controller through a verb was not refused: %v", err) + } + if _, err := assign(ctx, nil, "laptop", "mesh-controller"); err == nil || !strings.Contains(err.Error(), "terminal") { + t.Fatalf("assigning the controller through a verb was not refused: %v", err) + } + t.Setenv("MESH_VERB", "unassign") + if _, err := unassign(ctx, nil, "control", "mesh-controller"); err == nil || !strings.Contains(err.Error(), "terminal") { + t.Fatalf("unassigning the controller through a verb was not refused: %v", err) + } + // Another module through a verb, and the controller's at the terminal, are not refused for it. + if err := refusedMovingTheController([]string{"zsh"}); err != nil { + t.Fatalf("another module was refused: %v", err) + } + t.Setenv("MESH_VERB", "") + if err := refusedMovingTheController([]string{"mesh-controller"}); err != nil { + t.Fatalf("the terminal was refused: %v", err) + } +} + +func TestEveryMeshCLILineIsSaidInTheJournal(t *testing.T) { + t.Setenv(echoEnvironment, "1") + var said []string + was := cliJournal + cliJournal = func(line string) { said = append(said, line) } + t.Cleanup(func() { cliJournal = was }) + ctx := link.WithCallID(context.Background(), "call-1") + runForMeshCLI(ctx, "control", cliAsked("operator", 1000, "settings", "set", "x", `{"password":"s3cret"}`), + cliVerdict{terminal: true, why: "the terminal"}) + runForMeshCLI(ctx, "control", cliAsked("agent", 1001, "status"), cliVerdict{refused: "agent is not answered"}) + all := strings.Join(said, "\n") + if len(said) != 2 || !strings.Contains(all, "call-1") || !strings.Contains(all, "operator on control") || + !strings.Contains(all, "as the controller's terminal") || !strings.Contains(all, "refused") { + t.Fatalf("the journal said %q", said) + } + if strings.Contains(all, "s3cret") { + t.Fatalf("the journal carries the line's values: %q", said) + } +} + +func TestAnOrdinaryLineRunsNothingTheCommandVerbWouldRefuse(t *testing.T) { + t.Setenv(echoEnvironment, "1") + for _, line := range [][]string{ + {"node", "account", "control", "x"}, + {"node", "agent-account", "control", "x", "--clear"}, + {"token", "issue", "laptop"}, + {"secret", "export", "x"}, + {"operator", "key", "set", "x"}, + {"assign", "laptop", "zsh"}, + } { + if _, err := ordinaryLine(line); err == nil { + t.Errorf("%q composed as an ordinary line", line) + } + a := runForMeshCLI(context.Background(), "laptop", cliAsked("operator", 1000, line...), cliVerdict{why: "not the terminal"}) + if a.Refused == "" || len(a.Stdout) != 0 { + t.Errorf("%q ran as an ordinary line: %+v", line, a) + } + } + if argv, err := ordinaryLine([]string{"status"}); err != nil || argv[0] != "status" { + t.Fatalf("a read was refused: %v", err) + } +} + +func TestTheCallsVerbNeverShowsAMeshCLILinesAnswer(t *testing.T) { + conn, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + stop, err := link.OverNATS{Conn: conn}.ServeCLI(func(context.Context, string, link.CLIAsked) link.CLIAnswer { + return link.CLIAnswer{Stdout: []byte("s3cret-join")} + }, nil) + if err != nil { + t.Fatal(err) + } + defer stop() + body, _ := json.Marshal(link.CLIAsked{Line: []string{"token", "issue", "x"}, Account: "operator"}) + msg, err := conn.Request(link.CLISubject("control"), body, 5*time.Second) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(msg.Data), base64.StdEncoding.EncodeToString([]byte("s3cret-join"))) { + t.Fatalf("the asker was not given its answer: %s", msg.Data) + } + recent, _ := link.Calls.Recent() + var id string + for _, c := range recent { + if c.Seat == link.CLISeat { + id = c.ID + break + } + } + shown, err := callsAnswer(link.Calls, id) + if err != nil { + t.Fatal(err) + } + said, _ := json.Marshal(shown) + if strings.Contains(string(said), "s3cret-join") || + strings.Contains(string(said), base64.StdEncoding.EncodeToString([]byte("s3cret-join"))) { + t.Fatalf("calls showed a mesh-cli line's answer: %s", said) + } +} + +func TestTheTerminalsMarkIsStrippedFromEveryOtherLine(t *testing.T) { + t.Setenv(echoEnvironment, "1") + t.Setenv(cliTerminalVar, "1") + t.Setenv(servedVar, "1") + for _, env := range [][]string{commandEnvironment("someone", "status", false)} { + for _, kv := range env { + if strings.HasPrefix(kv, cliTerminalVar+"=") { + t.Fatalf("a verb's line carries the terminal's mark: %s", kv) + } + } + } + a := runForMeshCLI(context.Background(), "laptop", cliAsked("operator", 1000, "status"), cliVerdict{why: "not the terminal"}) + if got := string(a.Stdout); !strings.Contains(got, "terminal=false") || !strings.Contains(got, `verb="mesh-cli"`) { + t.Fatalf("an ordinary line with the mark in the serving environment ran as %s", got) + } + a = runForMeshCLI(context.Background(), "control", cliAsked("operator", 1000, "status"), cliVerdict{terminal: true}) + if got := string(a.Stdout); !strings.Contains(got, "terminal=true") { + t.Fatalf("the terminal's line ran as %s", got) + } +} diff --git a/cmd/mesh-controller/meshcli_reads_integration_test.go b/cmd/mesh-controller/meshcli_reads_integration_test.go new file mode 100644 index 00000000..5a76dfd2 --- /dev/null +++ b/cmd/mesh-controller/meshcli_reads_integration_test.go @@ -0,0 +1,66 @@ +//go:build integration + +package main + +import ( + "context" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/link" +) + +func TestAReadCallWritesNoJournalLine(t *testing.T) { + said := journalled(t) + ctx := link.WithCallID(context.Background(), "call-1") + ordinary := cliVerdict{why: "not the terminal"} + for _, line := range [][]string{{"plans"}, {"status"}, {"conditions"}, {"node", "list"}, {"builds"}, {"queue"}, + {"settings", "show", "zsh"}, {"plans", "--json"}} { + if a := runForMeshCLI(ctx, "g14", cliAsked("jochen", 1000, line...), ordinary); a.Exit != 0 || a.Refused != "" { + t.Fatalf("%q did not run: %+v", line, a) + } + } + runForMeshCLI(ctx, "control", cliAsked("jochen", 1000, "plans"), cliVerdict{terminal: true, why: "the terminal"}) + if len(*said) != 0 { + t.Fatalf("read calls wrote journal lines: %q", *said) + } + runForMeshCLI(ctx, "control", cliAsked("jochen", 1000, "assign", "g14", "zsh"), cliVerdict{terminal: true, why: "the terminal"}) + if len(*said) != 1 || !strings.Contains((*said)[0], `asked "assign"`) { + t.Fatalf("a call that changes something was not said: %q", *said) + } +} + +func TestAFailedOrRefusedReadIsSaid(t *testing.T) { + said := journalled(t) + runForMeshCLI(context.Background(), "g14", cliAsked("root", 0, "plans"), cliVerdict{refused: "root is not answered"}) + t.Setenv(echoFails, "1") + a := runForMeshCLI(context.Background(), "g14", cliAsked("jochen", 1000, "plans"), cliVerdict{why: "not the terminal"}) + if a.Exit == 0 { + t.Fatalf("the read did not fail: %+v", a) + } + if len(*said) != 2 || !strings.Contains((*said)[0], "refused") || !strings.Contains((*said)[1], "failed (exit 3)") { + t.Fatalf("the journal said %q", *said) + } + if line := cliReads.said(readCallsEvery); line != "" { + t.Fatalf("a failed read was counted as a read: %q", line) + } +} + +func TestTheReadCallsAreCountedPerCaller(t *testing.T) { + said := journalled(t) + ordinary := cliVerdict{why: "not the terminal"} + for range 3 { + runForMeshCLI(context.Background(), "g14", cliAsked("jochen", 1000, "plans"), ordinary) + } + runForMeshCLI(context.Background(), "ace", cliAsked("jochen", 1000, "status"), ordinary) + if len(*said) != 0 { + t.Fatalf("read calls wrote lines: %q", *said) + } + line := cliReads.said(readCallsEvery) + if !strings.Contains(line, "jochen on ace 1, jochen on g14 3") || !strings.Contains(line, "1h0m0s") { + t.Fatalf("the hourly line said %q", line) + } + if again := cliReads.said(readCallsEvery); again != "" { + t.Fatalf("an hour with no read calls said %q", again) + } +} diff --git a/cmd/mesh-controller/meshcli_reads_test.go b/cmd/mesh-controller/meshcli_reads_test.go index f56e2534..e4c82681 100644 --- a/cmd/mesh-controller/meshcli_reads_test.go +++ b/cmd/mesh-controller/meshcli_reads_test.go @@ -1,11 +1,7 @@ package main import ( - "context" - "strings" "testing" - - "github.com/novox/mesh-controller/internal/link" ) // echoFails makes the echoing test binary end with a failure, as a read that failed would. @@ -24,42 +20,8 @@ func journalled(t *testing.T) *[]string { } // **A call that only reads writes no journal line; one that changes something does** (novox/hq issue 762). -func TestAReadCallWritesNoJournalLine(t *testing.T) { - said := journalled(t) - ctx := link.WithCallID(context.Background(), "call-1") - ordinary := cliVerdict{why: "not the terminal"} - for _, line := range [][]string{{"plans"}, {"status"}, {"conditions"}, {"node", "list"}, {"builds"}, {"queue"}, - {"settings", "show", "zsh"}, {"plans", "--json"}} { - if a := runForMeshCLI(ctx, "g14", cliAsked("jochen", 1000, line...), ordinary); a.Exit != 0 || a.Refused != "" { - t.Fatalf("%q did not run: %+v", line, a) - } - } - runForMeshCLI(ctx, "control", cliAsked("jochen", 1000, "plans"), cliVerdict{terminal: true, why: "the terminal"}) - if len(*said) != 0 { - t.Fatalf("read calls wrote journal lines: %q", *said) - } - runForMeshCLI(ctx, "control", cliAsked("jochen", 1000, "assign", "g14", "zsh"), cliVerdict{terminal: true, why: "the terminal"}) - if len(*said) != 1 || !strings.Contains((*said)[0], `asked "assign"`) { - t.Fatalf("a call that changes something was not said: %q", *said) - } -} // **A refused or failed read keeps its line** (novox/hq issue 762). -func TestAFailedOrRefusedReadIsSaid(t *testing.T) { - said := journalled(t) - runForMeshCLI(context.Background(), "g14", cliAsked("root", 0, "plans"), cliVerdict{refused: "root is not answered"}) - t.Setenv(echoFails, "1") - a := runForMeshCLI(context.Background(), "g14", cliAsked("jochen", 1000, "plans"), cliVerdict{why: "not the terminal"}) - if a.Exit == 0 { - t.Fatalf("the read did not fail: %+v", a) - } - if len(*said) != 2 || !strings.Contains((*said)[0], "refused") || !strings.Contains((*said)[1], "failed (exit 3)") { - t.Fatalf("the journal said %q", *said) - } - if line := cliReads.said(readCallsEvery); line != "" { - t.Fatalf("a failed read was counted as a read: %q", line) - } -} // **A verb is classed by what the call does, not its name** (novox/hq issue 762): the same command word reads with // some arguments and writes with others, and a line no reading form names is said. @@ -103,21 +65,3 @@ func TestAVerbIsClassedByItsArguments(t *testing.T) { } // **Once an hour the read calls are counted per caller, only if there were any** (novox/hq issue 762). -func TestTheReadCallsAreCountedPerCaller(t *testing.T) { - said := journalled(t) - ordinary := cliVerdict{why: "not the terminal"} - for range 3 { - runForMeshCLI(context.Background(), "g14", cliAsked("jochen", 1000, "plans"), ordinary) - } - runForMeshCLI(context.Background(), "ace", cliAsked("jochen", 1000, "status"), ordinary) - if len(*said) != 0 { - t.Fatalf("read calls wrote lines: %q", *said) - } - line := cliReads.said(readCallsEvery) - if !strings.Contains(line, "jochen on ace 1, jochen on g14 3") || !strings.Contains(line, "1h0m0s") { - t.Fatalf("the hourly line said %q", line) - } - if again := cliReads.said(readCallsEvery); again != "" { - t.Fatalf("an hour with no read calls said %q", again) - } -} diff --git a/cmd/mesh-controller/meshcli_stdin_test.go b/cmd/mesh-controller/meshcli_stdin_test.go index 78971630..78193011 100644 --- a/cmd/mesh-controller/meshcli_stdin_test.go +++ b/cmd/mesh-controller/meshcli_stdin_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/meshcli_test.go b/cmd/mesh-controller/meshcli_test.go index 7968f0c8..cc279de1 100644 --- a/cmd/mesh-controller/meshcli_test.go +++ b/cmd/mesh-controller/meshcli_test.go @@ -1,18 +1,11 @@ package main import ( - "context" - "encoding/base64" - "encoding/json" "strings" "testing" - "time" - - "github.com/nats-io/nats.go" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" ) // echoEnvironment makes the test binary, run as a command line, say the verb and caller it was given (TestMain). @@ -63,82 +56,12 @@ func TestMeshCLIIsTheTerminalOnlyForTheControlNodesOperator(t *testing.T) { // The terminal runs its line without MESH_VERB, even where this process carries one; an ordinary call names // mesh-cli; both record who asked through mesh-cli. -func TestTheTerminalRunsWithoutAVerbAndAnOrdinaryCallNamesMeshCLI(t *testing.T) { - t.Setenv(echoEnvironment, "1") - t.Setenv("MESH_VERB", "leaked-from-the-serving-process") - // The serving controller marks itself (ADR 0266); its terminal line for mesh-cli is still the terminal's. - t.Setenv(servedVar, "1") - ctx := context.Background() - - a := runForMeshCLI(ctx, "control", cliAsked("operator", 1000, "status"), cliVerdict{terminal: true, why: "the terminal"}) - if a.Exit != 0 || a.Refused != "" || !a.Terminal { - t.Fatalf("the terminal's line did not run: %+v", a) - } - if got := string(a.Stdout); !strings.Contains(got, `verb=""`) || !strings.Contains(got, "operator through mesh-cli on control") || - !strings.Contains(got, "terminal=true") { - t.Fatalf("the terminal's line ran with %s", got) - } - - a = runForMeshCLI(ctx, "laptop", cliAsked("operator", 1000, "status"), cliVerdict{why: "not the terminal"}) - if a.Exit != 0 || a.Terminal || a.Why != "not the terminal" { - t.Fatalf("an ordinary line did not run as one: %+v", a) - } - if got := string(a.Stdout); !strings.Contains(got, `verb="mesh-cli"`) || !strings.Contains(got, "terminal=false") { - t.Fatalf("an ordinary line ran with %s", got) - } -} // An ordinary call meets every refusal of the generic command verb, and nothing runs; a server is never run. -func TestAnOrdinaryCallMeetsTheCommandVerbsRefusals(t *testing.T) { - t.Setenv(echoEnvironment, "1") - ctx := context.Background() - ordinary := cliVerdict{why: "not the terminal"} - a := runForMeshCLI(ctx, "laptop", cliAsked("operator", 1000, "cleanup", "delete", "x"), ordinary) - if a.Refused == "" || len(a.Stdout) != 0 || a.Exit != 1 || a.Why != "not the terminal" { - t.Fatalf("a repair without --why ran as an ordinary call: %+v", a) - } - a = runForMeshCLI(ctx, "laptop", cliAsked("operator", 1000, "settings", "set", "claude-code", "{}"), ordinary) - if a.Refused != "" || !strings.Contains(string(a.Stdout), `verb="mesh-cli"`) { - t.Fatalf("an ordinary settings set did not run through the settings verb's path with MESH_VERB set: %+v", a) - } - for _, server := range []string{"serve", "api", "board"} { - a := runForMeshCLI(ctx, "control", cliAsked("operator", 1000, server), cliVerdict{terminal: true}) - if a.Refused == "" || len(a.Stdout) != 0 { - t.Fatalf("%s was run for mesh-cli: %+v", server, a) - } - } - a = runForMeshCLI(ctx, "control", cliAsked("agent", 1001, "status"), cliVerdict{refused: "agent is not answered"}) - if a.Refused != "agent is not answered" || len(a.Stdout) != 0 { - t.Fatalf("a refused line ran: %+v", a) - } -} // **Which node is the terminal does not follow a verb** (review of ADR 0272): the controller's module is assigned // and unassigned at the terminal alone, so no caller of `assign` can move the terminal to a node of its choosing. // Asked through the acts themselves, as the verbs ask them; refused before any store is touched (none is given). -func TestTheControllersModuleIsMovedAtTheTerminalAlone(t *testing.T) { - t.Setenv("MESH_VERB", "assign") - ctx := context.Background() - if _, err := assignWith(ctx, nil, "laptop", assignOptions{}, "zsh", "mesh-controller"); err == nil || - !strings.Contains(err.Error(), "terminal") { - t.Fatalf("assigning the controller through a verb was not refused: %v", err) - } - if _, err := assign(ctx, nil, "laptop", "mesh-controller"); err == nil || !strings.Contains(err.Error(), "terminal") { - t.Fatalf("assigning the controller through a verb was not refused: %v", err) - } - t.Setenv("MESH_VERB", "unassign") - if _, err := unassign(ctx, nil, "control", "mesh-controller"); err == nil || !strings.Contains(err.Error(), "terminal") { - t.Fatalf("unassigning the controller through a verb was not refused: %v", err) - } - // Another module through a verb, and the controller's at the terminal, are not refused for it. - if err := refusedMovingTheController([]string{"zsh"}); err != nil { - t.Fatalf("another module was refused: %v", err) - } - t.Setenv("MESH_VERB", "") - if err := refusedMovingTheController([]string{"mesh-controller"}); err != nil { - t.Fatalf("the terminal was refused: %v", err) - } -} // An ordinary `settings set|clear` goes down the settings verb's own path: composed as that verb composes it, and // run with MESH_VERB set, so its refusals — the terminal-only keys among them — are the settings command's own, @@ -177,117 +100,17 @@ func TestAnOrdinarySettingsLineTakesTheSettingsVerbsPath(t *testing.T) { // Every line is said in the controller's journal, with its call, who asked where and how it ran — its command word // only, never the rest of the line (review of ADR 0272). -func TestEveryMeshCLILineIsSaidInTheJournal(t *testing.T) { - t.Setenv(echoEnvironment, "1") - var said []string - was := cliJournal - cliJournal = func(line string) { said = append(said, line) } - t.Cleanup(func() { cliJournal = was }) - ctx := link.WithCallID(context.Background(), "call-1") - runForMeshCLI(ctx, "control", cliAsked("operator", 1000, "settings", "set", "x", `{"password":"s3cret"}`), - cliVerdict{terminal: true, why: "the terminal"}) - runForMeshCLI(ctx, "control", cliAsked("agent", 1001, "status"), cliVerdict{refused: "agent is not answered"}) - all := strings.Join(said, "\n") - if len(said) != 2 || !strings.Contains(all, "call-1") || !strings.Contains(all, "operator on control") || - !strings.Contains(all, "as the controller's terminal") || !strings.Contains(all, "refused") { - t.Fatalf("the journal said %q", said) - } - if strings.Contains(all, "s3cret") { - t.Fatalf("the journal carries the line's values: %q", said) - } -} // **An ordinary line runs only what the generic command verb would** (review of ADR 0272, ADR 0266): its reading // forms, and never a command that is the terminal's alone. Each of these, from another node's operator, is refused // and runs nothing. -func TestAnOrdinaryLineRunsNothingTheCommandVerbWouldRefuse(t *testing.T) { - t.Setenv(echoEnvironment, "1") - for _, line := range [][]string{ - {"node", "account", "control", "x"}, - {"node", "agent-account", "control", "x", "--clear"}, - {"token", "issue", "laptop"}, - {"secret", "export", "x"}, - {"operator", "key", "set", "x"}, - {"assign", "laptop", "zsh"}, - } { - if _, err := ordinaryLine(line); err == nil { - t.Errorf("%q composed as an ordinary line", line) - } - a := runForMeshCLI(context.Background(), "laptop", cliAsked("operator", 1000, line...), cliVerdict{why: "not the terminal"}) - if a.Refused == "" || len(a.Stdout) != 0 { - t.Errorf("%q ran as an ordinary line: %+v", line, a) - } - } - if argv, err := ordinaryLine([]string{"status"}); err != nil || argv[0] != "status" { - t.Fatalf("a read was refused: %v", err) - } -} // **`calls` never shows a mesh-cli line's answer** (review of ADR 0272): the verb's own answer is read for a line // served over a bus, and neither the answer's text nor the base64 JSON writes its bytes in is there. -func TestTheCallsVerbNeverShowsAMeshCLILinesAnswer(t *testing.T) { - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - stop, err := link.OverNATS{Conn: conn}.ServeCLI(func(context.Context, string, link.CLIAsked) link.CLIAnswer { - return link.CLIAnswer{Stdout: []byte("s3cret-join")} - }, nil) - if err != nil { - t.Fatal(err) - } - defer stop() - body, _ := json.Marshal(link.CLIAsked{Line: []string{"token", "issue", "x"}, Account: "operator"}) - msg, err := conn.Request(link.CLISubject("control"), body, 5*time.Second) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(string(msg.Data), base64.StdEncoding.EncodeToString([]byte("s3cret-join"))) { - t.Fatalf("the asker was not given its answer: %s", msg.Data) - } - recent, _ := link.Calls.Recent() - var id string - for _, c := range recent { - if c.Seat == link.CLISeat { - id = c.ID - break - } - } - shown, err := callsAnswer(link.Calls, id) - if err != nil { - t.Fatal(err) - } - said, _ := json.Marshal(shown) - if strings.Contains(string(said), "s3cret-join") || - strings.Contains(string(said), base64.StdEncoding.EncodeToString([]byte("s3cret-join"))) { - t.Fatalf("calls showed a mesh-cli line's answer: %s", said) - } -} // **Only the terminal's line carries the terminal's mark** (review of ADR 0272): a mark the serving controller's // environment holds — leaked, or set by anything — is stripped from every other command line it runs, a verb's and // an ordinary mesh-cli line alike, so neither reads as the terminal. -func TestTheTerminalsMarkIsStrippedFromEveryOtherLine(t *testing.T) { - t.Setenv(echoEnvironment, "1") - t.Setenv(cliTerminalVar, "1") - t.Setenv(servedVar, "1") - for _, env := range [][]string{commandEnvironment("someone", "status", false)} { - for _, kv := range env { - if strings.HasPrefix(kv, cliTerminalVar+"=") { - t.Fatalf("a verb's line carries the terminal's mark: %s", kv) - } - } - } - a := runForMeshCLI(context.Background(), "laptop", cliAsked("operator", 1000, "status"), cliVerdict{why: "not the terminal"}) - if got := string(a.Stdout); !strings.Contains(got, "terminal=false") || !strings.Contains(got, `verb="mesh-cli"`) { - t.Fatalf("an ordinary line with the mark in the serving environment ran as %s", got) - } - a = runForMeshCLI(context.Background(), "control", cliAsked("operator", 1000, "status"), cliVerdict{terminal: true}) - if got := string(a.Stdout); !strings.Contains(got, "terminal=true") { - t.Fatalf("the terminal's line ran as %s", got) - } -} // **Only a login session is the terminal** (review of ADR 0272): the operator's account on the control-node, asking // from a service — the tool runner, a user unit — and not a login session, is an ordinary call. diff --git a/cmd/mesh-controller/mirrors_integration_test.go b/cmd/mesh-controller/mirrors_integration_test.go new file mode 100644 index 00000000..b2e5cfc7 --- /dev/null +++ b/cmd/mesh-controller/mirrors_integration_test.go @@ -0,0 +1,127 @@ +//go:build integration + +package main + +import ( + "fmt" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestRecordingACopyTakesWhatTheStoreHoldsAndDeletesNothing(t *testing.T) { + inv := inventory.ForTest(t) + held := ref("route-proxy", "on-go_base", 1) + absent := ref("route-proxy", "on-go_base", 2) + known := ref("nats", "on-nats_base", 3) + notACopy := ref("route-proxy", "server", 4) + if err := inv.RecordMirrored(t.Context(), "b1", "", []string{known}); err != nil { + t.Fatal(err) + } + store, asked := heldStore(t, fmt.Sprintf("%064x", 1)) + given := []string{held, absent, known, notACopy} + mirrored, err := inv.Mirrored(t.Context()) + if err != nil { + t.Fatal(err) + } + + // A dry run says, and records nothing. + dry, err := recordMirrors(t.Context(), inv, store, given, mirrored, "", false) + if err != nil { + t.Fatal(err) + } + if !dry.DryRun || !slices.Equal(dry.Recorded, []string{held}) || !slices.Equal(dry.AlreadyRecorded, []string{known}) || + len(dry.Refused) != 2 || dry.Refused[absent] == "" || dry.Refused[notACopy] == "" { + t.Fatalf("a dry run answered %+v", dry) + } + if !strings.Contains(dry.Then, "would become eligible") || !strings.Contains(dry.Then, "next sweep") { + t.Fatalf("a dry run did not say what recording does: %q", dry.Then) + } + if again, _ := inv.Mirrored(t.Context()); again[held] { + t.Fatal("a dry run recorded a copy") + } + + // A real one records what the store holds, with the person's why, and the sweep may now decide. + real, err := recordMirrors(t.Context(), inv, store, given, mirrored, "copied before copies were recorded", true) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(real.Recorded, []string{held}) { + t.Fatalf("recorded %v", real.Recorded) + } + left, err := inv.ToCollect(t.Context()) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(left, held) { + t.Fatalf("a recorded copy no kept build stood on is not offered to collect: %v", left) + } + for _, r := range *asked { + if !strings.HasPrefix(r, "HEAD ") { + t.Fatalf("recording a copy asked the store %s", r) + } + } +} + +func TestAConfirmedCollectLetsAnIndexGoWithItsOwnPlatformsOnly(t *testing.T) { + inv := inventory.ForTest(t) + _, eligible, reg := twoCopies(t, inv) + store := reg.serve(t) + references, err := inv.ToCollect(t.Context()) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(references, []string{eligible}) { + t.Fatalf("offered %v, want the failed build's copy", references) + } + a := runCollect(t.Context(), inv, store, references, nil, true, + sweepBounds{most: 10, budget: 5 * time.Second, platforms: true}) + if !slices.Equal(a.LetGo, []string{eligible}) || a.Stopped != "" { + t.Fatalf("let go %v, stopped %q", a.LetGo, a.Stopped) + } + if !slices.Equal(reg.deleted, []string{copyDigest(13), copyDigest(2)}) { + t.Fatalf("deleted %v; want its own platform, then the index", reg.deleted) + } +} + +func TestTheSweepAfterABuildLeavesAnIndexForAConfirmedCollect(t *testing.T) { + inv := inventory.ForTest(t) + _, eligible, reg := twoCopies(t, inv) + reg.held[copyDigest(5)] = true + image := catalogue.ArtifactStoreScheme + "upstream/docker.io/library/golang@" + copyDigest(5) + store := reg.serve(t) + r := sweep(t.Context(), inv, store, []string{eligible, image}, nil, afterBuild) + if r.Indexes != 1 || !slices.Equal(r.LetGo, []string{image}) || !slices.Equal(reg.deleted, []string{copyDigest(5)}) { + t.Fatalf("after a build: %d indexes left, let go %v, deleted %v; want the index left and the image gone", + r.Indexes, r.LetGo, reg.deleted) + } + left, err := inv.ToCollect(t.Context()) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(left, []string{eligible}) { + t.Fatalf("after the sweep the records offer %v; want the index still eligible", left) + } + a := runCollect(t.Context(), inv, store, left, nil, true, + sweepBounds{most: 10, budget: 5 * time.Second, platforms: true}) + if !slices.Equal(a.LetGo, []string{eligible}) || + !slices.Equal(reg.deleted, []string{copyDigest(5), copyDigest(13), copyDigest(2)}) { + t.Fatalf("a confirmed collect let go %v, deleted %v; want the index with its own platform", a.LetGo, reg.deleted) + } +} + +func TestASpareListThatCannotBeReadStopsTheSweepBeforeAnyDelete(t *testing.T) { + inv := inventory.ForTest(t) + _, eligible, reg := twoCopies(t, inv) + reg.fail[copyDigest(1)] = true + store := reg.serve(t) + a := runCollect(t.Context(), inv, store, []string{eligible}, nil, true, + sweepBounds{most: 10, budget: 5 * time.Second, platforms: true}) + if len(a.LetGo) != 0 || len(reg.deleted) != 0 || !strings.Contains(a.Stopped, "nothing was let go") { + t.Fatalf("let go %v, deleted %v, stopped %q", a.LetGo, reg.deleted, a.Stopped) + } +} diff --git a/cmd/mesh-controller/mirrors_test.go b/cmd/mesh-controller/mirrors_test.go index de85038c..fee175df 100644 --- a/cmd/mesh-controller/mirrors_test.go +++ b/cmd/mesh-controller/mirrors_test.go @@ -8,7 +8,6 @@ import ( "strings" "sync" "testing" - "time" "github.com/novox/mesh-controller/internal/artifacts" "github.com/novox/mesh-controller/internal/catalogue" @@ -88,60 +87,6 @@ func heldStore(t *testing.T, holds ...string) (artifacts.Store, *[]string) { return artifacts.Store{Address: strings.TrimPrefix(server.URL, "http://")}, &asked } -func TestRecordingACopyTakesWhatTheStoreHoldsAndDeletesNothing(t *testing.T) { - inv := inventory.ForTest(t) - held := ref("route-proxy", "on-go_base", 1) - absent := ref("route-proxy", "on-go_base", 2) - known := ref("nats", "on-nats_base", 3) - notACopy := ref("route-proxy", "server", 4) - if err := inv.RecordMirrored(t.Context(), "b1", "", []string{known}); err != nil { - t.Fatal(err) - } - store, asked := heldStore(t, fmt.Sprintf("%064x", 1)) - given := []string{held, absent, known, notACopy} - mirrored, err := inv.Mirrored(t.Context()) - if err != nil { - t.Fatal(err) - } - - // A dry run says, and records nothing. - dry, err := recordMirrors(t.Context(), inv, store, given, mirrored, "", false) - if err != nil { - t.Fatal(err) - } - if !dry.DryRun || !slices.Equal(dry.Recorded, []string{held}) || !slices.Equal(dry.AlreadyRecorded, []string{known}) || - len(dry.Refused) != 2 || dry.Refused[absent] == "" || dry.Refused[notACopy] == "" { - t.Fatalf("a dry run answered %+v", dry) - } - if !strings.Contains(dry.Then, "would become eligible") || !strings.Contains(dry.Then, "next sweep") { - t.Fatalf("a dry run did not say what recording does: %q", dry.Then) - } - if again, _ := inv.Mirrored(t.Context()); again[held] { - t.Fatal("a dry run recorded a copy") - } - - // A real one records what the store holds, with the person's why, and the sweep may now decide. - real, err := recordMirrors(t.Context(), inv, store, given, mirrored, "copied before copies were recorded", true) - if err != nil { - t.Fatal(err) - } - if !slices.Equal(real.Recorded, []string{held}) { - t.Fatalf("recorded %v", real.Recorded) - } - left, err := inv.ToCollect(t.Context()) - if err != nil { - t.Fatal(err) - } - if !slices.Contains(left, held) { - t.Fatalf("a recorded copy no kept build stood on is not offered to collect: %v", left) - } - for _, r := range *asked { - if !strings.HasPrefix(r, "HEAD ") { - t.Fatalf("recording a copy asked the store %s", r) - } - } -} - // indexRegistry holds indexes and platforms of one image's repository, answers GETs with their // documents, refuses GETs for the digests in fail, and records every delete. type indexRegistry struct { @@ -234,65 +179,9 @@ func twoCopies(t *testing.T, inv *inventory.Inventory) (kept, eligible string, r // Through the records: a confirmed collect lets the eligible index go with the platform only it names, // and leaves the platform the kept index names. -func TestAConfirmedCollectLetsAnIndexGoWithItsOwnPlatformsOnly(t *testing.T) { - inv := inventory.ForTest(t) - _, eligible, reg := twoCopies(t, inv) - store := reg.serve(t) - references, err := inv.ToCollect(t.Context()) - if err != nil { - t.Fatal(err) - } - if !slices.Equal(references, []string{eligible}) { - t.Fatalf("offered %v, want the failed build's copy", references) - } - a := runCollect(t.Context(), inv, store, references, nil, true, - sweepBounds{most: 10, budget: 5 * time.Second, platforms: true}) - if !slices.Equal(a.LetGo, []string{eligible}) || a.Stopped != "" { - t.Fatalf("let go %v, stopped %q", a.LetGo, a.Stopped) - } - if !slices.Equal(reg.deleted, []string{copyDigest(13), copyDigest(2)}) { - t.Fatalf("deleted %v; want its own platform, then the index", reg.deleted) - } -} // The sweep after a build leaves an eligible index in place and eligible: let go of alone, its // platforms would stay for ever under a record that says collected. A later collect a person confirms // takes it with the platform only it names. An image the same sweep reaches is let go of as before. -func TestTheSweepAfterABuildLeavesAnIndexForAConfirmedCollect(t *testing.T) { - inv := inventory.ForTest(t) - _, eligible, reg := twoCopies(t, inv) - reg.held[copyDigest(5)] = true - image := catalogue.ArtifactStoreScheme + "upstream/docker.io/library/golang@" + copyDigest(5) - store := reg.serve(t) - r := sweep(t.Context(), inv, store, []string{eligible, image}, nil, afterBuild) - if r.Indexes != 1 || !slices.Equal(r.LetGo, []string{image}) || !slices.Equal(reg.deleted, []string{copyDigest(5)}) { - t.Fatalf("after a build: %d indexes left, let go %v, deleted %v; want the index left and the image gone", - r.Indexes, r.LetGo, reg.deleted) - } - left, err := inv.ToCollect(t.Context()) - if err != nil { - t.Fatal(err) - } - if !slices.Equal(left, []string{eligible}) { - t.Fatalf("after the sweep the records offer %v; want the index still eligible", left) - } - a := runCollect(t.Context(), inv, store, left, nil, true, - sweepBounds{most: 10, budget: 5 * time.Second, platforms: true}) - if !slices.Equal(a.LetGo, []string{eligible}) || - !slices.Equal(reg.deleted, []string{copyDigest(5), copyDigest(13), copyDigest(2)}) { - t.Fatalf("a confirmed collect let go %v, deleted %v; want the index with its own platform", a.LetGo, reg.deleted) - } -} // A kept index the store will not answer for stops a confirmed collect before its first delete. -func TestASpareListThatCannotBeReadStopsTheSweepBeforeAnyDelete(t *testing.T) { - inv := inventory.ForTest(t) - _, eligible, reg := twoCopies(t, inv) - reg.fail[copyDigest(1)] = true - store := reg.serve(t) - a := runCollect(t.Context(), inv, store, []string{eligible}, nil, true, - sweepBounds{most: 10, budget: 5 * time.Second, platforms: true}) - if len(a.LetGo) != 0 || len(reg.deleted) != 0 || !strings.Contains(a.Stopped, "nothing was let go") { - t.Fatalf("let go %v, deleted %v, stopped %q", a.LetGo, reg.deleted, a.Stopped) - } -} diff --git a/cmd/mesh-controller/module_health_test.go b/cmd/mesh-controller/module_health_test.go index dba65381..f3706247 100644 --- a/cmd/mesh-controller/module_health_test.go +++ b/cmd/mesh-controller/module_health_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/moved_bus_test.go b/cmd/mesh-controller/moved_bus_test.go index c793ba96..d1fad9de 100644 --- a/cmd/mesh-controller/moved_bus_test.go +++ b/cmd/mesh-controller/moved_bus_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/network_test.go b/cmd/mesh-controller/network_test.go index 2c773cd8..de917559 100644 --- a/cmd/mesh-controller/network_test.go +++ b/cmd/mesh-controller/network_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/new_module_on_merge_test.go b/cmd/mesh-controller/new_module_on_merge_test.go index f5080df1..9bd9623d 100644 --- a/cmd/mesh-controller/new_module_on_merge_test.go +++ b/cmd/mesh-controller/new_module_on_merge_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/nodes_test.go b/cmd/mesh-controller/nodes_test.go index 86b95588..89ad569b 100644 --- a/cmd/mesh-controller/nodes_test.go +++ b/cmd/mesh-controller/nodes_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/one_resolution_integration_test.go b/cmd/mesh-controller/one_resolution_integration_test.go new file mode 100644 index 00000000..989f3ca2 --- /dev/null +++ b/cmd/mesh-controller/one_resolution_integration_test.go @@ -0,0 +1,92 @@ +//go:build integration + +package main + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" +) + +func TestAnUnassignThatLeavesItsNodeUnresolvedSaysSoOnceInItsAnswer(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + twoRivalsAndANote(t, open) + // Straight into the store, as a node that came to this before the act. + for _, m := range []string{"rival-one", "rival-two", "notes"} { + if _, err := open.inventory.Assign(ctx, "laptop", m); err != nil { + t.Fatal(err) + } + } + + said := answered(t, func() error { return assignCommand(ctx, "unassign", []string{"laptop", "notes"}) }) + + if _, _, err := planFor(ctx, open, "laptop"); err == nil || !unresolvable(err) { + t.Fatalf("laptop resolves with two claimants of one seat: %v", err) + } + if n := strings.Count(said, "laptop does not resolve now"); n != 1 { + t.Errorf("the answer says laptop does not resolve %d times, want once:\n%s", n, said) + } + if strings.Contains(said, "is left out of the rest of the mesh") { + t.Errorf("the answer carries a line from a read that is not its resolution:\n%s", said) + } +} + +func TestAnUnassignThatLeavesItsNodeResolvedSaysNothingOfIt(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + aProviderAndItsConsumer(t, open) + if _, err := assign(ctx, open, "laptop", "db", "app"); err != nil { + t.Fatal(err) + } + said, err := unassign(ctx, open, "laptop", "app") + if err != nil { + t.Fatal(err) + } + if _, _, err := planFor(ctx, open, "laptop"); err != nil { + t.Fatal(err) + } + if strings.Contains(said, "does not resolve") || strings.Contains(said, "could not be read") { + t.Errorf("a node that resolves is said not to:\n%s", said) + } +} + +func TestAnAssignThatLeavesItsNodeUnresolvedSaysOnlyThePlansRefusal(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + twoRivalsAndANote(t, open) + if _, err := assign(ctx, open, "laptop", "rival-one"); err != nil { + t.Fatal(err) + } + // Kept and not resolvable: a second claimant of the seat, which assignment keeps. + said, err := assign(ctx, open, "laptop", "rival-two") + if err == nil { + t.Fatalf("two claimants of one node seat resolved:\n%s", said) + } + _, _, planErr := planFor(ctx, open, "laptop") + if planErr == nil || planErr.Error() != err.Error() { + t.Errorf("the act's refusal is not the plan's right after:\nact: %v\nplan: %v", err, planErr) + } + if strings.Contains(said, "now has") { + t.Errorf("the answer says a seat is now held on a node that does not resolve:\n%s", said) + } +} + +// The other readers of the whole mesh still say a machine they leave out, on standard error (novox/hq +// issue 188); an act sends it elsewhere. +func TestAMachineLeftOutIsSaidWhereTheContextSays(t *testing.T) { + var buf bytes.Buffer + sayLeftOut(leftOutSaidBy(context.Background(), &buf), "laptop", errors.New("why")) + if buf.String() != "laptop is left out of the rest of the mesh: it does not resolve: why\n" { + t.Errorf("said %q", buf.String()) + } + got := answered(t, func() error { + sayLeftOut(context.Background(), "laptop", errors.New("why")) + return nil + }) + if !strings.Contains(got, "laptop is left out") { + t.Errorf("without an act's writer the line did not reach standard error: %q", got) + } +} diff --git a/cmd/mesh-controller/one_resolution_test.go b/cmd/mesh-controller/one_resolution_test.go index c49b91c6..82a873a9 100644 --- a/cmd/mesh-controller/one_resolution_test.go +++ b/cmd/mesh-controller/one_resolution_test.go @@ -1,10 +1,6 @@ package main import ( - "bytes" - "context" - "errors" - "strings" "testing" "github.com/novox/mesh-controller/internal/catalogue" @@ -29,84 +25,3 @@ func twoRivalsAndANote(t *testing.T, open *stores) { register(t, open, two) register(t, open, catalogue.Manifest{Module: "notes", Version: "1"}) } - -func TestAnUnassignThatLeavesItsNodeUnresolvedSaysSoOnceInItsAnswer(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - twoRivalsAndANote(t, open) - // Straight into the store, as a node that came to this before the act. - for _, m := range []string{"rival-one", "rival-two", "notes"} { - if _, err := open.inventory.Assign(ctx, "laptop", m); err != nil { - t.Fatal(err) - } - } - - said := answered(t, func() error { return assignCommand(ctx, "unassign", []string{"laptop", "notes"}) }) - - if _, _, err := planFor(ctx, open, "laptop"); err == nil || !unresolvable(err) { - t.Fatalf("laptop resolves with two claimants of one seat: %v", err) - } - if n := strings.Count(said, "laptop does not resolve now"); n != 1 { - t.Errorf("the answer says laptop does not resolve %d times, want once:\n%s", n, said) - } - if strings.Contains(said, "is left out of the rest of the mesh") { - t.Errorf("the answer carries a line from a read that is not its resolution:\n%s", said) - } -} - -func TestAnUnassignThatLeavesItsNodeResolvedSaysNothingOfIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - aProviderAndItsConsumer(t, open) - if _, err := assign(ctx, open, "laptop", "db", "app"); err != nil { - t.Fatal(err) - } - said, err := unassign(ctx, open, "laptop", "app") - if err != nil { - t.Fatal(err) - } - if _, _, err := planFor(ctx, open, "laptop"); err != nil { - t.Fatal(err) - } - if strings.Contains(said, "does not resolve") || strings.Contains(said, "could not be read") { - t.Errorf("a node that resolves is said not to:\n%s", said) - } -} - -func TestAnAssignThatLeavesItsNodeUnresolvedSaysOnlyThePlansRefusal(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - twoRivalsAndANote(t, open) - if _, err := assign(ctx, open, "laptop", "rival-one"); err != nil { - t.Fatal(err) - } - // Kept and not resolvable: a second claimant of the seat, which assignment keeps. - said, err := assign(ctx, open, "laptop", "rival-two") - if err == nil { - t.Fatalf("two claimants of one node seat resolved:\n%s", said) - } - _, _, planErr := planFor(ctx, open, "laptop") - if planErr == nil || planErr.Error() != err.Error() { - t.Errorf("the act's refusal is not the plan's right after:\nact: %v\nplan: %v", err, planErr) - } - if strings.Contains(said, "now has") { - t.Errorf("the answer says a seat is now held on a node that does not resolve:\n%s", said) - } -} - -// The other readers of the whole mesh still say a machine they leave out, on standard error (novox/hq -// issue 188); an act sends it elsewhere. -func TestAMachineLeftOutIsSaidWhereTheContextSays(t *testing.T) { - var buf bytes.Buffer - sayLeftOut(leftOutSaidBy(context.Background(), &buf), "laptop", errors.New("why")) - if buf.String() != "laptop is left out of the rest of the mesh: it does not resolve: why\n" { - t.Errorf("said %q", buf.String()) - } - got := answered(t, func() error { - sayLeftOut(context.Background(), "laptop", errors.New("why")) - return nil - }) - if !strings.Contains(got, "laptop is left out") { - t.Errorf("without an act's writer the line did not reach standard error: %q", got) - } -} diff --git a/cmd/mesh-controller/operator_wait_integration_test.go b/cmd/mesh-controller/operator_wait_integration_test.go new file mode 100644 index 00000000..98a468ef --- /dev/null +++ b/cmd/mesh-controller/operator_wait_integration_test.go @@ -0,0 +1,133 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestTheNeedsOperatorConditionNamesTheAct(t *testing.T) { + k, _ := withConditionsInMemory(t) + ctx := t.Context() + rs := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}} + if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 1}, time.Now()); err != nil { + t.Fatal(err) + } + if got, _ := needsOperatorOpen(t, k); got != nil { + t.Fatal("raised on one statement") + } + if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 2}, time.Now()); err != nil { + t.Fatal(err) + } + got, open := needsOperatorOpen(t, k) + if got == nil { + t.Fatalf("not raised on two statements: %+v", open) + } + for _, c := range open { + if c.Kind == kindModuleUnhealthy { + t.Fatalf("raised as a fault too: %+v", c) + } + } + if got.Kind != kindNeedsOperator || got.Resolver != conditions.ResolverOperator || got.Severity != conditions.Warning { + t.Fatalf("the condition: %+v", got) + } + if !strings.Contains(got.Needs, "type the password of the source games at workstation's desk prompt") || + !strings.Contains(got.Explanation, "hidden prompt opens at the desk") { + t.Fatalf("its needs do not name the act: %q", got.Needs) + } + if strings.Contains(strings.ToLower(got.Explanation), "nothing for you") || !strings.Contains(got.Explanation, "nothing was undone") { + t.Fatalf("its explanation: %q", got.Explanation) + } + if !strings.Contains(got.Summary, "smb-password-games") || !strings.Contains(got.Summary, "nox secret ask workstation mounts smb-password-games") { + t.Fatalf("its summary does not name the secret and the line: %q", got.Summary) + } + // Long open is still a warning: only the operator can end it. + if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 3}, time.Now().Add(48*time.Hour)); err != nil { + t.Fatal(err) + } + if got, _ := needsOperatorOpen(t, k); got == nil || got.Severity == conditions.Urgent { + t.Fatalf("after two days: %+v", got) + } + // Given: the next statement does not say it, and it clears. + if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil { + t.Fatal(err) + } + if got, _ := needsOperatorOpen(t, k); got != nil { + t.Fatal("not cleared once given") + } +} + +func TestASettingsWaitAsksForTheApproval(t *testing.T) { + k, _ := withConditionsInMemory(t) + ctx := t.Context() + rs := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(usernameWait)}} + for i := 1; i <= 2; i++ { + if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": i}, time.Now()); err != nil { + t.Fatal(err) + } + } + got, _ := needsOperatorOpen(t, k) + if got == nil || !strings.Contains(got.Needs, "approve smb-users of mounts on workstation when it is proposed to you") { + t.Fatalf("the condition: %+v", got) + } +} + +func TestAWaitThatFailsItsCheckRaisesUnhealthy(t *testing.T) { + k, _ := withConditionsInMemory(t) + ctx := t.Context() + checked := checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait)}, + factsGiven(map[string]time.Time{"smb-password-games": time.Now()})) + rs := map[string][]inventory.ResourceHealth{"mounts": checked} + for i := 1; i <= 2; i++ { + if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": i}, time.Now()); err != nil { + t.Fatal(err) + } + } + got, open := needsOperatorOpen(t, k) + if got != nil { + t.Fatalf("a wait for a secret given raised needs-operator: %+v", got) + } + unhealthy := false + for _, c := range open { + unhealthy = unhealthy || c.Key == moduleUnhealthyKey("mounts", "workstation") + } + if !unhealthy { + t.Fatalf("not raised as unhealthy: %+v", open) + } +} + +func TestANeedsOperatorThatBecameUnhealthySaysSo(t *testing.T) { + k, _ := withConditionsInMemory(t) + ctx := t.Context() + waiting := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}} + for i := 1; i <= 2; i++ { + if err := judgeModuleHealth(ctx, nil, k, "workstation", waiting, map[string]int{"mounts": i}, time.Now()); err != nil { + t.Fatal(err) + } + } + broken := waitingResource() + broken.State, broken.Reason, broken.Waits = link.StateUnhealthy, "the source games refused its login", nil + for i := 3; i <= 4; i++ { + if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": {broken}}, + map[string]int{"mounts": i}, time.Now()); err != nil { + t.Fatal(err) + } + } + got, open := needsOperatorOpen(t, k) + if got != nil { + t.Fatal("needs-operator still open after it became a fault") + } + found := false + for _, c := range open { + found = found || c.Key == moduleUnhealthyKey("mounts", "workstation") + } + if !found { + t.Fatalf("the fault is not raised: %+v", open) + } +} diff --git a/cmd/mesh-controller/operator_wait_test.go b/cmd/mesh-controller/operator_wait_test.go index ff98a88b..1961f821 100644 --- a/cmd/mesh-controller/operator_wait_test.go +++ b/cmd/mesh-controller/operator_wait_test.go @@ -141,124 +141,7 @@ func needsOperatorOpen(t *testing.T, k *conditions.Keeper) (*conditions.Conditio // Rule 5: two statements of an excused wait raise needs-operator, the operator's, a warning however long, naming the // act; a statement without it clears it. -func TestTheNeedsOperatorConditionNamesTheAct(t *testing.T) { - k, _ := withConditionsInMemory(t) - ctx := t.Context() - rs := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}} - if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 1}, time.Now()); err != nil { - t.Fatal(err) - } - if got, _ := needsOperatorOpen(t, k); got != nil { - t.Fatal("raised on one statement") - } - if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 2}, time.Now()); err != nil { - t.Fatal(err) - } - got, open := needsOperatorOpen(t, k) - if got == nil { - t.Fatalf("not raised on two statements: %+v", open) - } - for _, c := range open { - if c.Kind == kindModuleUnhealthy { - t.Fatalf("raised as a fault too: %+v", c) - } - } - if got.Kind != kindNeedsOperator || got.Resolver != conditions.ResolverOperator || got.Severity != conditions.Warning { - t.Fatalf("the condition: %+v", got) - } - if !strings.Contains(got.Needs, "type the password of the source games at workstation's desk prompt") || - !strings.Contains(got.Explanation, "hidden prompt opens at the desk") { - t.Fatalf("its needs do not name the act: %q", got.Needs) - } - if strings.Contains(strings.ToLower(got.Explanation), "nothing for you") || !strings.Contains(got.Explanation, "nothing was undone") { - t.Fatalf("its explanation: %q", got.Explanation) - } - if !strings.Contains(got.Summary, "smb-password-games") || !strings.Contains(got.Summary, "nox secret ask workstation mounts smb-password-games") { - t.Fatalf("its summary does not name the secret and the line: %q", got.Summary) - } - // Long open is still a warning: only the operator can end it. - if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 3}, time.Now().Add(48*time.Hour)); err != nil { - t.Fatal(err) - } - if got, _ := needsOperatorOpen(t, k); got == nil || got.Severity == conditions.Urgent { - t.Fatalf("after two days: %+v", got) - } - // Given: the next statement does not say it, and it clears. - if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil { - t.Fatal(err) - } - if got, _ := needsOperatorOpen(t, k); got != nil { - t.Fatal("not cleared once given") - } -} - -func TestASettingsWaitAsksForTheApproval(t *testing.T) { - k, _ := withConditionsInMemory(t) - ctx := t.Context() - rs := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(usernameWait)}} - for i := 1; i <= 2; i++ { - if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": i}, time.Now()); err != nil { - t.Fatal(err) - } - } - got, _ := needsOperatorOpen(t, k) - if got == nil || !strings.Contains(got.Needs, "approve smb-users of mounts on workstation when it is proposed to you") { - t.Fatalf("the condition: %+v", got) - } -} // A wait that fails its check is the module's own fault: unhealthy, with why, and no needs-operator. -func TestAWaitThatFailsItsCheckRaisesUnhealthy(t *testing.T) { - k, _ := withConditionsInMemory(t) - ctx := t.Context() - checked := checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait)}, - factsGiven(map[string]time.Time{"smb-password-games": time.Now()})) - rs := map[string][]inventory.ResourceHealth{"mounts": checked} - for i := 1; i <= 2; i++ { - if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": i}, time.Now()); err != nil { - t.Fatal(err) - } - } - got, open := needsOperatorOpen(t, k) - if got != nil { - t.Fatalf("a wait for a secret given raised needs-operator: %+v", got) - } - unhealthy := false - for _, c := range open { - unhealthy = unhealthy || c.Key == moduleUnhealthyKey("mounts", "workstation") - } - if !unhealthy { - t.Fatalf("not raised as unhealthy: %+v", open) - } -} // A module that waited and is then broken says so when the wait clears, and the other way round. -func TestANeedsOperatorThatBecameUnhealthySaysSo(t *testing.T) { - k, _ := withConditionsInMemory(t) - ctx := t.Context() - waiting := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}} - for i := 1; i <= 2; i++ { - if err := judgeModuleHealth(ctx, nil, k, "workstation", waiting, map[string]int{"mounts": i}, time.Now()); err != nil { - t.Fatal(err) - } - } - broken := waitingResource() - broken.State, broken.Reason, broken.Waits = link.StateUnhealthy, "the source games refused its login", nil - for i := 3; i <= 4; i++ { - if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": {broken}}, - map[string]int{"mounts": i}, time.Now()); err != nil { - t.Fatal(err) - } - } - got, open := needsOperatorOpen(t, k) - if got != nil { - t.Fatal("needs-operator still open after it became a fault") - } - found := false - for _, c := range open { - found = found || c.Key == moduleUnhealthyKey("mounts", "workstation") - } - if !found { - t.Fatalf("the fault is not raised: %+v", open) - } -} diff --git a/cmd/mesh-controller/outside_accept_test.go b/cmd/mesh-controller/outside_accept_test.go index b3baff33..bf314b1c 100644 --- a/cmd/mesh-controller/outside_accept_test.go +++ b/cmd/mesh-controller/outside_accept_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/outside_not_given_test.go b/cmd/mesh-controller/outside_not_given_test.go index 11a096c0..74421aa9 100644 --- a/cmd/mesh-controller/outside_not_given_test.go +++ b/cmd/mesh-controller/outside_not_given_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/over_budget_integration_test.go b/cmd/mesh-controller/over_budget_integration_test.go new file mode 100644 index 00000000..38fc57ab --- /dev/null +++ b/cmd/mesh-controller/over_budget_integration_test.go @@ -0,0 +1,50 @@ +//go:build integration + +package main + +import ( + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestAWalksClassAndWindowAreReadAtItsCut(t *testing.T) { + entries := []inventory.Entry{ + {Manifest: catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}}, + {Manifest: catalogue.Manifest{Module: "gitea"}}, + } + walk := func(modules ...string) inventory.Plan { + p := inventory.Plan{Modules: map[string]*inventory.PlanModule{}} + for _, m := range modules { + p.Modules[m] = &inventory.PlanModule{} + } + return p + } + for want, w := range map[string]inventory.Plan{ + inventory.ClassLeaf: walk("gitea"), inventory.ClassCore: walk("gitea", "mesh-controller"), + } { + if got := classOf(w, entries); got != want { + t.Errorf("%v: %s, want %s", w.Modules, got, want) + } + } + if got := classOf(walk("dnsmasq"), entries); got != inventory.ClassCore { + t.Errorf("the resolver's holder is core: %s", got) + } + now := time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC) + batch := inventory.Plan{Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{ + ClosesAt: now.Add(-20 * time.Second), AtMost: now.Add(5 * time.Minute)}}} + times := walkTimesAtCut(batch, walk("gitea"), entries, now) + if times.Cut == nil || !times.Cut.Equal(now) || times.WindowClosed == nil || !times.WindowClosed.Equal(now.Add(-20*time.Second)) || + times.Class != inventory.ClassLeaf { + t.Fatalf("times at the cut: %+v", times) + } + batch.Delivery.Batch.AtMost = now.Add(-time.Minute) + if times := walkTimesAtCut(batch, walk("gitea"), entries, now); !times.WindowClosed.Equal(now.Add(-time.Minute)) { + t.Fatalf("a window closed at its maximum: %v", times.WindowClosed) + } + if times := walkTimesAtCut(inventory.Plan{Delivery: &inventory.PlanDelivery{Alone: true}}, walk("gitea"), entries, now); times.WindowClosed != nil { + t.Fatalf("a merge walked alone had no window: %v", times.WindowClosed) + } +} diff --git a/cmd/mesh-controller/over_budget_test.go b/cmd/mesh-controller/over_budget_test.go index 3131e9b8..52773bbc 100644 --- a/cmd/mesh-controller/over_budget_test.go +++ b/cmd/mesh-controller/over_budget_test.go @@ -61,44 +61,6 @@ func TestTheControllerMayAskForTheDeliveryTimes(t *testing.T) { // A walk's class is core when it walks a module of the controller's own path or one holding the mesh's resolver, // leaf otherwise; its window closed at its batch's window, or its maximum, never after its cut. -func TestAWalksClassAndWindowAreReadAtItsCut(t *testing.T) { - entries := []inventory.Entry{ - {Manifest: catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}}, - {Manifest: catalogue.Manifest{Module: "gitea"}}, - } - walk := func(modules ...string) inventory.Plan { - p := inventory.Plan{Modules: map[string]*inventory.PlanModule{}} - for _, m := range modules { - p.Modules[m] = &inventory.PlanModule{} - } - return p - } - for want, w := range map[string]inventory.Plan{ - inventory.ClassLeaf: walk("gitea"), inventory.ClassCore: walk("gitea", "mesh-controller"), - } { - if got := classOf(w, entries); got != want { - t.Errorf("%v: %s, want %s", w.Modules, got, want) - } - } - if got := classOf(walk("dnsmasq"), entries); got != inventory.ClassCore { - t.Errorf("the resolver's holder is core: %s", got) - } - now := time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC) - batch := inventory.Plan{Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{ - ClosesAt: now.Add(-20 * time.Second), AtMost: now.Add(5 * time.Minute)}}} - times := walkTimesAtCut(batch, walk("gitea"), entries, now) - if times.Cut == nil || !times.Cut.Equal(now) || times.WindowClosed == nil || !times.WindowClosed.Equal(now.Add(-20*time.Second)) || - times.Class != inventory.ClassLeaf { - t.Fatalf("times at the cut: %+v", times) - } - batch.Delivery.Batch.AtMost = now.Add(-time.Minute) - if times := walkTimesAtCut(batch, walk("gitea"), entries, now); !times.WindowClosed.Equal(now.Add(-time.Minute)) { - t.Fatalf("a window closed at its maximum: %v", times.WindowClosed) - } - if times := walkTimesAtCut(inventory.Plan{Delivery: &inventory.PlanDelivery{Alone: true}}, walk("gitea"), entries, now); times.WindowClosed != nil { - t.Fatalf("a merge walked alone had no window: %v", times.WindowClosed) - } -} // What each machine of the rest was sent is kept only for the machines the send reached. func TestTheRestIsKeptForTheMachinesTheSendReached(t *testing.T) { diff --git a/cmd/mesh-controller/pending_integration_test.go b/cmd/mesh-controller/pending_integration_test.go new file mode 100644 index 00000000..bbb10749 --- /dev/null +++ b/cmd/mesh-controller/pending_integration_test.go @@ -0,0 +1,733 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func failedBuild(t *testing.T, open *stores, id string) { + t.Helper() + if err := (builds{open.inventory, open}).Built(t.Context(), outcome(id, "modules/sensors", "3da80a4b00aa", + "", "boom")); err != nil { + t.Fatal(err) + } +} + +func aCatalogueMesh(t *testing.T) *stores { + t.Helper() + open := aMesh(t) + if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: "networkmanager", Version: "1"}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main", + BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + return open +} + +func mergeAdding(t *testing.T, open *stores, dir, commit string) { + t.Helper() + m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, + Paths: []string{dir + "/module.json"}, ModuleDirs: []string{dir}, ModuleDirsSaid: true} + if err := (following{open: open}).SourceMoved(t.Context(), m); err != nil { + t.Fatal(err) + } +} + +func TestAnAssignmentWaitsForTheBuildInFlight(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + asksWithPaths(t) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + + requests, err := open.inventory.RequestsNamed(ctx, "sensors") + if err != nil || len(requests) != 1 || requests[0].For != "merge" || requests[0].Commit != "3da80a4b00aa" { + t.Fatalf("the merge's build request: %+v %v", requests, err) + } + + said, err := assign(ctx, open, "laptop", "sensors") + if err != nil { + t.Fatalf("an assignment while its build runs was refused: %v", err) + } + t.Logf("assign laptop sensors, while its build runs:\n%s", said) + for _, want := range []string{"being built from novox/mesh-catalog@3da80a4b (modules/sensors)", + "build b-modules/sensors", "kept as pending", "the controller assigns sensors to laptop", "`unassign laptop sensors`"} { + if !strings.Contains(said, want) { + t.Fatalf("the answer does not say %q:\n%s", want, said) + } + } + if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { + t.Fatal("a module not registered was assigned") + } + pending := pendingOf(t, open) + if len(pending) != 1 || pending[0].State != inventory.PendingWaiting || pending[0].Build != "b-modules/sensors" { + t.Fatalf("pending: %+v", pending) + } + if again, err := assign(ctx, open, "laptop", "sensors"); err != nil || !strings.Contains(again, "already waits") { + t.Fatalf("a second assignment: %q %v", again, err) + } + + asked, err := theThreeQuestions(ctx, open) + if err != nil { + t.Fatal(err) + } + if asked.well() { + t.Fatal("status called the mesh well while an assignment waits") + } + body, err := statusAsJSON(asked) + if err != nil { + t.Fatal(err) + } + var doc meshStatus + if err := json.Unmarshal(body, &doc); err != nil { + t.Fatal(err) + } + if len(doc.Pending) != 1 || doc.Pending[0].State != "waiting" || doc.Pending[0].Module != "sensors" { + t.Fatalf("status says pending %+v", doc.Pending) + } + + if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", + "3da80a4b00aa", "sensors", "")); err != nil { + t.Fatal(err) + } + if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") { + t.Fatal("the build registered sensors and the pending assignment was not made") + } + pending = pendingOf(t, open) + if len(pending) != 1 || pending[0].State != inventory.PendingApplied || + !strings.HasSuffix(pending[0].Note, "`push laptop` sends it") { + t.Fatalf("pending after the build: %+v", pending) + } +} + +func TestAStatusReadSettlesNothing(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + asksWithPaths(t) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"}) + for range 2 { + if _, err := theThreeQuestions(ctx, open); err != nil { + t.Fatal(err) + } + } + if slices.Contains(assignedTo(t, open, "laptop"), "sensors") || pendingOf(t, open)[0].State != inventory.PendingWaiting { + t.Fatalf("a status read settled a pending assignment: %+v", pendingOf(t, open)) + } + said := settlePending(ctx, open, time.Now()) + if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") || pendingOf(t, open)[0].State != inventory.PendingApplied { + t.Fatalf("the tick did not make it: %v %+v", said, pendingOf(t, open)) + } +} + +func TestAPendingAssignmentExpiresWhenItsBuildFails(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + asksWithPaths(t) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", + "3da80a4b00aa", "", "go build: undefined: sensorsRead")); err != nil { + t.Fatal(err) + } + pending := pendingOf(t, open) + if len(pending) != 1 || pending[0].State != inventory.PendingExpired { + t.Fatalf("pending after a failed build: %+v", pending) + } + for _, want := range []string{"expired", "build b-modules/sensors of modules/sensors failed", + "undefined: sensorsRead", "nothing was assigned", "build \"true\""} { + if !strings.Contains(pending[0].Note, want) { + t.Fatalf("the expiry does not say %q: %s", want, pending[0].Note) + } + } + if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { + t.Fatal("a failed build's module was assigned") + } + + settlePending(ctx, open, time.Now()) + key := pendingObservation(pending[0]).Key() + c, found, err := conditionsFrom.Get(ctx, key) + if err != nil || !found || c.Kind != kindPendingEnded { + t.Fatalf("no condition %s for the assignment not made: %+v %v %v", key, c, found, err) + } + if asked, err := theThreeQuestions(ctx, open); err != nil || asked.well() || len(asked.conditions) == 0 { + t.Fatalf("status does not hold the open condition: %v", err) + } + + _, err = assign(ctx, open, "laptop", "sensors") + if !errors.Is(err, inventory.ErrNoSuchModule) { + t.Fatalf("a module whose build failed: %v", err) + } + for _, want := range []string{"sensors is known and not registered", "its last build, b-modules/sensors", + "failed: go build: undefined: sensorsRead", "build \"true\"", "repository novox/mesh-catalog, path modules/sensors"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("the refusal does not say %q:\n%v", want, err) + } + } + + said, err := unassign(ctx, open, "laptop", "sensors") + if err != nil || !strings.Contains(said, "taken back") { + t.Fatalf("unassign of an expired pending assignment: %q %v", said, err) + } + settlePending(ctx, open, time.Now()) + if _, found, _ := conditionsFrom.Get(ctx, key); found { + t.Fatal("the condition stayed open after the assignment was taken back") + } + // Nothing of it keeps status from being well any more: no open pending assignment, no open condition of it. + asked, err := theThreeQuestions(ctx, open) + if err != nil || pendingOpen(asked.pending) { + t.Fatalf("status still counts the pending assignment: %+v %v", asked.pending, err) + } + for _, c := range asked.conditions { + if c.Kind == kindPendingEnded { + t.Fatalf("status still holds the condition: %+v", c) + } + } +} + +func TestTheConditionClearsWhenTheModuleIsAssignedLater(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + asksWithPaths(t) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", + "3da80a4b00aa", "", "boom")); err != nil { + t.Fatal(err) + } + settlePending(ctx, open, time.Now()) + key := pendingObservation(pendingOf(t, open)[0]).Key() + register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"}) + if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + settlePending(ctx, open, time.Now()) + if _, found, _ := conditionsFrom.Get(ctx, key); found { + t.Fatal("the condition stayed open after the module was assigned") + } +} + +func TestAssignWithBuildAsksForAKnownModule(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-old", Repository: "novox/mesh-catalog", + Seat: "git", Path: "modules/sensors", Ref: "main", For: "build", At: time.Now().Add(-time.Hour)}); err != nil { + t.Fatal(err) + } + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-old", Repository: "novox/mesh-catalog", Ref: "main", + Path: "modules/sensors", On: "anchor", Failed: "no space left on device"}); err != nil { + t.Fatal(err) + } + asked := asksWithPaths(t) + said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors") + if err != nil { + t.Fatal(err) + } + if len(*asked) != 1 || (*asked)[0] != [3]string{"novox/mesh-catalog", "modules/sensors", "main"} { + t.Fatalf("asked %v", *asked) + } + for _, want := range []string{"known and not registered", "no space left on device", + "build b-modules/sensors of novox/mesh-catalog@main (modules/sensors) is asked for now", "kept as pending"} { + if !strings.Contains(said, want) { + t.Fatalf("the answer does not say %q:\n%s", want, said) + } + } + pending := pendingOf(t, open) + if len(pending) != 1 || pending[0].Build != "b-modules/sensors" { + t.Fatalf("pending: %+v", pending) + } + requests, _ := inv.RequestsNamed(ctx, "sensors") + if len(requests) != 2 || requests[0].ID != "b-modules/sensors" || requests[0].For != "assign" { + t.Fatalf("the request is not kept as assign's: %+v", requests) + } +} + +func TestAssignWithBuildRepointsTheWaitingAssignment(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-old", Repository: "novox/mesh-catalog", + Seat: "git", Path: "modules/sensors", Ref: "main", For: "merge", At: time.Now().Add(-time.Hour)}); err != nil { + t.Fatal(err) + } + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-old", Repository: "novox/mesh-catalog", Ref: "main", + Path: "modules/sensors", On: "anchor", Failed: "boom"}); err != nil { + t.Fatal(err) + } + // Waiting on the failed build, its expiry not yet settled. + if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "laptop", Module: "sensors", + Build: "build-old", Repository: "novox/mesh-catalog", Path: "modules/sensors"}); err != nil { + t.Fatal(err) + } + _, err := assign(ctx, open, "laptop", "sensors") + if err == nil || !strings.Contains(err.Error(), "already has a pending assignment of sensors, waiting for build build-old") { + t.Fatalf("known and not built with a pending assignment waiting: %v", err) + } + asked := asksWithPaths(t) + said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors") + if err != nil || !strings.Contains(said, "which waited for build build-old, now waits for build b-modules/sensors") { + t.Fatalf("assign with build: %q %v", said, err) + } + if len(*asked) != 1 { + t.Fatalf("asked %v", *asked) + } + pending := pendingOf(t, open) + if len(pending) != 1 || pending[0].Build != "b-modules/sensors" || pending[0].State != inventory.PendingWaiting { + t.Fatalf("pending: %+v", pending) + } + // The new build in flight now: build "true" again asks nothing. + if said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors"); err != nil || + !strings.Contains(said, "no second build was asked") || len(*asked) != 1 { + t.Fatalf("a second build true while the build runs: %q %v, asked %v", said, err, *asked) + } +} + +func TestAFailedAskIsNotABuildInFlight(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + was := askABuild + askABuild = func(context.Context, buildSource, string, string) (string, error) { + return "", errors.New("cannot submit a build: nats: timeout") + } + t.Cleanup(func() { askABuild = was }) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + _, err := assign(ctx, open, "laptop", "sensors") + if err == nil || !strings.Contains(err.Error(), "the merge that added it (3da80a4b) could not ask for its build: cannot submit") { + t.Fatalf("a merge that could not ask: %v", err) + } + if len(pendingOf(t, open)) != 0 { + t.Fatal("a pending assignment waits on a build never asked") + } + + if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-waited", Repository: "novox/mesh-catalog", + Seat: "git", Path: "modules/gauges", Ref: "main", For: "build"}); err != nil { + t.Fatal(err) + } + if err := inv.MarkNotAsked(ctx, "build-waited", "cannot submit a build: no responders"); err != nil { + t.Fatal(err) + } + _, err = assign(ctx, open, "laptop", "gauges") + if err == nil || !strings.Contains(err.Error(), "build build-waited, asked by build, was not handed over") { + t.Fatalf("a build not handed over: %v", err) + } + // Heard first, the outcome stands: marking it afterwards changes nothing. + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-waited", Repository: "novox/mesh-catalog", Ref: "main", + Path: "modules/gauges", On: "anchor", Failed: "the real failure"}); err != nil { + t.Fatal(err) + } + _, err = assign(ctx, open, "laptop", "gauges") + if err == nil || !strings.Contains(err.Error(), "failed: the real failure") { + t.Fatalf("an outcome heard after a failed wait: %v", err) + } +} + +func TestAPlansBuildRequestIsThePlans(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + asksWithPaths(t) + if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main", + BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1aaaaaaaa", + Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true} + if err := (following{open: open}).SourceMoved(ctx, m); err != nil { + t.Fatal(err) + } + requests, err := open.inventory.RequestsNamed(ctx, "app") + if err != nil || len(requests) != 1 || requests[0].For != "plan" { + t.Fatalf("the plan's request: %+v %v", requests, err) + } +} + +func TestABuildBeingRegisteredIsInFlight(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-done", Repository: "novox/mesh-catalog", + Seat: "git", Path: "modules/sensors", Ref: "main", For: "merge"}); err != nil { + t.Fatal(err) + } + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-done", Repository: "novox/mesh-catalog", Ref: "main", + Path: "modules/sensors", Module: "sensors", On: "anchor"}); err != nil { + t.Fatal(err) + } + said, err := assign(ctx, open, "laptop", "sensors") + if err != nil || !strings.Contains(said, "built and being registered") || !strings.Contains(said, "kept as pending") { + t.Fatalf("a build being registered: %q %v", said, err) + } + settlePending(ctx, open, time.Now().Add(registerGrace+time.Minute)) + p := pendingOf(t, open) + if len(p) != 1 || p[0].State != inventory.PendingExpired || !strings.Contains(p[0].Note, "recorded and not registered") { + t.Fatalf("past the grace: %+v", p) + } +} + +func TestAnUnknownModuleIsRefusedWithTheClosestNames(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "sensord", Version: "1"}) + asked := asksWithPaths(t) + _, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors") + if !errors.Is(err, inventory.ErrNoSuchModule) { + t.Fatalf("an unknown module: %v", err) + } + for _, want := range []string{"no module of that name: sensors", "the catalogue holds no module of that name", + "no build request the controller kept (the last 30 days)", "the closest names it holds: sensord", + "asks for nothing here"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("the refusal does not say %q:\n%v", want, err) + } + } + if strings.Contains(err.Error(), "merge") { + t.Fatalf("the refusal claims a merge it never looked at: %v", err) + } + if len(*asked) != 0 || len(pendingOf(t, open)) != 0 { + t.Fatalf("an unknown module asked %v, pending %+v", *asked, pendingOf(t, open)) + } +} + +func TestAnActWithAModuleNotRegisteredIsRefusedWhole(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + asksWithPaths(t) + register(t, open, catalogue.Manifest{Module: "known", Version: "1"}) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + _, err := assign(ctx, open, "laptop", "known", "sensors") + if err == nil || !strings.Contains(err.Error(), "nothing was assigned") || !strings.Contains(err.Error(), "being built") { + t.Fatalf("an act naming a module in flight: %v", err) + } + if slices.Contains(assignedTo(t, open, "laptop"), "known") || len(pendingOf(t, open)) != 0 { + t.Fatal("an act refused whole was made in part") + } +} + +func TestUnassignWithdrawsAPendingAssignment(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + asksWithPaths(t) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + said, err := unassign(ctx, open, "laptop", "sensors") + if err != nil || !strings.Contains(said, "withdrawn") { + t.Fatalf("unassign of a pending assignment: %q %v", said, err) + } + if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", + "3da80a4b00aa", "sensors", "")); err != nil { + t.Fatal(err) + } + settlePending(ctx, open, time.Now()) + if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { + t.Fatal("a withdrawn assignment was made") + } +} + +func TestAWithdrawalNeverOverridesAClaim(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + asksWithPaths(t) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + p := pendingOf(t, open)[0] + if ok, err := inv.ClaimPending(ctx, p.ID); err != nil || !ok { + t.Fatalf("claim: %v %v", ok, err) + } + _, err := unassign(ctx, open, "laptop", "sensors") + if err == nil || !strings.Contains(err.Error(), "is being assigned sensors now") { + t.Fatalf("unassign of a claimed pending assignment: %v", err) + } + if got := pendingOf(t, open)[0]; got.State != inventory.PendingApplying { + t.Fatalf("the claim was overridden: %+v", got) + } + // Pending(zero) is the waiting ones alone (review point 7). + if waiting, err := inv.Pending(ctx, time.Time{}); err != nil || len(waiting) != 0 { + t.Fatalf("Pending(zero) read a claimed row: %+v %v", waiting, err) + } + + // Left by a controller that stopped: back to waiting, since sensors is not assigned. + settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute)) + if got := pendingOf(t, open)[0]; got.State != inventory.PendingWaiting { + t.Fatalf("a stale claim was not released: %+v", got) + } + + // Withdrawn, then the build registers it: applyPending finds nothing to claim. + if _, err := unassign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"}) + if line := applyPending(ctx, open, p, "b-modules/sensors"); line != "" { + t.Fatalf("a withdrawn pending assignment was made: %s", line) + } + if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { + t.Fatal("a withdrawn pending assignment was assigned") + } + + // A stale claim whose module was assigned is applied. + q, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "sensors", Build: "b-x"}) + if err != nil { + t.Fatal(err) + } + if ok, _ := inv.ClaimPending(ctx, q.ID); !ok { + t.Fatal("claim") + } + if _, err := inv.Assign(ctx, "anchor", "sensors"); err != nil { + t.Fatal(err) + } + settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute)) + rows, _ := inv.PendingFor(ctx, "anchor", "sensors") + if len(rows) != 1 || rows[0].State != inventory.PendingApplied { + t.Fatalf("a stale claim of an assigned module: %+v", rows) + } +} + +func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + for _, r := range []inventory.BuildRequest{ + {ID: "build-lost", Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/lost", Ref: "main", + For: "merge", At: time.Now().Add(-buildRequestBound / 2)}, + {ID: "build-heard", Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/heard", Ref: "main", + For: "merge", At: time.Now()}, + } { + if err := inv.RecordBuildRequest(ctx, r); err != nil { + t.Fatal(err) + } + } + for _, m := range []string{"lost", "heard"} { + if _, err := assign(ctx, open, "laptop", m); err != nil { + t.Fatal(err) + } + } + register(t, open, catalogue.Manifest{Module: "heard", Version: "1"}) + + settlePending(ctx, open, time.Now().Add(buildRequestBound)) + if !slices.Contains(assignedTo(t, open, "laptop"), "heard") { + t.Fatal("a module registered meanwhile was not assigned at the next tick") + } + byModule := map[string]inventory.PendingAssignment{} + for _, p := range pendingOf(t, open) { + byModule[p.Module] = p + } + if p := byModule["lost"]; p.State != inventory.PendingExpired || !strings.Contains(p.Note, "no outcome of build build-lost") { + t.Fatalf("a build that said nothing: %+v", p) + } + if p := byModule["heard"]; p.State != inventory.PendingApplied { + t.Fatalf("a module registered meanwhile: %+v", p) + } + + // Ended rows are deleted after KeptFor (review point 7); open ones never, nor one whose condition is + // still open (second review, bug B). + if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "lost", Build: "build-lost"}); err != nil { + t.Fatal(err) + } + said := settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour)) + rows, err := inv.Pending(ctx, time.Unix(0, 0)) + if err != nil { + t.Fatal(err) + } + for _, r := range rows { + if !r.Open() && (r.Raised == nil || r.Cleared != nil) { + t.Fatalf("an ended pending assignment with no open condition outlived %s: %+v (%v)", inventory.KeptFor, r, said) + } + } + if !strings.Contains(strings.Join(said, "\n"), "deleted 1 pending assignment(s)") { + t.Fatalf("the applied row was not deleted: %v", said) + } +} + +func TestASecondAssignmentNotMadeKeepsItsCondition(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + asked := 0 + was := askABuild + askABuild = func(context.Context, buildSource, string, string) (string, error) { + asked++ + return fmt.Sprintf("b-%d", asked), nil + } + t.Cleanup(func() { askABuild = was }) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + failedBuild(t, open, "b-1") + settlePending(ctx, open, time.Now()) + first := pendingOf(t, open)[0] + if !conditionOpen(t, first) { + t.Fatal("the first assignment not made raised nothing") + } + + if _, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors"); err != nil { + t.Fatal(err) + } + failedBuild(t, open, "b-2") + settlePending(ctx, open, time.Now()) + + var second inventory.PendingAssignment + for _, p := range pendingOf(t, open) { + if p.Build == "b-2" { + second = p + } + } + if second.State != inventory.PendingExpired { + t.Fatalf("the second: %+v", second) + } + if !conditionOpen(t, second) { + t.Fatal("the second assignment's condition was cleared in the tick that raised it") + } + if !conditionOpen(t, rowOf(t, open, first.ID)) { + t.Fatal("the first was judged answered by a second that itself was not made") + } + // unassign takes both back, and the next tick clears both. + if said, err := unassign(ctx, open, "laptop", "sensors"); err != nil || !strings.Contains(said, "b-1") || + !strings.Contains(said, "b-2") { + t.Fatalf("unassign: %q %v", said, err) + } + settlePending(ctx, open, time.Now()) + if conditionOpen(t, first) || conditionOpen(t, second) { + t.Fatal("a condition stayed open after both were taken back") + } +} + +func TestARaisedRowIsKeptUntilItsConditionClears(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + asksWithPaths(t) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + failedBuild(t, open, "b-modules/sensors") + settlePending(ctx, open, time.Now()) + row := pendingOf(t, open)[0] + + settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour)) + if rowOf(t, open, row.ID).Raised == nil || !conditionOpen(t, row) { + t.Fatal("a raised row was pruned, or its condition closed, while the condition was open") + } + if _, err := unassign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + settlePending(ctx, open, time.Now()) + settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour)) + if known, err := inv.PendingKnown(ctx, []int64{row.ID}); err != nil || known[row.ID] { + t.Fatalf("a cleared row outlived %s: %v", inventory.KeptFor, err) + } + + // On anchor, then anchor removed: the row goes, and its condition with it at the next tick. + if _, err := assign(ctx, open, "anchor", "sensors"); err == nil { + t.Fatal("known and not built was not refused") + } + q, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "sensors", Build: "b-x"}) + if err != nil { + t.Fatal(err) + } + if _, err := inv.SettlePending(ctx, q.ID, inventory.PendingWaiting, inventory.PendingExpired, "boom"); err != nil { + t.Fatal(err) + } + settlePending(ctx, open, time.Now()) + q.Node = "anchor" + if !conditionOpen(t, q) { + t.Fatal("not raised") + } + if _, err := inv.RemoveNodeForTest(ctx, "anchor"); err != nil { + t.Fatal(err) + } + settlePending(ctx, open, time.Now()) + if conditionOpen(t, q) { + t.Fatal("a removed machine's condition stayed open") + } +} + +func TestAStaleClaimIsSettledFromWhatTheMeshHolds(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + claimed := func(node string) inventory.PendingAssignment { + p, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: node, Module: "sensors", Build: "b-x"}) + if err != nil { + t.Fatal(err) + } + if ok, err := inv.ClaimPending(ctx, p.ID); err != nil || !ok { + t.Fatalf("claim: %v %v", ok, err) + } + return p + } + notAssigned, assigned := claimed("laptop"), claimed("anchor") + register(t, open, catalogue.Manifest{Module: "unrelated", Version: "1"}) + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "sensors", Version: "1"}, inventory.Source{}); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, "anchor", "sensors"); err != nil { + t.Fatal(err) + } + + settlePending(ctx, open, time.Now()) + if rowOf(t, open, notAssigned.ID).State != inventory.PendingApplying || rowOf(t, open, assigned.ID).State != inventory.PendingApplying { + t.Fatal("a fresh claim was settled") + } + settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute)) + if got := rowOf(t, open, assigned.ID); got.State != inventory.PendingApplied { + t.Fatalf("a stale claim of an assigned module: %+v", got) + } + // Released to waiting, and in the same tick made, since sensors is registered now. + if got := rowOf(t, open, notAssigned.ID); got.State != inventory.PendingApplied || + !slices.Contains(assignedTo(t, open, "laptop"), "sensors") { + t.Fatalf("a stale claim of a module not assigned: %+v", got) + } +} + +func TestABuildNoLongerWaitedForIsStillInFlight(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + for _, id := range []string{"build-timeout", "build-nothandedover"} { + dir := "modules/" + strings.TrimPrefix(id, "build-") + if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-catalog", + Seat: "git", Path: dir, Ref: "main", For: "build"}); err != nil { + t.Fatal(err) + } + } + markWaitFailed(ctx, "build-timeout", errors.New("no build machine answered within 10m0s")) + markWaitFailed(ctx, "build-nothandedover", fmt.Errorf("%w: cannot submit a build: nats: timeout", link.ErrNotHandedOver)) + + said, err := assign(ctx, open, "laptop", "timeout") + if err != nil || !strings.Contains(said, "being built") || !strings.Contains(said, "kept as pending") { + t.Fatalf("a build no longer waited for: %q %v", said, err) + } + if _, err := assign(ctx, open, "laptop", "nothandedover"); err == nil || !strings.Contains(err.Error(), "was not handed over") { + t.Fatalf("a build never handed over: %v", err) + } + settlePending(ctx, open, time.Now().Add(buildRequestBound+time.Minute)) + rows, _ := inv.PendingFor(ctx, "laptop", "timeout") + if len(rows) != 1 || rows[0].State != inventory.PendingExpired || !strings.Contains(rows[0].Note, "its asker stopped waiting") { + t.Fatalf("past the bound: %+v", rows) + } +} diff --git a/cmd/mesh-controller/pending_test.go b/cmd/mesh-controller/pending_test.go index 3efc8053..d949c02c 100644 --- a/cmd/mesh-controller/pending_test.go +++ b/cmd/mesh-controller/pending_test.go @@ -1,12 +1,8 @@ package main import ( - "context" "encoding/json" - "errors" - "fmt" "slices" - "strings" "testing" "time" @@ -17,13 +13,6 @@ import ( ) // failedBuild settles the merge's build of modules/sensors as failed. -func failedBuild(t *testing.T, open *stores, id string) { - t.Helper() - if err := (builds{open.inventory, open}).Built(t.Context(), outcome(id, "modules/sensors", "3da80a4b00aa", - "", "boom")); err != nil { - t.Fatal(err) - } -} func conditionOpen(t *testing.T, p inventory.PendingAssignment) bool { t.Helper() @@ -54,26 +43,8 @@ func rowOf(t *testing.T, open *stores, id int64) inventory.PendingAssignment { // closest names) — and a pending assignment is made when its build registers the module, or ends with why. // aCatalogueMesh is aMesh with one module held from the catalogue, so a merge of it is acted on. -func aCatalogueMesh(t *testing.T) *stores { - t.Helper() - open := aMesh(t) - if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: "networkmanager", Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - return open -} // mergeAdding is the catalogue's merge adding one module's directory, acted on as the bus hands it over. -func mergeAdding(t *testing.T, open *stores, dir, commit string) { - t.Helper() - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, - Paths: []string{dir + "/module.json"}, ModuleDirs: []string{dir}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(t.Context(), m); err != nil { - t.Fatal(err) - } -} // outcome is a build's outcome as the build seat announces it: registered as module, or failed. func outcome(id, dir, commit, module, failed string) link.BuildResult { @@ -107,554 +78,51 @@ func pendingOf(t *testing.T, open *stores) []inventory.PendingAssignment { // A build in flight: the merge asked for it, the request is kept as the merge's, the assignment is kept // pending and said so with the build, status reads it without settling anything, and the build's outcome // makes it — saying that a push sends it, and nothing more. -func TestAnAssignmentWaitsForTheBuildInFlight(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - - requests, err := open.inventory.RequestsNamed(ctx, "sensors") - if err != nil || len(requests) != 1 || requests[0].For != "merge" || requests[0].Commit != "3da80a4b00aa" { - t.Fatalf("the merge's build request: %+v %v", requests, err) - } - - said, err := assign(ctx, open, "laptop", "sensors") - if err != nil { - t.Fatalf("an assignment while its build runs was refused: %v", err) - } - t.Logf("assign laptop sensors, while its build runs:\n%s", said) - for _, want := range []string{"being built from novox/mesh-catalog@3da80a4b (modules/sensors)", - "build b-modules/sensors", "kept as pending", "the controller assigns sensors to laptop", "`unassign laptop sensors`"} { - if !strings.Contains(said, want) { - t.Fatalf("the answer does not say %q:\n%s", want, said) - } - } - if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatal("a module not registered was assigned") - } - pending := pendingOf(t, open) - if len(pending) != 1 || pending[0].State != inventory.PendingWaiting || pending[0].Build != "b-modules/sensors" { - t.Fatalf("pending: %+v", pending) - } - if again, err := assign(ctx, open, "laptop", "sensors"); err != nil || !strings.Contains(again, "already waits") { - t.Fatalf("a second assignment: %q %v", again, err) - } - - asked, err := theThreeQuestions(ctx, open) - if err != nil { - t.Fatal(err) - } - if asked.well() { - t.Fatal("status called the mesh well while an assignment waits") - } - body, err := statusAsJSON(asked) - if err != nil { - t.Fatal(err) - } - var doc meshStatus - if err := json.Unmarshal(body, &doc); err != nil { - t.Fatal(err) - } - if len(doc.Pending) != 1 || doc.Pending[0].State != "waiting" || doc.Pending[0].Module != "sensors" { - t.Fatalf("status says pending %+v", doc.Pending) - } - - if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", - "3da80a4b00aa", "sensors", "")); err != nil { - t.Fatal(err) - } - if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatal("the build registered sensors and the pending assignment was not made") - } - pending = pendingOf(t, open) - if len(pending) != 1 || pending[0].State != inventory.PendingApplied || - !strings.HasSuffix(pending[0].Note, "`push laptop` sends it") { - t.Fatalf("pending after the build: %+v", pending) - } -} // **A read settles nothing** (review of #150, point 6): status — and so the board, the summary and the // probe, which compose from the same reading — leaves a pending assignment whose module is registered as it // is; the controller's tick makes it. -func TestAStatusReadSettlesNothing(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"}) - for range 2 { - if _, err := theThreeQuestions(ctx, open); err != nil { - t.Fatal(err) - } - } - if slices.Contains(assignedTo(t, open, "laptop"), "sensors") || pendingOf(t, open)[0].State != inventory.PendingWaiting { - t.Fatalf("a status read settled a pending assignment: %+v", pendingOf(t, open)) - } - said := settlePending(ctx, open, time.Now()) - if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") || pendingOf(t, open)[0].State != inventory.PendingApplied { - t.Fatalf("the tick did not make it: %v %+v", said, pendingOf(t, open)) - } -} // The build of a pending assignment fails: it expires with the build's words and nothing is assigned; the // tick raises it as a condition once; it then reads as known and not built; and `unassign` takes it back, // after which the tick clears the condition. Status is not called well while the condition is open, and is // again once it clears — no fixed day of "not well" (review point 6). -func TestAPendingAssignmentExpiresWhenItsBuildFails(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", - "3da80a4b00aa", "", "go build: undefined: sensorsRead")); err != nil { - t.Fatal(err) - } - pending := pendingOf(t, open) - if len(pending) != 1 || pending[0].State != inventory.PendingExpired { - t.Fatalf("pending after a failed build: %+v", pending) - } - for _, want := range []string{"expired", "build b-modules/sensors of modules/sensors failed", - "undefined: sensorsRead", "nothing was assigned", "build \"true\""} { - if !strings.Contains(pending[0].Note, want) { - t.Fatalf("the expiry does not say %q: %s", want, pending[0].Note) - } - } - if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatal("a failed build's module was assigned") - } - - settlePending(ctx, open, time.Now()) - key := pendingObservation(pending[0]).Key() - c, found, err := conditionsFrom.Get(ctx, key) - if err != nil || !found || c.Kind != kindPendingEnded { - t.Fatalf("no condition %s for the assignment not made: %+v %v %v", key, c, found, err) - } - if asked, err := theThreeQuestions(ctx, open); err != nil || asked.well() || len(asked.conditions) == 0 { - t.Fatalf("status does not hold the open condition: %v", err) - } - - _, err = assign(ctx, open, "laptop", "sensors") - if !errors.Is(err, inventory.ErrNoSuchModule) { - t.Fatalf("a module whose build failed: %v", err) - } - for _, want := range []string{"sensors is known and not registered", "its last build, b-modules/sensors", - "failed: go build: undefined: sensorsRead", "build \"true\"", "repository novox/mesh-catalog, path modules/sensors"} { - if !strings.Contains(err.Error(), want) { - t.Fatalf("the refusal does not say %q:\n%v", want, err) - } - } - - said, err := unassign(ctx, open, "laptop", "sensors") - if err != nil || !strings.Contains(said, "taken back") { - t.Fatalf("unassign of an expired pending assignment: %q %v", said, err) - } - settlePending(ctx, open, time.Now()) - if _, found, _ := conditionsFrom.Get(ctx, key); found { - t.Fatal("the condition stayed open after the assignment was taken back") - } - // Nothing of it keeps status from being well any more: no open pending assignment, no open condition of it. - asked, err := theThreeQuestions(ctx, open) - if err != nil || pendingOpen(asked.pending) { - t.Fatalf("status still counts the pending assignment: %+v %v", asked.pending, err) - } - for _, c := range asked.conditions { - if c.Kind == kindPendingEnded { - t.Fatalf("status still holds the condition: %+v", c) - } - } -} // The condition also clears when the module is later assigned to the machine. -func TestTheConditionClearsWhenTheModuleIsAssignedLater(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", - "3da80a4b00aa", "", "boom")); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now()) - key := pendingObservation(pendingOf(t, open)[0]).Key() - register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"}) - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now()) - if _, found, _ := conditionsFrom.Get(ctx, key); found { - t.Fatal("the condition stayed open after the module was assigned") - } -} // Known and not built, asked with build: the build is asked from where the last one was, the request kept as // assign's, and the assignment kept pending on the new build. -func TestAssignWithBuildAsksForAKnownModule(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-old", Repository: "novox/mesh-catalog", - Seat: "git", Path: "modules/sensors", Ref: "main", For: "build", At: time.Now().Add(-time.Hour)}); err != nil { - t.Fatal(err) - } - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-old", Repository: "novox/mesh-catalog", Ref: "main", - Path: "modules/sensors", On: "anchor", Failed: "no space left on device"}); err != nil { - t.Fatal(err) - } - asked := asksWithPaths(t) - said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors") - if err != nil { - t.Fatal(err) - } - if len(*asked) != 1 || (*asked)[0] != [3]string{"novox/mesh-catalog", "modules/sensors", "main"} { - t.Fatalf("asked %v", *asked) - } - for _, want := range []string{"known and not registered", "no space left on device", - "build b-modules/sensors of novox/mesh-catalog@main (modules/sensors) is asked for now", "kept as pending"} { - if !strings.Contains(said, want) { - t.Fatalf("the answer does not say %q:\n%s", want, said) - } - } - pending := pendingOf(t, open) - if len(pending) != 1 || pending[0].Build != "b-modules/sensors" { - t.Fatalf("pending: %+v", pending) - } - requests, _ := inv.RequestsNamed(ctx, "sensors") - if len(requests) != 2 || requests[0].ID != "b-modules/sensors" || requests[0].For != "assign" { - t.Fatalf("the request is not kept as assign's: %+v", requests) - } -} // **build "true" with a pending assignment already waiting** (review point 5): the waiting one is made to // wait for the new build, and no build is asked that nothing waits on. -func TestAssignWithBuildRepointsTheWaitingAssignment(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-old", Repository: "novox/mesh-catalog", - Seat: "git", Path: "modules/sensors", Ref: "main", For: "merge", At: time.Now().Add(-time.Hour)}); err != nil { - t.Fatal(err) - } - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-old", Repository: "novox/mesh-catalog", Ref: "main", - Path: "modules/sensors", On: "anchor", Failed: "boom"}); err != nil { - t.Fatal(err) - } - // Waiting on the failed build, its expiry not yet settled. - if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "laptop", Module: "sensors", - Build: "build-old", Repository: "novox/mesh-catalog", Path: "modules/sensors"}); err != nil { - t.Fatal(err) - } - _, err := assign(ctx, open, "laptop", "sensors") - if err == nil || !strings.Contains(err.Error(), "already has a pending assignment of sensors, waiting for build build-old") { - t.Fatalf("known and not built with a pending assignment waiting: %v", err) - } - asked := asksWithPaths(t) - said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors") - if err != nil || !strings.Contains(said, "which waited for build build-old, now waits for build b-modules/sensors") { - t.Fatalf("assign with build: %q %v", said, err) - } - if len(*asked) != 1 { - t.Fatalf("asked %v", *asked) - } - pending := pendingOf(t, open) - if len(pending) != 1 || pending[0].Build != "b-modules/sensors" || pending[0].State != inventory.PendingWaiting { - t.Fatalf("pending: %+v", pending) - } - // The new build in flight now: build "true" again asks nothing. - if said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors"); err != nil || - !strings.Contains(said, "no second build was asked") || len(*asked) != 1 { - t.Fatalf("a second build true while the build runs: %q %v, asked %v", said, err, *asked) - } -} // **A failed ask never reads as in flight** (review point 2): a merge whose ask could not be made keeps the // request as not asked, and assign says the merge could not ask; a request whose asker could not hand it over // or stopped waiting reads the same, unless its outcome was heard. -func TestAFailedAskIsNotABuildInFlight(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - was := askABuild - askABuild = func(context.Context, buildSource, string, string) (string, error) { - return "", errors.New("cannot submit a build: nats: timeout") - } - t.Cleanup(func() { askABuild = was }) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - _, err := assign(ctx, open, "laptop", "sensors") - if err == nil || !strings.Contains(err.Error(), "the merge that added it (3da80a4b) could not ask for its build: cannot submit") { - t.Fatalf("a merge that could not ask: %v", err) - } - if len(pendingOf(t, open)) != 0 { - t.Fatal("a pending assignment waits on a build never asked") - } - - if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-waited", Repository: "novox/mesh-catalog", - Seat: "git", Path: "modules/gauges", Ref: "main", For: "build"}); err != nil { - t.Fatal(err) - } - if err := inv.MarkNotAsked(ctx, "build-waited", "cannot submit a build: no responders"); err != nil { - t.Fatal(err) - } - _, err = assign(ctx, open, "laptop", "gauges") - if err == nil || !strings.Contains(err.Error(), "build build-waited, asked by build, was not handed over") { - t.Fatalf("a build not handed over: %v", err) - } - // Heard first, the outcome stands: marking it afterwards changes nothing. - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-waited", Repository: "novox/mesh-catalog", Ref: "main", - Path: "modules/gauges", On: "anchor", Failed: "the real failure"}); err != nil { - t.Fatal(err) - } - _, err = assign(ctx, open, "laptop", "gauges") - if err == nil || !strings.Contains(err.Error(), "failed: the real failure") { - t.Fatalf("an outcome heard after a failed wait: %v", err) - } -} // A plan's tier keeps its requests as the plan's (review point 3). -func TestAPlansBuildRequestIsThePlans(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1aaaaaaaa", - Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - requests, err := open.inventory.RequestsNamed(ctx, "app") - if err != nil || len(requests) != 1 || requests[0].For != "plan" { - t.Fatalf("the plan's request: %+v %v", requests, err) - } -} // **A build heard as built and not registered yet is in flight** (review point 4): the outcome is recorded // before the module is registered; for that moment assign keeps the assignment pending. Past the grace it is // recorded and not registered, and the tick ends the pending assignment with that. -func TestABuildBeingRegisteredIsInFlight(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-done", Repository: "novox/mesh-catalog", - Seat: "git", Path: "modules/sensors", Ref: "main", For: "merge"}); err != nil { - t.Fatal(err) - } - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-done", Repository: "novox/mesh-catalog", Ref: "main", - Path: "modules/sensors", Module: "sensors", On: "anchor"}); err != nil { - t.Fatal(err) - } - said, err := assign(ctx, open, "laptop", "sensors") - if err != nil || !strings.Contains(said, "built and being registered") || !strings.Contains(said, "kept as pending") { - t.Fatalf("a build being registered: %q %v", said, err) - } - settlePending(ctx, open, time.Now().Add(registerGrace+time.Minute)) - p := pendingOf(t, open) - if len(p) != 1 || p[0].State != inventory.PendingExpired || !strings.Contains(p[0].Note, "recorded and not registered") { - t.Fatalf("past the grace: %+v", p) - } -} // Unknown: refused as no module of that name, claiming only what was looked at, with the closest names; build // asks for nothing. -func TestAnUnknownModuleIsRefusedWithTheClosestNames(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "sensord", Version: "1"}) - asked := asksWithPaths(t) - _, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors") - if !errors.Is(err, inventory.ErrNoSuchModule) { - t.Fatalf("an unknown module: %v", err) - } - for _, want := range []string{"no module of that name: sensors", "the catalogue holds no module of that name", - "no build request the controller kept (the last 30 days)", "the closest names it holds: sensord", - "asks for nothing here"} { - if !strings.Contains(err.Error(), want) { - t.Fatalf("the refusal does not say %q:\n%v", want, err) - } - } - if strings.Contains(err.Error(), "merge") { - t.Fatalf("the refusal claims a merge it never looked at: %v", err) - } - if len(*asked) != 0 || len(pendingOf(t, open)) != 0 { - t.Fatalf("an unknown module asked %v, pending %+v", *asked, pendingOf(t, open)) - } -} // Several modules in one act, one of them not registered: nothing is assigned and nothing kept pending, and // each missing one is said. -func TestAnActWithAModuleNotRegisteredIsRefusedWhole(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - register(t, open, catalogue.Manifest{Module: "known", Version: "1"}) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - _, err := assign(ctx, open, "laptop", "known", "sensors") - if err == nil || !strings.Contains(err.Error(), "nothing was assigned") || !strings.Contains(err.Error(), "being built") { - t.Fatalf("an act naming a module in flight: %v", err) - } - if slices.Contains(assignedTo(t, open, "laptop"), "known") || len(pendingOf(t, open)) != 0 { - t.Fatal("an act refused whole was made in part") - } -} // unassign withdraws a waiting pending assignment; the build goes on and registers the module assigned // nowhere: a withdrawn row is never claimed. -func TestUnassignWithdrawsAPendingAssignment(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - said, err := unassign(ctx, open, "laptop", "sensors") - if err != nil || !strings.Contains(said, "withdrawn") { - t.Fatalf("unassign of a pending assignment: %q %v", said, err) - } - if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", - "3da80a4b00aa", "sensors", "")); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now()) - if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatal("a withdrawn assignment was made") - } -} // **A claim is never overridden** (review point 1): a pending assignment being made is claimed // (waiting → applying); an unassign meanwhile is refused and says so, and leaves the claim; making a row // already withdrawn makes nothing. A claim left by a controller that stopped is settled by the tick from what // the mesh holds: back to waiting when the module is not assigned, applied when it is. -func TestAWithdrawalNeverOverridesAClaim(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - p := pendingOf(t, open)[0] - if ok, err := inv.ClaimPending(ctx, p.ID); err != nil || !ok { - t.Fatalf("claim: %v %v", ok, err) - } - _, err := unassign(ctx, open, "laptop", "sensors") - if err == nil || !strings.Contains(err.Error(), "is being assigned sensors now") { - t.Fatalf("unassign of a claimed pending assignment: %v", err) - } - if got := pendingOf(t, open)[0]; got.State != inventory.PendingApplying { - t.Fatalf("the claim was overridden: %+v", got) - } - // Pending(zero) is the waiting ones alone (review point 7). - if waiting, err := inv.Pending(ctx, time.Time{}); err != nil || len(waiting) != 0 { - t.Fatalf("Pending(zero) read a claimed row: %+v %v", waiting, err) - } - - // Left by a controller that stopped: back to waiting, since sensors is not assigned. - settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute)) - if got := pendingOf(t, open)[0]; got.State != inventory.PendingWaiting { - t.Fatalf("a stale claim was not released: %+v", got) - } - - // Withdrawn, then the build registers it: applyPending finds nothing to claim. - if _, err := unassign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"}) - if line := applyPending(ctx, open, p, "b-modules/sensors"); line != "" { - t.Fatalf("a withdrawn pending assignment was made: %s", line) - } - if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatal("a withdrawn pending assignment was assigned") - } - - // A stale claim whose module was assigned is applied. - q, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "sensors", Build: "b-x"}) - if err != nil { - t.Fatal(err) - } - if ok, _ := inv.ClaimPending(ctx, q.ID); !ok { - t.Fatal("claim") - } - if _, err := inv.Assign(ctx, "anchor", "sensors"); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute)) - rows, _ := inv.PendingFor(ctx, "anchor", "sensors") - if len(rows) != 1 || rows[0].State != inventory.PendingApplied { - t.Fatalf("a stale claim of an assigned module: %+v", rows) - } -} // A build that says nothing within the bound: the pending assignment expires at the next tick, saying so, // rather than waiting for ever; and a module registered by a process that kept no pending assignment is // assigned at the next tick. -func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - for _, r := range []inventory.BuildRequest{ - {ID: "build-lost", Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/lost", Ref: "main", - For: "merge", At: time.Now().Add(-buildRequestBound / 2)}, - {ID: "build-heard", Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/heard", Ref: "main", - For: "merge", At: time.Now()}, - } { - if err := inv.RecordBuildRequest(ctx, r); err != nil { - t.Fatal(err) - } - } - for _, m := range []string{"lost", "heard"} { - if _, err := assign(ctx, open, "laptop", m); err != nil { - t.Fatal(err) - } - } - register(t, open, catalogue.Manifest{Module: "heard", Version: "1"}) - - settlePending(ctx, open, time.Now().Add(buildRequestBound)) - if !slices.Contains(assignedTo(t, open, "laptop"), "heard") { - t.Fatal("a module registered meanwhile was not assigned at the next tick") - } - byModule := map[string]inventory.PendingAssignment{} - for _, p := range pendingOf(t, open) { - byModule[p.Module] = p - } - if p := byModule["lost"]; p.State != inventory.PendingExpired || !strings.Contains(p.Note, "no outcome of build build-lost") { - t.Fatalf("a build that said nothing: %+v", p) - } - if p := byModule["heard"]; p.State != inventory.PendingApplied { - t.Fatalf("a module registered meanwhile: %+v", p) - } - - // Ended rows are deleted after KeptFor (review point 7); open ones never, nor one whose condition is - // still open (second review, bug B). - if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "lost", Build: "build-lost"}); err != nil { - t.Fatal(err) - } - said := settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour)) - rows, err := inv.Pending(ctx, time.Unix(0, 0)) - if err != nil { - t.Fatal(err) - } - for _, r := range rows { - if !r.Open() && (r.Raised == nil || r.Cleared != nil) { - t.Fatalf("an ended pending assignment with no open condition outlived %s: %+v (%v)", inventory.KeptFor, r, said) - } - } - if !strings.Contains(strings.Join(said, "\n"), "deleted 1 pending assignment(s)") { - t.Fatalf("the applied row was not deleted: %v", said) - } -} // The seat's assign passes build on; unassign takes none. func TestTheSeatsAssignPassesBuild(t *testing.T) { @@ -685,180 +153,14 @@ func TestAnAssignmentNotMadeIsSaidPlainly(t *testing.T) { // pending assignment; its build fails too before the next tick. That tick raises the second, and must not // clear it by judging the first answered by the second: each row has its own condition, and a newer row that // itself ended unmade answers nothing. -func TestASecondAssignmentNotMadeKeepsItsCondition(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asked := 0 - was := askABuild - askABuild = func(context.Context, buildSource, string, string) (string, error) { - asked++ - return fmt.Sprintf("b-%d", asked), nil - } - t.Cleanup(func() { askABuild = was }) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - failedBuild(t, open, "b-1") - settlePending(ctx, open, time.Now()) - first := pendingOf(t, open)[0] - if !conditionOpen(t, first) { - t.Fatal("the first assignment not made raised nothing") - } - - if _, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors"); err != nil { - t.Fatal(err) - } - failedBuild(t, open, "b-2") - settlePending(ctx, open, time.Now()) - - var second inventory.PendingAssignment - for _, p := range pendingOf(t, open) { - if p.Build == "b-2" { - second = p - } - } - if second.State != inventory.PendingExpired { - t.Fatalf("the second: %+v", second) - } - if !conditionOpen(t, second) { - t.Fatal("the second assignment's condition was cleared in the tick that raised it") - } - if !conditionOpen(t, rowOf(t, open, first.ID)) { - t.Fatal("the first was judged answered by a second that itself was not made") - } - // unassign takes both back, and the next tick clears both. - if said, err := unassign(ctx, open, "laptop", "sensors"); err != nil || !strings.Contains(said, "b-1") || - !strings.Contains(said, "b-2") { - t.Fatalf("unassign: %q %v", said, err) - } - settlePending(ctx, open, time.Now()) - if conditionOpen(t, first) || conditionOpen(t, second) { - t.Fatal("a condition stayed open after both were taken back") - } -} // **A raised row outlives the pruning until its condition clears** (second review, bug B); and a machine's // removal, which takes its rows with it, clears their conditions at the next tick. -func TestARaisedRowIsKeptUntilItsConditionClears(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - failedBuild(t, open, "b-modules/sensors") - settlePending(ctx, open, time.Now()) - row := pendingOf(t, open)[0] - - settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour)) - if rowOf(t, open, row.ID).Raised == nil || !conditionOpen(t, row) { - t.Fatal("a raised row was pruned, or its condition closed, while the condition was open") - } - if _, err := unassign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now()) - settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour)) - if known, err := inv.PendingKnown(ctx, []int64{row.ID}); err != nil || known[row.ID] { - t.Fatalf("a cleared row outlived %s: %v", inventory.KeptFor, err) - } - - // On anchor, then anchor removed: the row goes, and its condition with it at the next tick. - if _, err := assign(ctx, open, "anchor", "sensors"); err == nil { - t.Fatal("known and not built was not refused") - } - q, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "sensors", Build: "b-x"}) - if err != nil { - t.Fatal(err) - } - if _, err := inv.SettlePending(ctx, q.ID, inventory.PendingWaiting, inventory.PendingExpired, "boom"); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now()) - q.Node = "anchor" - if !conditionOpen(t, q) { - t.Fatal("not raised") - } - if _, err := inv.RemoveNodeForTest(ctx, "anchor"); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now()) - if conditionOpen(t, q) { - t.Fatal("a removed machine's condition stayed open") - } -} // **A claim left by a controller that stopped** (second review): the tick settles it from what the mesh // holds — back to waiting when the module is not assigned there, applied when it is — and leaves a fresh // claim alone. -func TestAStaleClaimIsSettledFromWhatTheMeshHolds(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - claimed := func(node string) inventory.PendingAssignment { - p, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: node, Module: "sensors", Build: "b-x"}) - if err != nil { - t.Fatal(err) - } - if ok, err := inv.ClaimPending(ctx, p.ID); err != nil || !ok { - t.Fatalf("claim: %v %v", ok, err) - } - return p - } - notAssigned, assigned := claimed("laptop"), claimed("anchor") - register(t, open, catalogue.Manifest{Module: "unrelated", Version: "1"}) - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "sensors", Version: "1"}, inventory.Source{}); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, "anchor", "sensors"); err != nil { - t.Fatal(err) - } - - settlePending(ctx, open, time.Now()) - if rowOf(t, open, notAssigned.ID).State != inventory.PendingApplying || rowOf(t, open, assigned.ID).State != inventory.PendingApplying { - t.Fatal("a fresh claim was settled") - } - settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute)) - if got := rowOf(t, open, assigned.ID); got.State != inventory.PendingApplied { - t.Fatalf("a stale claim of an assigned module: %+v", got) - } - // Released to waiting, and in the same tick made, since sensors is registered now. - if got := rowOf(t, open, notAssigned.ID); got.State != inventory.PendingApplied || - !slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatalf("a stale claim of a module not assigned: %+v", got) - } -} // **A waited build whose asker stopped waiting is asked, outcome unknown** (second review): it may still run, // so it reads as in flight until its outcome or its bound, never as not asked; a build never handed over // is not asked. -func TestABuildNoLongerWaitedForIsStillInFlight(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - for _, id := range []string{"build-timeout", "build-nothandedover"} { - dir := "modules/" + strings.TrimPrefix(id, "build-") - if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-catalog", - Seat: "git", Path: dir, Ref: "main", For: "build"}); err != nil { - t.Fatal(err) - } - } - markWaitFailed(ctx, "build-timeout", errors.New("no build machine answered within 10m0s")) - markWaitFailed(ctx, "build-nothandedover", fmt.Errorf("%w: cannot submit a build: nats: timeout", link.ErrNotHandedOver)) - - said, err := assign(ctx, open, "laptop", "timeout") - if err != nil || !strings.Contains(said, "being built") || !strings.Contains(said, "kept as pending") { - t.Fatalf("a build no longer waited for: %q %v", said, err) - } - if _, err := assign(ctx, open, "laptop", "nothandedover"); err == nil || !strings.Contains(err.Error(), "was not handed over") { - t.Fatalf("a build never handed over: %v", err) - } - settlePending(ctx, open, time.Now().Add(buildRequestBound+time.Minute)) - rows, _ := inv.PendingFor(ctx, "laptop", "timeout") - if len(rows) != 1 || rows[0].State != inventory.PendingExpired || !strings.Contains(rows[0].Note, "its asker stopped waiting") { - t.Fatalf("past the bound: %+v", rows) - } -} diff --git a/cmd/mesh-controller/person_wait_integration_test.go b/cmd/mesh-controller/person_wait_integration_test.go new file mode 100644 index 00000000..72bb5dd6 --- /dev/null +++ b/cmd/mesh-controller/person_wait_integration_test.go @@ -0,0 +1,165 @@ +//go:build integration + +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + open := aMesh(t) + ctx := t.Context() + k, _ := withConditionsInMemory(t) + at := h0 + say := func(rs ...link.ResourceHealth) { + t.Helper() + at = at.Add(time.Second) + for i := range rs { + rs[i].Since = at + } + if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, + Resources: rs}, at); err != nil { + t.Fatal(err) + } + } + asLink := func(r inventory.ResourceHealth) link.ResourceHealth { + return link.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target, State: r.State, + Reason: r.Reason, Account: r.Account} + } + account, unit := asLink(relogin("lights", "operator")), asLink(userUnit("lights", "operator")) + openNow := func() []conditions.Condition { + t.Helper() + list, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + return list + } + say(account, unit) + say(account, unit) + list := openNow() + if len(list) != 1 || list[0].Key != "module.lights.laptop.relogin-needed" { + t.Fatalf("raised %+v; want lights' relogin-needed alone", list) + } + c := list[0] + if c.Severity != conditions.Warning || c.Resolver != conditions.ResolverOperator || + !strings.HasPrefix(c.Summary, "relogin needed on laptop:") { + t.Fatalf("the condition is %s %s %q", c.Severity, c.Resolver, c.Summary) + } + // After the new login the account is healthy, and the unit, still failing, is the module's own fault. + account.State, account.Reason = link.StateHealthy, "" + say(account, unit) + list = openNow() + if len(list) != 1 || list[0].Key != "module.lights.laptop.unhealthy" { + t.Fatalf("after the login: %+v; want lights' own unhealthy condition alone", list) + } + unit.State, unit.Reason = link.StateHealthy, "" + say(account, unit) + if list = openNow(); len(list) != 0 { + t.Fatalf("after the unit runs: %+v; want nothing open", list) + } +} + +func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + releaseHeard = func(context.Context, *stores) (map[string]bool, error) { + return map[string]bool{"laptop": true}, nil + } + backlogFacts := gatherGateFacts + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + f, err := backlogFacts(ctx, open, component) + f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: time.Now(), + Resources: []inventory.ResourceHealth{ + {Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy}, + {Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, + Reason: "down"}}}} + return f, err + } + gateEvery, gateBound = 0, 300*time.Millisecond + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + advancePlans(ctx, b.open) + if p := b.release(t); p.State != inventory.PlanRolling { + break + } + time.Sleep(20 * time.Millisecond) + } + p := b.release(t) + if p.State != inventory.PlanFailed { + t.Fatalf("the walk is %s: %s; want it failed on late", p.State, p.Note) + } + if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed || + !strings.Contains(v.Why, "on its own") { + t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err) + } + if failed, _ := inv.GateFailed(ctx, "build-late-c2"); !failed { + t.Fatal("late's build is not marked failed at its gate") + } + current, _ := inv.CurrentBuilds(ctx) + if current["app"].Commit != "c2" || current["late"].Commit != "c1" { + t.Fatalf("registered: app %s, late %s; want app kept at c2 and late put back to c1", current["app"].Commit, + current["late"].Commit) + } + if !strings.Contains(p.Note, "passed on their own and kept: app") { + t.Errorf("the walk does not say what kept its pass: %s", p.Note) + } +} + +func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + open := aMesh(t) + ctx := t.Context() + store := conditions.NewInMemory() + told := &conditions.Told{} + k := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store, Teller: told}) + t.Cleanup(func() { k.Close(context.Background()) }) + at := h0 + say := func(rs ...link.ResourceHealth) { + t.Helper() + at = at.Add(time.Second) + for i := range rs { + rs[i].Since = at + } + if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, + Resources: rs}, at); err != nil { + t.Fatal(err) + } + } + unit := link.ResourceHealth{Module: "lights", Resource: "lights.daemon", Kind: link.KindUnit, Target: "lights.service", + Account: "operator", State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"} + account := link.ResourceHealth{Module: "lights", Resource: "lights.operator", Kind: link.KindAccount, Target: "operator", + Account: "operator", State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": operator is in the group lights"} + say(unit) + say(unit) // lights not working + say(account, unit) + say(account, unit) // now it waits for a login + var resolved []string + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline) && len(resolved) == 0; { + time.Sleep(20 * time.Millisecond) + for _, e := range told.Said() { + if e.Change == conditions.ChangeCleared { + resolved = append(resolved, e.Condition.Key+": "+e.Condition.Resolved) + // The clearing line is held to the plain rule too (ADR 0253). + w := conditions.Words{Headline: e.Condition.Headline, Explanation: e.Condition.Explanation, + Resolved: e.Condition.Resolved, Needs: e.Condition.Needs} + if why, ok := conditions.PlainWords(w, "laptop"); !ok { + t.Errorf("%s: its clearing words are not plain: %s", e.Condition.Key, why) + } + } + } + } + if len(resolved) != 1 || !strings.Contains(resolved[0], "module.lights.laptop.unhealthy: lights on laptop now waits only for a new login") { + t.Fatalf("cleared %v; want the not-working condition cleared as now waiting for a login", resolved) + } +} diff --git a/cmd/mesh-controller/person_wait_test.go b/cmd/mesh-controller/person_wait_test.go index d5337899..cd61536e 100644 --- a/cmd/mesh-controller/person_wait_test.go +++ b/cmd/mesh-controller/person_wait_test.go @@ -1,7 +1,6 @@ package main import ( - "context" "strings" "testing" "time" @@ -103,110 +102,9 @@ func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) { // The module's condition says the wait in one sentence for a person, and clears on the first statement that // no longer says it — said as the module's own fault when its unit still fails after the new login. -func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - k, _ := withConditionsInMemory(t) - at := h0 - say := func(rs ...link.ResourceHealth) { - t.Helper() - at = at.Add(time.Second) - for i := range rs { - rs[i].Since = at - } - if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, - Resources: rs}, at); err != nil { - t.Fatal(err) - } - } - asLink := func(r inventory.ResourceHealth) link.ResourceHealth { - return link.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target, State: r.State, - Reason: r.Reason, Account: r.Account} - } - account, unit := asLink(relogin("lights", "operator")), asLink(userUnit("lights", "operator")) - openNow := func() []conditions.Condition { - t.Helper() - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - return list - } - say(account, unit) - say(account, unit) - list := openNow() - if len(list) != 1 || list[0].Key != "module.lights.laptop.relogin-needed" { - t.Fatalf("raised %+v; want lights' relogin-needed alone", list) - } - c := list[0] - if c.Severity != conditions.Warning || c.Resolver != conditions.ResolverOperator || - !strings.HasPrefix(c.Summary, "relogin needed on laptop:") { - t.Fatalf("the condition is %s %s %q", c.Severity, c.Resolver, c.Summary) - } - // After the new login the account is healthy, and the unit, still failing, is the module's own fault. - account.State, account.Reason = link.StateHealthy, "" - say(account, unit) - list = openNow() - if len(list) != 1 || list[0].Key != "module.lights.laptop.unhealthy" { - t.Fatalf("after the login: %+v; want lights' own unhealthy condition alone", list) - } - unit.State, unit.Reason = link.StateHealthy, "" - say(account, unit) - if list = openNow(); len(list) != 0 { - t.Fatalf("after the unit runs: %+v; want nothing open", list) - } -} // A module that is healthy on its own keeps its pass when another module of the same send fails at the // bound: it is neither put back nor left without a verdict for every other walk to wait on. -func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - releaseHeard = func(context.Context, *stores) (map[string]bool, error) { - return map[string]bool{"laptop": true}, nil - } - backlogFacts := gatherGateFacts - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f, err := backlogFacts(ctx, open, component) - f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: time.Now(), - Resources: []inventory.ResourceHealth{ - {Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy}, - {Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, - Reason: "down"}}}} - return f, err - } - gateEvery, gateBound = 0, 300*time.Millisecond - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - advancePlans(ctx, b.open) - if p := b.release(t); p.State != inventory.PlanRolling { - break - } - time.Sleep(20 * time.Millisecond) - } - p := b.release(t) - if p.State != inventory.PlanFailed { - t.Fatalf("the walk is %s: %s; want it failed on late", p.State, p.Note) - } - if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed || - !strings.Contains(v.Why, "on its own") { - t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err) - } - if failed, _ := inv.GateFailed(ctx, "build-late-c2"); !failed { - t.Fatal("late's build is not marked failed at its gate") - } - current, _ := inv.CurrentBuilds(ctx) - if current["app"].Commit != "c2" || current["late"].Commit != "c1" { - t.Fatalf("registered: app %s, late %s; want app kept at c2 and late put back to c1", current["app"].Commit, - current["late"].Commit) - } - if !strings.Contains(p.Note, "passed on their own and kept: app") { - t.Errorf("the walk does not say what kept its pass: %s", p.Note) - } -} // What the operator reads of the wait (ADR 0253, ADR 0254): it needs them, so it is never quiet, and it offers // no button, since nothing but their own new login can do it. @@ -242,53 +140,6 @@ func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) { // When a module not working turns into one waiting for a new login, and back, the clearing line says what it // became, never that it works again (issue 318 review). -func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - store := conditions.NewInMemory() - told := &conditions.Told{} - k := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store, Teller: told}) - t.Cleanup(func() { k.Close(context.Background()) }) - at := h0 - say := func(rs ...link.ResourceHealth) { - t.Helper() - at = at.Add(time.Second) - for i := range rs { - rs[i].Since = at - } - if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, - Resources: rs}, at); err != nil { - t.Fatal(err) - } - } - unit := link.ResourceHealth{Module: "lights", Resource: "lights.daemon", Kind: link.KindUnit, Target: "lights.service", - Account: "operator", State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"} - account := link.ResourceHealth{Module: "lights", Resource: "lights.operator", Kind: link.KindAccount, Target: "operator", - Account: "operator", State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": operator is in the group lights"} - say(unit) - say(unit) // lights not working - say(account, unit) - say(account, unit) // now it waits for a login - var resolved []string - for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline) && len(resolved) == 0; { - time.Sleep(20 * time.Millisecond) - for _, e := range told.Said() { - if e.Change == conditions.ChangeCleared { - resolved = append(resolved, e.Condition.Key+": "+e.Condition.Resolved) - // The clearing line is held to the plain rule too (ADR 0253). - w := conditions.Words{Headline: e.Condition.Headline, Explanation: e.Condition.Explanation, - Resolved: e.Condition.Resolved, Needs: e.Condition.Needs} - if why, ok := conditions.PlainWords(w, "laptop"); !ok { - t.Errorf("%s: its clearing words are not plain: %s", e.Condition.Key, why) - } - } - } - } - if len(resolved) != 1 || !strings.Contains(resolved[0], "module.lights.laptop.unhealthy: lights on laptop now waits only for a new login") { - t.Fatalf("cleared %v; want the not-working condition cleared as now waiting for a login", resolved) - } -} // Which moves add an account group, read from the builds' manifests (issue 318 review). func TestOnlyAMoveThatAddsAnAccountGroupCanBringAWait(t *testing.T) { diff --git a/cmd/mesh-controller/placement_moved_test.go b/cmd/mesh-controller/placement_moved_test.go index 5831cc56..f4a8938b 100644 --- a/cmd/mesh-controller/placement_moved_test.go +++ b/cmd/mesh-controller/placement_moved_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/placement_words_moved_integration_test.go b/cmd/mesh-controller/placement_words_moved_integration_test.go new file mode 100644 index 00000000..f707f74d --- /dev/null +++ b/cmd/mesh-controller/placement_words_moved_integration_test.go @@ -0,0 +1,92 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func aMeshWithTheWordsInItsLayers(t *testing.T) (*stores, wordedMesh) { + t.Helper() + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + w := theWordedMesh(t) + image := "registry.invalid:5000/library@sha256:" + strings.Repeat("a", 64) + register(t, open, catalogue.Manifest{Module: "proxy", Version: "1", Provides: catalogue.FromAnywhere("route"), + Receives: map[string]string{"route": "/var/lib/proxy/routes.json"}}) + for _, text := range w.Modules { + m, err := catalogue.ParseManifest([]byte(strings.ReplaceAll(string(text), "IMAGE", image))) + if err != nil { + t.Fatal(err) + } + register(t, open, m) + } + on := map[string][]string{"anchor": {"proxy"}, "laptop": {"proxy"}} + for _, l := range w.Layers { + settings, err := json.Marshal(l.Settings) + if err != nil { + t.Fatal(err) + } + inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`, + l.Node, l.Module, string(settings)) + if l.Placement != nil { + placement, err := json.Marshal(l.Placement) + if err != nil { + t.Fatal(err) + } + inv.ExecForTest(t, `insert into placement (node, module, values) select id, $2, $3 from node where name = $1`, + l.Node, l.Module, string(placement)) + } + on[l.Node] = append(on[l.Node], l.Module) + } + for _, node := range []string{"anchor", "laptop"} { + if err := inv.SetPublicDomain(ctx, node, node+".example.test"); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, node, on[node]...); err != nil { + t.Fatalf("assign %s %v: %v", node, on[node], err) + } + } + return open, w +} + +func TestADeclarationIsTheSameWhileTheWordsAreStillInTheLayers(t *testing.T) { + open, _ := aMeshWithTheWordsInItsLayers(t) + sameAsTheGolden(t, open, "before the move") +} + +func TestADeclarationIsTheSameAfterTheMoveOfPlacement(t *testing.T) { + open, w := aMeshWithTheWordsInItsLayers(t) + ctx := t.Context() + inv := open.inventory + asked, err := runMovePlacement(ctx, inv, false) + if err != nil || !asked.DryRun || len(asked.Moved) != len(w.Layers) { + t.Fatalf("the move, not confirmed, says %+v (%v); %d layers hold the words", asked, err, len(w.Layers)) + } + if held := inv.QueryForTest(t, `select module from settings where values ?| $1`, catalogue.PlacementKeys()); len(held) != len(w.Layers) { + t.Fatalf("the move, not confirmed, moved something: %d layers still hold the words", len(held)) + } + sameAsTheGolden(t, open, "after the move was asked without confirming") + + moved, err := runMovePlacement(ctx, inv, true) + if err != nil || moved.DryRun || !reflect.DeepEqual(moved.Moved, asked.Moved) { + t.Fatalf("the move says %+v (%v), it said it would move %+v", moved, err, asked.Moved) + } + if held := inv.QueryForTest(t, `select module from settings where values ?| $1`, catalogue.PlacementKeys()); len(held) != 0 { + t.Fatalf("a settings layer still holds a placement key: %v", held) + } + sameAsTheGolden(t, open, "after the move") + + if again, err := runMovePlacement(ctx, inv, true); err != nil || len(again.Moved) != 0 { + t.Fatalf("asked again, the move moved %+v (%v)", again, err) + } + if said := describeMovePlacement(movePlacementAnswer{}); !strings.Contains(said, "nothing to move") { + t.Fatalf("an empty move says %q", said) + } +} diff --git a/cmd/mesh-controller/placement_words_moved_test.go b/cmd/mesh-controller/placement_words_moved_test.go index 63aa3185..f6562433 100644 --- a/cmd/mesh-controller/placement_words_moved_test.go +++ b/cmd/mesh-controller/placement_words_moved_test.go @@ -4,12 +4,10 @@ import ( "encoding/json" "os" "path/filepath" - "reflect" "sort" "strings" "testing" - "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/link" ) @@ -45,50 +43,6 @@ func theWordedMesh(t *testing.T) wordedMesh { // aMeshWithTheWordsInItsLayers is aMesh with every module of the worded mesh registered, a route provider on both // machines, and its layers held as the store holds them before the four words move. -func aMeshWithTheWordsInItsLayers(t *testing.T) (*stores, wordedMesh) { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - w := theWordedMesh(t) - image := "registry.invalid:5000/library@sha256:" + strings.Repeat("a", 64) - register(t, open, catalogue.Manifest{Module: "proxy", Version: "1", Provides: catalogue.FromAnywhere("route"), - Receives: map[string]string{"route": "/var/lib/proxy/routes.json"}}) - for _, text := range w.Modules { - m, err := catalogue.ParseManifest([]byte(strings.ReplaceAll(string(text), "IMAGE", image))) - if err != nil { - t.Fatal(err) - } - register(t, open, m) - } - on := map[string][]string{"anchor": {"proxy"}, "laptop": {"proxy"}} - for _, l := range w.Layers { - settings, err := json.Marshal(l.Settings) - if err != nil { - t.Fatal(err) - } - inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`, - l.Node, l.Module, string(settings)) - if l.Placement != nil { - placement, err := json.Marshal(l.Placement) - if err != nil { - t.Fatal(err) - } - inv.ExecForTest(t, `insert into placement (node, module, values) select id, $2, $3 from node where name = $1`, - l.Node, l.Module, string(placement)) - } - on[l.Node] = append(on[l.Node], l.Module) - } - for _, node := range []string{"anchor", "laptop"} { - if err := inv.SetPublicDomain(ctx, node, node+".example.test"); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, node, on[node]...); err != nil { - t.Fatalf("assign %s %v: %v", node, on[node], err) - } - } - return open, w -} // toldOfTheWords is what a machine is told of the modules whose layers say the four words, and of the route // provider that serves their names: those modules' resources and the filter's rules, as JSON a person can diff. @@ -166,44 +120,11 @@ func sameAsTheGolden(t *testing.T, open *stores, when string) { // issue 736). Every live layer that says `endpoints`, `expose`, `reach` or `ports` is seeded as the store holds it // after delivery, on a module shaped as its definition is, and every machine's declaration is composed. What each // is told is byte for byte what the controller before this change composed from the same seed. -func TestADeclarationIsTheSameWhileTheWordsAreStillInTheLayers(t *testing.T) { - open, _ := aMeshWithTheWordsInItsLayers(t) - sameAsTheGolden(t, open, "before the move") -} // **And after the move, still nothing new** (issue 736). The move of placement, asked without confirming, says // every layer that holds one of the four words and moves nothing; confirmed, it moves them all, no settings layer // holds a placement key after, and each machine is told byte for byte what it was told before the move — and what // the controller before this change told it. Asked again, it moves nothing. -func TestADeclarationIsTheSameAfterTheMoveOfPlacement(t *testing.T) { - open, w := aMeshWithTheWordsInItsLayers(t) - ctx := t.Context() - inv := open.inventory - asked, err := runMovePlacement(ctx, inv, false) - if err != nil || !asked.DryRun || len(asked.Moved) != len(w.Layers) { - t.Fatalf("the move, not confirmed, says %+v (%v); %d layers hold the words", asked, err, len(w.Layers)) - } - if held := inv.QueryForTest(t, `select module from settings where values ?| $1`, catalogue.PlacementKeys()); len(held) != len(w.Layers) { - t.Fatalf("the move, not confirmed, moved something: %d layers still hold the words", len(held)) - } - sameAsTheGolden(t, open, "after the move was asked without confirming") - - moved, err := runMovePlacement(ctx, inv, true) - if err != nil || moved.DryRun || !reflect.DeepEqual(moved.Moved, asked.Moved) { - t.Fatalf("the move says %+v (%v), it said it would move %+v", moved, err, asked.Moved) - } - if held := inv.QueryForTest(t, `select module from settings where values ?| $1`, catalogue.PlacementKeys()); len(held) != 0 { - t.Fatalf("a settings layer still holds a placement key: %v", held) - } - sameAsTheGolden(t, open, "after the move") - - if again, err := runMovePlacement(ctx, inv, true); err != nil || len(again.Moved) != 0 { - t.Fatalf("asked again, the move moved %+v (%v)", again, err) - } - if said := describeMovePlacement(movePlacementAnswer{}); !strings.Contains(said, "nothing to move") { - t.Fatalf("an empty move says %q", said) - } -} // The verb's command line: a dry run unless confirmed, and a confirmed move says why, recorded as a person's step // of the release, never a repair (issue 736). diff --git a/cmd/mesh-controller/plan_clock_integration_test.go b/cmd/mesh-controller/plan_clock_integration_test.go new file mode 100644 index 00000000..8581e2e3 --- /dev/null +++ b/cmd/mesh-controller/plan_clock_integration_test.go @@ -0,0 +1,52 @@ +//go:build integration + +package main + +import ( + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestAPlanStandingStillIsNotSavedAgain(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + asked := asksRecorded(t) + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main", + BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1aaaaaaaa", + Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true} + if err := (following{open: open}).SourceMoved(ctx, m); err != nil { + t.Fatal(err) + } + advanceHeld(ctx, open) + recent, err := inv.RecentPlans(ctx, 1) + if err != nil || len(recent) != 1 || len(*asked) != 1 { + t.Fatalf("no plan building: %v %v, asked %v", recent, err, *asked) + } + before := recent[0] + if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" { + t.Fatalf("the plan is not waiting on its build: %+v", before) + } + + for range 3 { + advanceHeld(ctx, open) + } + after, err := inv.PlanByID(ctx, before.ID) + if err != nil { + t.Fatal(err) + } + if after.Revision != before.Revision || !after.Updated.Equal(before.Updated) { + t.Fatalf("a plan that did not move was saved again: revision %d → %d, saved %s → %s", + before.Revision, after.Revision, before.Updated, after.Updated) + } + if len(*asked) != 1 { + t.Fatalf("advancing asked again: %v", *asked) + } +} diff --git a/cmd/mesh-controller/plan_clock_test.go b/cmd/mesh-controller/plan_clock_test.go index 26dc6143..dde106be 100644 --- a/cmd/mesh-controller/plan_clock_test.go +++ b/cmd/mesh-controller/plan_clock_test.go @@ -7,7 +7,6 @@ import ( "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" ) // novox/hq issue 296: a plan saved a few seconds ago that has stood in its tier for forty minutes says @@ -79,43 +78,3 @@ func TestAPlansLineCountsFromItsTierNotFromItsLastSave(t *testing.T) { // novox/hq issue 296: a plan whose tier stands still is not written again by every advance — its revision, // its last save and its `plan-moved` stay where its last change left them. -func TestAPlanStandingStillIsNotSavedAgain(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - asked := asksRecorded(t) - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1aaaaaaaa", - Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - advanceHeld(ctx, open) - recent, err := inv.RecentPlans(ctx, 1) - if err != nil || len(recent) != 1 || len(*asked) != 1 { - t.Fatalf("no plan building: %v %v, asked %v", recent, err, *asked) - } - before := recent[0] - if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" { - t.Fatalf("the plan is not waiting on its build: %+v", before) - } - - for range 3 { - advanceHeld(ctx, open) - } - after, err := inv.PlanByID(ctx, before.ID) - if err != nil { - t.Fatal(err) - } - if after.Revision != before.Revision || !after.Updated.Equal(before.Updated) { - t.Fatalf("a plan that did not move was saved again: revision %d → %d, saved %s → %s", - before.Revision, after.Revision, before.Updated, after.Updated) - } - if len(*asked) != 1 { - t.Fatalf("advancing asked again: %v", *asked) - } -} diff --git a/cmd/mesh-controller/plan_looks_integration_test.go b/cmd/mesh-controller/plan_looks_integration_test.go new file mode 100644 index 00000000..df7d3c9f --- /dev/null +++ b/cmd/mesh-controller/plan_looks_integration_test.go @@ -0,0 +1,188 @@ +//go:build integration + +package main + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestAsksMadeWhileALookRunsAreOneLookAfterIt(t *testing.T) { + var failures, looks atomic.Int32 + block := make(chan struct{}) + l := lookerCounting(&failures, &looks, block) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go l.run(ctx) + l.ask() + eventually(t, "the first look starts", func() bool { return looks.Load() == 1 }) + for i := 0; i < 20; i++ { + l.ask() + } + block <- struct{}{} // the first look ends + eventually(t, "the asks made meanwhile are looked at", func() bool { return looks.Load() == 2 }) + block <- struct{}{} + time.Sleep(60 * time.Millisecond) + if n := looks.Load(); n != 2 { + t.Errorf("%d looks for 21 asks; want 2", n) + } +} + +func TestAFailedLookIsLookedAtAgainAndAGoodOneEndsTheRetries(t *testing.T) { + var failures, looks atomic.Int32 + failures.Store(3) // three looks fail, then one is good + l := lookerCounting(&failures, &looks, nil) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go l.run(ctx) + l.ask() + eventually(t, "four looks", func() bool { return looks.Load() == 4 }) + time.Sleep(200 * time.Millisecond) + if n := looks.Load(); n != 4 { + t.Errorf("%d looks; the good one should have ended the retries at 4", n) + } + l.mu.Lock() + armed, backoff := l.retry != nil, l.backoff + l.mu.Unlock() + if armed || backoff != 0 { + t.Errorf("a retry is armed (%v) or the wait kept (%v) after a good look", armed, backoff) + } +} + +func TestAPlanSavedAsksForALookAndADoneWalkIsArmedToBeRecorded(t *testing.T) { + var failures, looks atomic.Int32 + l := lookerCounting(&failures, &looks, nil) + lookerMu.Lock() + theLooker = l + lookerMu.Unlock() + t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() }) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go l.run(ctx) + + planKept(inventory.Plan{ID: "walk-1", State: inventory.PlanDone}) + t.Cleanup(func() { + walkEnds.mu.Lock() + defer walkEnds.mu.Unlock() + if t := walkEnds.timers["walk-1"]; t != nil { + t.Stop() + } + delete(walkEnds.timers, "walk-1") + }) + eventually(t, "a look", func() bool { return looks.Load() == 1 }) + walkEnds.mu.Lock() + _, armed := walkEnds.timers["walk-1"] + walkEnds.mu.Unlock() + if !armed { + t.Error("a walk saved done is not armed to be recorded") + } + askToLook() // no panic, no looker, once it is taken away +} + +func TestAnExpiredSilentKeyAsksForALookAndAReportsDeleteDoesNot(t *testing.T) { + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + var failures, looks atomic.Int32 + l := lookerCounting(&failures, &looks, nil) + lookerMu.Lock() + theLooker = l + lookerMu.Unlock() + t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock(); setSilentClock(nil) }) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go l.run(ctx) + if err := startRolloutSilent(ctx, js); err != nil { + t.Fatal(err) + } + c := silentClockOf() + w := silentWait{Plan: "plan-5", Tier: 0, Node: "ace"} + if err := c.Arm(ctx, w, time.Second, time.Now()); err != nil { + t.Fatal(err) + } + if err := c.Reported(ctx, w); err != nil { + t.Fatal(err) + } + time.Sleep(1800 * time.Millisecond) + if n := looks.Load(); n != 0 { + t.Fatalf("a key a report deleted asked for %d looks", n) + } + if err := c.Arm(ctx, w, time.Second, time.Now()); err != nil { + t.Fatal(err) + } + eventually(t, "the expiry asks for a look", func() bool { return looks.Load() == 1 }) +} + +func TestAProvisionerClientsWordAsksForALook(t *testing.T) { + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + var failures, looks atomic.Int32 + l := lookerCounting(&failures, &looks, nil) + lookerMu.Lock() + theLooker = l + lookerMu.Unlock() + t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() }) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go l.run(ctx) + if err := startProvAppliedLooks(ctx, js); err != nil { + t.Fatal(err) + } + legacy, err := js.Conn().JetStream() + if err != nil { + t.Fatal(err) + } + if _, err := legacy.Publish(broker.ProvisionerAppliedSubject("ace", "plex"), []byte(`{"status":"applied"}`)); err != nil { + t.Fatal(err) + } + eventually(t, "a look on the client's word", func() bool { return looks.Load() >= 1 }) +} + +func TestAPlanMovedByAnotherProcessAsksForALook(t *testing.T) { + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + var failures, looks atomic.Int32 + l := lookerCounting(&failures, &looks, nil) + lookerMu.Lock() + theLooker = l + lookerMu.Unlock() + t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() }) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go l.run(ctx) + if err := startPlanMovedLooks(ctx, js); err != nil { + t.Fatal(err) + } + legacy, err := js.Conn().JetStream() + if err != nil { + t.Fatal(err) + } + if _, err := legacy.Publish(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), []byte(`{}`)); err != nil { + t.Fatal(err) + } + eventually(t, "a look on the plan moved", func() bool { return looks.Load() >= 1 }) +} diff --git a/cmd/mesh-controller/plan_looks_test.go b/cmd/mesh-controller/plan_looks_test.go index c2076a4b..be81219a 100644 --- a/cmd/mesh-controller/plan_looks_test.go +++ b/cmd/mesh-controller/plan_looks_test.go @@ -7,10 +7,7 @@ import ( "testing" "time" - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" ) func rolloutOf(sentAgo time.Duration, reported bool) (inventory.Plan, []inventory.Reported) { @@ -112,49 +109,9 @@ func eventually(t *testing.T, what string, ok func() bool) { } // Asks made while a look runs are not lost and are one look after it, not one each. -func TestAsksMadeWhileALookRunsAreOneLookAfterIt(t *testing.T) { - var failures, looks atomic.Int32 - block := make(chan struct{}) - l := lookerCounting(&failures, &looks, block) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - l.ask() - eventually(t, "the first look starts", func() bool { return looks.Load() == 1 }) - for i := 0; i < 20; i++ { - l.ask() - } - block <- struct{}{} // the first look ends - eventually(t, "the asks made meanwhile are looked at", func() bool { return looks.Load() == 2 }) - block <- struct{}{} - time.Sleep(60 * time.Millisecond) - if n := looks.Load(); n != 2 { - t.Errorf("%d looks for 21 asks; want 2", n) - } -} // A look that failed a step is looked at again, one at a time with a longer wait each time, and a look that fails // nothing ends the retries (novox/hq ADR 0406 §5; the one named retry). -func TestAFailedLookIsLookedAtAgainAndAGoodOneEndsTheRetries(t *testing.T) { - var failures, looks atomic.Int32 - failures.Store(3) // three looks fail, then one is good - l := lookerCounting(&failures, &looks, nil) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - l.ask() - eventually(t, "four looks", func() bool { return looks.Load() == 4 }) - time.Sleep(200 * time.Millisecond) - if n := looks.Load(); n != 4 { - t.Errorf("%d looks; the good one should have ended the retries at 4", n) - } - l.mu.Lock() - armed, backoff := l.retry != nil, l.backoff - l.mu.Unlock() - if armed || backoff != 0 { - t.Errorf("a retry is armed (%v) or the wait kept (%v) after a good look", armed, backoff) - } -} func TestTheRetryWaitDoublesToItsMost(t *testing.T) { l := newLooker(func(context.Context) bool { return true }) @@ -175,35 +132,6 @@ func TestTheRetryWaitDoublesToItsMost(t *testing.T) { } // A plan saved asks for a look, and a walk saved done is armed to be recorded once its end can be known. -func TestAPlanSavedAsksForALookAndADoneWalkIsArmedToBeRecorded(t *testing.T) { - var failures, looks atomic.Int32 - l := lookerCounting(&failures, &looks, nil) - lookerMu.Lock() - theLooker = l - lookerMu.Unlock() - t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() }) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - - planKept(inventory.Plan{ID: "walk-1", State: inventory.PlanDone}) - t.Cleanup(func() { - walkEnds.mu.Lock() - defer walkEnds.mu.Unlock() - if t := walkEnds.timers["walk-1"]; t != nil { - t.Stop() - } - delete(walkEnds.timers, "walk-1") - }) - eventually(t, "a look", func() bool { return looks.Load() == 1 }) - walkEnds.mu.Lock() - _, armed := walkEnds.timers["walk-1"] - walkEnds.mu.Unlock() - if !armed { - t.Error("a walk saved done is not armed to be recorded") - } - askToLook() // no panic, no looker, once it is taken away -} // The tier-sends a step made are said once each, per machine, with the modules it was sent. func TestTheSendsAStepMadeAreSaidPerMachine(t *testing.T) { @@ -221,77 +149,9 @@ func TestTheSendsAStepMadeAreSaidPerMachine(t *testing.T) { } // A silent key that expires asks for a look; a key a report deleted does not (issue 974, novox/hq ADR 0406 §5). -func TestAnExpiredSilentKeyAsksForALookAndAReportsDeleteDoesNot(t *testing.T) { - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - var failures, looks atomic.Int32 - l := lookerCounting(&failures, &looks, nil) - lookerMu.Lock() - theLooker = l - lookerMu.Unlock() - t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock(); setSilentClock(nil) }) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - if err := startRolloutSilent(ctx, js); err != nil { - t.Fatal(err) - } - c := silentClockOf() - w := silentWait{Plan: "plan-5", Tier: 0, Node: "ace"} - if err := c.Arm(ctx, w, time.Second, time.Now()); err != nil { - t.Fatal(err) - } - if err := c.Reported(ctx, w); err != nil { - t.Fatal(err) - } - time.Sleep(1800 * time.Millisecond) - if n := looks.Load(); n != 0 { - t.Fatalf("a key a report deleted asked for %d looks", n) - } - if err := c.Arm(ctx, w, time.Second, time.Now()); err != nil { - t.Fatal(err) - } - eventually(t, "the expiry asks for a look", func() bool { return looks.Load() == 1 }) -} // A provisioner client's word on a module's files asks for a look, as a send that handed it modules is applied only // once it has said so (issue 974, review 1): no walk waits for its silent key for what the client already said. -func TestAProvisionerClientsWordAsksForALook(t *testing.T) { - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - var failures, looks atomic.Int32 - l := lookerCounting(&failures, &looks, nil) - lookerMu.Lock() - theLooker = l - lookerMu.Unlock() - t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() }) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - if err := startProvAppliedLooks(ctx, js); err != nil { - t.Fatal(err) - } - legacy, err := js.Conn().JetStream() - if err != nil { - t.Fatal(err) - } - if _, err := legacy.Publish(broker.ProvisionerAppliedSubject("ace", "plex"), []byte(`{"status":"applied"}`)); err != nil { - t.Fatal(err) - } - eventually(t, "a look on the client's word", func() bool { return looks.Load() >= 1 }) -} type brokenClock struct{ silentClock } @@ -319,33 +179,3 @@ func TestKeysThatCannotBeReadAreWatchedByTimersAndTheLookFails(t *testing.T) { // A plan moved by a process the serving controller did not spawn — a verb run at the control node's terminal — is // looked at: the controller follows its own plan-moved (issue 974, round 1). -func TestAPlanMovedByAnotherProcessAsksForALook(t *testing.T) { - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - var failures, looks atomic.Int32 - l := lookerCounting(&failures, &looks, nil) - lookerMu.Lock() - theLooker = l - lookerMu.Unlock() - t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() }) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - if err := startPlanMovedLooks(ctx, js); err != nil { - t.Fatal(err) - } - legacy, err := js.Conn().JetStream() - if err != nil { - t.Fatal(err) - } - if _, err := legacy.Publish(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), []byte(`{}`)); err != nil { - t.Fatal(err) - } - eventually(t, "a look on the plan moved", func() bool { return looks.Load() >= 1 }) -} diff --git a/cmd/mesh-controller/plan_moved_every_writer_integration_test.go b/cmd/mesh-controller/plan_moved_every_writer_integration_test.go new file mode 100644 index 00000000..2d376641 --- /dev/null +++ b/cmd/mesh-controller/plan_moved_every_writer_integration_test.go @@ -0,0 +1,37 @@ +//go:build integration + +package main + +import ( + "testing" + "time" +) + +func TestEveryMoveAMergeAndTheCutMakeOfAPlanIsSaid(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + asksWithPaths(t) + heard := sayingEverySave(t) + hear(t, open, catalogueMerge("553b7191claude", "app", t0), t0.Add(time.Second)) + if _, bs := walks(t, open); len(bs) != 1 { + t.Fatalf("%d batch(es) opened", len(bs)) + } + heard.noUnsaid(t, open) + cutAt(t, open, t0.Add(91*time.Second)) + if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 { + t.Fatalf("%d walk(s), %d batch(es) after the cut", len(ws), len(bs)) + } + heard.noUnsaid(t, open) + counted := 0 + for _, r := range heard.revs { + counted += len(r) + } + cutAt(t, open, t0.Add(92*time.Second)) + again := 0 + for _, r := range heard.revs { + again += len(r) + } + if again != counted { + t.Errorf("a look that changed nothing said %d more move(s)", again-counted) + } +} diff --git a/cmd/mesh-controller/plan_moved_every_writer_test.go b/cmd/mesh-controller/plan_moved_every_writer_test.go index 4be2f8aa..0973ae79 100644 --- a/cmd/mesh-controller/plan_moved_every_writer_test.go +++ b/cmd/mesh-controller/plan_moved_every_writer_test.go @@ -7,7 +7,6 @@ import ( "strings" "sync" "testing" - "time" "github.com/novox/mesh-controller/internal/inventory" ) @@ -110,31 +109,3 @@ func (s *movesSaid) noUnsaid(t *testing.T, open *stores) { // Live, with the store: the moves a merge makes of a plan — the batch it opens, the walk it is cut into, the tier that // is asked — are each said, and a look that changes nothing says nothing (novox/hq ADR 0406 §2). -func TestEveryMoveAMergeAndTheCutMakeOfAPlanIsSaid(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - heard := sayingEverySave(t) - hear(t, open, catalogueMerge("553b7191claude", "app", t0), t0.Add(time.Second)) - if _, bs := walks(t, open); len(bs) != 1 { - t.Fatalf("%d batch(es) opened", len(bs)) - } - heard.noUnsaid(t, open) - cutAt(t, open, t0.Add(91*time.Second)) - if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 { - t.Fatalf("%d walk(s), %d batch(es) after the cut", len(ws), len(bs)) - } - heard.noUnsaid(t, open) - counted := 0 - for _, r := range heard.revs { - counted += len(r) - } - cutAt(t, open, t0.Add(92*time.Second)) - again := 0 - for _, r := range heard.revs { - again += len(r) - } - if again != counted { - t.Errorf("a look that changed nothing said %d more move(s)", again-counted) - } -} diff --git a/cmd/mesh-controller/planner_records_test.go b/cmd/mesh-controller/planner_records_test.go index 9f56ef4c..8bebf2df 100644 --- a/cmd/mesh-controller/planner_records_test.go +++ b/cmd/mesh-controller/planner_records_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/plans_follow_integration_test.go b/cmd/mesh-controller/plans_follow_integration_test.go new file mode 100644 index 00000000..13968d55 --- /dev/null +++ b/cmd/mesh-controller/plans_follow_integration_test.go @@ -0,0 +1,62 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestAPlanFollowedFromTheStoreAnswersWhenItMoves(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + // Read every 50ms rather than every second, so the wait answers as soon as the plan moves and the bound + // below passes in a fraction of a second (novox/hq issue 591); the move still waits 1.5s, long enough + // for the wait to have read the plan before it. + was := planWaitEvery + planWaitEvery = 50 * time.Millisecond + t.Cleanup(func() { planWaitEvery = was }) + now := time.Now().UTC() + walk := inventory.Plan{ID: "plan-229", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c229c229", + Created: now, State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}}, + Modules: map[string]*inventory.PlanModule{"a": {}, "b": {}}} + if err := open.inventory.SavePlan(ctx, &walk); err != nil { + t.Fatal(err) + } + moved := make(chan error, 1) + go func() { + time.Sleep(1500 * time.Millisecond) + p, err := open.inventory.PlanByID(ctx, walk.ID) + if err == nil { + p.Tier, p.State = 1, inventory.PlanRolling + err = open.inventory.SavePlan(ctx, &p) + } + moved <- err + }() + began := time.Now() + var got planWaited + out := printed(t, func() error { + return plansCommand(ctx, []string{"wait", walk.ID, "--until", "tier", "--bound", "15s"}) + }) + if err := <-moved; err != nil { + t.Fatal(err) + } + if err := json.Unmarshal([]byte(out), &got); err != nil { + t.Fatalf("plans wait did not answer data: %v: %q", err, out) + } + if got.Came != "tier" || got.Plan.Tier != 1 || got.Plan.State != inventory.PlanRolling { + t.Fatalf("the wait answered %+v", got) + } + if waited := time.Since(began); waited > 3*time.Second { + t.Errorf("it answered after %s, not when the plan moved", waited) + } + out = printed(t, func() error { + return plansCommand(ctx, []string{"wait", walk.ID, "--bound", "300ms"}) + }) + if err := json.Unmarshal([]byte(out), &got); err != nil || got.Came != "bound" || got.Until != "moved" { + t.Fatalf("a plan that did not move answered %+v (%v)", got, err) + } +} diff --git a/cmd/mesh-controller/plans_follow_test.go b/cmd/mesh-controller/plans_follow_test.go index 0f25870d..0bd63bdc 100644 --- a/cmd/mesh-controller/plans_follow_test.go +++ b/cmd/mesh-controller/plans_follow_test.go @@ -135,56 +135,6 @@ func TestThePlansVerbComposesAWait(t *testing.T) { // Live, with the store: `plans wait` run as the verb runs it answers when the plan reaches its next tier, as // data, and says the bound passed when nothing moved (novox/hq issue 229). -func TestAPlanFollowedFromTheStoreAnswersWhenItMoves(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - // Read every 50ms rather than every second, so the wait answers as soon as the plan moves and the bound - // below passes in a fraction of a second (novox/hq issue 591); the move still waits 1.5s, long enough - // for the wait to have read the plan before it. - was := planWaitEvery - planWaitEvery = 50 * time.Millisecond - t.Cleanup(func() { planWaitEvery = was }) - now := time.Now().UTC() - walk := inventory.Plan{ID: "plan-229", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c229c229", - Created: now, State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}}, - Modules: map[string]*inventory.PlanModule{"a": {}, "b": {}}} - if err := open.inventory.SavePlan(ctx, &walk); err != nil { - t.Fatal(err) - } - moved := make(chan error, 1) - go func() { - time.Sleep(1500 * time.Millisecond) - p, err := open.inventory.PlanByID(ctx, walk.ID) - if err == nil { - p.Tier, p.State = 1, inventory.PlanRolling - err = open.inventory.SavePlan(ctx, &p) - } - moved <- err - }() - began := time.Now() - var got planWaited - out := printed(t, func() error { - return plansCommand(ctx, []string{"wait", walk.ID, "--until", "tier", "--bound", "15s"}) - }) - if err := <-moved; err != nil { - t.Fatal(err) - } - if err := json.Unmarshal([]byte(out), &got); err != nil { - t.Fatalf("plans wait did not answer data: %v: %q", err, out) - } - if got.Came != "tier" || got.Plan.Tier != 1 || got.Plan.State != inventory.PlanRolling { - t.Fatalf("the wait answered %+v", got) - } - if waited := time.Since(began); waited > 3*time.Second { - t.Errorf("it answered after %s, not when the plan moved", waited) - } - out = printed(t, func() error { - return plansCommand(ctx, []string{"wait", walk.ID, "--bound", "300ms"}) - }) - if err := json.Unmarshal([]byte(out), &got); err != nil || got.Came != "bound" || got.Until != "moved" { - t.Fatalf("a plan that did not move answered %+v (%v)", got, err) - } -} // The verb's schema says the longest bound in words, and the words are the value (review 75 on #266): the // catalogue cannot read maxPlanWait, so this holds them together. diff --git a/cmd/mesh-controller/plans_one_at_a_time_test.go b/cmd/mesh-controller/plans_one_at_a_time_test.go index 1eb4ec3c..3f3eef97 100644 --- a/cmd/mesh-controller/plans_one_at_a_time_test.go +++ b/cmd/mesh-controller/plans_one_at_a_time_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/predefined_integration_test.go b/cmd/mesh-controller/predefined_integration_test.go new file mode 100644 index 00000000..e0d28433 --- /dev/null +++ b/cmd/mesh-controller/predefined_integration_test.go @@ -0,0 +1,101 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestTheMergeGateJudgesEachManifestAgainstTheListOfTheTree(t *testing.T) { + withoutAPublishedList(t) + f, manifests := templatedMesh(t) + + v := gateJudged(t, f, writeTheList(t, aTree(t, withEdit(manifests, "notifier", + notifierFallingBackTo("font-size", "number"))), theFirstKeys), "modules/notifier/module.json") + if v.Verdict == "fail" || v.Modules["notifier"] != "changed" { + t.Fatalf("a module naming font-size on the tree's list failed the gate (%v):\n%s", v.Modules, v.Report()) + } + for _, c := range []struct{ manifest, list, says string }{ + {notifierFallingBackTo("font-scale", "number"), theFirstKeys, `"font-scale", which is not on the list`}, + {notifierFallingBackTo("font-size", "string"), theFirstKeys, "which is a number"}, + {inlineNotifier, `{"font-size": {"type": "number", "description": "points", "unit": "pt"}}`, `unknown field "unit"`}, + } { + v := gateJudged(t, f, writeTheList(t, aTree(t, withEdit(manifests, "notifier", c.manifest)), c.list), + "modules/notifier/module.json", catalogue.PredefinedFile) + if v.Verdict != "fail" || !strings.Contains(v.Report(), c.says) { + t.Errorf("wanted the gate to fail saying %q:\n%s", c.says, v.Report()) + } + } +} + +func TestRegistrationPublishesTheNewestTrunkListAndJudgesEveryModuleAgainstIt(t *testing.T) { + withoutAPublishedList(t) + open := aMesh(t) + ctx := t.Context() + + // A manifest naming font-size registers, and its build's list is published. + if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, + catalogueBuild("b-1", notifierFallingBackTo("font-size", "number"), theFirstKeys, 10))); err != nil { + t.Fatalf("a module naming font-size was not registered: %v", err) + } + kept, has, err := open.inventory.PredefinedKept(ctx) + if err != nil || !has || kept.Generation != 10 || kept.Build != "b-1" { + t.Fatalf("the list was not published: %+v %v %v", kept, has, err) + } + if got := catalogue.PublishedPredefined(); len(got) != 2 { + t.Fatalf("the controller judges against %v, not the list published", got) + } + + // One naming a key not on the list, or of another type, is refused and never enters the catalogue. + for i, c := range []struct{ manifest, says string }{ + {notifierFallingBackTo("font-scale", "number"), "not on the list"}, + {notifierFallingBackTo("font-size", "string"), "which is a number"}, + } { + id := []string{"b-2", "b-3"}[i] + if _, _, err := takeIn(ctx, open.inventory, catalogueBuild(id, c.manifest, theFirstKeys, 10)); err == nil || + !strings.Contains(err.Error(), c.says) { + t.Errorf("wanted the registration refused saying %q, got %v", c.says, err) + } + } + src, err := open.inventory.SourceOf(ctx, "notifier") + if err != nil || src.BuiltFrom != strings.Repeat("1", 12) { + t.Fatalf("a refused build replaced the registered one: %+v %v", src, err) + } + + // A newer build's list, which drops agent-managed-settings, replaces the published one. + smaller := `{"font-size": {"type": "number", "description": "points", "minimum": 6, "maximum": 72}}` + if _, _, err := takeIn(ctx, open.inventory, catalogueBuild("b-4", + notifierFallingBackTo("font-size", "number"), smaller, 12)); err != nil { + t.Fatal(err) + } + // The older build's outcome heard later registers nothing new about the list: the newer stands. + if _, _, err := takeIn(ctx, open.inventory, catalogueBuild("b-5", + notifierFallingBackTo("font-size", "number"), theFirstKeys, 11)); err != nil { + t.Fatal(err) + } + kept, _, _ = open.inventory.PredefinedKept(ctx) + if kept.Build != "b-4" || len(kept.Keys) != 1 || len(catalogue.PublishedPredefined()) != 1 { + t.Fatalf("an older build's list heard later replaced the newer one: %+v", kept) + } + + // A build of another repository publishes nothing, and its manifest is judged against the published list. + other := catalogueBuild("b-6", notifierFallingBackTo("font-size", "number"), theFirstKeys, 99) + other.Repository, other.Source.Repository = "http://forge.internal:20000/novox/notifier.git", "novox/notifier" + other.Manifest = json.RawMessage(strings.Replace(notifierFallingBackTo("font-size", "number"), `"notifier"`, + `"chimes"`, 1)) + if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, other)); err != nil { + t.Fatalf("a module of another repository naming a published key: %v", err) + } + if kept, _, _ = open.inventory.PredefinedKept(ctx); kept.Build != "b-4" { + t.Fatalf("a build of another repository published its list: %+v", kept) + } + other.ID, other.Manifest = "b-7", json.RawMessage(strings.Replace(notifierFallingBackTo("agent-managed-settings", + "number"), `"notifier"`, `"chimes"`, 1)) + if _, _, err := takeIn(ctx, open.inventory, other); err == nil || !strings.Contains(err.Error(), "not on the list") { + t.Fatalf("a key only another repository's list holds was registered: %v", err) + } +} diff --git a/cmd/mesh-controller/predefined_test.go b/cmd/mesh-controller/predefined_test.go index 25f09c50..c9cf3cd6 100644 --- a/cmd/mesh-controller/predefined_test.go +++ b/cmd/mesh-controller/predefined_test.go @@ -95,28 +95,6 @@ func writeTheList(t *testing.T, tree, list string) string { return tree } -func TestTheMergeGateJudgesEachManifestAgainstTheListOfTheTree(t *testing.T) { - withoutAPublishedList(t) - f, manifests := templatedMesh(t) - - v := gateJudged(t, f, writeTheList(t, aTree(t, withEdit(manifests, "notifier", - notifierFallingBackTo("font-size", "number"))), theFirstKeys), "modules/notifier/module.json") - if v.Verdict == "fail" || v.Modules["notifier"] != "changed" { - t.Fatalf("a module naming font-size on the tree's list failed the gate (%v):\n%s", v.Modules, v.Report()) - } - for _, c := range []struct{ manifest, list, says string }{ - {notifierFallingBackTo("font-scale", "number"), theFirstKeys, `"font-scale", which is not on the list`}, - {notifierFallingBackTo("font-size", "string"), theFirstKeys, "which is a number"}, - {inlineNotifier, `{"font-size": {"type": "number", "description": "points", "unit": "pt"}}`, `unknown field "unit"`}, - } { - v := gateJudged(t, f, writeTheList(t, aTree(t, withEdit(manifests, "notifier", c.manifest)), c.list), - "modules/notifier/module.json", catalogue.PredefinedFile) - if v.Verdict != "fail" || !strings.Contains(v.Report(), c.says) { - t.Errorf("wanted the gate to fail saying %q:\n%s", c.says, v.Report()) - } - } -} - // catalogueBuild is a trunk build of the catalogue carrying the list at a generation. func catalogueBuild(id, manifest, list string, generation int) link.BuildResult { r := link.BuildResult{ID: id, Repository: "http://forge.internal:20000/novox/mesh-catalog.git", @@ -127,71 +105,3 @@ func catalogueBuild(id, manifest, list string, generation int) link.BuildResult } return r } - -func TestRegistrationPublishesTheNewestTrunkListAndJudgesEveryModuleAgainstIt(t *testing.T) { - withoutAPublishedList(t) - open := aMesh(t) - ctx := t.Context() - - // A manifest naming font-size registers, and its build's list is published. - if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, - catalogueBuild("b-1", notifierFallingBackTo("font-size", "number"), theFirstKeys, 10))); err != nil { - t.Fatalf("a module naming font-size was not registered: %v", err) - } - kept, has, err := open.inventory.PredefinedKept(ctx) - if err != nil || !has || kept.Generation != 10 || kept.Build != "b-1" { - t.Fatalf("the list was not published: %+v %v %v", kept, has, err) - } - if got := catalogue.PublishedPredefined(); len(got) != 2 { - t.Fatalf("the controller judges against %v, not the list published", got) - } - - // One naming a key not on the list, or of another type, is refused and never enters the catalogue. - for i, c := range []struct{ manifest, says string }{ - {notifierFallingBackTo("font-scale", "number"), "not on the list"}, - {notifierFallingBackTo("font-size", "string"), "which is a number"}, - } { - id := []string{"b-2", "b-3"}[i] - if _, _, err := takeIn(ctx, open.inventory, catalogueBuild(id, c.manifest, theFirstKeys, 10)); err == nil || - !strings.Contains(err.Error(), c.says) { - t.Errorf("wanted the registration refused saying %q, got %v", c.says, err) - } - } - src, err := open.inventory.SourceOf(ctx, "notifier") - if err != nil || src.BuiltFrom != strings.Repeat("1", 12) { - t.Fatalf("a refused build replaced the registered one: %+v %v", src, err) - } - - // A newer build's list, which drops agent-managed-settings, replaces the published one. - smaller := `{"font-size": {"type": "number", "description": "points", "minimum": 6, "maximum": 72}}` - if _, _, err := takeIn(ctx, open.inventory, catalogueBuild("b-4", - notifierFallingBackTo("font-size", "number"), smaller, 12)); err != nil { - t.Fatal(err) - } - // The older build's outcome heard later registers nothing new about the list: the newer stands. - if _, _, err := takeIn(ctx, open.inventory, catalogueBuild("b-5", - notifierFallingBackTo("font-size", "number"), theFirstKeys, 11)); err != nil { - t.Fatal(err) - } - kept, _, _ = open.inventory.PredefinedKept(ctx) - if kept.Build != "b-4" || len(kept.Keys) != 1 || len(catalogue.PublishedPredefined()) != 1 { - t.Fatalf("an older build's list heard later replaced the newer one: %+v", kept) - } - - // A build of another repository publishes nothing, and its manifest is judged against the published list. - other := catalogueBuild("b-6", notifierFallingBackTo("font-size", "number"), theFirstKeys, 99) - other.Repository, other.Source.Repository = "http://forge.internal:20000/novox/notifier.git", "novox/notifier" - other.Manifest = json.RawMessage(strings.Replace(notifierFallingBackTo("font-size", "number"), `"notifier"`, - `"chimes"`, 1)) - if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, other)); err != nil { - t.Fatalf("a module of another repository naming a published key: %v", err) - } - if kept, _, _ = open.inventory.PredefinedKept(ctx); kept.Build != "b-4" { - t.Fatalf("a build of another repository published its list: %+v", kept) - } - other.ID, other.Manifest = "b-7", json.RawMessage(strings.Replace(notifierFallingBackTo("agent-managed-settings", - "number"), `"notifier"`, `"chimes"`, 1)) - if _, _, err := takeIn(ctx, open.inventory, other); err == nil || !strings.Contains(err.Error(), "not on the list") { - t.Fatalf("a key only another repository's list holds was registered: %v", err) - } -} diff --git a/cmd/mesh-controller/proposals_integration_test.go b/cmd/mesh-controller/proposals_integration_test.go new file mode 100644 index 00000000..7a58d0c2 --- /dev/null +++ b/cmd/mesh-controller/proposals_integration_test.go @@ -0,0 +1,135 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestSetOnAWarrantJudgesTheLayerAndKeepsWhoApprovedIt(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "notes", Version: "1", + // y is a preference with a default, so a layer may leave it out; x is the operator's own (ADR 0262). + Settings: map[string]catalogue.SettingDeclaration{"y": {Kind: "preference", Default: "10", Why: "a size"}}, + Resources: []map[string]any{{"id": "data", "type": "directory", "mode": "0755"}, + // A trusted file (unmarked), so its keys are the terminal's — and now the warrant's (novox/hq ADR 0277). + {"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644", "content": "x = ${setting:x}\ny = ${setting:y}\n"}}}) + if _, err := assign(ctx, open, "laptop", "notes"); err != nil { + t.Fatal(err) + } + set := setLayerIn(open) + now := time.Now() + first := map[string]any{"x": "1", "y": "2"} + p := settingsProposal{Module: "notes", Node: "laptop", Values: first, From: "g14/claude-code", At: now, + Digest: layerDigest(first), BeforeDigest: layerDigest(nil)} + changed, err := set(ctx, p, "s9", "approved by the operator, as telegram identity 42 via telegram at 14:05 (ask s9)") + if err != nil || !strings.Contains(changed, "+ x") { + t.Fatalf("%q %v", changed, err) + } + layer, has, err := open.inventory.Layer(ctx, "laptop", "notes") + if err != nil || !has || layer["x"] != "1" { + t.Fatalf("the layer: %v %v %v", layer, has, err) + } + setBy, _, has, err := open.inventory.LayerOrigin(ctx, "laptop", "notes") + if err != nil || !has || !strings.HasPrefix(setBy, "approved by the operator, as telegram identity 42 via telegram") { + t.Fatalf("who set it: %q %v", setBy, err) + } + // Shown by `settings show`, at the terminal and through the verb. + out := captureStdout(t, func() { + if err := atTheTerminal(t, "settings", "show", "notes", "--node", "laptop"); err != nil { + t.Fatal(err) + } + }) + if !strings.Contains(out, "approved by the operator, as telegram identity 42 via telegram") { + t.Errorf("settings show says:\n%s", out) + } + // The same proposal again: the layer is no longer the one the operator was shown it against. + if _, err := set(ctx, p, "s9", "approved …"); err == nil || !strings.Contains(err.Error(), "changed since the operator was shown") { + t.Errorf("a stale proposal: %v", err) + } + // A removal not meant is refused; one meant is taken, and the history says who had set the layer. + second := map[string]any{"x": "1"} + q := settingsProposal{Module: "notes", Node: "laptop", Values: second, Before: first, HadLayer: true, + From: "g14/claude-code", At: now, Digest: layerDigest(second), BeforeDigest: layerDigest(first)} + if _, err := set(ctx, q, "s10", "approved …"); err == nil || !strings.Contains(err.Error(), "removal was not meant") { + t.Errorf("a silent removal: %v", err) + } + // A layer the mesh refuses is refused here too, with the same words as at the terminal. + bad := q + bad.Values, bad.Digest = map[string]any{"x": "a\nb", "y": "2"}, layerDigest(map[string]any{"x": "a\nb", "y": "2"}) + if _, err := set(ctx, bad, "s11", "approved …"); err == nil || !strings.Contains(err.Error(), "line break") { + t.Errorf("a line break on a warrant: %v", err) + } + if layer, _, _ := open.inventory.Layer(ctx, "laptop", "notes"); layer["y"] != "2" { + t.Fatalf("a refused act changed the layer: %v", layer) + } + q.Replace = true + if _, err := set(ctx, q, "s10", "approved later"); err != nil { + t.Fatal(err) + } + past, err := open.inventory.SettingsHistory(ctx, "laptop", "notes") + if err != nil || len(past) != 1 || !strings.HasPrefix(past[0].SetBy, "approved by the operator") { + t.Errorf("the history: %+v %v", past, err) + } + // And a clear, keeping who cleared it with the copy. + c := settingsProposal{Module: "notes", Node: "laptop", Clear: true, Before: second, HadLayer: true, From: "x", At: now, + Digest: layerDigest(second), BeforeDigest: layerDigest(second)} + if _, err := set(ctx, c, "s12", "approved by the operator at 15:00"); err != nil { + t.Fatal(err) + } + if _, has, _ := open.inventory.Layer(ctx, "laptop", "notes"); has { + t.Error("the layer was not cleared") + } + past, _ = open.inventory.SettingsHistory(ctx, "laptop", "notes") + if len(past) != 2 || !strings.Contains(past[0].SetBy, "cleared approved by the operator at 15:00") { + t.Errorf("the history after a clear: %+v", past) + } +} + +func TestALayerSaysWhoSetIt(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "notes", Version: "1", + Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644", "trusted": false, + "content": "x = ${setting:x}\n"}}}) + if _, err := assign(ctx, open, "laptop", "notes"); err != nil { + t.Fatal(err) + } + if err := atTheTerminal(t, "settings", "set", "notes", `{"x":"1"}`, "--node", "laptop"); err != nil { + t.Fatal(err) + } + setBy, _, _, _ := open.inventory.LayerOrigin(ctx, "laptop", "notes") + if !strings.HasPrefix(setBy, "set at the controller's terminal by ") { + t.Errorf("at the terminal: %q", setBy) + } + if err := throughVerb(t, "settings", map[string]any{"module": "notes", "node": "laptop", "values": `{"x":"2"}`}); err != nil { + t.Fatal(err) + } + setBy, _, _, _ = open.inventory.LayerOrigin(ctx, "laptop", "notes") + if !strings.HasPrefix(setBy, "set by ") || !strings.Contains(setBy, "through settings") { + t.Errorf("through the verb: %q", setBy) + } +} + +func TestATrustedSettingThroughAVerbNamesTheProposal(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "notes", Version: "1", + Resources: []map[string]any{{"id": "data", "type": "directory", "mode": "0755"}}}) + if _, err := assign(ctx, open, "laptop", "notes"); err != nil { + t.Fatal(err) + } + err := throughVerb(t, "settings", map[string]any{"module": "notes", "node": "laptop", + "values": `{"places":{"data":{"path":"/srv/notes","owner":"1001:1001"}}}`}) + if err == nil || !strings.Contains(err.Error(), "issue 339") || !strings.Contains(err.Error(), "propose") { + t.Errorf("%v", err) + } + if _, has, _ := open.inventory.Layer(ctx, "laptop", "notes"); has { + t.Error("a layer was kept") + } +} diff --git a/cmd/mesh-controller/proposals_test.go b/cmd/mesh-controller/proposals_test.go index 5a6f759f..c791c9f3 100644 --- a/cmd/mesh-controller/proposals_test.go +++ b/cmd/mesh-controller/proposals_test.go @@ -14,7 +14,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/asks" - "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/outward" ) @@ -708,132 +707,11 @@ func TestTheSettingsVerbComposesProposeAndProposals(t *testing.T) { // setLayerIn sets the layer as the terminal does — judged, the removal meant, the history kept — with who approved it // beside it; and refuses once the layer is no longer the one the operator was shown the change against. -func TestSetOnAWarrantJudgesTheLayerAndKeepsWhoApprovedIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "notes", Version: "1", - // y is a preference with a default, so a layer may leave it out; x is the operator's own (ADR 0262). - Settings: map[string]catalogue.SettingDeclaration{"y": {Kind: "preference", Default: "10", Why: "a size"}}, - Resources: []map[string]any{{"id": "data", "type": "directory", "mode": "0755"}, - // A trusted file (unmarked), so its keys are the terminal's — and now the warrant's (novox/hq ADR 0277). - {"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644", "content": "x = ${setting:x}\ny = ${setting:y}\n"}}}) - if _, err := assign(ctx, open, "laptop", "notes"); err != nil { - t.Fatal(err) - } - set := setLayerIn(open) - now := time.Now() - first := map[string]any{"x": "1", "y": "2"} - p := settingsProposal{Module: "notes", Node: "laptop", Values: first, From: "g14/claude-code", At: now, - Digest: layerDigest(first), BeforeDigest: layerDigest(nil)} - changed, err := set(ctx, p, "s9", "approved by the operator, as telegram identity 42 via telegram at 14:05 (ask s9)") - if err != nil || !strings.Contains(changed, "+ x") { - t.Fatalf("%q %v", changed, err) - } - layer, has, err := open.inventory.Layer(ctx, "laptop", "notes") - if err != nil || !has || layer["x"] != "1" { - t.Fatalf("the layer: %v %v %v", layer, has, err) - } - setBy, _, has, err := open.inventory.LayerOrigin(ctx, "laptop", "notes") - if err != nil || !has || !strings.HasPrefix(setBy, "approved by the operator, as telegram identity 42 via telegram") { - t.Fatalf("who set it: %q %v", setBy, err) - } - // Shown by `settings show`, at the terminal and through the verb. - out := captureStdout(t, func() { - if err := atTheTerminal(t, "settings", "show", "notes", "--node", "laptop"); err != nil { - t.Fatal(err) - } - }) - if !strings.Contains(out, "approved by the operator, as telegram identity 42 via telegram") { - t.Errorf("settings show says:\n%s", out) - } - // The same proposal again: the layer is no longer the one the operator was shown it against. - if _, err := set(ctx, p, "s9", "approved …"); err == nil || !strings.Contains(err.Error(), "changed since the operator was shown") { - t.Errorf("a stale proposal: %v", err) - } - // A removal not meant is refused; one meant is taken, and the history says who had set the layer. - second := map[string]any{"x": "1"} - q := settingsProposal{Module: "notes", Node: "laptop", Values: second, Before: first, HadLayer: true, - From: "g14/claude-code", At: now, Digest: layerDigest(second), BeforeDigest: layerDigest(first)} - if _, err := set(ctx, q, "s10", "approved …"); err == nil || !strings.Contains(err.Error(), "removal was not meant") { - t.Errorf("a silent removal: %v", err) - } - // A layer the mesh refuses is refused here too, with the same words as at the terminal. - bad := q - bad.Values, bad.Digest = map[string]any{"x": "a\nb", "y": "2"}, layerDigest(map[string]any{"x": "a\nb", "y": "2"}) - if _, err := set(ctx, bad, "s11", "approved …"); err == nil || !strings.Contains(err.Error(), "line break") { - t.Errorf("a line break on a warrant: %v", err) - } - if layer, _, _ := open.inventory.Layer(ctx, "laptop", "notes"); layer["y"] != "2" { - t.Fatalf("a refused act changed the layer: %v", layer) - } - q.Replace = true - if _, err := set(ctx, q, "s10", "approved later"); err != nil { - t.Fatal(err) - } - past, err := open.inventory.SettingsHistory(ctx, "laptop", "notes") - if err != nil || len(past) != 1 || !strings.HasPrefix(past[0].SetBy, "approved by the operator") { - t.Errorf("the history: %+v %v", past, err) - } - // And a clear, keeping who cleared it with the copy. - c := settingsProposal{Module: "notes", Node: "laptop", Clear: true, Before: second, HadLayer: true, From: "x", At: now, - Digest: layerDigest(second), BeforeDigest: layerDigest(second)} - if _, err := set(ctx, c, "s12", "approved by the operator at 15:00"); err != nil { - t.Fatal(err) - } - if _, has, _ := open.inventory.Layer(ctx, "laptop", "notes"); has { - t.Error("the layer was not cleared") - } - past, _ = open.inventory.SettingsHistory(ctx, "laptop", "notes") - if len(past) != 2 || !strings.Contains(past[0].SetBy, "cleared approved by the operator at 15:00") { - t.Errorf("the history after a clear: %+v", past) - } -} // A layer set at the terminal says so, and one set through a verb names the verb (novox/hq ADR 0277). -func TestALayerSaysWhoSetIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "notes", Version: "1", - Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644", "trusted": false, - "content": "x = ${setting:x}\n"}}}) - if _, err := assign(ctx, open, "laptop", "notes"); err != nil { - t.Fatal(err) - } - if err := atTheTerminal(t, "settings", "set", "notes", `{"x":"1"}`, "--node", "laptop"); err != nil { - t.Fatal(err) - } - setBy, _, _, _ := open.inventory.LayerOrigin(ctx, "laptop", "notes") - if !strings.HasPrefix(setBy, "set at the controller's terminal by ") { - t.Errorf("at the terminal: %q", setBy) - } - if err := throughVerb(t, "settings", map[string]any{"module": "notes", "node": "laptop", "values": `{"x":"2"}`}); err != nil { - t.Fatal(err) - } - setBy, _, _, _ = open.inventory.LayerOrigin(ctx, "laptop", "notes") - if !strings.HasPrefix(setBy, "set by ") || !strings.Contains(setBy, "through settings") { - t.Errorf("through the verb: %q", setBy) - } -} // A trusted setting is still refused through the settings verb (novox/hq issue 339), and the refusal now names the // proposal as the way. -func TestATrustedSettingThroughAVerbNamesTheProposal(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "notes", Version: "1", - Resources: []map[string]any{{"id": "data", "type": "directory", "mode": "0755"}}}) - if _, err := assign(ctx, open, "laptop", "notes"); err != nil { - t.Fatal(err) - } - err := throughVerb(t, "settings", map[string]any{"module": "notes", "node": "laptop", - "values": `{"places":{"data":{"path":"/srv/notes","owner":"1001:1001"}}}`}) - if err == nil || !strings.Contains(err.Error(), "issue 339") || !strings.Contains(err.Error(), "propose") { - t.Errorf("%v", err) - } - if _, has, _ := open.inventory.Layer(ctx, "laptop", "notes"); has { - t.Error("a layer was kept") - } -} // captureStdout runs f and answers what it printed to standard output. func captureStdout(t *testing.T, f func()) string { diff --git a/cmd/mesh-controller/provider_hold_test.go b/cmd/mesh-controller/provider_hold_test.go index d07e9c7b..8597e133 100644 --- a/cmd/mesh-controller/provider_hold_test.go +++ b/cmd/mesh-controller/provider_hold_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/provisioner_account_integration_test.go b/cmd/mesh-controller/provisioner_account_integration_test.go new file mode 100644 index 00000000..75ede30b --- /dev/null +++ b/cmd/mesh-controller/provisioner_account_integration_test.go @@ -0,0 +1,79 @@ +//go:build integration + +package main + +import ( + "slices" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestTheProvisionerClientsAccountIsIssuedForItsServiceAndNeverInvented(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + m := aProvisioner() + register(t, open, m) + if _, err := assign(ctx, open, "laptop", m.Module); err != nil { + t.Fatal(err) + } + if err := busCredentialIssued(ctx, inv, "laptop", m.Module, broker.ProvisionerSecret); err == nil { + t.Fatal("a push would seal a placeholder for the client's account before it was issued") + } + + if err := issueProvisionerClient(ctx, inv, m, "laptop", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222"); err != nil { + t.Fatal(err) + } + if _, minted, err := inv.BusUserHash(ctx, "provisioner.laptop"); err != nil || !minted { + t.Fatalf("the client's account is not minted: %v %v", minted, err) + } + states, err := inv.SecretsOf(ctx, "laptop", m.Module) + if err != nil { + t.Fatal(err) + } + if !slices.ContainsFunc(states, func(s inventory.SecretState) bool { return s.Name == broker.ProvisionerSecret }) { + t.Fatalf("the client's account is not sealed to the machine for its service: %+v", states) + } + if err := busCredentialIssued(ctx, inv, "laptop", m.Module, broker.ProvisionerSecret); err != nil { + t.Fatalf("an issued client account is refused: %v", err) + } + if !runsTheProvisionerClient(m) { + t.Fatal("the module is not read as running the client") + } + unread := m + unread.OwnSecrets = catalogue.OwnSecrets{"broker": {Path: "/run/broker"}} + if runsTheProvisionerClient(unread) { + t.Fatal("a module with nowhere to read the client's account is read as running it") + } +} + +func TestAssigningTheClientIssuesItsAccount(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + client := aClientModule() + register(t, open, client) + out, err := assign(ctx, open, "laptop", client.Module) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out, "bus credential") { + t.Fatalf("assigning the client said nothing of its account:\n%s", out) + } +} + +// A module that runs the client and declares no broker of its own, as the catalogue's mesh-provisioner-client does, +// may be issued: the client's account alone (issue 917). Before, `module issue` refused it for having no broker, +// while a push refused it for having no account, so no machine could run the client. +func TestAClientWithoutABrokerOfItsOwnMayBeIssued(t *testing.T) { + if err := mayIssue(aClientModule()); err != nil { + t.Fatalf("the client alone is refused an account: %v", err) + } + neither := aBot("bot", false) + if err := mayIssue(neither); err == nil { + t.Fatal("a module with nowhere to read any account may be issued one") + } +} diff --git a/cmd/mesh-controller/provisioner_account_test.go b/cmd/mesh-controller/provisioner_account_test.go index 20dc7b15..7cc212d6 100644 --- a/cmd/mesh-controller/provisioner_account_test.go +++ b/cmd/mesh-controller/provisioner_account_test.go @@ -1,13 +1,8 @@ package main import ( - "slices" - "strings" - "testing" - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/catalogue" - "github.com/novox/mesh-controller/internal/inventory" ) func aProvisioner() catalogue.Manifest { @@ -23,70 +18,6 @@ func aProvisioner() catalogue.Manifest { // A machine's provisioner client is issued an account of its own, under its own name, sealed to the machine as the // module's own secret its service reads (novox/hq ADR 0349 clause 11, design 57 P3); until it is, a push refuses // to invent one, as it does for a module's `broker`. -func TestTheProvisionerClientsAccountIsIssuedForItsServiceAndNeverInvented(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - m := aProvisioner() - register(t, open, m) - if _, err := assign(ctx, open, "laptop", m.Module); err != nil { - t.Fatal(err) - } - if err := busCredentialIssued(ctx, inv, "laptop", m.Module, broker.ProvisionerSecret); err == nil { - t.Fatal("a push would seal a placeholder for the client's account before it was issued") - } - - if err := issueProvisionerClient(ctx, inv, m, "laptop", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222"); err != nil { - t.Fatal(err) - } - if _, minted, err := inv.BusUserHash(ctx, "provisioner.laptop"); err != nil || !minted { - t.Fatalf("the client's account is not minted: %v %v", minted, err) - } - states, err := inv.SecretsOf(ctx, "laptop", m.Module) - if err != nil { - t.Fatal(err) - } - if !slices.ContainsFunc(states, func(s inventory.SecretState) bool { return s.Name == broker.ProvisionerSecret }) { - t.Fatalf("the client's account is not sealed to the machine for its service: %+v", states) - } - if err := busCredentialIssued(ctx, inv, "laptop", m.Module, broker.ProvisionerSecret); err != nil { - t.Fatalf("an issued client account is refused: %v", err) - } - if !runsTheProvisionerClient(m) { - t.Fatal("the module is not read as running the client") - } - unread := m - unread.OwnSecrets = catalogue.OwnSecrets{"broker": {Path: "/run/broker"}} - if runsTheProvisionerClient(unread) { - t.Fatal("a module with nowhere to read the client's account is read as running it") - } -} - -// A module that runs the client and declares no broker of its own, as the catalogue's mesh-provisioner-client does, -// may be issued: the client's account alone (issue 917). Before, `module issue` refused it for having no broker, -// while a push refused it for having no account, so no machine could run the client. -func TestAClientWithoutABrokerOfItsOwnMayBeIssued(t *testing.T) { - if err := mayIssue(aClientModule()); err != nil { - t.Fatalf("the client alone is refused an account: %v", err) - } - neither := aBot("bot", false) - if err := mayIssue(neither); err == nil { - t.Fatal("a module with nowhere to read any account may be issued one") - } -} // Assigning the client tries to issue the client's account, as assigning any module issues its broker (issue 917): // without a bus to issue on, the line says so and names the verb, never nothing. -func TestAssigningTheClientIssuesItsAccount(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - client := aClientModule() - register(t, open, client) - out, err := assign(ctx, open, "laptop", client.Module) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(out, "bus credential") { - t.Fatalf("assigning the client said nothing of its account:\n%s", out) - } -} diff --git a/cmd/mesh-controller/push_recreates_test.go b/cmd/mesh-controller/push_recreates_test.go index 1fc486a0..2b73fd29 100644 --- a/cmd/mesh-controller/push_recreates_test.go +++ b/cmd/mesh-controller/push_recreates_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/queue_integration_test.go b/cmd/mesh-controller/queue_integration_test.go new file mode 100644 index 00000000..91d0d484 --- /dev/null +++ b/cmd/mesh-controller/queue_integration_test.go @@ -0,0 +1,492 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestAFailedPlanIsRetriedAndGoesOnThroughItsLaterTiers(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + asked := asksRecorded(t) + twoTiers(t, open) + before := time.Now().UTC().Add(-time.Hour) + failed := inventory.Plan{ID: "plan-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee", + Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}}, + Note: "a failed to build in tier 0", + Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", + Why: link.KilledByHand}}} + if err := open.inventory.SavePlan(ctx, &failed); err != nil { + t.Fatal(err) + } + + said, err := retryPlan(ctx, open, failed.ID) + if err != nil { + t.Fatal(err) + } + p, err := open.inventory.PlanByID(ctx, failed.ID) + if err != nil { + t.Fatal(err) + } + a := p.Modules["a"] + if p.State != inventory.PlanBuilding || a.State != "asked" || a.Build == "" || a.Build == "build-1" || a.Why != "" { + t.Fatalf("after retry the plan is %s and a is %+v", p.State, a) + } + if !strings.Contains(said, a.Build) { + t.Errorf("retry does not say the new id: %q", said) + } + if len(*asked) != 1 { + t.Fatalf("asked %v", *asked) + } + + // The killed build's own late outcome is not this ask's; the new one's is, and tier 1 follows. + planBuilt(ctx, open, "a", "c0ffee", link.KilledByHand, before, "build-1") + if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.State != inventory.PlanBuilding { + t.Fatalf("the old ask's outcome failed the retried plan: %s %q", p.State, p.Note) + } + asking, _ := link.BuildAskedAt(a.Build) + planBuilt(ctx, open, "a", "c0ffee", "", asking, a.Build) + p, err = open.inventory.PlanByID(ctx, failed.ID) + if err != nil { + t.Fatal(err) + } + if p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" || p.Modules["b"].Build == "" { + t.Fatalf("the retried plan did not go on to tier 1: tier %d, %s, b %+v", p.Tier, p.State, p.Modules["b"]) + } + if len(*asked) != 2 { + t.Fatalf("asked %v", *asked) + } +} + +func TestARebuildJoinsThePlanHoldingTheModule(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + asked := asksRecorded(t) + twoTiers(t, open) + before := time.Now().UTC().Add(-time.Hour) + failed := inventory.Plan{ID: "plan-join", Repository: "novox/a", Branch: "main", Commit: "c0ffee", + Created: before, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}}, + Note: "a failed to build in tier 0", + Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", Why: link.CancelledByHand}}} + if err := open.inventory.SavePlan(ctx, &failed); err != nil { + t.Fatal(err) + } + if err := rebuildCommand(ctx, []string{"a"}); err != nil { + t.Fatal(err) + } + p, err := open.inventory.PlanByID(ctx, failed.ID) + if err != nil { + t.Fatal(err) + } + if p.State != inventory.PlanBuilding || p.Modules["a"].State != "asked" || p.Modules["a"].Build == "build-1" { + t.Fatalf("the rebuild did not join the failed plan: %s %+v", p.State, p.Modules["a"]) + } + if len(*asked) != 1 { + t.Fatalf("asked %v", *asked) + } + // b is in a tier not yet reached: nothing holds it in its current tier, so it is asked alone. + if err := rebuildCommand(ctx, []string{"b"}); err != nil { + t.Fatal(err) + } + if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.Modules["b"] != nil { + t.Fatalf("a module the plan has not reached joined it: %+v", p.Modules["b"]) + } + if len(*asked) != 2 { + t.Fatalf("asked %v", *asked) + } +} + +func aBuildQueue(t *testing.T) (*broker.JetStream, func(id, repository string) link.BuildRequest) { + t.Helper() + url := testbus.URL(t) + t.Setenv(broker.NATSVar, url) + js, err := broker.Dial(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + seat := broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}, + Emits: []string{"started", "built", "log.*", "paused.*"}} + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + _ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT") + if err := broker.RaiseSeats(js, []broker.DeclaredSeat{seat}, map[string]broker.Holder{ + link.TheBuildMachine: {Node: "anchor", Module: "build-agent"}}); err != nil { + t.Fatal(err) + } + if err := broker.RaiseCancelledSets(js, []broker.DeclaredSeat{seat}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT") + _ = js.Context().DeleteKeyValue(broker.CancelledSetName(link.TheBuildMachine)) + _ = js.Context().PurgeStream(broker.EventsStream) + }) + ask := func(id, repository string) link.BuildRequest { + r := link.BuildRequest{ID: id, Repository: repository, Held: map[string]string{"x/y": "secret-ish"}} + body, _ := json.Marshal(r) + if _, err := js.Context().Publish(link.BuildWork(), body); err != nil { + t.Fatal(err) + } + return r + } + return js, ask +} + +func TestCancelDeletesTheAskAndFailsThePlanThatAskedIt(t *testing.T) { + js, ask := aBuildQueue(t) + open := aMesh(t) + ctx := t.Context() + twoTiers(t, open) + id := link.NewBuildID(time.Now()) + ask(id, "https://forge.example/novox/a.git") + asked, _ := link.BuildAskedAt(id) + plan := inventory.Plan{ID: "plan-cancel", Repository: "novox/a", Commit: "c0ffee", Created: asked, + State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}}, + Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: id}}} + if err := open.inventory.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + + q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) + if err != nil { + t.Fatal(err) + } + if len(q.Asks) != 1 || q.Asks[0].State != link.AskWaiting || q.Asks[0].ID != id { + t.Fatalf("the queue reads %+v", q) + } + if text := queueText(q, time.Now()); !strings.Contains(text, "1 waiting, 0 in flight, 0 dead") || + strings.Contains(text, "secret-ish") { + t.Errorf("the queue says:\n%s", text) + } + + if err := cancelCommand(ctx, []string{id}); err != nil { + t.Fatal(err) + } + if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 { + t.Fatalf("the ask is still queued: %+v", q.Asks) + } + if cancelled, err := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); err != nil || !cancelled { + t.Errorf("the cancelled set does not hold it: %v %v", cancelled, err) + } + b, found, err := open.inventory.BuildByID(ctx, id) + if err != nil || !found || b.Failed != link.CancelledByHand { + t.Fatalf("the cancel is recorded as %+v (%v %v)", b, found, err) + } + p, err := open.inventory.PlanByID(ctx, plan.ID) + if err != nil { + t.Fatal(err) + } + if p.State != inventory.PlanFailed || p.Modules["a"].State != "failed" || p.Modules["a"].Why != link.CancelledByHand { + t.Fatalf("the plan that asked is %s, a %+v", p.State, p.Modules["a"]) + } + if err := cancelCommand(ctx, []string{id}); err == nil { + t.Error("an ask cancelled already was cancelled again") + } +} + +func TestClearCancelsTheWaitingAndTheDeadOnlyWhenTold(t *testing.T) { + js, ask := aBuildQueue(t) + open := aMesh(t) + ctx := t.Context() + start := time.Now() + dead := link.NewBuildID(start) + ask(dead, "https://forge.example/novox/dead.git") + deadOne(t, js) + var waiting []string + for i := 1; i <= 2; i++ { + id := link.NewBuildID(start.Add(time.Duration(i) * time.Millisecond)) + waiting = append(waiting, id) + ask(id, fmt.Sprintf("https://forge.example/novox/w%d.git", i)) + } + + q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) + if err != nil { + t.Fatal(err) + } + if len(q.Of(link.AskDead)) != 1 || len(q.Of(link.AskWaiting)) != 2 || q.MaxDeliver != 5 { + t.Fatalf("the queue reads %+v", q) + } + said, err := clearQueue(ctx, js, open, link.TheBuildMachine, false) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(said, "2 waiting ask(s) cancelled") || !strings.Contains(said, "1 dead, left") { + t.Errorf("clear said:\n%s", said) + } + q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine) + if len(q.Asks) != 1 || q.Asks[0].ID != dead || q.Asks[0].State != link.AskDead { + t.Fatalf("after clear the queue is %+v", q.Asks) + } + if _, err := clearQueue(ctx, js, open, link.TheBuildMachine, true); err != nil { + t.Fatal(err) + } + if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 { + t.Fatalf("clear --dead left %+v", q.Asks) + } + for _, id := range append(waiting, dead) { + if b, found, _ := open.inventory.BuildByID(ctx, id); !found || b.Failed != link.CancelledByHand { + t.Errorf("%s is recorded as %+v", id, b) + } + } +} + +func TestCancelRefusesAnAskInFlight(t *testing.T) { + js, ask := aBuildQueue(t) + open := aMesh(t) + ctx := t.Context() + id := link.NewBuildID(time.Now()) + ask(id, "https://forge.example/novox/a.git") + worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}}) + sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = sub.Unsubscribe() }() + if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil { + t.Fatal(err) + } + started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)}) + if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil { + t.Fatal(err) + } + q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) + if err != nil { + t.Fatal(err) + } + a, _ := q.Find(id) + if a.State != link.AskInFlight || a.On != "ace" { + t.Fatalf("the taken ask reads %+v", a) + } + _, err = cancelAsk(ctx, js, open, link.TheBuildMachine, a) + if err == nil || !strings.Contains(err.Error(), "kill "+id) { + t.Fatalf("an ask in flight was cancelled: %v", err) + } + if _, found, _ := open.inventory.BuildByID(ctx, id); found { + t.Error("a refused cancel recorded an outcome") + } +} + +func TestKillAndPauseAskTheHolderOnTheMachine(t *testing.T) { + js, _ := aBuildQueue(t) + ctx := t.Context() + asked := map[string]string{} + handlers := map[string]link.ToolHandler{ + "kill": func(_ context.Context, raw json.RawMessage) (any, error) { + var args struct{ ID string } + _ = json.Unmarshal(raw, &args) + asked["kill"] = args.ID + if args.ID != "build-running" { + return nil, fmt.Errorf("ace is not building %s", args.ID) + } + return map[string]any{"said": "killed " + args.ID}, nil + }, + "pause": func(context.Context, json.RawMessage) (any, error) { + asked["pause"] = "ace" + return map[string]any{"said": "ace is paused"}, nil + }, + } + stop, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(link.TheBuildMachine, "ace", handlers, nil) + if err != nil { + t.Fatal(err) + } + defer stop() + for _, id := range []string{"build-running", "build-other"} { + started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)}) + if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil { + t.Fatal(err) + } + } + if err := killCommand(ctx, []string{"build-running"}); err != nil { + t.Fatal(err) + } + if asked["kill"] != "build-running" { + t.Fatalf("the holder was asked %v", asked) + } + if err := killCommand(ctx, []string{"build-other"}); err == nil { + t.Error("the holder's refusal was not the command's") + } + if err := killCommand(ctx, []string{"build-never"}); err == nil || !strings.Contains(err.Error(), "cancel build-never") { + t.Errorf("a build nobody started: %v", err) + } + if err := pauseCommand(ctx, "pause", []string{"ace"}); err != nil || asked["pause"] != "ace" { + t.Fatalf("pause: %v %v", err, asked) + } + if err := pauseCommand(ctx, "resume", []string{"g14"}); err == nil { + t.Error("a machine nothing answers on was resumed") + } +} + +func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) { + withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) + open := aMesh(t) + ctx := t.Context() + asked := asksRecorded(t) + twoTiers(t, open) + var sentTo [][]string + was := sendRollout + sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) { + sentTo = append(sentTo, names) + return names, nil + } + t.Cleanup(func() { sendRollout = was }) + + // a records: a person's choice, since the default rolls out (novox/hq ADR 0236). + if err := open.inventory.SetUpgradeOf(ctx, "a", inventory.Upgrade{Why: "test"}); err != nil { + t.Fatal(err) + } + long := time.Now().UTC().Add(-2 * time.Hour) + stopped := inventory.Plan{ID: "plan-rollout", Repository: "novox/a", Branch: "main", Commit: "c0ffee", + Created: long, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}}, + Note: "a stopped at its first machine in tier 0: laptop refused what it was sent", + // Asked by the plan, as every module a plan holds built is (novox/hq issue 754). + Modules: map[string]*inventory.PlanModule{"a": {State: "built", AskedAt: &long, Build: "build-a", BuiltAt: &long, + Commit: "c0ffee", First: []string{"laptop"}, FirstAt: &long, Why: "laptop refused what it was sent"}}} + if err := open.inventory.SavePlan(ctx, &stopped); err != nil { + t.Fatal(err) + } + + // A newer plan holding a refuses it: sending the older build would put it back. + newer := inventory.Plan{ID: "plan-newer", Repository: "novox/other", Commit: "d00d", Created: long.Add(time.Hour), + State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}} + if err := open.inventory.SavePlan(ctx, &newer); err != nil { + t.Fatal(err) + } + if _, err := retryPlan(ctx, open, stopped.ID); err == nil || !strings.Contains(err.Error(), "plan-newer") { + t.Fatalf("retried under a newer plan: %v", err) + } + newer.State = inventory.PlanSuperseded + if err := open.inventory.SavePlan(ctx, &newer); err != nil { + t.Fatal(err) + } + + said, err := retryPlan(ctx, open, stopped.ID) + if err != nil { + t.Fatal(err) + } + if len(sentTo) != 1 || !reflect.DeepEqual(sentTo[0], []string{"laptop"}) || !strings.Contains(said, "laptop") { + t.Fatalf("sent %v; said %q", sentTo, said) + } + p, err := open.inventory.PlanByID(ctx, stopped.ID) + if err != nil { + t.Fatal(err) + } + a := p.Modules["a"] + if p.State != inventory.PlanRolling || a.FirstAt == nil || !a.FirstAt.After(long) || a.Why != "" { + t.Fatalf("after retry the plan is %s, a %+v", p.State, a) + } + // And it goes on: a records (its policy sends nothing more), so the next tier is asked. + advancePlans(ctx, open) + if p, _ = open.inventory.PlanByID(ctx, stopped.ID); p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" { + t.Fatalf("the retried plan did not go on: tier %d %s %+v", p.Tier, p.State, p.Modules["b"]) + } + if len(*asked) != 1 { + t.Fatalf("asked %v", *asked) + } +} + +func TestAPlanIsAnsweredOnlyByTheBuildItAskedFor(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + asked := time.Now().UTC().Add(-time.Minute) + plan := inventory.Plan{ID: "plan-own", Repository: "novox/a", Commit: "c0ffee", Created: asked, + State: inventory.PlanBuilding, Tiers: [][]string{{"a"}}, + Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: "build-own"}}} + if err := open.inventory.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + planBuilt(ctx, open, "a", "0ldc0mm1t", "", time.Now().UTC(), "build-replay") + p, _ := open.inventory.PlanByID(ctx, plan.ID) + if p.Modules["a"].State != "asked" { + t.Fatalf("a replay asked after the plan settled it: %+v", p.Modules["a"]) + } + // From the records too: a later build of the module recorded is not the plan's. + recorded := map[string][]inventory.Build{"a": {{ID: "build-replay", Commit: "0ldc0mm1t", Asked: time.Now(), At: time.Now()}}} + if settleFromRecords(&p, p.Tiers[0], recorded, nil) { + t.Fatalf("the records settled it with another build: %+v", p.Modules["a"]) + } + planBuilt(ctx, open, "a", "c0ffee", "", asked, "build-own") + if p, _ = open.inventory.PlanByID(ctx, plan.ID); p.Modules["a"].State != "built" || p.Modules["a"].Commit != "c0ffee" { + t.Fatalf("its own build did not settle it: %+v", p.Modules["a"]) + } +} + +func TestARebuildOfACommitAsksWhatTheModuleFollows(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + asked := asksRecorded(t) + twoTiers(t, open) + var refs []string + was := askABuild + askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) { + refs = append(refs, ref) + return was(c, source, path, ref) + } + if err := open.inventory.RecordBuild(ctx, inventory.Build{ID: "build-at-commit", Repository: "novox/a", + Ref: "0123456789abcdef0123456789abcdef01234567", Module: "a", Commit: "0123456789abcdef0123456789abcdef01234567"}); err != nil { + t.Fatal(err) + } + if err := rebuildCommand(ctx, []string{"build-at-commit"}); err != nil { + t.Fatal(err) + } + if len(refs) != 1 || refs[0] != "main" || len(*asked) != 1 { + t.Fatalf("asked %v at %v", *asked, refs) + } +} + +func TestCancelOfAnAskNobodySaidIsWithdrawnWhenItStarts(t *testing.T) { + js, ask := aBuildQueue(t) + open := aMesh(t) + ctx := t.Context() + was := holderLooks + holderLooks = 300 * time.Millisecond + t.Cleanup(func() { holderLooks = was }) + id := link.NewBuildID(time.Now()) + ask(id, "https://forge.example/novox/a.git") + worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}}) + sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = sub.Unsubscribe() }() + if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil { + t.Fatal(err) + } + q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) + if err != nil { + t.Fatal(err) + } + a, _ := q.Find(id) + if a.State != link.AskInFlight || a.On != "" { + t.Fatalf("the taken ask reads %+v", a) + } + // The holder that took it says it started, while the cancel waits. + go func() { + time.Sleep(100 * time.Millisecond) + started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)}) + _, _ = js.Context().Publish(link.BuildStarted(), started) + }() + if _, err := cancelAsk(ctx, js, open, link.TheBuildMachine, a); err == nil || !strings.Contains(err.Error(), "started on ace") { + t.Fatalf("a build that started was cancelled: %v", err) + } + if cancelled, _ := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); cancelled { + t.Error("the withdrawn cancel is still marked") + } + if _, found, _ := open.inventory.BuildByID(ctx, id); found { + t.Error("a withdrawn cancel recorded an outcome") + } +} diff --git a/cmd/mesh-controller/queue_test.go b/cmd/mesh-controller/queue_test.go index 0a4a13fd..d7dc6cb2 100644 --- a/cmd/mesh-controller/queue_test.go +++ b/cmd/mesh-controller/queue_test.go @@ -2,8 +2,6 @@ package main import ( "context" - "encoding/json" - "fmt" "reflect" "strings" "testing" @@ -15,8 +13,6 @@ import ( "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" - - "github.com/novox/mesh-controller/internal/testbus" ) // The build queue, controlled by hand (novox/hq ADR 0219), and the plans that follow it. @@ -52,58 +48,6 @@ func twoTiers(t *testing.T, open *stores) { // A failed plan is retried: its failed module asked again under a new id, the plan building at that // tier, and when that build comes in the plan goes on and asks its next tier. -func TestAFailedPlanIsRetriedAndGoesOnThroughItsLaterTiers(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - asked := asksRecorded(t) - twoTiers(t, open) - before := time.Now().UTC().Add(-time.Hour) - failed := inventory.Plan{ID: "plan-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee", - Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}}, - Note: "a failed to build in tier 0", - Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", - Why: link.KilledByHand}}} - if err := open.inventory.SavePlan(ctx, &failed); err != nil { - t.Fatal(err) - } - - said, err := retryPlan(ctx, open, failed.ID) - if err != nil { - t.Fatal(err) - } - p, err := open.inventory.PlanByID(ctx, failed.ID) - if err != nil { - t.Fatal(err) - } - a := p.Modules["a"] - if p.State != inventory.PlanBuilding || a.State != "asked" || a.Build == "" || a.Build == "build-1" || a.Why != "" { - t.Fatalf("after retry the plan is %s and a is %+v", p.State, a) - } - if !strings.Contains(said, a.Build) { - t.Errorf("retry does not say the new id: %q", said) - } - if len(*asked) != 1 { - t.Fatalf("asked %v", *asked) - } - - // The killed build's own late outcome is not this ask's; the new one's is, and tier 1 follows. - planBuilt(ctx, open, "a", "c0ffee", link.KilledByHand, before, "build-1") - if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.State != inventory.PlanBuilding { - t.Fatalf("the old ask's outcome failed the retried plan: %s %q", p.State, p.Note) - } - asking, _ := link.BuildAskedAt(a.Build) - planBuilt(ctx, open, "a", "c0ffee", "", asking, a.Build) - p, err = open.inventory.PlanByID(ctx, failed.ID) - if err != nil { - t.Fatal(err) - } - if p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" || p.Modules["b"].Build == "" { - t.Fatalf("the retried plan did not go on to tier 1: tier %d, %s, b %+v", p.Tier, p.State, p.Modules["b"]) - } - if len(*asked) != 2 { - t.Fatalf("asked %v", *asked) - } -} // What retry refuses, and says why. func TestRetryRefusesWhatItCannotResume(t *testing.T) { @@ -152,43 +96,6 @@ func TestRetryRefusesWhatItCannotResume(t *testing.T) { } // `rebuild` of a module a failed plan holds joins that plan; one held by nothing runs alone. -func TestARebuildJoinsThePlanHoldingTheModule(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - asked := asksRecorded(t) - twoTiers(t, open) - before := time.Now().UTC().Add(-time.Hour) - failed := inventory.Plan{ID: "plan-join", Repository: "novox/a", Branch: "main", Commit: "c0ffee", - Created: before, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}}, - Note: "a failed to build in tier 0", - Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", Why: link.CancelledByHand}}} - if err := open.inventory.SavePlan(ctx, &failed); err != nil { - t.Fatal(err) - } - if err := rebuildCommand(ctx, []string{"a"}); err != nil { - t.Fatal(err) - } - p, err := open.inventory.PlanByID(ctx, failed.ID) - if err != nil { - t.Fatal(err) - } - if p.State != inventory.PlanBuilding || p.Modules["a"].State != "asked" || p.Modules["a"].Build == "build-1" { - t.Fatalf("the rebuild did not join the failed plan: %s %+v", p.State, p.Modules["a"]) - } - if len(*asked) != 1 { - t.Fatalf("asked %v", *asked) - } - // b is in a tier not yet reached: nothing holds it in its current tier, so it is asked alone. - if err := rebuildCommand(ctx, []string{"b"}); err != nil { - t.Fatal(err) - } - if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.Modules["b"] != nil { - t.Fatalf("a module the plan has not reached joined it: %+v", p.Modules["b"]) - } - if len(*asked) != 2 { - t.Fatalf("asked %v", *asked) - } -} // A failed plan an open plan of its repository supersedes is not joined: the open one holds the module. func TestARebuildJoinsTheOpenPlanBeforeASupersededFailedOne(t *testing.T) { @@ -361,43 +268,6 @@ func TestTheQueueVerbsRunTheirCommands(t *testing.T) { // aBuildQueue is the build seat's queue, worker and cancelled set on a real server, the controller // pointed at it, and a function that asks the seat one build. -func aBuildQueue(t *testing.T) (*broker.JetStream, func(id, repository string) link.BuildRequest) { - t.Helper() - url := testbus.URL(t) - t.Setenv(broker.NATSVar, url) - js, err := broker.Dial(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - seat := broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}, - Emits: []string{"started", "built", "log.*", "paused.*"}} - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - _ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT") - if err := broker.RaiseSeats(js, []broker.DeclaredSeat{seat}, map[string]broker.Holder{ - link.TheBuildMachine: {Node: "anchor", Module: "build-agent"}}); err != nil { - t.Fatal(err) - } - if err := broker.RaiseCancelledSets(js, []broker.DeclaredSeat{seat}); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { - _ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT") - _ = js.Context().DeleteKeyValue(broker.CancelledSetName(link.TheBuildMachine)) - _ = js.Context().PurgeStream(broker.EventsStream) - }) - ask := func(id, repository string) link.BuildRequest { - r := link.BuildRequest{ID: id, Repository: repository, Held: map[string]string{"x/y": "secret-ish"}} - body, _ := json.Marshal(r) - if _, err := js.Context().Publish(link.BuildWork(), body); err != nil { - t.Fatal(err) - } - return r - } - return js, ask -} // deadOne takes the oldest ask from the worker and hands it back as often as the worker allows. func deadOne(t *testing.T, js *broker.JetStream) { @@ -425,357 +295,21 @@ func deadOne(t *testing.T, js *broker.JetStream) { // cancel drops a waiting ask from the bus and records it failed, cancelled by hand — and the plan // that asked for it, matched by the id, fails with it; an ask the plan's records name no module for // is still found. -func TestCancelDeletesTheAskAndFailsThePlanThatAskedIt(t *testing.T) { - js, ask := aBuildQueue(t) - open := aMesh(t) - ctx := t.Context() - twoTiers(t, open) - id := link.NewBuildID(time.Now()) - ask(id, "https://forge.example/novox/a.git") - asked, _ := link.BuildAskedAt(id) - plan := inventory.Plan{ID: "plan-cancel", Repository: "novox/a", Commit: "c0ffee", Created: asked, - State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}}, - Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: id}}} - if err := open.inventory.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - - q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) - if err != nil { - t.Fatal(err) - } - if len(q.Asks) != 1 || q.Asks[0].State != link.AskWaiting || q.Asks[0].ID != id { - t.Fatalf("the queue reads %+v", q) - } - if text := queueText(q, time.Now()); !strings.Contains(text, "1 waiting, 0 in flight, 0 dead") || - strings.Contains(text, "secret-ish") { - t.Errorf("the queue says:\n%s", text) - } - - if err := cancelCommand(ctx, []string{id}); err != nil { - t.Fatal(err) - } - if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 { - t.Fatalf("the ask is still queued: %+v", q.Asks) - } - if cancelled, err := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); err != nil || !cancelled { - t.Errorf("the cancelled set does not hold it: %v %v", cancelled, err) - } - b, found, err := open.inventory.BuildByID(ctx, id) - if err != nil || !found || b.Failed != link.CancelledByHand { - t.Fatalf("the cancel is recorded as %+v (%v %v)", b, found, err) - } - p, err := open.inventory.PlanByID(ctx, plan.ID) - if err != nil { - t.Fatal(err) - } - if p.State != inventory.PlanFailed || p.Modules["a"].State != "failed" || p.Modules["a"].Why != link.CancelledByHand { - t.Fatalf("the plan that asked is %s, a %+v", p.State, p.Modules["a"]) - } - if err := cancelCommand(ctx, []string{id}); err == nil { - t.Error("an ask cancelled already was cancelled again") - } -} // clear cancels every waiting ask and leaves the dead ones unless told, and never one in flight. -func TestClearCancelsTheWaitingAndTheDeadOnlyWhenTold(t *testing.T) { - js, ask := aBuildQueue(t) - open := aMesh(t) - ctx := t.Context() - start := time.Now() - dead := link.NewBuildID(start) - ask(dead, "https://forge.example/novox/dead.git") - deadOne(t, js) - var waiting []string - for i := 1; i <= 2; i++ { - id := link.NewBuildID(start.Add(time.Duration(i) * time.Millisecond)) - waiting = append(waiting, id) - ask(id, fmt.Sprintf("https://forge.example/novox/w%d.git", i)) - } - - q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) - if err != nil { - t.Fatal(err) - } - if len(q.Of(link.AskDead)) != 1 || len(q.Of(link.AskWaiting)) != 2 || q.MaxDeliver != 5 { - t.Fatalf("the queue reads %+v", q) - } - said, err := clearQueue(ctx, js, open, link.TheBuildMachine, false) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(said, "2 waiting ask(s) cancelled") || !strings.Contains(said, "1 dead, left") { - t.Errorf("clear said:\n%s", said) - } - q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine) - if len(q.Asks) != 1 || q.Asks[0].ID != dead || q.Asks[0].State != link.AskDead { - t.Fatalf("after clear the queue is %+v", q.Asks) - } - if _, err := clearQueue(ctx, js, open, link.TheBuildMachine, true); err != nil { - t.Fatal(err) - } - if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 { - t.Fatalf("clear --dead left %+v", q.Asks) - } - for _, id := range append(waiting, dead) { - if b, found, _ := open.inventory.BuildByID(ctx, id); !found || b.Failed != link.CancelledByHand { - t.Errorf("%s is recorded as %+v", id, b) - } - } -} // An ask in flight is not cancelled: kill ends it where it runs. -func TestCancelRefusesAnAskInFlight(t *testing.T) { - js, ask := aBuildQueue(t) - open := aMesh(t) - ctx := t.Context() - id := link.NewBuildID(time.Now()) - ask(id, "https://forge.example/novox/a.git") - worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}}) - sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck()) - if err != nil { - t.Fatal(err) - } - defer func() { _ = sub.Unsubscribe() }() - if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil { - t.Fatal(err) - } - started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)}) - if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil { - t.Fatal(err) - } - q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) - if err != nil { - t.Fatal(err) - } - a, _ := q.Find(id) - if a.State != link.AskInFlight || a.On != "ace" { - t.Fatalf("the taken ask reads %+v", a) - } - _, err = cancelAsk(ctx, js, open, link.TheBuildMachine, a) - if err == nil || !strings.Contains(err.Error(), "kill "+id) { - t.Fatalf("an ask in flight was cancelled: %v", err) - } - if _, found, _ := open.inventory.BuildByID(ctx, id); found { - t.Error("a refused cancel recorded an outcome") - } -} // kill finds the machine from the build's start and asks that machine's holder; pause asks the // machine named. Each prints what the holder answered, and a refusal is the command's failure. -func TestKillAndPauseAskTheHolderOnTheMachine(t *testing.T) { - js, _ := aBuildQueue(t) - ctx := t.Context() - asked := map[string]string{} - handlers := map[string]link.ToolHandler{ - "kill": func(_ context.Context, raw json.RawMessage) (any, error) { - var args struct{ ID string } - _ = json.Unmarshal(raw, &args) - asked["kill"] = args.ID - if args.ID != "build-running" { - return nil, fmt.Errorf("ace is not building %s", args.ID) - } - return map[string]any{"said": "killed " + args.ID}, nil - }, - "pause": func(context.Context, json.RawMessage) (any, error) { - asked["pause"] = "ace" - return map[string]any{"said": "ace is paused"}, nil - }, - } - stop, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(link.TheBuildMachine, "ace", handlers, nil) - if err != nil { - t.Fatal(err) - } - defer stop() - for _, id := range []string{"build-running", "build-other"} { - started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)}) - if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil { - t.Fatal(err) - } - } - if err := killCommand(ctx, []string{"build-running"}); err != nil { - t.Fatal(err) - } - if asked["kill"] != "build-running" { - t.Fatalf("the holder was asked %v", asked) - } - if err := killCommand(ctx, []string{"build-other"}); err == nil { - t.Error("the holder's refusal was not the command's") - } - if err := killCommand(ctx, []string{"build-never"}); err == nil || !strings.Contains(err.Error(), "cancel build-never") { - t.Errorf("a build nobody started: %v", err) - } - if err := pauseCommand(ctx, "pause", []string{"ace"}); err != nil || asked["pause"] != "ace" { - t.Fatalf("pause: %v %v", err, asked) - } - if err := pauseCommand(ctx, "resume", []string{"g14"}); err == nil { - t.Error("a machine nothing answers on was resumed") - } -} // A plan that stopped at its first machine is retried: the module sent to that machine again, the // send recorded as the first anew, and the plan goes on — unless a newer plan holds the module. -func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) { - withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - asked := asksRecorded(t) - twoTiers(t, open) - var sentTo [][]string - was := sendRollout - sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) { - sentTo = append(sentTo, names) - return names, nil - } - t.Cleanup(func() { sendRollout = was }) - - // a records: a person's choice, since the default rolls out (novox/hq ADR 0236). - if err := open.inventory.SetUpgradeOf(ctx, "a", inventory.Upgrade{Why: "test"}); err != nil { - t.Fatal(err) - } - long := time.Now().UTC().Add(-2 * time.Hour) - stopped := inventory.Plan{ID: "plan-rollout", Repository: "novox/a", Branch: "main", Commit: "c0ffee", - Created: long, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}}, - Note: "a stopped at its first machine in tier 0: laptop refused what it was sent", - // Asked by the plan, as every module a plan holds built is (novox/hq issue 754). - Modules: map[string]*inventory.PlanModule{"a": {State: "built", AskedAt: &long, Build: "build-a", BuiltAt: &long, - Commit: "c0ffee", First: []string{"laptop"}, FirstAt: &long, Why: "laptop refused what it was sent"}}} - if err := open.inventory.SavePlan(ctx, &stopped); err != nil { - t.Fatal(err) - } - - // A newer plan holding a refuses it: sending the older build would put it back. - newer := inventory.Plan{ID: "plan-newer", Repository: "novox/other", Commit: "d00d", Created: long.Add(time.Hour), - State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}} - if err := open.inventory.SavePlan(ctx, &newer); err != nil { - t.Fatal(err) - } - if _, err := retryPlan(ctx, open, stopped.ID); err == nil || !strings.Contains(err.Error(), "plan-newer") { - t.Fatalf("retried under a newer plan: %v", err) - } - newer.State = inventory.PlanSuperseded - if err := open.inventory.SavePlan(ctx, &newer); err != nil { - t.Fatal(err) - } - - said, err := retryPlan(ctx, open, stopped.ID) - if err != nil { - t.Fatal(err) - } - if len(sentTo) != 1 || !reflect.DeepEqual(sentTo[0], []string{"laptop"}) || !strings.Contains(said, "laptop") { - t.Fatalf("sent %v; said %q", sentTo, said) - } - p, err := open.inventory.PlanByID(ctx, stopped.ID) - if err != nil { - t.Fatal(err) - } - a := p.Modules["a"] - if p.State != inventory.PlanRolling || a.FirstAt == nil || !a.FirstAt.After(long) || a.Why != "" { - t.Fatalf("after retry the plan is %s, a %+v", p.State, a) - } - // And it goes on: a records (its policy sends nothing more), so the next tier is asked. - advancePlans(ctx, open) - if p, _ = open.inventory.PlanByID(ctx, stopped.ID); p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" { - t.Fatalf("the retried plan did not go on: tier %d %s %+v", p.Tier, p.State, p.Modules["b"]) - } - if len(*asked) != 1 { - t.Fatalf("asked %v", *asked) - } -} // A plan module asked under an id is settled by that id's outcome alone: a replay or a rebuild beside // the plan, asked later, never answers it (novox/hq ADR 0219). -func TestAPlanIsAnsweredOnlyByTheBuildItAskedFor(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - asked := time.Now().UTC().Add(-time.Minute) - plan := inventory.Plan{ID: "plan-own", Repository: "novox/a", Commit: "c0ffee", Created: asked, - State: inventory.PlanBuilding, Tiers: [][]string{{"a"}}, - Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: "build-own"}}} - if err := open.inventory.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - planBuilt(ctx, open, "a", "0ldc0mm1t", "", time.Now().UTC(), "build-replay") - p, _ := open.inventory.PlanByID(ctx, plan.ID) - if p.Modules["a"].State != "asked" { - t.Fatalf("a replay asked after the plan settled it: %+v", p.Modules["a"]) - } - // From the records too: a later build of the module recorded is not the plan's. - recorded := map[string][]inventory.Build{"a": {{ID: "build-replay", Commit: "0ldc0mm1t", Asked: time.Now(), At: time.Now()}}} - if settleFromRecords(&p, p.Tiers[0], recorded, nil) { - t.Fatalf("the records settled it with another build: %+v", p.Modules["a"]) - } - planBuilt(ctx, open, "a", "c0ffee", "", asked, "build-own") - if p, _ = open.inventory.PlanByID(ctx, plan.ID); p.Modules["a"].State != "built" || p.Modules["a"].Commit != "c0ffee" { - t.Fatalf("its own build did not settle it: %+v", p.Modules["a"]) - } -} // rebuild of a build made at a commit asks what the module follows now, never the commit. -func TestARebuildOfACommitAsksWhatTheModuleFollows(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - asked := asksRecorded(t) - twoTiers(t, open) - var refs []string - was := askABuild - askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) { - refs = append(refs, ref) - return was(c, source, path, ref) - } - if err := open.inventory.RecordBuild(ctx, inventory.Build{ID: "build-at-commit", Repository: "novox/a", - Ref: "0123456789abcdef0123456789abcdef01234567", Module: "a", Commit: "0123456789abcdef0123456789abcdef01234567"}); err != nil { - t.Fatal(err) - } - if err := rebuildCommand(ctx, []string{"build-at-commit"}); err != nil { - t.Fatal(err) - } - if len(refs) != 1 || refs[0] != "main" || len(*asked) != 1 { - t.Fatalf("asked %v at %v", *asked, refs) - } -} // An ask the worker counts out that no machine said it started is cancelled only if no start comes // while a holder looks: one that does withdraws the cancel, and kill is what ends it. -func TestCancelOfAnAskNobodySaidIsWithdrawnWhenItStarts(t *testing.T) { - js, ask := aBuildQueue(t) - open := aMesh(t) - ctx := t.Context() - was := holderLooks - holderLooks = 300 * time.Millisecond - t.Cleanup(func() { holderLooks = was }) - id := link.NewBuildID(time.Now()) - ask(id, "https://forge.example/novox/a.git") - worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}}) - sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck()) - if err != nil { - t.Fatal(err) - } - defer func() { _ = sub.Unsubscribe() }() - if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil { - t.Fatal(err) - } - q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) - if err != nil { - t.Fatal(err) - } - a, _ := q.Find(id) - if a.State != link.AskInFlight || a.On != "" { - t.Fatalf("the taken ask reads %+v", a) - } - // The holder that took it says it started, while the cancel waits. - go func() { - time.Sleep(100 * time.Millisecond) - started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)}) - _, _ = js.Context().Publish(link.BuildStarted(), started) - }() - if _, err := cancelAsk(ctx, js, open, link.TheBuildMachine, a); err == nil || !strings.Contains(err.Error(), "started on ace") { - t.Fatalf("a build that started was cancelled: %v", err) - } - if cancelled, _ := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); cancelled { - t.Error("the withdrawn cancel is still marked") - } - if _, found, _ := open.inventory.BuildByID(ctx, id); found { - t.Error("a withdrawn cancel recorded an outcome") - } -} diff --git a/cmd/mesh-controller/reconnects_integration_test.go b/cmd/mesh-controller/reconnects_integration_test.go new file mode 100644 index 00000000..6cf7e7d3 --- /dev/null +++ b/cmd/mesh-controller/reconnects_integration_test.go @@ -0,0 +1,140 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestTheServingControllerLendsItsOwnConnection(t *testing.T) { + bus := testbus.Start(t) + serving, err := broker.Dial(bus.ClientURL()) + if err != nil { + t.Fatal(err) + } + defer serving.Close() + if err := serving.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + was := servingBus.Load() + servingBus.Store(serving) + defer servingBus.Store(was) + t.Setenv(broker.NATSVar, bus.ClientURL()) + + before, _ := bus.Varz(nil) + lent, err := aBus() + if err != nil { + t.Fatal(err) + } + lent.Close() + if !serving.Conn().IsConnected() { + t.Fatal("closing a lent connection closed the serving controller's") + } + if err := onTheBus(func(*nats.Conn) error { return nil }); err != nil { + t.Fatal(err) + } + handlers, _, err := seatToolHandlers() + if err != nil { + t.Fatal(err) + } + answer, err := handlers["hand-acts"](context.Background(), json.RawMessage(`{}`)) + if err != nil { + t.Fatal(err) + } + if a := answer.(verbAnswer); !a.OK || a.Answer == nil { + t.Fatalf("hand-acts answered %+v", a) + } + _, _ = handlers["queue"](context.Background(), json.RawMessage(`{}`)) + after, _ := bus.Varz(nil) + if opened := after.TotalConnections - before.TotalConnections; opened != 0 { + t.Fatalf("the serving controller opened %d connection(s) of its own", opened) + } +} + +func TestAVerbsOwnProcessNamesItsConnection(t *testing.T) { + bus := testbus.Start(t) + setup, err := broker.Dial(bus.ClientURL()) + if err != nil { + t.Fatal(err) + } + if err := setup.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + setup.Close() + asAProcess(t, bus.ClientURL()) + handlers, _, err := seatToolHandlers() + if err != nil { + t.Fatal(err) + } + // A silence acts, so it is still its own process; it dials, and finds no such condition. + _, _ = handlers["conditions"](context.Background(), + json.RawMessage(`{"silence":"bus.nothing.here","for":"1h","why":"a test"}`)) + names := closedNames(t, bus, "mesh-controller verb conditions") + found := false + for _, n := range names { + found = found || n == "mesh-controller verb conditions" + } + if !found { + t.Fatalf("the verb's connection was named %q", names) + } +} + +func TestTheBusModuleIsAskedWhoClosedConnections(t *testing.T) { + bus := testbus.Start(t) + conn, err := nats.Connect(bus.ClientURL()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + sub, err := conn.Subscribe(link.ModuleToolOn("nats", "nats_closed_connections", "anchor"), func(m *nats.Msg) { + _ = m.Respond([]byte(`{"result":{"hours":1,"reaches":true,"users":[{"user":"controller","closed":2,"dropped":0}]}}`)) + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = sub.Unsubscribe() }() + read, err := askClosed(context.Background(), conn, "anchor") + if err != nil { + t.Fatal(err) + } + if len(read.Users) != 1 || read.Users[0].Closed != 2 { + t.Fatalf("%+v", read) + } +} + +func TestAnOlderBusModuleIsPassedOverQuietly(t *testing.T) { + bus := testbus.Start(t) + conn, err := nats.Connect(bus.ClientURL()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + found, err := reconnectsOn(context.Background(), conn, "anchor") + if err != nil || len(found) != 0 { + t.Fatalf("a bus module without the tool: %v %v", found, err) + } +} + +func TestAFlagErrorIsSaidInTheAnswer(t *testing.T) { + bus := testbus.Start(t) + serving, err := broker.Dial(bus.ClientURL()) + if err != nil { + t.Fatal(err) + } + defer serving.Close() + was := servingBus.Load() + servingBus.Store(serving) + defer servingBus.Store(was) + answer, read := readHere(context.Background(), []string{"hand-acts", "--bogus"}) + if !read || answer.OK || !strings.Contains(answer.Output, "flag provided but not defined") { + t.Fatalf("answered %+v", answer) + } +} diff --git a/cmd/mesh-controller/reconnects_test.go b/cmd/mesh-controller/reconnects_test.go index 4f43f5e3..3c708898 100644 --- a/cmd/mesh-controller/reconnects_test.go +++ b/cmd/mesh-controller/reconnects_test.go @@ -1,94 +1,17 @@ package main import ( - "context" "encoding/json" "strings" "testing" - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" ) // The serving controller's own connection serves what it does for a moment: no connection is opened, // and closing what it was lent leaves the serving one open (novox/hq issue 327). -func TestTheServingControllerLendsItsOwnConnection(t *testing.T) { - bus := testbus.Start(t) - serving, err := broker.Dial(bus.ClientURL()) - if err != nil { - t.Fatal(err) - } - defer serving.Close() - if err := serving.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - was := servingBus.Load() - servingBus.Store(serving) - defer servingBus.Store(was) - t.Setenv(broker.NATSVar, bus.ClientURL()) - - before, _ := bus.Varz(nil) - lent, err := aBus() - if err != nil { - t.Fatal(err) - } - lent.Close() - if !serving.Conn().IsConnected() { - t.Fatal("closing a lent connection closed the serving controller's") - } - if err := onTheBus(func(*nats.Conn) error { return nil }); err != nil { - t.Fatal(err) - } - handlers, _, err := seatToolHandlers() - if err != nil { - t.Fatal(err) - } - answer, err := handlers["hand-acts"](context.Background(), json.RawMessage(`{}`)) - if err != nil { - t.Fatal(err) - } - if a := answer.(verbAnswer); !a.OK || a.Answer == nil { - t.Fatalf("hand-acts answered %+v", a) - } - _, _ = handlers["queue"](context.Background(), json.RawMessage(`{}`)) - after, _ := bus.Varz(nil) - if opened := after.TotalConnections - before.TotalConnections; opened != 0 { - t.Fatalf("the serving controller opened %d connection(s) of its own", opened) - } -} // A verb that still runs as a process of its own says which in the bus's list of connections. -func TestAVerbsOwnProcessNamesItsConnection(t *testing.T) { - bus := testbus.Start(t) - setup, err := broker.Dial(bus.ClientURL()) - if err != nil { - t.Fatal(err) - } - if err := setup.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - setup.Close() - asAProcess(t, bus.ClientURL()) - handlers, _, err := seatToolHandlers() - if err != nil { - t.Fatal(err) - } - // A silence acts, so it is still its own process; it dials, and finds no such condition. - _, _ = handlers["conditions"](context.Background(), - json.RawMessage(`{"silence":"bus.nothing.here","for":"1h","why":"a test"}`)) - names := closedNames(t, bus, "mesh-controller verb conditions") - found := false - for _, n := range names { - found = found || n == "mesh-controller verb conditions" - } - if !found { - t.Fatalf("the verb's connection was named %q", names) - } -} func TestEachProcessNamesItsConnectionsForWhatItIs(t *testing.T) { for _, c := range []struct{ command, verb, want string }{ @@ -134,56 +57,7 @@ func TestAClientReconnectingInALoopIsSaid(t *testing.T) { } // The bus's module is asked on its machine, over the bus. -func TestTheBusModuleIsAskedWhoClosedConnections(t *testing.T) { - bus := testbus.Start(t) - conn, err := nats.Connect(bus.ClientURL()) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - sub, err := conn.Subscribe(link.ModuleToolOn("nats", "nats_closed_connections", "anchor"), func(m *nats.Msg) { - _ = m.Respond([]byte(`{"result":{"hours":1,"reaches":true,"users":[{"user":"controller","closed":2,"dropped":0}]}}`)) - }) - if err != nil { - t.Fatal(err) - } - defer func() { _ = sub.Unsubscribe() }() - read, err := askClosed(context.Background(), conn, "anchor") - if err != nil { - t.Fatal(err) - } - if len(read.Users) != 1 || read.Users[0].Closed != 2 { - t.Fatalf("%+v", read) - } -} // A bus module older than the tool answers nothing to the question: the probe passes over it quietly. -func TestAnOlderBusModuleIsPassedOverQuietly(t *testing.T) { - bus := testbus.Start(t) - conn, err := nats.Connect(bus.ClientURL()) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - found, err := reconnectsOn(context.Background(), conn, "anchor") - if err != nil || len(found) != 0 { - t.Fatalf("a bus module without the tool: %v %v", found, err) - } -} // A verb answered in the serving controller says its flag errors in its answer, not in the controller's log. -func TestAFlagErrorIsSaidInTheAnswer(t *testing.T) { - bus := testbus.Start(t) - serving, err := broker.Dial(bus.ClientURL()) - if err != nil { - t.Fatal(err) - } - defer serving.Close() - was := servingBus.Load() - servingBus.Store(serving) - defer servingBus.Store(was) - answer, read := readHere(context.Background(), []string{"hand-acts", "--bogus"}) - if !read || answer.OK || !strings.Contains(answer.Output, "flag provided but not defined") { - t.Fatalf("answered %+v", answer) - } -} diff --git a/cmd/mesh-controller/recorded_kept_integration_test.go b/cmd/mesh-controller/recorded_kept_integration_test.go new file mode 100644 index 00000000..50b76e46 --- /dev/null +++ b/cmd/mesh-controller/recorded_kept_integration_test.go @@ -0,0 +1,144 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestARecordedBuildIsCarriedOnlyByAPersonsPush(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + start := time.Now().Add(-time.Hour) + pgImage := "registry.invalid:5000/postgres/server@sha256:" + strings.Repeat("a", 64) + mailImage := "registry.invalid:5000/mailu/smtp@sha256:" + strings.Repeat("c", 64) + for _, b := range []link.BuildResult{ + aContainerBuild(t, "postgres", "c1111111", catalogue.PolicyRecord, start, + map[string][2]string{"server": {pgImage, ""}}), + aContainerBuild(t, "mailu", "c1111111", "", start.Add(time.Second), + map[string][2]string{"smtp": {mailImage, ""}, "imap": {mailImage, ""}}), + } { + if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil { + t.Fatal(err) + } + } + for _, m := range []string{"postgres", "mailu"} { + if _, err := inv.Assign(ctx, "anchor", m); err != nil { + t.Fatal(err) + } + } + if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", + map[string]string{"postgres": "c1111111", "mailu": "c1111111"}); err != nil { + t.Fatal(err) + } + // The merge: both adopt a health check; the images are the same. + for _, b := range []link.BuildResult{ + aContainerBuild(t, "postgres", "c2222222", catalogue.PolicyRecord, start.Add(time.Minute), + map[string][2]string{"server": {pgImage, "runtime"}}), + aContainerBuild(t, "mailu", "c2222222", "", start.Add(time.Minute+time.Second), + map[string][2]string{"smtp": {mailImage, "runtime"}, "imap": {mailImage, "runtime"}}), + } { + if _, _, err := takeIn(ctx, inv, b); err != nil { + t.Fatal(err) + } + } + + healthOf := func(plan catalogue.Resolution, module, id string) any { + for _, m := range plan.Modules { + for _, r := range m.Resources { + if m.Module == module && r["id"] == id { + return r["health"] + } + } + } + return nil + } + + // A plan's, a release plan's, a healer's send: the database is kept as it runs, mail moves. + kept := keepingRecorded(ctx) + plan, settings, err := planFor(kept, open, "anchor") + if err != nil { + t.Fatal(err) + } + if healthOf(plan, "postgres", "server") != nil { + t.Fatal("a send that is not a person's push composed the recorded module's new build") + } + if healthOf(plan, "mailu", "smtp") == nil { + t.Fatal("the module that rolls out was not composed at its new build") + } + if imageOf(t, plan, "postgres", "server") != pgImage { + t.Fatal("the recorded module's container lost its image") + } + gens, err := generators(kept, open) + if err != nil { + t.Fatal(err) + } + declared, err := declarationWith(kept, open, "anchor", plan, settings, gens, Allocating) + if err != nil { + t.Fatal(err) + } + if declared.Builds["postgres"] != "c1111111" || declared.Builds["mailu"] != "c2222222" { + t.Fatalf("the send records it carries %v; want postgres still at c1111111 and mailu at c2222222", declared.Builds) + } + + // A person's push: the recorded module's new build. + plan, _, err = planFor(ctx, open, "anchor") + if err != nil { + t.Fatal(err) + } + if healthOf(plan, "postgres", "server") == nil { + t.Fatal("a person's push did not compose the recorded module's new build") + } + + // The bus step moves the bus alone: a recorded module it is told it may move moves, the others stay. + plan, _, err = planFor(keepingRecorded(ctx, "postgres"), open, "anchor") + if err != nil { + t.Fatal(err) + } + if healthOf(plan, "postgres", "server") == nil { + t.Fatal("the module a send is let move was kept") + } + + // What a send composes under (sendToEach): every recorded module kept; on the bus step, the bus moves. + if under, err := sendKeeps(ctx, inv); err != nil { + t.Fatal(err) + } else if skip, on := keptExcept(under); !on || len(skip) != 0 { + t.Fatalf("an ordinary send keeps %v %v", on, skip) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "nats", Version: "2", + Provides: []catalogue.Offer{{Name: "mesh-bus"}}}, inventory.Source{Repository: "novox/mesh-catalog", + Seat: "git", Path: "modules/nats", BuiltFrom: "n2", Head: "n2"}); err != nil { + t.Fatal(err) + } + if under, err := sendKeeps(withBusStep(ctx), inv); err != nil { + t.Fatal(err) + } else if skip, on := keptExcept(under); !on || !skip["nats"] || len(skip) != 1 { + t.Fatalf("the bus step keeps %v %v; want everything recorded but the bus", on, skip) + } + + // And a recorded module whose kept build the records no longer hold refuses the send, said. + if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", + map[string]string{"postgres": "c0000000", "mailu": "c2222222"}); err != nil { + t.Fatal(err) + } + if _, _, err := planFor(kept, open, "anchor"); err == nil || !strings.Contains(err.Error(), "push anchor") { + t.Fatalf("a recorded build that cannot be kept did not refuse the send with its remedy: %v", err) + } + + // And the gated send for mail says what it recreates: both containers, no new image, at once. + moves := []inventory.CarriedMove{{Module: "mailu", Node: "anchor", From: "c1111111", To: "c2222222"}} + sayRecreations(ctx, open, moves) + if !strings.Contains(moves[0].Recreates, "recreates 2 of mailu's 2") || !strings.Contains(moves[0].Recreates, "no new image") { + t.Fatalf("the send says %q of mail's containers", moves[0].Recreates) + } + if said := recreationsSaid(moves); !strings.HasPrefix(said, "on anchor recreates") { + t.Fatalf("the plan's note says %q", said) + } +} diff --git a/cmd/mesh-controller/recorded_kept_test.go b/cmd/mesh-controller/recorded_kept_test.go index 07bce8bb..38d626b4 100644 --- a/cmd/mesh-controller/recorded_kept_test.go +++ b/cmd/mesh-controller/recorded_kept_test.go @@ -7,7 +7,6 @@ import ( "time" "github.com/novox/mesh-controller/internal/catalogue" - "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -58,136 +57,6 @@ func imageOf(t *testing.T, plan catalogue.Resolution, module, id string) string // database's new build to the control node and recreated it, unjudged, with nobody's word. A send that // is not a person's push now composes the database at the build the machine runs and records that it // still carries it; a person's push composes the new one; the bus step moves the bus alone. -func TestARecordedBuildIsCarriedOnlyByAPersonsPush(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - start := time.Now().Add(-time.Hour) - pgImage := "registry.invalid:5000/postgres/server@sha256:" + strings.Repeat("a", 64) - mailImage := "registry.invalid:5000/mailu/smtp@sha256:" + strings.Repeat("c", 64) - for _, b := range []link.BuildResult{ - aContainerBuild(t, "postgres", "c1111111", catalogue.PolicyRecord, start, - map[string][2]string{"server": {pgImage, ""}}), - aContainerBuild(t, "mailu", "c1111111", "", start.Add(time.Second), - map[string][2]string{"smtp": {mailImage, ""}, "imap": {mailImage, ""}}), - } { - if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil { - t.Fatal(err) - } - } - for _, m := range []string{"postgres", "mailu"} { - if _, err := inv.Assign(ctx, "anchor", m); err != nil { - t.Fatal(err) - } - } - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", - map[string]string{"postgres": "c1111111", "mailu": "c1111111"}); err != nil { - t.Fatal(err) - } - // The merge: both adopt a health check; the images are the same. - for _, b := range []link.BuildResult{ - aContainerBuild(t, "postgres", "c2222222", catalogue.PolicyRecord, start.Add(time.Minute), - map[string][2]string{"server": {pgImage, "runtime"}}), - aContainerBuild(t, "mailu", "c2222222", "", start.Add(time.Minute+time.Second), - map[string][2]string{"smtp": {mailImage, "runtime"}, "imap": {mailImage, "runtime"}}), - } { - if _, _, err := takeIn(ctx, inv, b); err != nil { - t.Fatal(err) - } - } - - healthOf := func(plan catalogue.Resolution, module, id string) any { - for _, m := range plan.Modules { - for _, r := range m.Resources { - if m.Module == module && r["id"] == id { - return r["health"] - } - } - } - return nil - } - - // A plan's, a release plan's, a healer's send: the database is kept as it runs, mail moves. - kept := keepingRecorded(ctx) - plan, settings, err := planFor(kept, open, "anchor") - if err != nil { - t.Fatal(err) - } - if healthOf(plan, "postgres", "server") != nil { - t.Fatal("a send that is not a person's push composed the recorded module's new build") - } - if healthOf(plan, "mailu", "smtp") == nil { - t.Fatal("the module that rolls out was not composed at its new build") - } - if imageOf(t, plan, "postgres", "server") != pgImage { - t.Fatal("the recorded module's container lost its image") - } - gens, err := generators(kept, open) - if err != nil { - t.Fatal(err) - } - declared, err := declarationWith(kept, open, "anchor", plan, settings, gens, Allocating) - if err != nil { - t.Fatal(err) - } - if declared.Builds["postgres"] != "c1111111" || declared.Builds["mailu"] != "c2222222" { - t.Fatalf("the send records it carries %v; want postgres still at c1111111 and mailu at c2222222", declared.Builds) - } - - // A person's push: the recorded module's new build. - plan, _, err = planFor(ctx, open, "anchor") - if err != nil { - t.Fatal(err) - } - if healthOf(plan, "postgres", "server") == nil { - t.Fatal("a person's push did not compose the recorded module's new build") - } - - // The bus step moves the bus alone: a recorded module it is told it may move moves, the others stay. - plan, _, err = planFor(keepingRecorded(ctx, "postgres"), open, "anchor") - if err != nil { - t.Fatal(err) - } - if healthOf(plan, "postgres", "server") == nil { - t.Fatal("the module a send is let move was kept") - } - - // What a send composes under (sendToEach): every recorded module kept; on the bus step, the bus moves. - if under, err := sendKeeps(ctx, inv); err != nil { - t.Fatal(err) - } else if skip, on := keptExcept(under); !on || len(skip) != 0 { - t.Fatalf("an ordinary send keeps %v %v", on, skip) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "nats", Version: "2", - Provides: []catalogue.Offer{{Name: "mesh-bus"}}}, inventory.Source{Repository: "novox/mesh-catalog", - Seat: "git", Path: "modules/nats", BuiltFrom: "n2", Head: "n2"}); err != nil { - t.Fatal(err) - } - if under, err := sendKeeps(withBusStep(ctx), inv); err != nil { - t.Fatal(err) - } else if skip, on := keptExcept(under); !on || !skip["nats"] || len(skip) != 1 { - t.Fatalf("the bus step keeps %v %v; want everything recorded but the bus", on, skip) - } - - // And a recorded module whose kept build the records no longer hold refuses the send, said. - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", - map[string]string{"postgres": "c0000000", "mailu": "c2222222"}); err != nil { - t.Fatal(err) - } - if _, _, err := planFor(kept, open, "anchor"); err == nil || !strings.Contains(err.Error(), "push anchor") { - t.Fatalf("a recorded build that cannot be kept did not refuse the send with its remedy: %v", err) - } - - // And the gated send for mail says what it recreates: both containers, no new image, at once. - moves := []inventory.CarriedMove{{Module: "mailu", Node: "anchor", From: "c1111111", To: "c2222222"}} - sayRecreations(ctx, open, moves) - if !strings.Contains(moves[0].Recreates, "recreates 2 of mailu's 2") || !strings.Contains(moves[0].Recreates, "no new image") { - t.Fatalf("the send says %q of mail's containers", moves[0].Recreates) - } - if said := recreationsSaid(moves); !strings.HasPrefix(said, "on anchor recreates") { - t.Fatalf("the plan's note says %q", said) - } -} // The send says what it recreates (ADR 0245): mail's health checks adopted recreate both its // containers, with no new image, every one at once. diff --git a/cmd/mesh-controller/recorded_moves_test.go b/cmd/mesh-controller/recorded_moves_test.go index cc11869a..a464c940 100644 --- a/cmd/mesh-controller/recorded_moves_test.go +++ b/cmd/mesh-controller/recorded_moves_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/registry_verbs_integration_test.go b/cmd/mesh-controller/registry_verbs_integration_test.go new file mode 100644 index 00000000..1de9c6be --- /dev/null +++ b/cmd/mesh-controller/registry_verbs_integration_test.go @@ -0,0 +1,75 @@ +//go:build integration + +package main + +import ( + "context" + "fmt" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestADryRunCollectAsksOnlyAndChangesNothing(t *testing.T) { + f := &fakeStore{} + store := f.serve(t) + kept := []string{archiveRef("web", "tools", 1)} + eligible := []string{ref("web", "app", 2), archiveRef("web", "tools", 3)} + a := runCollect(context.Background(), nil, store, eligible, kept, false, + sweepBounds{most: 10, budget: 5 * time.Second}) + if !a.DryRun || a.Eligible != 2 || !slices.Equal(a.WouldLetGo, eligible) || len(a.LetGo) != 0 { + t.Errorf("a dry run answered %+v", a) + } + if a.KeptArchives != 1 || a.Held != 0 || a.Unheld != 1 { + t.Errorf("the kept archive is unheld in the fake store, and was said as %+v", a) + } + for _, r := range f.seen() { + if !strings.HasPrefix(r, "HEAD ") { + t.Errorf("a dry run asked the store %s", r) + } + } +} + +func TestARealCollectHoldsEveryKeptArchiveBeforeItLetsAnythingGo(t *testing.T) { + inv := inventory.ForTest(t) + f := &fakeStore{} + store := f.serve(t) + kept := []string{archiveRef("web", "tools", 1)} + eligible := []string{ref("web", "app", 2), ref("web", "app", 3), ref("web", "app", 4)} + madeBy(t, inv, eligible) + a := runCollect(t.Context(), inv, store, eligible, kept, true, sweepBounds{most: 2, budget: 5 * time.Second}) + if a.DryRun || a.HoldersWritten != 1 || a.Held != 1 { + t.Errorf("the kept archive was not held first: %+v", a) + } + if !slices.Equal(a.LetGo, eligible[:2]) || a.Left != 1 || a.Stopped == "" { + t.Errorf("bounded at two: let go %v, left %d, stopped %q", a.LetGo, a.Left, a.Stopped) + } + requests := f.seen() + firstPut := slices.IndexFunc(requests, func(r string) bool { return strings.HasPrefix(r, "PUT ") && strings.Contains(r, "/manifests/") }) + firstDelete := slices.IndexFunc(requests, func(r string) bool { return strings.HasPrefix(r, "DELETE ") }) + if firstPut < 0 || firstDelete < 0 || firstPut > firstDelete { + t.Errorf("the holder was not put before the first delete: %v", requests) + } + // What was let go is recorded collected: the sweep no longer offers it. + left, err := inv.ToCollect(t.Context()) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(left, eligible[2:]) { + t.Errorf("after letting go of two, the records offer %v", left) + } +} + +func TestARealCollectStopsAtTheStoresFirstRefusal(t *testing.T) { + inv := inventory.ForTest(t) + f := &fakeStore{refuse: fmt.Sprintf("%064x", 3)} + store := f.serve(t) + eligible := []string{ref("web", "app", 2), ref("web", "app", 3), ref("web", "app", 4)} + a := runCollect(t.Context(), inv, store, eligible, nil, true, sweepBounds{most: 10, budget: 5 * time.Second}) + if !slices.Equal(a.LetGo, eligible[:1]) || a.Left != 2 || !strings.Contains(a.Stopped, "kept") { + t.Errorf("after a refusal: let go %v, left %d, stopped %q", a.LetGo, a.Left, a.Stopped) + } +} diff --git a/cmd/mesh-controller/registry_verbs_test.go b/cmd/mesh-controller/registry_verbs_test.go index da61e37c..3c0ea218 100644 --- a/cmd/mesh-controller/registry_verbs_test.go +++ b/cmd/mesh-controller/registry_verbs_test.go @@ -160,26 +160,6 @@ func (f *fakeStore) seen() []string { return slices.Clone(f.requests) } -func TestADryRunCollectAsksOnlyAndChangesNothing(t *testing.T) { - f := &fakeStore{} - store := f.serve(t) - kept := []string{archiveRef("web", "tools", 1)} - eligible := []string{ref("web", "app", 2), archiveRef("web", "tools", 3)} - a := runCollect(context.Background(), nil, store, eligible, kept, false, - sweepBounds{most: 10, budget: 5 * time.Second}) - if !a.DryRun || a.Eligible != 2 || !slices.Equal(a.WouldLetGo, eligible) || len(a.LetGo) != 0 { - t.Errorf("a dry run answered %+v", a) - } - if a.KeptArchives != 1 || a.Held != 0 || a.Unheld != 1 { - t.Errorf("the kept archive is unheld in the fake store, and was said as %+v", a) - } - for _, r := range f.seen() { - if !strings.HasPrefix(r, "HEAD ") { - t.Errorf("a dry run asked the store %s", r) - } - } -} - func TestCollectSaysWhatItDoesNotListAndWithNoStoreAsksNothing(t *testing.T) { var many []string for i := range mostListed + 5 { @@ -192,36 +172,6 @@ func TestCollectSaysWhatItDoesNotListAndWithNoStoreAsksNothing(t *testing.T) { } } -func TestARealCollectHoldsEveryKeptArchiveBeforeItLetsAnythingGo(t *testing.T) { - inv := inventory.ForTest(t) - f := &fakeStore{} - store := f.serve(t) - kept := []string{archiveRef("web", "tools", 1)} - eligible := []string{ref("web", "app", 2), ref("web", "app", 3), ref("web", "app", 4)} - madeBy(t, inv, eligible) - a := runCollect(t.Context(), inv, store, eligible, kept, true, sweepBounds{most: 2, budget: 5 * time.Second}) - if a.DryRun || a.HoldersWritten != 1 || a.Held != 1 { - t.Errorf("the kept archive was not held first: %+v", a) - } - if !slices.Equal(a.LetGo, eligible[:2]) || a.Left != 1 || a.Stopped == "" { - t.Errorf("bounded at two: let go %v, left %d, stopped %q", a.LetGo, a.Left, a.Stopped) - } - requests := f.seen() - firstPut := slices.IndexFunc(requests, func(r string) bool { return strings.HasPrefix(r, "PUT ") && strings.Contains(r, "/manifests/") }) - firstDelete := slices.IndexFunc(requests, func(r string) bool { return strings.HasPrefix(r, "DELETE ") }) - if firstPut < 0 || firstDelete < 0 || firstPut > firstDelete { - t.Errorf("the holder was not put before the first delete: %v", requests) - } - // What was let go is recorded collected: the sweep no longer offers it. - left, err := inv.ToCollect(t.Context()) - if err != nil { - t.Fatal(err) - } - if !slices.Equal(left, eligible[2:]) { - t.Errorf("after letting go of two, the records offer %v", left) - } -} - // madeBy records one successful build, of a module the mesh does not hold, that made these images: // eligible, every one. func madeBy(t *testing.T, inv *inventory.Inventory, references []string) { @@ -236,17 +186,6 @@ func madeBy(t *testing.T, inv *inventory.Inventory, references []string) { } } -func TestARealCollectStopsAtTheStoresFirstRefusal(t *testing.T) { - inv := inventory.ForTest(t) - f := &fakeStore{refuse: fmt.Sprintf("%064x", 3)} - store := f.serve(t) - eligible := []string{ref("web", "app", 2), ref("web", "app", 3), ref("web", "app", 4)} - a := runCollect(t.Context(), inv, store, eligible, nil, true, sweepBounds{most: 10, budget: 5 * time.Second}) - if !slices.Equal(a.LetGo, eligible[:1]) || a.Left != 2 || !strings.Contains(a.Stopped, "kept") { - t.Errorf("after a refusal: let go %v, left %d, stopped %q", a.LetGo, a.Left, a.Stopped) - } -} - func TestImagesAreEveryContainerImageOfTheDeclarationAndOfWhatWasSent(t *testing.T) { body := []byte(`{"declaration":1,"resources":[ {"id":"web.server","type":"container","image":"store.invalid/web/server@sha256:aa"}, diff --git a/cmd/mesh-controller/rehearse_test.go b/cmd/mesh-controller/rehearse_test.go index df738810..5e934395 100644 --- a/cmd/mesh-controller/rehearse_test.go +++ b/cmd/mesh-controller/rehearse_test.go @@ -2,11 +2,12 @@ package main import ( "context" - "github.com/novox/mesh-controller/internal/link" "strings" "testing" "time" + "github.com/novox/mesh-controller/internal/link" + "git.novox.be/novox/mesh-sdk/go/asks" ) diff --git a/cmd/mesh-controller/reissue_verified_sender_integration_test.go b/cmd/mesh-controller/reissue_verified_sender_integration_test.go new file mode 100644 index 00000000..3dcc45d4 --- /dev/null +++ b/cmd/mesh-controller/reissue_verified_sender_integration_test.go @@ -0,0 +1,57 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "testing" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestNatsAMembershipIssuedWithoutItsVerifiedSenderIsIssuedAgainWhenTheJudgementPasses(t *testing.T) { + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + bus := link.OverNATS{Conn: js.Conn(), JS: js.Context()} + + failed, router := verifiedSenderRecords(nil) + body, err := json.Marshal(broker.MembershipFor("novox", router, broker.PlacementsOf(failed, nil))) + if err != nil { + t.Fatal(err) + } + if err := bus.PublishMembership(t.Context(), "novox", "messenger", body); err != nil { + t.Fatal(err) + } + issued := func() []byte { + m, err := js.Context().GetLastMsg(broker.AssignmentsStream, broker.MembershipSubject("novox", "messenger")) + if err != nil { + t.Fatal(err) + } + return m.Data + } + if hasVerified(t, issued()) { + t.Fatal("issued with a failed judgement, and the verified sender is kept: the test proves nothing") + } + if n := reissueVerifiedSendersOn(t.Context(), bus, failed, broker.PlacementsOf(failed, nil)); n != 0 { + t.Fatalf("issued %d memberships while the judgement still fails", n) + } + passing, _ := verifiedSenderRecords(map[string]bool{"novox": true}) + where := broker.PlacementsOf(passing, nil) + if n := reissueVerifiedSendersOn(t.Context(), bus, passing, where); n != 1 { + t.Fatalf("issued %d memberships after the judgement passed, want 1", n) + } + if !hasVerified(t, issued()) { + t.Fatal("the membership on the bus lacks the verified sender after the judgement passed") + } + if n := reissueVerifiedSendersOn(t.Context(), bus, passing, where); n != 0 { + t.Fatalf("a second pass issued %d memberships", n) + } +} diff --git a/cmd/mesh-controller/reissue_verified_sender_test.go b/cmd/mesh-controller/reissue_verified_sender_test.go index 0f00fdaf..57503c88 100644 --- a/cmd/mesh-controller/reissue_verified_sender_test.go +++ b/cmd/mesh-controller/reissue_verified_sender_test.go @@ -9,8 +9,6 @@ import ( "github.com/nats-io/nats.go" "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" ) func verifiedSenderRecords(free map[string]bool) (broker.Records, broker.Declared) { @@ -124,50 +122,6 @@ func TestAMembershipNeverIssuedIsLeftToItsOwnSend(t *testing.T) { // **On a real bus** (novox/hq issue 930): a router's membership issued without the verified sender because the // judgement failed is, once the judgement passes, published again to the AssignmentsStream with it in its seat // traffic's kinds, and a second pass publishes nothing. -func TestNatsAMembershipIssuedWithoutItsVerifiedSenderIsIssuedAgainWhenTheJudgementPasses(t *testing.T) { - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - bus := link.OverNATS{Conn: js.Conn(), JS: js.Context()} - - failed, router := verifiedSenderRecords(nil) - body, err := json.Marshal(broker.MembershipFor("novox", router, broker.PlacementsOf(failed, nil))) - if err != nil { - t.Fatal(err) - } - if err := bus.PublishMembership(t.Context(), "novox", "messenger", body); err != nil { - t.Fatal(err) - } - issued := func() []byte { - m, err := js.Context().GetLastMsg(broker.AssignmentsStream, broker.MembershipSubject("novox", "messenger")) - if err != nil { - t.Fatal(err) - } - return m.Data - } - if hasVerified(t, issued()) { - t.Fatal("issued with a failed judgement, and the verified sender is kept: the test proves nothing") - } - if n := reissueVerifiedSendersOn(t.Context(), bus, failed, broker.PlacementsOf(failed, nil)); n != 0 { - t.Fatalf("issued %d memberships while the judgement still fails", n) - } - passing, _ := verifiedSenderRecords(map[string]bool{"novox": true}) - where := broker.PlacementsOf(passing, nil) - if n := reissueVerifiedSendersOn(t.Context(), bus, passing, where); n != 1 { - t.Fatalf("issued %d memberships after the judgement passed, want 1", n) - } - if !hasVerified(t, issued()) { - t.Fatal("the membership on the bus lacks the verified sender after the judgement passed") - } - if n := reissueVerifiedSendersOn(t.Context(), bus, passing, where); n != 0 { - t.Fatalf("a second pass issued %d memberships", n) - } -} // A push issuing a newer membership between the self-check's read and its write is not overwritten (issue 930). func TestANewerMembershipIssuedMeanwhileIsNotOverwritten(t *testing.T) { diff --git a/cmd/mesh-controller/related_integration_test.go b/cmd/mesh-controller/related_integration_test.go new file mode 100644 index 00000000..b29fb967 --- /dev/null +++ b/cmd/mesh-controller/related_integration_test.go @@ -0,0 +1,52 @@ +//go:build integration + +package main + +import ( + "testing" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestRelatedOpenNamesTheFirstOpenWalkAChangeIsRelatedTo(t *testing.T) { + edges := []inventory.Edge{{From: "nzbget", To: "base-image", Kind: inventory.EdgeStandsOn}} + walk := func(id, repo string, own bool, modules ...string) openWalk { + return openWalk{id: id, fp: footprintOfWalk(walkOf(repo, modules...), nil), own: own} + } + one, two := walk("plan-1", "novox/one", false, "notes"), walk("plan-2", "novox/two", false, "nzbget") + cand := func(repo, head string, moves ...string) footprint { + return footprintOfBatch([]inventory.BatchedMerge{heardMerge(repo, head, moves...)}, edges) + } + cases := []struct { + name string + fp footprint + own bool + open []openWalk + behind string + why string + }{ + {"nothing open", cand("novox/x", "fix/a", "app"), false, nil, "", ""}, + {"unrelated to every open walk", cand("novox/x", "fix/a", "app"), false, []openWalk{one, two}, "", ""}, + {"a repository", cand("novox/ONE", "fix/a", "app"), false, []openWalk{two, one}, "plan-1", "both change novox/one"}, + {"a module through its dependents", cand("novox/x", "fix/a", "base-image"), false, []openWalk{one, two}, + "plan-2", "both move nzbget"}, + {"a graph edge", cand("novox/x", "fix/a", "nzbget"), + false, []openWalk{one, walk("plan-3", "novox/y", false, "base-image")}, "plan-3", + "nzbget depends on base-image (stands-on)"}, + {"moves not known", cand("novox/x", "fix/a"), false, []openWalk{one, two}, "plan-1", + "the modules a merge of novox/x moves are not known until its batch is cut"}, + {"the candidate on the own path", cand("novox/x", "fix/a", "app"), true, []openWalk{one}, "plan-1", + "it is on the controller's own path"}, + {"an open walk on the own path", cand("novox/x", "fix/a", "app"), false, + []openWalk{one, walk("plan-4", "novox/mesh-controller", true, "mesh-controller")}, "plan-4", + "plan-4 is on the controller's own path"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + behind, why := relatedOpen(c.fp, c.own, c.open, edges) + if behind != c.behind || why != c.why { + t.Errorf("relatedOpen = %q, %q; want %q, %q", behind, why, c.behind, c.why) + } + }) + } +} diff --git a/cmd/mesh-controller/related_test.go b/cmd/mesh-controller/related_test.go index 90a062e8..40f80802 100644 --- a/cmd/mesh-controller/related_test.go +++ b/cmd/mesh-controller/related_test.go @@ -116,48 +116,6 @@ func TestABatchWhoseMovesAreNotKnownIsRelatedToEverything(t *testing.T) { // held behind none (issue 973, ADR 0407 decision 4); one that shares a repository, a module, a graph edge or a // group head is held behind the first walk it shares it with, with the reason; one whose moves are not known // is held behind the first; a walk or change on the controller's own path is related to every other. -func TestRelatedOpenNamesTheFirstOpenWalkAChangeIsRelatedTo(t *testing.T) { - edges := []inventory.Edge{{From: "nzbget", To: "base-image", Kind: inventory.EdgeStandsOn}} - walk := func(id, repo string, own bool, modules ...string) openWalk { - return openWalk{id: id, fp: footprintOfWalk(walkOf(repo, modules...), nil), own: own} - } - one, two := walk("plan-1", "novox/one", false, "notes"), walk("plan-2", "novox/two", false, "nzbget") - cand := func(repo, head string, moves ...string) footprint { - return footprintOfBatch([]inventory.BatchedMerge{heardMerge(repo, head, moves...)}, edges) - } - cases := []struct { - name string - fp footprint - own bool - open []openWalk - behind string - why string - }{ - {"nothing open", cand("novox/x", "fix/a", "app"), false, nil, "", ""}, - {"unrelated to every open walk", cand("novox/x", "fix/a", "app"), false, []openWalk{one, two}, "", ""}, - {"a repository", cand("novox/ONE", "fix/a", "app"), false, []openWalk{two, one}, "plan-1", "both change novox/one"}, - {"a module through its dependents", cand("novox/x", "fix/a", "base-image"), false, []openWalk{one, two}, - "plan-2", "both move nzbget"}, - {"a graph edge", cand("novox/x", "fix/a", "nzbget"), - false, []openWalk{one, walk("plan-3", "novox/y", false, "base-image")}, "plan-3", - "nzbget depends on base-image (stands-on)"}, - {"moves not known", cand("novox/x", "fix/a"), false, []openWalk{one, two}, "plan-1", - "the modules a merge of novox/x moves are not known until its batch is cut"}, - {"the candidate on the own path", cand("novox/x", "fix/a", "app"), true, []openWalk{one}, "plan-1", - "it is on the controller's own path"}, - {"an open walk on the own path", cand("novox/x", "fix/a", "app"), false, - []openWalk{one, walk("plan-4", "novox/mesh-controller", true, "mesh-controller")}, "plan-4", - "plan-4 is on the controller's own path"}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - behind, why := relatedOpen(c.fp, c.own, c.open, edges) - if behind != c.behind || why != c.why { - t.Errorf("relatedOpen = %q, %q; want %q, %q", behind, why, c.behind, c.why) - } - }) - } -} // A walk is on the controller's own path when its batch says so or it plans a module of that path (issue 973). func TestAWalkOnTheControllersPathIsKnownByItsFlagOrItsModules(t *testing.T) { diff --git a/cmd/mesh-controller/release_provisioned_integration_test.go b/cmd/mesh-controller/release_provisioned_integration_test.go new file mode 100644 index 00000000..547e0b73 --- /dev/null +++ b/cmd/mesh-controller/release_provisioned_integration_test.go @@ -0,0 +1,145 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func stepRelease(t *testing.T, b *backlogMesh, n int) inventory.Plan { + t.Helper() + for i := 0; i < n; i++ { + advancePlans(t.Context(), b.open) + } + return b.release(t) +} + +func TestTheRestGateWaitsForTheProvisioner(t *testing.T) { + judgeLeavesAsTheCore(t) + applied := inventory.ProvReport{Status: inventory.ProvApplied, Templates: "t1", At: time.Now()} + + t.Run("a engine applied, the client has not said: no verdict, the release stays open", func(t *testing.T) { + b := aBacklog(t) + handingAnchor(t, b, map[string]string{"app": "t1"}) + p := stepRelease(t, b, 8) + g := p.Release.Gates["anchor"] + if !p.Open() || g == nil || g.Verdict != "" || g.Silent || slices.Contains(p.Release.Done, "anchor") { + t.Fatalf("anchor is judged before its provisioner reported: %s %s %+v", p.State, p.Note, g) + } + if !slices.Contains(p.Release.Done, "laptop") { + t.Fatalf("laptop hands nothing and passes on the engine's report: %+v", p.Release) + } + }) + t.Run("b the client failed: a failed apply named with the reason", func(t *testing.T) { + b := aBacklog(t) + says := handingAnchor(t, b, map[string]string{"app": "t1"}) + says["anchor/app"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t1", Reason: "no value", At: time.Now()} + p := stepRelease(t, b, 8) + g := p.Release.Gates["anchor"] + if g == nil || g.Verdict != inventory.GateFailed || !slices.Contains(p.Release.Failed, "anchor") || + !strings.Contains(p.Note, "provisioning failed (app: no value)") { + t.Fatalf("the failure is not named: %s %q %+v %+v", p.State, p.Note, g, p.Release) + } + }) + t.Run("c no client report past ApplySilentAfter: silent, never done", func(t *testing.T) { + b := aBacklog(t) + handingAnchor(t, b, map[string]string{"app": "t1"}) + p := stepRelease(t, b, 8) + g := p.Release.Gates["anchor"] + if g == nil { + t.Fatalf("no gate: %+v", p.Release) + } + since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) + g.Since = &since + if err := b.open.inventory.SavePlan(t.Context(), &p); err != nil { + t.Fatal(err) + } + p = stepRelease(t, b, 1) + if !slices.Equal(p.Release.Silent, []string{"anchor"}) || slices.Contains(p.Release.Done, "anchor") { + t.Fatalf("anchor is not silent: %s %s %+v", p.State, p.Note, p.Release) + } + }) + t.Run("d both applied: the gate passes", func(t *testing.T) { + b := aBacklog(t) + says := handingAnchor(t, b, map[string]string{"app": "t1"}) + says["anchor/app"] = applied + p := stepRelease(t, b, 8) + if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") || len(p.Release.Failed) > 0 { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + }) + t.Run("e a send handing nothing passes on the engine's report", func(t *testing.T) { + b := aBacklog(t) + p := stepRelease(t, b, 8) + if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + }) + t.Run("f a module the delivery does not carry does not touch its end", func(t *testing.T) { + // anchor's declaration also hands `same`, rebuilt with the very bytes and so not carried: its failed report + // is none of this delivery's. + b := aBacklog(t) + says := handingAnchor(t, b, map[string]string{"app": "t1", "same": "t9"}) + says["anchor/app"] = applied + says["anchor/same"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t9", Reason: "stuck", At: time.Now()} + p := stepRelease(t, b, 8) + if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") || len(p.Release.Failed) > 0 { + t.Fatalf("another module's failure ended this delivery: %s %s %+v", p.State, p.Note, p.Release) + } + }) +} + +// handingAnchor makes every send to anchor hand the provisioner the given modules (module -> templates digest), and +// the clients' words the given reports (key `node/module`), as the stream would give them (issue 840, ADR 0362). +func handingAnchor(t *testing.T, b *backlogMesh, handed map[string]string) map[string]inventory.ProvReport { + t.Helper() + says := map[string]inventory.ProvReport{} + wasSays := provAppliedOn + provAppliedOn = func(_ context.Context, node, module string) (inventory.ProvReport, bool, error) { + r, ok := says[node+"/"+module] + return r, ok, nil + } + inner := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + got, err := inner(ctx, open, names) + if err != nil { + return got, err + } + for _, n := range names { + if n != "anchor" { + continue + } + inv := open.inventory + s, _, err := inv.SentTo(ctx, n) + if err != nil { + return nil, err + } + builds, _, err := inv.SentBuilds(ctx, n) + if err != nil { + return nil, err + } + plans := map[string]inventory.Provisioning{} + for m, d := range handed { + plans[m] = inventory.Provisioning{Templates: inventory.TemplatesArchive{Digest: d}} + } + raw, err := json.Marshal(plans) + if err != nil { + return nil, err + } + id := nodeID(t, open, n) + if err := inv.RecordSentWith(ctx, id, s.Digest, builds, 0, 0, inventory.Send{Node: id, Provisioning: raw}); err != nil { + return nil, err + } + } + return got, nil + } + t.Cleanup(func() { sendRollout, provAppliedOn = inner, wasSays }) + return says +} diff --git a/cmd/mesh-controller/release_provisioned_test.go b/cmd/mesh-controller/release_provisioned_test.go index 52241dcf..da197dfa 100644 --- a/cmd/mesh-controller/release_provisioned_test.go +++ b/cmd/mesh-controller/release_provisioned_test.go @@ -1,9 +1,6 @@ package main import ( - "context" - "encoding/json" - "slices" "strings" "testing" "time" @@ -11,139 +8,9 @@ import ( "github.com/novox/mesh-controller/internal/inventory" ) -// handingAnchor makes every send to anchor hand the provisioner the given modules (module -> templates digest), and -// the clients' words the given reports (key `node/module`), as the stream would give them (issue 840, ADR 0362). -func handingAnchor(t *testing.T, b *backlogMesh, handed map[string]string) map[string]inventory.ProvReport { - t.Helper() - says := map[string]inventory.ProvReport{} - wasSays := provAppliedOn - provAppliedOn = func(_ context.Context, node, module string) (inventory.ProvReport, bool, error) { - r, ok := says[node+"/"+module] - return r, ok, nil - } - inner := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - got, err := inner(ctx, open, names) - if err != nil { - return got, err - } - for _, n := range names { - if n != "anchor" { - continue - } - inv := open.inventory - s, _, err := inv.SentTo(ctx, n) - if err != nil { - return nil, err - } - builds, _, err := inv.SentBuilds(ctx, n) - if err != nil { - return nil, err - } - plans := map[string]inventory.Provisioning{} - for m, d := range handed { - plans[m] = inventory.Provisioning{Templates: inventory.TemplatesArchive{Digest: d}} - } - raw, err := json.Marshal(plans) - if err != nil { - return nil, err - } - id := nodeID(t, open, n) - if err := inv.RecordSentWith(ctx, id, s.Digest, builds, 0, 0, inventory.Send{Node: id, Provisioning: raw}); err != nil { - return nil, err - } - } - return got, nil - } - t.Cleanup(func() { sendRollout, provAppliedOn = inner, wasSays }) - return says -} - -func stepRelease(t *testing.T, b *backlogMesh, n int) inventory.Plan { - t.Helper() - for i := 0; i < n; i++ { - advancePlans(t.Context(), b.open) - } - return b.release(t) -} - // Issue 840 (novox/hq ADR 0362 §1-5): the release's rest gate, which every delivery passes since ADR 0360, counts a // machine's send of a provisioned module applied only once the engine and the machine's provisioner client both said // so; a failed client is a failed apply named with its reason; none past ApplySilentAfter is silent, never done. -func TestTheRestGateWaitsForTheProvisioner(t *testing.T) { - judgeLeavesAsTheCore(t) - applied := inventory.ProvReport{Status: inventory.ProvApplied, Templates: "t1", At: time.Now()} - - t.Run("a engine applied, the client has not said: no verdict, the release stays open", func(t *testing.T) { - b := aBacklog(t) - handingAnchor(t, b, map[string]string{"app": "t1"}) - p := stepRelease(t, b, 8) - g := p.Release.Gates["anchor"] - if !p.Open() || g == nil || g.Verdict != "" || g.Silent || slices.Contains(p.Release.Done, "anchor") { - t.Fatalf("anchor is judged before its provisioner reported: %s %s %+v", p.State, p.Note, g) - } - if !slices.Contains(p.Release.Done, "laptop") { - t.Fatalf("laptop hands nothing and passes on the engine's report: %+v", p.Release) - } - }) - t.Run("b the client failed: a failed apply named with the reason", func(t *testing.T) { - b := aBacklog(t) - says := handingAnchor(t, b, map[string]string{"app": "t1"}) - says["anchor/app"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t1", Reason: "no value", At: time.Now()} - p := stepRelease(t, b, 8) - g := p.Release.Gates["anchor"] - if g == nil || g.Verdict != inventory.GateFailed || !slices.Contains(p.Release.Failed, "anchor") || - !strings.Contains(p.Note, "provisioning failed (app: no value)") { - t.Fatalf("the failure is not named: %s %q %+v %+v", p.State, p.Note, g, p.Release) - } - }) - t.Run("c no client report past ApplySilentAfter: silent, never done", func(t *testing.T) { - b := aBacklog(t) - handingAnchor(t, b, map[string]string{"app": "t1"}) - p := stepRelease(t, b, 8) - g := p.Release.Gates["anchor"] - if g == nil { - t.Fatalf("no gate: %+v", p.Release) - } - since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) - g.Since = &since - if err := b.open.inventory.SavePlan(t.Context(), &p); err != nil { - t.Fatal(err) - } - p = stepRelease(t, b, 1) - if !slices.Equal(p.Release.Silent, []string{"anchor"}) || slices.Contains(p.Release.Done, "anchor") { - t.Fatalf("anchor is not silent: %s %s %+v", p.State, p.Note, p.Release) - } - }) - t.Run("d both applied: the gate passes", func(t *testing.T) { - b := aBacklog(t) - says := handingAnchor(t, b, map[string]string{"app": "t1"}) - says["anchor/app"] = applied - p := stepRelease(t, b, 8) - if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") || len(p.Release.Failed) > 0 { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - }) - t.Run("e a send handing nothing passes on the engine's report", func(t *testing.T) { - b := aBacklog(t) - p := stepRelease(t, b, 8) - if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - }) - t.Run("f a module the delivery does not carry does not touch its end", func(t *testing.T) { - // anchor's declaration also hands `same`, rebuilt with the very bytes and so not carried: its failed report - // is none of this delivery's. - b := aBacklog(t) - says := handingAnchor(t, b, map[string]string{"app": "t1", "same": "t9"}) - says["anchor/app"] = applied - says["anchor/same"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t9", Reason: "stuck", At: time.Now()} - p := stepRelease(t, b, 8) - if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") || len(p.Release.Failed) > 0 { - t.Fatalf("another module's failure ended this delivery: %s %s %+v", p.State, p.Note, p.Release) - } - }) -} // Issue 840: a walk's end waits on the modules it sent to a machine, not on every module that machine's declaration // hands to the provisioner. diff --git a/cmd/mesh-controller/release_test.go b/cmd/mesh-controller/release_test.go index a2fd2589..b80429be 100644 --- a/cmd/mesh-controller/release_test.go +++ b/cmd/mesh-controller/release_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay164_test.go b/cmd/mesh-controller/replay164_test.go index 34175bda..78f0e6a8 100644 --- a/cmd/mesh-controller/replay164_test.go +++ b/cmd/mesh-controller/replay164_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay201_test.go b/cmd/mesh-controller/replay201_test.go index c852ef20..f7559eee 100644 --- a/cmd/mesh-controller/replay201_test.go +++ b/cmd/mesh-controller/replay201_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay237_test.go b/cmd/mesh-controller/replay237_test.go index 85f7c071..beafe80a 100644 --- a/cmd/mesh-controller/replay237_test.go +++ b/cmd/mesh-controller/replay237_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay239_test.go b/cmd/mesh-controller/replay239_test.go index 46282a41..ed9b9ffa 100644 --- a/cmd/mesh-controller/replay239_test.go +++ b/cmd/mesh-controller/replay239_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay318_test.go b/cmd/mesh-controller/replay318_test.go index df8a33b6..a84699b8 100644 --- a/cmd/mesh-controller/replay318_test.go +++ b/cmd/mesh-controller/replay318_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay325_test.go b/cmd/mesh-controller/replay325_test.go index d3f0c6f1..c058c2d1 100644 --- a/cmd/mesh-controller/replay325_test.go +++ b/cmd/mesh-controller/replay325_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay327_test.go b/cmd/mesh-controller/replay327_test.go index bac8296d..2410455b 100644 --- a/cmd/mesh-controller/replay327_test.go +++ b/cmd/mesh-controller/replay327_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay335_integration_test.go b/cmd/mesh-controller/replay335_integration_test.go new file mode 100644 index 00000000..98d142a2 --- /dev/null +++ b/cmd/mesh-controller/replay335_integration_test.go @@ -0,0 +1,68 @@ +//go:build integration + +package main + +import ( + "reflect" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + judgeLeavesAsTheCore(t) + tm := aTierMesh(t, "m01", "m02") + ctx := t.Context() + inv := tm.open.inventory + + advancePlans(ctx, tm.open) + if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) { + t.Fatalf("sent %v: the first machine once, for the tier", tm.sent) + } + p := tm.plan(t) + lead, carried := p.Modules["m01"], p.Modules["m02"] + if lead.Gate == nil || carried.GatedBy != "m01" { + t.Fatalf("m02 is not carried by m01's send: lead %+v, carried %+v", lead.Gate, carried) + } + // The lead passed; the carried module's turn did not come. The first send is past the wait's bound. + sent := time.Now().UTC().Add(-planWaitBound - time.Minute) + judged := sent.Add(2 * time.Minute) + lead.FirstAt, carried.FirstAt, lead.Gate.Since = &sent, &sent, &sent + lead.Gate.Verdict, lead.Gate.Why, lead.Gate.JudgedAt, lead.Gate.Kept = inventory.GatePassed, + "healthy 3 times over 2m5s", &judged, true + carried.Gate = nil + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + // Another walk's send reached anchor, which has not reported on it. + if err := inv.RecordSent(ctx, nodeID(t, tm.open, "anchor"), "d-anchor-elsewhere", + map[string]string{"m01": "c2", "m02": "c2"}); err != nil { + t.Fatal(err) + } + + advancePlans(ctx, tm.open) + p = tm.plan(t) + if p.State == inventory.PlanFailed { + t.Fatalf("the plan failed after its gate passed: %s", p.Note) + } + if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) { + t.Fatalf("sent %v: the rest once, as the pass said", tm.sent) + } + current, err := inv.CurrentBuilds(ctx) + if err != nil { + t.Fatal(err) + } + for _, m := range []string{"m01", "m02"} { + if current[m].Commit != "c2" { + t.Errorf("%s was put back to %s after its gate passed", m, current[m].Commit) + } + if failed, _ := inv.GateFailed(ctx, "build-"+m+"-2"); failed { + t.Errorf("%s's build was marked failed at its gate after the gate passed", m) + } + } + if g := p.Modules["m02"].Gate; g == nil || g.Verdict != inventory.GatePassed { + t.Errorf("m02 did not take over m01's pass: %+v", g) + } +} diff --git a/cmd/mesh-controller/replay335_test.go b/cmd/mesh-controller/replay335_test.go index 27031906..5cd51649 100644 --- a/cmd/mesh-controller/replay335_test.go +++ b/cmd/mesh-controller/replay335_test.go @@ -77,59 +77,3 @@ func testAWalkStoppedAfterItsGatePassedKeepsThePass(t *testing.T) { // no gate of its own yet. Since then another send reached the first machine and it has not reported on it, and // the first send is older than the wait for a first machine's report. Read before the pass is taken over, the // carried module's step said the first machine never applied it, and the passed build was put back. -func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - tm := aTierMesh(t, "m01", "m02") - ctx := t.Context() - inv := tm.open.inventory - - advancePlans(ctx, tm.open) - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: the first machine once, for the tier", tm.sent) - } - p := tm.plan(t) - lead, carried := p.Modules["m01"], p.Modules["m02"] - if lead.Gate == nil || carried.GatedBy != "m01" { - t.Fatalf("m02 is not carried by m01's send: lead %+v, carried %+v", lead.Gate, carried) - } - // The lead passed; the carried module's turn did not come. The first send is past the wait's bound. - sent := time.Now().UTC().Add(-planWaitBound - time.Minute) - judged := sent.Add(2 * time.Minute) - lead.FirstAt, carried.FirstAt, lead.Gate.Since = &sent, &sent, &sent - lead.Gate.Verdict, lead.Gate.Why, lead.Gate.JudgedAt, lead.Gate.Kept = inventory.GatePassed, - "healthy 3 times over 2m5s", &judged, true - carried.Gate = nil - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - // Another walk's send reached anchor, which has not reported on it. - if err := inv.RecordSent(ctx, nodeID(t, tm.open, "anchor"), "d-anchor-elsewhere", - map[string]string{"m01": "c2", "m02": "c2"}); err != nil { - t.Fatal(err) - } - - advancePlans(ctx, tm.open) - p = tm.plan(t) - if p.State == inventory.PlanFailed { - t.Fatalf("the plan failed after its gate passed: %s", p.Note) - } - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) { - t.Fatalf("sent %v: the rest once, as the pass said", tm.sent) - } - current, err := inv.CurrentBuilds(ctx) - if err != nil { - t.Fatal(err) - } - for _, m := range []string{"m01", "m02"} { - if current[m].Commit != "c2" { - t.Errorf("%s was put back to %s after its gate passed", m, current[m].Commit) - } - if failed, _ := inv.GateFailed(ctx, "build-"+m+"-2"); failed { - t.Errorf("%s's build was marked failed at its gate after the gate passed", m) - } - } - if g := p.Modules["m02"].Gate; g == nil || g.Verdict != inventory.GatePassed { - t.Errorf("m02 did not take over m01's pass: %+v", g) - } -} diff --git a/cmd/mesh-controller/replay336_test.go b/cmd/mesh-controller/replay336_test.go index 20fa6dec..7e6b70f3 100644 --- a/cmd/mesh-controller/replay336_test.go +++ b/cmd/mesh-controller/replay336_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay340_test.go b/cmd/mesh-controller/replay340_test.go index 225c0c20..a1b2df52 100644 --- a/cmd/mesh-controller/replay340_test.go +++ b/cmd/mesh-controller/replay340_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay348_test.go b/cmd/mesh-controller/replay348_test.go index 97fcfdb4..30f23918 100644 --- a/cmd/mesh-controller/replay348_test.go +++ b/cmd/mesh-controller/replay348_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay349_test.go b/cmd/mesh-controller/replay349_test.go index fae683a2..ae71bb92 100644 --- a/cmd/mesh-controller/replay349_test.go +++ b/cmd/mesh-controller/replay349_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay401_test.go b/cmd/mesh-controller/replay401_test.go index 0c4fd94b..dfe1135f 100644 --- a/cmd/mesh-controller/replay401_test.go +++ b/cmd/mesh-controller/replay401_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay614_test.go b/cmd/mesh-controller/replay614_test.go index bd256104..411330c8 100644 --- a/cmd/mesh-controller/replay614_test.go +++ b/cmd/mesh-controller/replay614_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay867_test.go b/cmd/mesh-controller/replay867_test.go index 4aa03f3d..49a5fc0f 100644 --- a/cmd/mesh-controller/replay867_test.go +++ b/cmd/mesh-controller/replay867_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replays_integration_test.go b/cmd/mesh-controller/replays_integration_test.go new file mode 100644 index 00000000..d0127410 --- /dev/null +++ b/cmd/mesh-controller/replays_integration_test.go @@ -0,0 +1,501 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "os" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestReplay263AnIdentityTooLongNeverRefusesItsProvidersMachine(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "resolver", Version: "1", + Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}}) + register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1", + Requires: []string{"wildcard-resolution"}}) + for _, a := range [][2]string{{"anchor", "resolver"}, {"laptop", "networkmanager"}} { + if _, err := assign(ctx, open, a[0], a[1]); err != nil { + t.Fatalf("assign %s %s: %v", a[0], a[1], err) + } + } + // The consumer's machine composes first, as a push does: what it is sent is what its provider grants. + for _, node := range []string{"laptop", "anchor"} { + plan, settings, err := planFor(ctx, open, node) + if err != nil { + t.Fatalf("%s does not resolve: %v", node, err) + } + if _, err := declarationFor(ctx, open, node, plan, settings); err != nil { + t.Fatalf("%s cannot be sent anything — the consumer networkmanager on laptop, identified "+ + "mesh_laptop_networkmanager (26 characters), refused it: %v", node, err) + } + } +} + +func TestReplay273AConsumerBesideItsStoreStaysBoundToIt(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { + t.Fatal(err) + } + for _, m := range []catalogue.Manifest{ + {Module: "store", Version: "1", + Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}, + Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}}, + Serves: map[string]map[string]any{"postgres-database": {"port": 5432}}, + Grants: map[string]string{"postgres-database": "/var/lib/mesh/store/grants"}}, + {Module: "resolver", Version: "1", + Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}, + Claims: []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}}, + {Module: "network", Version: "1", Requires: []string{"wildcard-resolution"}}, + {Module: "board", Version: "1", Requires: []string{"postgres-database"}}, + } { + register(t, open, m) + } + assignAll := func(pairs ...[2]string) { + for _, a := range pairs { + if _, err := assign(ctx, open, a[0], a[1]); err != nil { + t.Fatalf("assign %s %s: %v", a[0], a[1], err) + } + } + } + // The control node holds the mesh's store and resolver seats; the home server runs its own of each. + assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"}) + for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} { + if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil { + t.Fatal(err) + } + } + assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"}, + [2]string{"laptop", "board"}) + + plan, _, err := planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + var board, network string + for _, n := range plan.Needs { + switch { + case n.For == "board" && n.Name == "postgres-database": + board = n.From + case n.For == "network" && n.Name == "wildcard-resolution": + network = n.From + } + } + if board != "laptop" { + t.Fatalf("the consumer beside its store was bound to the store on %q, which would make it a new, empty "+ + "database there (issue 273)", board) + } + if network != "anchor" { + t.Fatalf("the resolver was bound to %q; its seat is held on anchor (issue 258)", network) + } +} + +func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) { + withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + + crashLoop := []byte(`{"contract":1,"at":"2026-10-07T00:00:00Z","resources":[{"module":"app","resource":"app.server",` + + `"kind":"container","target":"app-server","state":"unhealthy","reason":"restarting","since":"2026-10-07T00:00:00Z",` + + `"streak":5,"restarts":2}]}`) + if path := os.Getenv("MESH_REPLAY_STATEMENT"); path != "" { + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("the engine's statement of the crash loop: %v", err) + } + crashLoop = raw + } + // `null` is an engine that states nothing — older than the judging — and its reports carry no health. + var stated map[string]any + if err := json.Unmarshal(crashLoop, &stated); err != nil { + t.Fatal(err) + } + + for _, b := range []inventory.Build{ + {ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app", + Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)}, + {ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app", + Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)}, + } { + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit}) + b.Manifest = manifest + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + } + registerAt := func(commit string, asked time.Time) { + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit, + Head: commit, Asked: asked}); err != nil { + t.Fatal(err) + } + } + registerAt("c1", time.Now().Add(-2*time.Hour)) + for _, n := range []string{"anchor", "laptop"} { + if _, err := inv.Assign(ctx, n, "app"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil { + t.Fatal(err) + } + } + registerAt("c2", time.Now().Add(-time.Minute)) + + // The machine: what it is sent is applied, and its report says what runs is starting. + listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}} + sequence := int64(0) + say := func(node, digest, state string) { + t.Helper() + sequence++ + health := map[string]any{} + for k, v := range stated { + health[k] = v + } + health["at"] = time.Now().UTC().Format(time.RFC3339Nano) + if state != "" && stated != nil { + resources := []any{} + for _, r := range stated["resources"].([]any) { + kept := map[string]any{} + for k, v := range r.(map[string]any) { + kept[k] = v + } + kept["state"], kept["reason"] = state, "" + resources = append(resources, kept) + } + health["resources"] = resources + } + said := map[string]any{"node": node, "applied": []string{"app.server"}, "declared": digest, + "report_sequence": sequence} + if stated != nil { + said["health"] = health + } + body, _ := json.Marshal(said) + var report link.Report + if err := json.Unmarshal(body, &report); err != nil { + t.Fatal(err) + } + if _, err := listener.Heard(ctx, report); err != nil { + t.Fatal(err) + } + } + var sent [][]string + last := map[string]string{} + n := 0 + wasSend := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + sent = append(sent, append([]string(nil), names...)) + current, err := open.inventory.CurrentBuilds(ctx) + if err != nil { + return nil, err + } + for _, node := range names { + n++ + digest := fmt.Sprintf("d-%s-%d", node, n) + if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil { + return nil, err + } + last[node] = digest + say(node, digest, "starting") + } + return names, nil + } + t.Cleanup(func() { sendRollout = wasSend }) + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) + gateWithin, gateEvery = time.Hour, 0 + t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) + + built := time.Now().UTC() + plan := inventory.Plan{ID: "plan-crashloop", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", + Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, + Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-2"}}} + if err := inv.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + + advancePlans(ctx, open) // the first machine is sent the new build, and starts it + if len(sent) == 0 || !slices.Equal(sent[0], []string{"anchor"}) { + t.Fatalf("sent %v, not the first machine first", sent) + } + advancePlans(ctx, open) // judged while it starts + // Its container exits at start, and the runtime restarts it: the machine says so, again and again. + for i := 0; i < 6 && len(sent) == 1; i++ { + say("anchor", last["anchor"], "") + advancePlans(ctx, open) + } + gateBound = -time.Second // and the bound passes + advancePlans(ctx, open) + + p, err := inv.PlanByID(ctx, "plan-crashloop") + if err != nil { + t.Fatal(err) + } + gate := p.Modules["app"].Gate + for _, names := range sent { + if slices.Contains(names, "laptop") { + t.Fatalf("the crash loop passed its gate on anchor and was sent to laptop: sent %v, the gate %+v", sent, gate) + } + } + if gate == nil || gate.Verdict != inventory.GateFailed || p.State != inventory.PlanFailed { + t.Fatalf("a crash-looping build did not fail its gate on the first machine: the plan is %s (%s), its gate %+v", + p.State, p.Note, gate) + } + if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" { + t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit) + } +} + +func TestReplaySilentWebAppIsRaisedWithinTwoLooks(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + silent := []byte(`{"contract":2,"at":"2026-10-07T00:00:00Z","resources":[{"module":"app","resource":"app.server",` + + `"kind":"container","target":"app-server","state":"unhealthy","reason":"http / on web: no answer within 5s",` + + `"since":"2026-10-07T00:00:00Z","streak":3,"check":"http"}]}`) + if path := os.Getenv("MESH_REPLAY_STATEMENT"); path != "" { + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("the engine's statement of the silent web application: %v", err) + } + silent = raw + } + var h link.Health + if err := json.Unmarshal(silent, &h); err != nil { + t.Fatal(err) + } + if h.Contract == 0 { + t.Fatal("the engine states nothing of the silent web application: it is older than the judging") + } + open1 := func() []conditions.Condition { + t.Helper() + list, err := conditionsFrom.Open(ctx) + if err != nil { + t.Fatal(err) + } + return list + } + for look := 1; look <= 2; look++ { + at := time.Now().Add(time.Duration(look) * time.Second) + h.At = at + if err := stateHealth(ctx, open.inventory, conditionsFrom, "anchor", h, at); err != nil { + t.Fatal(err) + } + raised := open1() + switch { + case look == 1 && len(raised) != 0: + t.Fatalf("one look raised %v", raised[0].Key) + case look == 2 && (len(raised) != 1 || raised[0].Key != "module.app.anchor.unhealthy"): + t.Fatalf("two looks in a row did not raise the module's condition: %v", raised) + case look == 2: + t.Logf("raised on the second look: %s", raised[0].Summary) + } + } +} + +func TestReplay296APlanStandingStillInItsTierIsNotSavedAgain(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + asked := asksRecorded(t) + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main", + BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "3ad22356aa", + Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true} + if err := (following{open: open}).SourceMoved(ctx, m); err != nil { + t.Fatal(err) + } + advanceHeld(ctx, open) + recent, err := inv.RecentPlans(ctx, 1) + if err != nil || len(recent) != 1 || len(*asked) != 1 { + t.Fatalf("no plan waiting on its build: %v %v, asked %v", recent, err, *asked) + } + before := recent[0] + if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" { + t.Fatalf("the plan is not waiting on its build: %+v", before) + } + for range 3 { + advanceHeld(ctx, open) + } + after, err := inv.PlanByID(ctx, before.ID) + if err != nil { + t.Fatal(err) + } + if after.Revision != before.Revision || !after.Updated.Equal(before.Updated) { + t.Fatalf("a plan that did not move was saved again: revision %d → %d, saved %s → %s", + before.Revision, after.Revision, before.Updated, after.Updated) + } +} + +func TestReplay299AHolderOfASeatWhoseVerbsAreAllOptionalIsNotSilent(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + catchingUp := catalogue.Seat{Name: "node-catching-up", Scope: catalogue.ScopeNode, Decision: "novox/hq issue 299", + Serves: []catalogue.Verb{{Name: "first", Optional: true}, {Name: "second", Optional: true}}} + if _, err := inv.SeedSeats(ctx, append(catalogue.DefaultSeats(), catchingUp)); err != nil { + t.Fatal(err) + } + rows, err := inv.Seats(ctx) + if err != nil { + t.Fatal(err) + } + // The working set as the serving controller holds it: the store's rows, the compiled seats' marks. + // A seat the compiled set does not know keeps its mark only as defined here. + for i := range rows { + if rows[i].Name == catchingUp.Name { + rows[i].Serves = catchingUp.Serves + } + } + catalogue.UseSeats(rows) + t.Cleanup(func() { catalogue.UseSeats(catalogue.DefaultSeats()) }) + + seats := []string{catchingUp.Name} + if uplink, ok := catalogue.SeatNamed("node-uplink"); ok && len(uplink.Serves) > 0 { + allOptional := true + for _, v := range uplink.Serves { + allOptional = allOptional && v.Optional + } + if allOptional { + seats = append(seats, "node-uplink") + } else { + t.Logf("node-uplink's verbs are required at this commit; the live seat is not replayed, its kind is") + } + } + + register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1"}) + now := time.Now() + var heard []machineFacts + for _, node := range []string{"anchor", "laptop"} { + if _, err := inv.Assign(ctx, node, "networkmanager"); err != nil { + t.Fatal(err) + } + for _, seat := range seats { + if err := inv.HoldSeat(ctx, seat, catalogue.ScopeNode, node, "networkmanager"); err != nil { + t.Fatal(err) + } + } + heard = append(heard, machineFacts{name: node, lastHeard: now}) + } + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + d := &doctor{open: open, js: js, watchdogs: &watchdogs{last: &signalFacts{now: now, machines: heard}}} + + found, err := probeHolders(ctx, d) + if err != nil { + t.Fatal(err) + } + for _, o := range found { + for _, seat := range seats { + if strings.HasPrefix(o.ID, seat+".") { + t.Errorf("a holder serving none of a seat's optional verbs was said silent: %s (%s)", o.Summary, o.Said) + } + } + } +} + +func TestReplay300AMergeAddingAModuleAsksForItsBuild(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + var asked [][3]string + was := askABuild + askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) { + asked = append(asked, [3]string{source.Repository, path, ref}) + return "build-" + path, nil + } + t.Cleanup(func() { askABuild = was }) + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "networkmanager", Version: "1"}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main", + BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "3da80a4b00", + Paths: []string{"modules/systemd-resolved/module.json", "modules/systemd-resolved/cmd/main.go"}, + ModuleDirs: []string{"modules/systemd-resolved"}, ModuleDirsSaid: true} + if err := (following{open: open}).SourceMoved(ctx, m); err != nil { + t.Fatal(err) + } + want := [3]string{"novox/mesh-catalog", "modules/systemd-resolved", "main"} + if len(asked) != 1 || asked[0] != want { + t.Fatalf("the merge adding modules/systemd-resolved asked the build seat for %v, not %v", asked, want) + } + if plans, err := inv.OpenPlans(ctx); err != nil || len(plans) != 0 { + t.Fatalf("a merge moving nothing the mesh holds opened a plan: %+v %v", plans, err) + } +} + +func TestReplay301APersonsPushOfAHeldRecordedBuildIsNoRepair(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + start := time.Now().Add(-time.Hour) + image := "registry.invalid:5000/resolver/server@sha256:" + strings.Repeat("e", 64) + if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, aContainerBuild(t, "resolver", "c1111111", catalogue.PolicyRecord, start, + map[string][2]string{"server": {image, ""}}))); err != nil { + t.Fatal(err) + } + for _, node := range []string{"anchor", "laptop"} { + if _, err := inv.Assign(ctx, node, "resolver"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, open, node), "d-"+node, map[string]string{"resolver": "c1111111"}); err != nil { + t.Fatal(err) + } + } + // The merge: the resolver's new build, registered and held for a person's word. + if _, _, err := takeIn(ctx, inv, aContainerBuild(t, "resolver", "c2222222", catalogue.PolicyRecord, + start.Add(time.Minute), map[string][2]string{"server": {image, "runtime"}})); err != nil { + t.Fatal(err) + } + + conn := onATestBus(t) + // The operator's yes, machine by machine, as the seat's push records it before it sends. + for _, node := range []string{"anchor", "laptop"} { + set := flag.NewFlagSet("push", flag.ContinueOnError) + why := addHandActFlags(set) + if err := set.Parse([]string{"--why", "operator's yes: the resolver's new build on " + node, + "--cause", "split-dns"}); err != nil { + t.Fatal(err) + } + why.record(ctx, "push", []string{node}) + } + acts, err := link.HandActs(ctx, conn, start) + if err != nil { + t.Fatal(err) + } + if len(acts) != 2 { + t.Fatalf("the two pushes were recorded as %d act(s): %+v", len(acts), acts) + } + f := calm(time.Now()) + f.handActs = acts + for _, o := range watchHandActs(f) { + if strings.Contains(o.ID, "split-dns") { + t.Fatalf("a person's push of a held recorded build was counted as a repair: %s — %s", o.ID, o.Summary) + } + } +} diff --git a/cmd/mesh-controller/replays_test.go b/cmd/mesh-controller/replays_test.go index f8a95346..b9c93fe2 100644 --- a/cmd/mesh-controller/replays_test.go +++ b/cmd/mesh-controller/replays_test.go @@ -1,22 +1,11 @@ package main import ( - "context" - "encoding/json" - "flag" - "fmt" - "os" - "slices" "strings" "testing" "time" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/catalogue" - "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" ) // The replays of the controller's incidents (novox/hq to-be 45 §9, M9): each a scripted replay of what @@ -34,30 +23,6 @@ import ( // consumer's grant — could not compose, and no push to it could go through. The outcome asserted: the // provider's machine composes whatever its consumers are called, and a provision that mints no // credential holds no consumer to a key's length. -func TestReplay263AnIdentityTooLongNeverRefusesItsProvidersMachine(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "resolver", Version: "1", - Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}}) - register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1", - Requires: []string{"wildcard-resolution"}}) - for _, a := range [][2]string{{"anchor", "resolver"}, {"laptop", "networkmanager"}} { - if _, err := assign(ctx, open, a[0], a[1]); err != nil { - t.Fatalf("assign %s %s: %v", a[0], a[1], err) - } - } - // The consumer's machine composes first, as a push does: what it is sent is what its provider grants. - for _, node := range []string{"laptop", "anchor"} { - plan, settings, err := planFor(ctx, open, node) - if err != nil { - t.Fatalf("%s does not resolve: %v", node, err) - } - if _, err := declarationFor(ctx, open, node, plan, settings); err != nil { - t.Fatalf("%s cannot be sent anything — the consumer networkmanager on laptop, identified "+ - "mesh_laptop_networkmanager (26 characters), refused it: %v", node, err) - } - } -} // **R273 — a binding to a consumer's data does not move by itself.** On 2026-10-05 a rule written for // the resolver re-bound every database consumer on the home server — which runs its own store, beside @@ -65,65 +30,6 @@ func TestReplay263AnIdentityTooLongNeverRefusesItsProvidersMachine(t *testing.T) // database; five applications ran on empty data for twenty hours. The outcome asserted: a consumer on a // machine running its own store stays bound to it while another machine holds the store's seat; the // resolver, which every holder answers alike, follows its seat. -func TestReplay273AConsumerBesideItsStoreStaysBoundToIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { - t.Fatal(err) - } - for _, m := range []catalogue.Manifest{ - {Module: "store", Version: "1", - Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}, - Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}}, - Serves: map[string]map[string]any{"postgres-database": {"port": 5432}}, - Grants: map[string]string{"postgres-database": "/var/lib/mesh/store/grants"}}, - {Module: "resolver", Version: "1", - Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}, - Claims: []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}}, - {Module: "network", Version: "1", Requires: []string{"wildcard-resolution"}}, - {Module: "board", Version: "1", Requires: []string{"postgres-database"}}, - } { - register(t, open, m) - } - assignAll := func(pairs ...[2]string) { - for _, a := range pairs { - if _, err := assign(ctx, open, a[0], a[1]); err != nil { - t.Fatalf("assign %s %s: %v", a[0], a[1], err) - } - } - } - // The control node holds the mesh's store and resolver seats; the home server runs its own of each. - assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"}) - for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} { - if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil { - t.Fatal(err) - } - } - assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"}, - [2]string{"laptop", "board"}) - - plan, _, err := planFor(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - var board, network string - for _, n := range plan.Needs { - switch { - case n.For == "board" && n.Name == "postgres-database": - board = n.From - case n.For == "network" && n.Name == "wildcard-resolution": - network = n.From - } - } - if board != "laptop" { - t.Fatalf("the consumer beside its store was bound to the store on %q, which would make it a new, empty "+ - "database there (issue 273)", board) - } - if network != "anchor" { - t.Fatalf("the resolver was bound to %q; its seat is held on anchor (issue 258)", network) - } -} // **R-crashloop — a module that crash-loops after it applied fails its gate on its first machine** (novox/hq // ADR 0240; research 032 §6). On the home server the agent server restarted about a hundred times while the @@ -138,164 +44,6 @@ func TestReplay273AConsumerBesideItsStoreStaysBoundToIt(t *testing.T) { // engine against a real container (mesh-lab replays, R-crashloop), and otherwise what the engine said of // one, kept below. Heard as bytes, so an older controller reads them as it reads any report — this file is // written only with what the controller had before the judging, for the prover to lay over that commit. -func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) { - withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - - crashLoop := []byte(`{"contract":1,"at":"2026-10-07T00:00:00Z","resources":[{"module":"app","resource":"app.server",` + - `"kind":"container","target":"app-server","state":"unhealthy","reason":"restarting","since":"2026-10-07T00:00:00Z",` + - `"streak":5,"restarts":2}]}`) - if path := os.Getenv("MESH_REPLAY_STATEMENT"); path != "" { - raw, err := os.ReadFile(path) - if err != nil { - t.Fatalf("the engine's statement of the crash loop: %v", err) - } - crashLoop = raw - } - // `null` is an engine that states nothing — older than the judging — and its reports carry no health. - var stated map[string]any - if err := json.Unmarshal(crashLoop, &stated); err != nil { - t.Fatal(err) - } - - for _, b := range []inventory.Build{ - {ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app", - Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)}, - {ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app", - Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)}, - } { - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit}) - b.Manifest = manifest - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - registerAt := func(commit string, asked time.Time) { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit, - Head: commit, Asked: asked}); err != nil { - t.Fatal(err) - } - } - registerAt("c1", time.Now().Add(-2*time.Hour)) - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, "app"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil { - t.Fatal(err) - } - } - registerAt("c2", time.Now().Add(-time.Minute)) - - // The machine: what it is sent is applied, and its report says what runs is starting. - listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}} - sequence := int64(0) - say := func(node, digest, state string) { - t.Helper() - sequence++ - health := map[string]any{} - for k, v := range stated { - health[k] = v - } - health["at"] = time.Now().UTC().Format(time.RFC3339Nano) - if state != "" && stated != nil { - resources := []any{} - for _, r := range stated["resources"].([]any) { - kept := map[string]any{} - for k, v := range r.(map[string]any) { - kept[k] = v - } - kept["state"], kept["reason"] = state, "" - resources = append(resources, kept) - } - health["resources"] = resources - } - said := map[string]any{"node": node, "applied": []string{"app.server"}, "declared": digest, - "report_sequence": sequence} - if stated != nil { - said["health"] = health - } - body, _ := json.Marshal(said) - var report link.Report - if err := json.Unmarshal(body, &report); err != nil { - t.Fatal(err) - } - if _, err := listener.Heard(ctx, report); err != nil { - t.Fatal(err) - } - } - var sent [][]string - last := map[string]string{} - n := 0 - wasSend := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - sent = append(sent, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - for _, node := range names { - n++ - digest := fmt.Sprintf("d-%s-%d", node, n) - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil { - return nil, err - } - last[node] = digest - say(node, digest, "starting") - } - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - gateWithin, gateEvery = time.Hour, 0 - t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) - - built := time.Now().UTC() - plan := inventory.Plan{ID: "plan-crashloop", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", - Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-2"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - - advancePlans(ctx, open) // the first machine is sent the new build, and starts it - if len(sent) == 0 || !slices.Equal(sent[0], []string{"anchor"}) { - t.Fatalf("sent %v, not the first machine first", sent) - } - advancePlans(ctx, open) // judged while it starts - // Its container exits at start, and the runtime restarts it: the machine says so, again and again. - for i := 0; i < 6 && len(sent) == 1; i++ { - say("anchor", last["anchor"], "") - advancePlans(ctx, open) - } - gateBound = -time.Second // and the bound passes - advancePlans(ctx, open) - - p, err := inv.PlanByID(ctx, "plan-crashloop") - if err != nil { - t.Fatal(err) - } - gate := p.Modules["app"].Gate - for _, names := range sent { - if slices.Contains(names, "laptop") { - t.Fatalf("the crash loop passed its gate on anchor and was sent to laptop: sent %v, the gate %+v", sent, gate) - } - } - if gate == nil || gate.Verdict != inventory.GateFailed || p.State != inventory.PlanFailed { - t.Fatalf("a crash-looping build did not fail its gate on the first machine: the plan is %s (%s), its gate %+v", - p.State, p.Note, gate) - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" { - t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit) - } -} // **R145 — a web application that accepts TCP and answers nothing is raised within two looks** (novox/hq // ADR 0240 rule 4 and Phase B, issue 145). For eleven hours a web application's port was open and its @@ -305,51 +53,6 @@ func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) { // found looking at such a program; here the controller hears that statement on two looks in a row and // raises the module's condition — the second, never the first. `null` is an engine older than the // readiness check: it states the program alive, and nothing is raised. -func TestReplaySilentWebAppIsRaisedWithinTwoLooks(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - silent := []byte(`{"contract":2,"at":"2026-10-07T00:00:00Z","resources":[{"module":"app","resource":"app.server",` + - `"kind":"container","target":"app-server","state":"unhealthy","reason":"http / on web: no answer within 5s",` + - `"since":"2026-10-07T00:00:00Z","streak":3,"check":"http"}]}`) - if path := os.Getenv("MESH_REPLAY_STATEMENT"); path != "" { - raw, err := os.ReadFile(path) - if err != nil { - t.Fatalf("the engine's statement of the silent web application: %v", err) - } - silent = raw - } - var h link.Health - if err := json.Unmarshal(silent, &h); err != nil { - t.Fatal(err) - } - if h.Contract == 0 { - t.Fatal("the engine states nothing of the silent web application: it is older than the judging") - } - open1 := func() []conditions.Condition { - t.Helper() - list, err := conditionsFrom.Open(ctx) - if err != nil { - t.Fatal(err) - } - return list - } - for look := 1; look <= 2; look++ { - at := time.Now().Add(time.Duration(look) * time.Second) - h.At = at - if err := stateHealth(ctx, open.inventory, conditionsFrom, "anchor", h, at); err != nil { - t.Fatal(err) - } - raised := open1() - switch { - case look == 1 && len(raised) != 0: - t.Fatalf("one look raised %v", raised[0].Key) - case look == 2 && (len(raised) != 1 || raised[0].Key != "module.app.anchor.unhealthy"): - t.Fatalf("two looks in a row did not raise the module's condition: %v", raised) - case look == 2: - t.Logf("raised on the second look: %s", raised[0].Summary) - } - } -} // **R296 — a plan's clock counts from its tier, not from its last save.** On 2026-10-07 a plan for a // merge to the catalogue, its one module's build asked at 19:48:41 and queued at the build seat for @@ -375,42 +78,6 @@ func TestReplay296APlansLineCountsFromWhenItEnteredItsTier(t *testing.T) { // still waits is advanced as the tick and the mesh's build outcomes advance it, and is not written again — // its revision and its last save stay where its last change left them. Before the fix each advance saved // it, and its revision went from 3 to 6. -func TestReplay296APlanStandingStillInItsTierIsNotSavedAgain(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - asked := asksRecorded(t) - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "3ad22356aa", - Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - advanceHeld(ctx, open) - recent, err := inv.RecentPlans(ctx, 1) - if err != nil || len(recent) != 1 || len(*asked) != 1 { - t.Fatalf("no plan waiting on its build: %v %v, asked %v", recent, err, *asked) - } - before := recent[0] - if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" { - t.Fatalf("the plan is not waiting on its build: %+v", before) - } - for range 3 { - advanceHeld(ctx, open) - } - after, err := inv.PlanByID(ctx, before.ID) - if err != nil { - t.Fatal(err) - } - if after.Revision != before.Revision || !after.Updated.Equal(before.Updated) { - t.Fatalf("a plan that did not move was saved again: revision %d → %d, saved %s → %s", - before.Revision, after.Revision, before.Updated, after.Updated) - } -} // **R299 — a holder of a seat whose verbs are all optional is not silent.** On 2026-10-07 the // node-uplink seat gained its first verbs, `resolvers` and `links`, both optional while its holders catch @@ -422,75 +89,6 @@ func TestReplay296APlanStandingStillInItsTierIsNotSavedAgain(t *testing.T) { // // The live seat is replayed while its verbs are all optional; beside it, a seat whose verbs are all // optional by definition, so the outcome is still held once node-uplink's verbs are required (step 3). -func TestReplay299AHolderOfASeatWhoseVerbsAreAllOptionalIsNotSilent(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - catchingUp := catalogue.Seat{Name: "node-catching-up", Scope: catalogue.ScopeNode, Decision: "novox/hq issue 299", - Serves: []catalogue.Verb{{Name: "first", Optional: true}, {Name: "second", Optional: true}}} - if _, err := inv.SeedSeats(ctx, append(catalogue.DefaultSeats(), catchingUp)); err != nil { - t.Fatal(err) - } - rows, err := inv.Seats(ctx) - if err != nil { - t.Fatal(err) - } - // The working set as the serving controller holds it: the store's rows, the compiled seats' marks. - // A seat the compiled set does not know keeps its mark only as defined here. - for i := range rows { - if rows[i].Name == catchingUp.Name { - rows[i].Serves = catchingUp.Serves - } - } - catalogue.UseSeats(rows) - t.Cleanup(func() { catalogue.UseSeats(catalogue.DefaultSeats()) }) - - seats := []string{catchingUp.Name} - if uplink, ok := catalogue.SeatNamed("node-uplink"); ok && len(uplink.Serves) > 0 { - allOptional := true - for _, v := range uplink.Serves { - allOptional = allOptional && v.Optional - } - if allOptional { - seats = append(seats, "node-uplink") - } else { - t.Logf("node-uplink's verbs are required at this commit; the live seat is not replayed, its kind is") - } - } - - register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1"}) - now := time.Now() - var heard []machineFacts - for _, node := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, node, "networkmanager"); err != nil { - t.Fatal(err) - } - for _, seat := range seats { - if err := inv.HoldSeat(ctx, seat, catalogue.ScopeNode, node, "networkmanager"); err != nil { - t.Fatal(err) - } - } - heard = append(heard, machineFacts{name: node, lastHeard: now}) - } - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - d := &doctor{open: open, js: js, watchdogs: &watchdogs{last: &signalFacts{now: now, machines: heard}}} - - found, err := probeHolders(ctx, d) - if err != nil { - t.Fatal(err) - } - for _, o := range found { - for _, seat := range seats { - if strings.HasPrefix(o.ID, seat+".") { - t.Errorf("a holder serving none of a seat's optional verbs was said silent: %s (%s)", o.Summary, o.Said) - } - } - } -} // **R300 — a merge that adds a module builds it.** On 2026-10-07 a pull request to the catalogue added // one module, systemd-resolved; its delivery plan said "builds new: modules/systemd-resolved, sent @@ -498,36 +96,6 @@ func TestReplay299AHolderOfASeatWhoseVerbsAreAllOptionalIsNotSilent(t *testing.T // from" and built nothing, so the module was never registered and `assign` refused it until a person // built it by hand. The outcome asserted: the merge asks the build seat for the new module's directory, at // the branch merged into, from the repository as the mesh spells it, and opens no plan for it. -func TestReplay300AMergeAddingAModuleAsksForItsBuild(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - var asked [][3]string - was := askABuild - askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) { - asked = append(asked, [3]string{source.Repository, path, ref}) - return "build-" + path, nil - } - t.Cleanup(func() { askABuild = was }) - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "networkmanager", Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "3da80a4b00", - Paths: []string{"modules/systemd-resolved/module.json", "modules/systemd-resolved/cmd/main.go"}, - ModuleDirs: []string{"modules/systemd-resolved"}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - want := [3]string{"novox/mesh-catalog", "modules/systemd-resolved", "main"} - if len(asked) != 1 || asked[0] != want { - t.Fatalf("the merge adding modules/systemd-resolved asked the build seat for %v, not %v", asked, want) - } - if plans, err := inv.OpenPlans(ctx); err != nil || len(plans) != 0 { - t.Fatalf("a merge moving nothing the mesh holds opened a plan: %+v %v", plans, err) - } -} // **R301 — a person's push of a held recorded build is no repair.** On 2026-10-07 the operator approved, // machine by machine, new builds of modules whose upgrade policy records rather than rolls out — the @@ -538,53 +106,3 @@ func TestReplay300AMergeAddingAModuleAsksForItsBuild(t *testing.T) { // days … a healer is wanted"), and the same for `words` and `uplink-verbs`. The outcome asserted: a push // that carries only held recorded builds, recorded as the push records it, wants no healer however // often the person gives that word — through S15 itself, reading the log as the watchdog reads it. -func TestReplay301APersonsPushOfAHeldRecordedBuildIsNoRepair(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - start := time.Now().Add(-time.Hour) - image := "registry.invalid:5000/resolver/server@sha256:" + strings.Repeat("e", 64) - if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, aContainerBuild(t, "resolver", "c1111111", catalogue.PolicyRecord, start, - map[string][2]string{"server": {image, ""}}))); err != nil { - t.Fatal(err) - } - for _, node := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, node, "resolver"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, node), "d-"+node, map[string]string{"resolver": "c1111111"}); err != nil { - t.Fatal(err) - } - } - // The merge: the resolver's new build, registered and held for a person's word. - if _, _, err := takeIn(ctx, inv, aContainerBuild(t, "resolver", "c2222222", catalogue.PolicyRecord, - start.Add(time.Minute), map[string][2]string{"server": {image, "runtime"}})); err != nil { - t.Fatal(err) - } - - conn := onATestBus(t) - // The operator's yes, machine by machine, as the seat's push records it before it sends. - for _, node := range []string{"anchor", "laptop"} { - set := flag.NewFlagSet("push", flag.ContinueOnError) - why := addHandActFlags(set) - if err := set.Parse([]string{"--why", "operator's yes: the resolver's new build on " + node, - "--cause", "split-dns"}); err != nil { - t.Fatal(err) - } - why.record(ctx, "push", []string{node}) - } - acts, err := link.HandActs(ctx, conn, start) - if err != nil { - t.Fatal(err) - } - if len(acts) != 2 { - t.Fatalf("the two pushes were recorded as %d act(s): %+v", len(acts), acts) - } - f := calm(time.Now()) - f.handActs = acts - for _, o := range watchHandActs(f) { - if strings.Contains(o.ID, "split-dns") { - t.Fatalf("a person's push of a held recorded build was counted as a repair: %s — %s", o.ID, o.Summary) - } - } -} diff --git a/cmd/mesh-controller/rest_at_once_integration_test.go b/cmd/mesh-controller/rest_at_once_integration_test.go new file mode 100644 index 00000000..1aabc267 --- /dev/null +++ b/cmd/mesh-controller/rest_at_once_integration_test.go @@ -0,0 +1,293 @@ +//go:build integration + +package main + +import ( + "context" + "slices" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +func aBacklogOnThree(t *testing.T) *backlogMesh { + t.Helper() + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + if _, err := inv.AddNode(ctx, "desk"); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, "desk", "app"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, b.open, "desk"), "d-desk", map[string]string{"app": "c1"}); err != nil { + t.Fatal(err) + } + wasHeard := releaseHeard + releaseHeard = func(context.Context, *stores) (map[string]bool, error) { + return map[string]bool{"anchor": true, "laptop": true, "desk": true}, nil + } + t.Cleanup(func() { releaseHeard = wasHeard }) + return b +} + +func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) { + withTheGate(t) + judgeLeavesAsTheCore(t) + b := aBacklogOnThree(t) + ctx := t.Context() + said := events(t) + gateEvery = 0 + for i := 0; i < 12; i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + if p.State != inventory.PlanDone || len(p.Release.Done) != 3 { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + desk, laptop := slices.Index(*said, "send desk"), slices.Index(*said, "send laptop") + if desk < 0 || laptop < 0 || slices.Index(*said, "send anchor") > min(desk, laptop) { + t.Fatalf("sends and judgings: %v", *said) + } + if slices.Contains((*said)[min(desk, laptop):max(desk, laptop)], "judge") { + t.Fatalf("the rest were sent one at a time, each judged before the next: %v", *said) + } + if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) + } +} + +func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T) { + withTheGate(t) + judgeLeavesAsTheCore(t) + b := aBacklogOnThree(t) + ctx := t.Context() + inv := b.open.inventory + gateEvery = 0 + b.broken["desk"] = true // desk fails what it is sent in the rest's step + advancePlans(ctx, b.open) + for i := 0; i < 12 && b.release(t).Open(); i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + if !slices.ContainsFunc(b.sent, func(s []string) bool { return slices.Equal(s, []string{"laptop"}) }) { + t.Fatalf("laptop was never sent: %v", b.sent) + } + if !slices.Contains(p.Release.Done, "laptop") { + t.Fatalf("laptop has no verdict of its own: %+v", p.Release) + } + // late waited on laptop alone and was healthy there: its pass is kept, whatever desk did. + if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) + } + // app failed on desk: marked, and put back wherever it was sent (ADR 0236). + if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed { + t.Fatal("app's build is not marked failed") + } + if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c1" { + t.Fatalf("desk was not put back: %v", sent) + } +} + +func TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere(t *testing.T) { + withTheGate(t) + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + advancePlans(ctx, b.open) // the release sends anchor app c2, and reads it once + release := b.release(t) + judging := release.Release.Gate + if judging == nil || judging.Pending || judging.Passes == 0 { + t.Fatalf("the release is not judging anchor: %+v", judging) + } + built := time.Now().UTC() + plan := inventory.Plan{ID: "plan-app", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", + Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, + Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", + Build: "build-app-c2"}}} + if err := inv.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + advancePlans(ctx, b.open) + if len(b.sent) != 1 { + t.Fatalf("anchor was sent again what it runs: %v", b.sent) + } + p, err := inv.PlanByID(ctx, "plan-app") + if err != nil { + t.Fatal(err) + } + g := p.Modules["app"].Gate + if g == nil || g.Since == nil || !g.Since.Equal(*judging.Since) || g.Passes < judging.Passes || g.TookOver != release.ID { + t.Fatalf("the walk's gate starts from zero: %+v (the release's: %+v)", g, judging) + } + if p.Modules["app"].Previous != "c1" { + t.Fatalf("what anchor ran before is %q, not c1", p.Modules["app"].Previous) + } +} + +func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) { + withTheGate(t) + judgeLeavesAsTheCore(t) + g := aGateMesh(t) + ctx := t.Context() + // The old controller sends anchor app c2 and is replaced before it keeps the send. + made := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + if _, err := made(ctx, open, names); err != nil { + return nil, err + } + return nil, context.Canceled + } + advancePlans(ctx, g.open) + sendRollout = made + if len(g.sent) != 1 || g.plan(t).Modules["app"].FirstAt != nil { + t.Fatalf("the fixture's send was not made, or was kept: %v", g.sent) + } + last, _, err := g.open.inventory.SentTo(ctx, "anchor") + if err != nil { + t.Fatal(err) + } + advancePlans(ctx, g.open) + if len(g.sent) != 1 { + t.Fatalf("the send was made again: %v", g.sent) + } + s := g.plan(t).Modules["app"] + if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) { + t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate) + } + // What anchor ran before was kept before the send, so a failure still puts it back (§5). + if s.Previous != "c1" || s.Gate.From != "c1" { + t.Fatalf("what anchor ran before is %q (gate from %q), not c1", s.Previous, s.Gate.From) + } +} + +func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) { + withTheGate(t) + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + p, err := releaseBacklog(ctx, b.open, "") + if err != nil || p == nil { + t.Fatalf("no release opened: %v", err) + } + if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gate == nil || !p.Release.Gate.Pending { + t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err) + } + if err := inv.SavePlan(ctx, p); err != nil { + t.Fatal(err) + } + // The send is made, and the controller is replaced before it keeps it. + if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil { + t.Fatal(err) + } + advancePlans(ctx, b.open) + if len(b.sent) != 1 { + t.Fatalf("anchor was sent again: %v", b.sent) + } + g := b.release(t).Release.Gate + if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 { + t.Fatalf("anchor is not judged on the send made: %+v", g) + } +} + +func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) { + withTheGate(t) + b := aBacklogOnThree(t) + ctx := t.Context() + gateEvery = 0 + b.broken["anchor"] = true + advancePlans(ctx, b.open) + for i := 0; i < 6 && b.release(t).Open(); i++ { + advancePlans(ctx, b.open) + } + if p := b.release(t); p.State != inventory.PlanFailed || p.Release.Rest { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + for _, s := range b.sent { + if !slices.Equal(s, []string{"anchor"}) { + t.Fatalf("sent %v: the rest after a failed first machine", b.sent) + } + } +} + +func TestTheControlNodeIsSentLastInAStep(t *testing.T) { + withTheGate(t) + judgeLeavesAsTheCore(t) + b := aBacklogOnThree(t) + ctx := t.Context() + register(t, b.open, catalogue.Manifest{Module: catalogue.ControllerSeatName, Version: "1"}) + if _, err := b.open.inventory.Assign(ctx, "anchor", catalogue.ControllerSeatName); err != nil { + t.Fatal(err) + } + said := events(t) + gateEvery = 0 + for i := 0; i < 12 && (len(b.sent) == 0 || b.release(t).Open()); i++ { + advancePlans(ctx, b.open) + } + if p := b.release(t); !slices.Equal(p.Release.Order, []string{"desk", "laptop", "anchor"}) { + t.Fatalf("the order is %v", p.Release.Order) + } + laptop, anchor := slices.Index(*said, "send laptop"), slices.Index(*said, "send anchor") + if laptop < 0 || anchor < laptop || slices.Contains((*said)[laptop:anchor], "judge") { + t.Fatalf("the control node is not sent last in the rest's step: %v", *said) + } +} + +func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) { + withTheGate(t) + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + reporting := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing + b.sent = append(b.sent, names) + current, _ := open.inventory.CurrentBuilds(ctx) + return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep", + map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit}) + } + return reporting(ctx, open, names) + } + t.Cleanup(func() { sendRollout = reporting }) + gateEvery = 0 + for i := 0; i < 8; i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + g := p.Release.Gates["laptop"] + if !p.Open() || g == nil || g.Verdict != "" || g.Silent { + t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g) + } + since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) + g.Since = &since + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + advancePlans(ctx, b.open) + p = b.release(t) + if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed { + t.Fatal("a silent machine's build was marked failed") + } + if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" { + t.Fatalf("laptop was put back: %v", sent) + } + obs, err := silentObservation(ctx, inv) + if err != nil { + t.Fatal(err) + } + if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) { + t.Fatalf("the silence is not said: %+v", obs) + } +} diff --git a/cmd/mesh-controller/rest_at_once_test.go b/cmd/mesh-controller/rest_at_once_test.go index 39a0cb55..ea332777 100644 --- a/cmd/mesh-controller/rest_at_once_test.go +++ b/cmd/mesh-controller/rest_at_once_test.go @@ -5,7 +5,6 @@ import ( "slices" "strings" "testing" - "time" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" @@ -20,27 +19,6 @@ import ( // had just applied, and judged it from its first reading. // aBacklogOnThree is the backlog fixture with a third machine, desk, running app at c1 like the others. -func aBacklogOnThree(t *testing.T) *backlogMesh { - t.Helper() - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - if _, err := inv.AddNode(ctx, "desk"); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, "desk", "app"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, b.open, "desk"), "d-desk", map[string]string{"app": "c1"}); err != nil { - t.Fatal(err) - } - wasHeard := releaseHeard - releaseHeard = func(context.Context, *stores) (map[string]bool, error) { - return map[string]bool{"anchor": true, "laptop": true, "desk": true}, nil - } - t.Cleanup(func() { releaseHeard = wasHeard }) - return b -} // events records every send and every judging, in order. func events(t *testing.T) *[]string { @@ -61,177 +39,18 @@ func events(t *testing.T) *[]string { // Once the first machine passed, the rest are both sent before either is judged (ADR 0353 §1); before, the second // was judged to its verdict before the third was sent. -func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - b := aBacklogOnThree(t) - ctx := t.Context() - said := events(t) - gateEvery = 0 - for i := 0; i < 12; i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - if p.State != inventory.PlanDone || len(p.Release.Done) != 3 { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - desk, laptop := slices.Index(*said, "send desk"), slices.Index(*said, "send laptop") - if desk < 0 || laptop < 0 || slices.Index(*said, "send anchor") > min(desk, laptop) { - t.Fatalf("sends and judgings: %v", *said) - } - if slices.Contains((*said)[min(desk, laptop):max(desk, laptop)], "judge") { - t.Fatalf("the rest were sent one at a time, each judged before the next: %v", *said) - } - if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) - } -} // A machine of the rest whose gate fails is put back at once, and the other machine of the step goes on to its own // verdict (ADR 0353 §3); before, the walk stopped at the failure and the machine after it was never sent. -func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - b := aBacklogOnThree(t) - ctx := t.Context() - inv := b.open.inventory - gateEvery = 0 - b.broken["desk"] = true // desk fails what it is sent in the rest's step - advancePlans(ctx, b.open) - for i := 0; i < 12 && b.release(t).Open(); i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - if !slices.ContainsFunc(b.sent, func(s []string) bool { return slices.Equal(s, []string{"laptop"}) }) { - t.Fatalf("laptop was never sent: %v", b.sent) - } - if !slices.Contains(p.Release.Done, "laptop") { - t.Fatalf("laptop has no verdict of its own: %+v", p.Release) - } - // late waited on laptop alone and was healthy there: its pass is kept, whatever desk did. - if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) - } - // app failed on desk: marked, and put back wherever it was sent (ADR 0236). - if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed { - t.Fatal("app's build is not marked failed") - } - if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c1" { - t.Fatalf("desk was not put back: %v", sent) - } -} // A walk whose first machine already runs its build under another walk's gate sends nothing and goes on from that // gate's readings (ADR 0353 §7); before, it sent the machine again and judged from its first reading. -func TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - advancePlans(ctx, b.open) // the release sends anchor app c2, and reads it once - release := b.release(t) - judging := release.Release.Gate - if judging == nil || judging.Pending || judging.Passes == 0 { - t.Fatalf("the release is not judging anchor: %+v", judging) - } - built := time.Now().UTC() - plan := inventory.Plan{ID: "plan-app", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", - Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", - Build: "build-app-c2"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - advancePlans(ctx, b.open) - if len(b.sent) != 1 { - t.Fatalf("anchor was sent again what it runs: %v", b.sent) - } - p, err := inv.PlanByID(ctx, "plan-app") - if err != nil { - t.Fatal(err) - } - g := p.Modules["app"].Gate - if g == nil || g.Since == nil || !g.Since.Equal(*judging.Since) || g.Passes < judging.Passes || g.TookOver != release.ID { - t.Fatalf("the walk's gate starts from zero: %+v (the release's: %+v)", g, judging) - } - if p.Modules["app"].Previous != "c1" { - t.Fatalf("what anchor ran before is %q, not c1", p.Modules["app"].Previous) - } -} // A walk's first send that was made and not kept — the controller replaced between the two — is not made again // by the controller that takes over: the gate counts from that send (ADR 0353 §5, §6). -func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - g := aGateMesh(t) - ctx := t.Context() - // The old controller sends anchor app c2 and is replaced before it keeps the send. - made := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - if _, err := made(ctx, open, names); err != nil { - return nil, err - } - return nil, context.Canceled - } - advancePlans(ctx, g.open) - sendRollout = made - if len(g.sent) != 1 || g.plan(t).Modules["app"].FirstAt != nil { - t.Fatalf("the fixture's send was not made, or was kept: %v", g.sent) - } - last, _, err := g.open.inventory.SentTo(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - advancePlans(ctx, g.open) - if len(g.sent) != 1 { - t.Fatalf("the send was made again: %v", g.sent) - } - s := g.plan(t).Modules["app"] - if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) { - t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate) - } - // What anchor ran before was kept before the send, so a failure still puts it back (§5). - if s.Previous != "c1" || s.Gate.From != "c1" { - t.Fatalf("what anchor ran before is %q (gate from %q), not c1", s.Previous, s.Gate.From) - } -} // A release's send kept before it was made, and made, is judged by the controller that takes over without being // made again (ADR 0353 §8). -func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - p, err := releaseBacklog(ctx, b.open, "") - if err != nil || p == nil { - t.Fatalf("no release opened: %v", err) - } - if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gate == nil || !p.Release.Gate.Pending { - t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err) - } - if err := inv.SavePlan(ctx, p); err != nil { - t.Fatal(err) - } - // The send is made, and the controller is replaced before it keeps it. - if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, b.open) - if len(b.sent) != 1 { - t.Fatalf("anchor was sent again: %v", b.sent) - } - g := b.release(t).Release.Gate - if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 { - t.Fatalf("anchor is not judged on the send made: %+v", g) - } -} // A build no gate has passed goes to one machine of a step, its first; a machine that would be its second waits // for the next step, and one whose every build passed goes now (ADR 0353 §2). @@ -283,100 +102,9 @@ func TestTheNodeEngineAndTheResolverGoToTheRestOneAtATime(t *testing.T) { } // A first machine that fails still stops the walk (ADR 0236 §5, kept by ADR 0353 §3): the rest are never sent. -func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) { - withTheGate(t) - b := aBacklogOnThree(t) - ctx := t.Context() - gateEvery = 0 - b.broken["anchor"] = true - advancePlans(ctx, b.open) - for i := 0; i < 6 && b.release(t).Open(); i++ { - advancePlans(ctx, b.open) - } - if p := b.release(t); p.State != inventory.PlanFailed || p.Release.Rest { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - for _, s := range b.sent { - if !slices.Equal(s, []string{"anchor"}) { - t.Fatalf("sent %v: the rest after a failed first machine", b.sent) - } - } -} // The control node is sent last in a step of the rest (ADR 0353 §1): the machine holding the controller is // ordered after every other. -func TestTheControlNodeIsSentLastInAStep(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - b := aBacklogOnThree(t) - ctx := t.Context() - register(t, b.open, catalogue.Manifest{Module: catalogue.ControllerSeatName, Version: "1"}) - if _, err := b.open.inventory.Assign(ctx, "anchor", catalogue.ControllerSeatName); err != nil { - t.Fatal(err) - } - said := events(t) - gateEvery = 0 - for i := 0; i < 12 && (len(b.sent) == 0 || b.release(t).Open()); i++ { - advancePlans(ctx, b.open) - } - if p := b.release(t); !slices.Equal(p.Release.Order, []string{"desk", "laptop", "anchor"}) { - t.Fatalf("the order is %v", p.Release.Order) - } - laptop, anchor := slices.Index(*said, "send laptop"), slices.Index(*said, "send anchor") - if laptop < 0 || anchor < laptop || slices.Contains((*said)[laptop:anchor], "judge") { - t.Fatalf("the control node is not sent last in the rest's step: %v", *said) - } -} // A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put // back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4). -func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - reporting := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing - b.sent = append(b.sent, names) - current, _ := open.inventory.CurrentBuilds(ctx) - return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep", - map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit}) - } - return reporting(ctx, open, names) - } - t.Cleanup(func() { sendRollout = reporting }) - gateEvery = 0 - for i := 0; i < 8; i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - g := p.Release.Gates["laptop"] - if !p.Open() || g == nil || g.Verdict != "" || g.Silent { - t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g) - } - since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) - g.Since = &since - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - advancePlans(ctx, b.open) - p = b.release(t) - if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed { - t.Fatal("a silent machine's build was marked failed") - } - if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" { - t.Fatalf("laptop was put back: %v", sent) - } - obs, err := silentObservation(ctx, inv) - if err != nil { - t.Fatal(err) - } - if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) { - t.Fatalf("the silence is not said: %+v", obs) - } -} diff --git a/cmd/mesh-controller/retirement_integration_test.go b/cmd/mesh-controller/retirement_integration_test.go new file mode 100644 index 00000000..62c80b86 --- /dev/null +++ b/cmd/mesh-controller/retirement_integration_test.go @@ -0,0 +1,191 @@ +//go:build integration + +package main + +import ( + "slices" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestARetirementConditionOfAnUnassignedProviderClears(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "pg", Version: "1", + Receives: map[string]string{"postgres-database": "/var/lib/mesh/pg/mesh.json"}}) + if _, err := assign(ctx, open, "anchor", "pg"); err != nil { + t.Fatal(err) + } + r := retirements{keeper: func() *conditions.Keeper { return conditionsFrom }} + for _, w := range []link.Retirement{waitingWord("pg", "anchor", "a", "b", "c", "d"), waitingWord("gone", "anchor", "a", "b", "c", "d")} { + if err := r.Retired(ctx, w); err != nil { + t.Fatal(err) + } + } + if err := conditionsFrom.Reconcile(ctx, "D11", []conditions.Observation{ + cleanupObservation(providerInstance{Node: "anchor", Module: "gone"}, + []link.RetiredConsumer{{Consumer: "a", RetiredAt: time.Now().Add(-40 * 24 * time.Hour).Format(time.RFC3339)}}, time.Now()), + }); err != nil { + t.Fatal(err) + } + all, _ := conditionsFrom.Open(ctx) + if len(all) != 3 { + t.Fatalf("%+v", all) + } + if err := unassignedProviders(ctx, open.inventory, conditionsFrom, providerConditions(all)); err != nil { + t.Fatal(err) + } + left := openKeys(t, conditionsFrom) + if len(left) != 1 || left["provider.pg.anchor.retire"].Key == "" { + t.Fatalf("only the assigned provider's waiting should stay: %+v", left) + } +} + +func onATestBus(t *testing.T) *nats.Conn { + t.Helper() + url := testbus.URL(t) + js, err := broker.Dial(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + before := handActConn + handActConn = js.Conn() + t.Cleanup(func() { handActConn = before }) + return js.Conn() +} + +func TestNatsApproveSendsTheExactSetTheProviderWaitsWith(t *testing.T) { + conn := onATestBus(t) + fake := &fakeProvider{} + fake.state.Waiting = &link.RetirementWaiting{Consumers: []link.RetiredConsumer{{Consumer: "d"}, {Consumer: "b"}, {Consumer: "a"}, {Consumer: "c"}}, Held: 6} + fake.serve(t, conn, "pg-approve", "anchor") + p := providerInstance{Node: "anchor", Module: "pg-approve"} + said := printed(t, func() error { return answerRetirement(t.Context(), conn, p, true, whyFlags(t, "moved them")) }) + if !slices.Equal(fake.approved, []string{"a", "b", "c", "d"}) || !strings.Contains(said, "retired d, b, a, c") { + t.Fatalf("approved %v; said %s", fake.approved, said) + } + acts := handActsBy(t, conn, "retire approve") + if len(acts) == 0 || acts[len(acts)-1].Cause != kindRetireWaiting || + acts[len(acts)-1].Condition != "provider.pg-approve.anchor.retire" { + t.Fatalf("the approval is not in the hand-act log: %+v", acts) + } + + // Reject, on another provider: the same set, and nothing approved. + other := &fakeProvider{state: fake.state} + other.serve(t, conn, "pg-reject", "anchor") + printed(t, func() error { + return answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-reject"}, false, + whyFlags(t, "still moving")) + }) + if !slices.Equal(other.rejected, []string{"a", "b", "c", "d"}) || other.approved != nil { + t.Fatalf("rejected %v approved %v", other.rejected, other.approved) + } + + // Nothing waiting: refused, nothing asked but the question. + idle := &fakeProvider{} + idle.serve(t, conn, "pg-idle", "anchor") + if err := answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-idle"}, true, + whyFlags(t, "x")); err == nil || !strings.Contains(err.Error(), "waits for nobody") { + t.Fatalf("%v", err) + } + if len(idle.asked) != 1 { + t.Fatalf("an idle provider was asked more than its state: %+v", idle.asked) + } +} + +func TestNatsCleanupDeletesOnlyTheNamedRetiredConsumer(t *testing.T) { + conn := onATestBus(t) + fake := &fakeProvider{} + fake.state.Held = []string{"active"} + fake.state.Retired = []link.RetiredConsumer{retiredDaysAgo("old", 40), retiredDaysAgo("young", 2)} + fake.serve(t, conn, "pg-clean", "anchor") + p := providerInstance{Node: "anchor", Module: "pg-clean"} + said := printed(t, func() error { return deleteRetired(t.Context(), conn, p, "old", whyFlags(t, "not needed")) }) + if !slices.Equal(fake.deleted, []string{"old"}) || !strings.Contains(said, "deleted old (1.0 KB freed)") { + t.Fatalf("deleted %v; said %s", fake.deleted, said) + } + // An active consumer, or one it does not hold, is refused before the provider is asked to delete. + for _, name := range []string{"active", "nobody"} { + if err := deleteRetired(t.Context(), conn, p, name, whyFlags(t, "x")); err == nil || + !strings.Contains(err.Error(), "only a retired consumer is deleted") { + t.Fatalf("%s: %v", name, err) + } + } + if !slices.Equal(fake.deleted, []string{"old"}) { + t.Fatalf("more was deleted: %v", fake.deleted) + } + if acts := handActsBy(t, conn, "cleanup delete"); len(acts) == 0 || acts[len(acts)-1].Args[2] != "old" { + t.Fatalf("the deletion is not in the hand-act log: %+v", acts) + } + + // Older than: listed, and nothing deleted without confirm; with it, only the old one. + listing := retiredOf(t.Context(), conn, []providerInstance{p}, time.Now()) + if len(listing.Retired) != 2 || listing.Retired[0].Consumer != "old" || listing.Retired[0].AgeDays != 40 { + t.Fatalf("%+v", listing) + } + fake.deleted = nil + said = printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, false, whyFlags(t, "tidy")) }) + if fake.deleted != nil || !strings.Contains(said, "nothing was deleted: add --confirm") || !strings.Contains(said, "old") || + strings.Contains(said, "young") { + t.Fatalf("deleted %v; said %s", fake.deleted, said) + } + printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, true, whyFlags(t, "tidy")) }) + if !slices.Equal(fake.deleted, []string{"old"}) { + t.Fatalf("confirmed, deleted %v", fake.deleted) + } +} + +func TestNatsD11SaysCleanupWaitsAfterThirtyDays(t *testing.T) { + conn := onATestBus(t) + old := &fakeProvider{} + old.state.Retired = []link.RetiredConsumer{retiredDaysAgo("mesh_a_letta", 31), retiredDaysAgo("fresh", 1)} + old.serve(t, conn, "pg-d11-old", "anchor") + young := &fakeProvider{} + young.state.Retired = []link.RetiredConsumer{retiredDaysAgo("x", 29)} + young.serve(t, conn, "pg-d11-young", "anchor") + instances := []providerInstance{{Node: "anchor", Module: "pg-d11-old"}, {Node: "anchor", Module: "pg-d11-young"}, + // Nothing serves this one: a provider older than the question is passed over, not a failure. + {Node: "anchor", Module: "pg-d11-predates"}} + found, err := retiredTooLong(t.Context(), conn, instances, time.Now()) + if err != nil { + t.Fatal(err) + } + if len(found) != 1 || found[0].Key() != "provider.pg-d11-old.anchor.cleanup" || found[0].Kind != kindCleanupWaiting || + found[0].Severity != conditions.Warning || !strings.Contains(found[0].Summary, "mesh_a_letta (31 days") || + strings.Contains(found[0].Summary, "fresh") { + t.Fatalf("%+v", found) + } +} + +func TestNatsAMarkOnlyRetirementIsSaidAsSuch(t *testing.T) { + conn := onATestBus(t) + fake := &fakeProvider{} + kept := retiredDaysAgo("ledger", 3) + kept.Access = "kept" + fake.state.Retired = []link.RetiredConsumer{kept} + fake.state.RetiresBy = "mark-only" + fake.state.Waiting = &link.RetirementWaiting{Consumers: []link.RetiredConsumer{{Consumer: "a"}, {Consumer: "b"}}, Held: 2} + fake.serve(t, conn, "vault-mark", "anchor") + p := providerInstance{Node: "anchor", Module: "vault-mark"} + listing := retiredOf(t.Context(), conn, []providerInstance{p}, time.Now()) + said := printed(t, func() error { return printRetired(listing, false) }) + if !strings.Contains(said, "MARK ONLY") || listing.Retired[0].Access != "kept" { + t.Fatalf("%s %+v", said, listing) + } + said = printed(t, func() error { return answerRetirement(t.Context(), conn, p, true, whyFlags(t, "gone")) }) + if !strings.Contains(said, "MARK ONLY") || strings.Contains(said, "access disabled") { + t.Fatal(said) + } +} diff --git a/cmd/mesh-controller/retirement_test.go b/cmd/mesh-controller/retirement_test.go index 0402eaab..7d6a17c6 100644 --- a/cmd/mesh-controller/retirement_test.go +++ b/cmd/mesh-controller/retirement_test.go @@ -13,12 +13,8 @@ import ( "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/link" - - "github.com/novox/mesh-controller/internal/testbus" ) // A consumer the mesh stops asking for is retired, not withdrawn, and deleted only by a person @@ -151,39 +147,6 @@ func TestARetirementWordIsKeptAsItsConditionNamingTheEmitterFromTheSubject(t *te } } -func TestARetirementConditionOfAnUnassignedProviderClears(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "pg", Version: "1", - Receives: map[string]string{"postgres-database": "/var/lib/mesh/pg/mesh.json"}}) - if _, err := assign(ctx, open, "anchor", "pg"); err != nil { - t.Fatal(err) - } - r := retirements{keeper: func() *conditions.Keeper { return conditionsFrom }} - for _, w := range []link.Retirement{waitingWord("pg", "anchor", "a", "b", "c", "d"), waitingWord("gone", "anchor", "a", "b", "c", "d")} { - if err := r.Retired(ctx, w); err != nil { - t.Fatal(err) - } - } - if err := conditionsFrom.Reconcile(ctx, "D11", []conditions.Observation{ - cleanupObservation(providerInstance{Node: "anchor", Module: "gone"}, - []link.RetiredConsumer{{Consumer: "a", RetiredAt: time.Now().Add(-40 * 24 * time.Hour).Format(time.RFC3339)}}, time.Now()), - }); err != nil { - t.Fatal(err) - } - all, _ := conditionsFrom.Open(ctx) - if len(all) != 3 { - t.Fatalf("%+v", all) - } - if err := unassignedProviders(ctx, open.inventory, conditionsFrom, providerConditions(all)); err != nil { - t.Fatal(err) - } - left := openKeys(t, conditionsFrom) - if len(left) != 1 || left["provider.pg.anchor.retire"].Key == "" { - t.Fatalf("only the assigned provider's waiting should stay: %+v", left) - } -} - // fakeProvider answers the retirement tools on one machine, over a real bus, keeping what it was asked. type fakeProvider struct { mu sync.Mutex @@ -260,23 +223,6 @@ func (f *fakeProvider) serve(t *testing.T, conn *nats.Conn, module, node string) } } -func onATestBus(t *testing.T) *nats.Conn { - t.Helper() - url := testbus.URL(t) - js, err := broker.Dial(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - before := handActConn - handActConn = js.Conn() - t.Cleanup(func() { handActConn = before }) - return js.Conn() -} - func whyFlags(t *testing.T, why string) handActFlags { t.Helper() set := flag.NewFlagSet("t", flag.ContinueOnError) @@ -308,109 +254,6 @@ func retiredDaysAgo(name string, days int) link.RetiredConsumer { RetiredAt: time.Now().Add(-time.Duration(days) * 24 * time.Hour).UTC().Format(time.RFC3339), SizeBytes: &size} } -func TestNatsApproveSendsTheExactSetTheProviderWaitsWith(t *testing.T) { - conn := onATestBus(t) - fake := &fakeProvider{} - fake.state.Waiting = &link.RetirementWaiting{Consumers: []link.RetiredConsumer{{Consumer: "d"}, {Consumer: "b"}, {Consumer: "a"}, {Consumer: "c"}}, Held: 6} - fake.serve(t, conn, "pg-approve", "anchor") - p := providerInstance{Node: "anchor", Module: "pg-approve"} - said := printed(t, func() error { return answerRetirement(t.Context(), conn, p, true, whyFlags(t, "moved them")) }) - if !slices.Equal(fake.approved, []string{"a", "b", "c", "d"}) || !strings.Contains(said, "retired d, b, a, c") { - t.Fatalf("approved %v; said %s", fake.approved, said) - } - acts := handActsBy(t, conn, "retire approve") - if len(acts) == 0 || acts[len(acts)-1].Cause != kindRetireWaiting || - acts[len(acts)-1].Condition != "provider.pg-approve.anchor.retire" { - t.Fatalf("the approval is not in the hand-act log: %+v", acts) - } - - // Reject, on another provider: the same set, and nothing approved. - other := &fakeProvider{state: fake.state} - other.serve(t, conn, "pg-reject", "anchor") - printed(t, func() error { - return answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-reject"}, false, - whyFlags(t, "still moving")) - }) - if !slices.Equal(other.rejected, []string{"a", "b", "c", "d"}) || other.approved != nil { - t.Fatalf("rejected %v approved %v", other.rejected, other.approved) - } - - // Nothing waiting: refused, nothing asked but the question. - idle := &fakeProvider{} - idle.serve(t, conn, "pg-idle", "anchor") - if err := answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-idle"}, true, - whyFlags(t, "x")); err == nil || !strings.Contains(err.Error(), "waits for nobody") { - t.Fatalf("%v", err) - } - if len(idle.asked) != 1 { - t.Fatalf("an idle provider was asked more than its state: %+v", idle.asked) - } -} - -func TestNatsCleanupDeletesOnlyTheNamedRetiredConsumer(t *testing.T) { - conn := onATestBus(t) - fake := &fakeProvider{} - fake.state.Held = []string{"active"} - fake.state.Retired = []link.RetiredConsumer{retiredDaysAgo("old", 40), retiredDaysAgo("young", 2)} - fake.serve(t, conn, "pg-clean", "anchor") - p := providerInstance{Node: "anchor", Module: "pg-clean"} - said := printed(t, func() error { return deleteRetired(t.Context(), conn, p, "old", whyFlags(t, "not needed")) }) - if !slices.Equal(fake.deleted, []string{"old"}) || !strings.Contains(said, "deleted old (1.0 KB freed)") { - t.Fatalf("deleted %v; said %s", fake.deleted, said) - } - // An active consumer, or one it does not hold, is refused before the provider is asked to delete. - for _, name := range []string{"active", "nobody"} { - if err := deleteRetired(t.Context(), conn, p, name, whyFlags(t, "x")); err == nil || - !strings.Contains(err.Error(), "only a retired consumer is deleted") { - t.Fatalf("%s: %v", name, err) - } - } - if !slices.Equal(fake.deleted, []string{"old"}) { - t.Fatalf("more was deleted: %v", fake.deleted) - } - if acts := handActsBy(t, conn, "cleanup delete"); len(acts) == 0 || acts[len(acts)-1].Args[2] != "old" { - t.Fatalf("the deletion is not in the hand-act log: %+v", acts) - } - - // Older than: listed, and nothing deleted without confirm; with it, only the old one. - listing := retiredOf(t.Context(), conn, []providerInstance{p}, time.Now()) - if len(listing.Retired) != 2 || listing.Retired[0].Consumer != "old" || listing.Retired[0].AgeDays != 40 { - t.Fatalf("%+v", listing) - } - fake.deleted = nil - said = printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, false, whyFlags(t, "tidy")) }) - if fake.deleted != nil || !strings.Contains(said, "nothing was deleted: add --confirm") || !strings.Contains(said, "old") || - strings.Contains(said, "young") { - t.Fatalf("deleted %v; said %s", fake.deleted, said) - } - printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, true, whyFlags(t, "tidy")) }) - if !slices.Equal(fake.deleted, []string{"old"}) { - t.Fatalf("confirmed, deleted %v", fake.deleted) - } -} - -func TestNatsD11SaysCleanupWaitsAfterThirtyDays(t *testing.T) { - conn := onATestBus(t) - old := &fakeProvider{} - old.state.Retired = []link.RetiredConsumer{retiredDaysAgo("mesh_a_letta", 31), retiredDaysAgo("fresh", 1)} - old.serve(t, conn, "pg-d11-old", "anchor") - young := &fakeProvider{} - young.state.Retired = []link.RetiredConsumer{retiredDaysAgo("x", 29)} - young.serve(t, conn, "pg-d11-young", "anchor") - instances := []providerInstance{{Node: "anchor", Module: "pg-d11-old"}, {Node: "anchor", Module: "pg-d11-young"}, - // Nothing serves this one: a provider older than the question is passed over, not a failure. - {Node: "anchor", Module: "pg-d11-predates"}} - found, err := retiredTooLong(t.Context(), conn, instances, time.Now()) - if err != nil { - t.Fatal(err) - } - if len(found) != 1 || found[0].Key() != "provider.pg-d11-old.anchor.cleanup" || found[0].Kind != kindCleanupWaiting || - found[0].Severity != conditions.Warning || !strings.Contains(found[0].Summary, "mesh_a_letta (31 days") || - strings.Contains(found[0].Summary, "fresh") { - t.Fatalf("%+v", found) - } -} - func TestTheRetireAndCleanupVerbsComposeTheirCommandLines(t *testing.T) { for _, c := range []struct { verb string @@ -459,23 +302,3 @@ func TestTheRetireAndCleanupVerbsComposeTheirCommandLines(t *testing.T) { // A mark-only provider's retired consumer keeps its access; the listing and the approval say so // rather than claiming it was disabled (ADR 0230). -func TestNatsAMarkOnlyRetirementIsSaidAsSuch(t *testing.T) { - conn := onATestBus(t) - fake := &fakeProvider{} - kept := retiredDaysAgo("ledger", 3) - kept.Access = "kept" - fake.state.Retired = []link.RetiredConsumer{kept} - fake.state.RetiresBy = "mark-only" - fake.state.Waiting = &link.RetirementWaiting{Consumers: []link.RetiredConsumer{{Consumer: "a"}, {Consumer: "b"}}, Held: 2} - fake.serve(t, conn, "vault-mark", "anchor") - p := providerInstance{Node: "anchor", Module: "vault-mark"} - listing := retiredOf(t.Context(), conn, []providerInstance{p}, time.Now()) - said := printed(t, func() error { return printRetired(listing, false) }) - if !strings.Contains(said, "MARK ONLY") || listing.Retired[0].Access != "kept" { - t.Fatalf("%s %+v", said, listing) - } - said = printed(t, func() error { return answerRetirement(t.Context(), conn, p, true, whyFlags(t, "gone")) }) - if !strings.Contains(said, "MARK ONLY") || strings.Contains(said, "access disabled") { - t.Fatal(said) - } -} diff --git a/cmd/mesh-controller/rollout_events_test.go b/cmd/mesh-controller/rollout_events_test.go index f3809bba..a90d359c 100644 --- a/cmd/mesh-controller/rollout_events_test.go +++ b/cmd/mesh-controller/rollout_events_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/rollout_silent_bus_integration_test.go b/cmd/mesh-controller/rollout_silent_bus_integration_test.go new file mode 100644 index 00000000..e3de4c3b --- /dev/null +++ b/cmd/mesh-controller/rollout_silent_bus_integration_test.go @@ -0,0 +1,92 @@ +//go:build integration + +package main + +import ( + "context" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/testbus" +) + +func silentOnABus(t *testing.T) (*busSilent, <-chan silentWait) { + t.Helper() + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + conn, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(conn.Close) + api, err := jetstream.New(conn) + if err != nil { + t.Fatal(err) + } + heard := make(chan silentWait, 16) + ctx, stop := context.WithCancel(context.Background()) + t.Cleanup(stop) + s, err := newBusSilent(ctx, api, func(w silentWait) { heard <- w }) + if err != nil { + t.Fatal(err) + } + return s, heard +} + +func TestAKeyLeftAloneExpiresAndTheWatcherNamesTheMachineSilent(t *testing.T) { + s, heard := silentOnABus(t) + want := silentWait{Plan: "plan.7", Tier: 2, Node: "ace"} + if err := s.Arm(t.Context(), want, time.Second, time.Now()); err != nil { + t.Fatal(err) + } + if got, ok := hearsSilent(t, heard, 10*time.Second); !ok || got != want { + t.Fatalf("heard %+v (%v); want %+v", got, ok, want) + } +} + +func TestAReportDeletesTheKeyAndNoOneIsNamedSilent(t *testing.T) { + s, heard := silentOnABus(t) + ctx := t.Context() + w := silentWait{Plan: "plan-1", Tier: 0, Node: "g14"} + if err := s.Arm(ctx, w, time.Second, time.Now()); err != nil { + t.Fatal(err) + } + if err := s.Reported(ctx, w); err != nil { + t.Fatal(err) + } + if got, ok := hearsSilent(t, heard, 2500*time.Millisecond); ok { + t.Fatalf("%+v was named silent after it reported", got) + } + // Reporting twice, or on a send that put no key, is no error. + if err := s.Reported(ctx, w); err != nil { + t.Fatalf("a second report: %v", err) + } +} + +func TestASendMadeAgainPutsTheKeyAgain(t *testing.T) { + s, heard := silentOnABus(t) + ctx := t.Context() + w := silentWait{Plan: "plan-3", Tier: 1, Node: "novox"} + if err := s.Arm(ctx, w, 2*time.Second, time.Now()); err != nil { + t.Fatal(err) + } + time.Sleep(1200 * time.Millisecond) + if err := s.Arm(ctx, w, 2*time.Second, time.Now()); err != nil { + t.Fatal(err) + } + if got, ok := hearsSilent(t, heard, 1500*time.Millisecond); ok { + t.Fatalf("%+v expired at the first send's time, not the second's", got) + } + if got, ok := hearsSilent(t, heard, 5*time.Second); !ok || got != w { + t.Fatalf("heard %+v (%v); want %+v after the second send's time", got, ok, w) + } +} diff --git a/cmd/mesh-controller/rollout_silent_bus_test.go b/cmd/mesh-controller/rollout_silent_bus_test.go index fef01437..aaf3aa85 100644 --- a/cmd/mesh-controller/rollout_silent_bus_test.go +++ b/cmd/mesh-controller/rollout_silent_bus_test.go @@ -1,46 +1,11 @@ package main import ( - "context" "testing" "time" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/testbus" ) // silentOnABus is the rollout's silent keys on a real bus, and the waits its watcher heard expire. -func silentOnABus(t *testing.T) (*busSilent, <-chan silentWait) { - t.Helper() - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - api, err := jetstream.New(conn) - if err != nil { - t.Fatal(err) - } - heard := make(chan silentWait, 16) - ctx, stop := context.WithCancel(context.Background()) - t.Cleanup(stop) - s, err := newBusSilent(ctx, api, func(w silentWait) { heard <- w }) - if err != nil { - t.Fatal(err) - } - return s, heard -} func hearsSilent(t *testing.T, heard <-chan silentWait, within time.Duration) (silentWait, bool) { t.Helper() @@ -54,56 +19,10 @@ func hearsSilent(t *testing.T, heard <-chan silentWait, within time.Duration) (s // A machine that never reports on its send is named silent when its key expires (issue 974, novox/hq ADR 0406 §5): // the watcher names the plan, the tier and the machine, which a plan id or a machine name with a dot does not blur. -func TestAKeyLeftAloneExpiresAndTheWatcherNamesTheMachineSilent(t *testing.T) { - s, heard := silentOnABus(t) - want := silentWait{Plan: "plan.7", Tier: 2, Node: "ace"} - if err := s.Arm(t.Context(), want, time.Second, time.Now()); err != nil { - t.Fatal(err) - } - if got, ok := hearsSilent(t, heard, 10*time.Second); !ok || got != want { - t.Fatalf("heard %+v (%v); want %+v", got, ok, want) - } -} // A machine that reports in time is never named silent: the report deletes the key, and a delete is not an expiry. -func TestAReportDeletesTheKeyAndNoOneIsNamedSilent(t *testing.T) { - s, heard := silentOnABus(t) - ctx := t.Context() - w := silentWait{Plan: "plan-1", Tier: 0, Node: "g14"} - if err := s.Arm(ctx, w, time.Second, time.Now()); err != nil { - t.Fatal(err) - } - if err := s.Reported(ctx, w); err != nil { - t.Fatal(err) - } - if got, ok := hearsSilent(t, heard, 2500*time.Millisecond); ok { - t.Fatalf("%+v was named silent after it reported", got) - } - // Reporting twice, or on a send that put no key, is no error. - if err := s.Reported(ctx, w); err != nil { - t.Fatalf("a second report: %v", err) - } -} // A send made again to a machine puts its key again with the full time, and only the last send's key can expire. -func TestASendMadeAgainPutsTheKeyAgain(t *testing.T) { - s, heard := silentOnABus(t) - ctx := t.Context() - w := silentWait{Plan: "plan-3", Tier: 1, Node: "novox"} - if err := s.Arm(ctx, w, 2*time.Second, time.Now()); err != nil { - t.Fatal(err) - } - time.Sleep(1200 * time.Millisecond) - if err := s.Arm(ctx, w, 2*time.Second, time.Now()); err != nil { - t.Fatal(err) - } - if got, ok := hearsSilent(t, heard, 1500*time.Millisecond); ok { - t.Fatalf("%+v expired at the first send's time, not the second's", got) - } - if got, ok := hearsSilent(t, heard, 5*time.Second); !ok || got != w { - t.Fatalf("heard %+v (%v); want %+v after the second send's time", got, ok, w) - } -} func TestASilentKeyIsParsedFromTheRightOfTheNode(t *testing.T) { for _, w := range []silentWait{{"plan-1", 0, "ace"}, {"a.b", 12, "g14"}} { diff --git a/cmd/mesh-controller/rollout_test.go b/cmd/mesh-controller/rollout_test.go index ba5ac2f5..409eb28e 100644 --- a/cmd/mesh-controller/rollout_test.go +++ b/cmd/mesh-controller/rollout_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/roster_failure_test.go b/cmd/mesh-controller/roster_failure_test.go index 2aa035d6..2479f38c 100644 --- a/cmd/mesh-controller/roster_failure_test.go +++ b/cmd/mesh-controller/roster_failure_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/runtime_health_test.go b/cmd/mesh-controller/runtime_health_test.go index 8bc63491..3856441c 100644 --- a/cmd/mesh-controller/runtime_health_test.go +++ b/cmd/mesh-controller/runtime_health_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/same_program_integration_test.go b/cmd/mesh-controller/same_program_integration_test.go new file mode 100644 index 00000000..294ae773 --- /dev/null +++ b/cmd/mesh-controller/same_program_integration_test.go @@ -0,0 +1,58 @@ +//go:build integration + +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestAWalkWhoseEveryBuildCameOutTheSameIsDone(t *testing.T) { + open := windowed(t) + endsItsLine(t) + asksRecorded(t) + ctx := t.Context() + inv := open.inventory + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-one", Repository: "novox/one", Ref: "main", + Module: "one", Commit: "c0", On: "builder", Asked: t0.Add(-time.Hour), + Made: []inventory.Artifact{{Name: "one", Kind: "image", Reference: "store/one@sha256:aaaa"}}, + Sources: []inventory.BuildSource{{Paths: []string{"main.go", "go.mod"}}}}); err != nil { + t.Fatal(err) + } + // What runs: the module registered with that build's image (RegisteredHeld reads the module table). + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "one", Version: "1", Resources: []map[string]any{ + {"id": "app", "type": "container", "name": "one", "image": "store/one@sha256:aaaa"}}}, + inventory.Source{Repository: "novox/one", Ref: "main", BuiltFrom: "c0", Asked: time.Now()}); err != nil { + t.Fatal(err) + } + pull := link.PullRef{Owner: "novox", Repo: "one", Number: 7} + was := checkedBuildsOf + checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) { + return []link.CheckedBuild{{Pull: pull, Head: "headaaaaaaaa", Built: link.CheckBuilt{Module: "one", Key: "k", + Made: []link.MadeArtifact{{Name: "one", Kind: "image", Reference: "scratch/one@sha256:bbbb"}}, + Same: "unchanged: its program is the same as running build one aaaa", + SameAs: map[string]string{"one": "10.0.0.1:5000/one@sha256:aaaa"}}}}, nil + } + t.Cleanup(func() { checkedBuildsOf = was }) + code := repoMerge("one", "one1aaaaaaaa", t0) + code.Number, code.HeadSHA = 7, "headaaaaaaaa1111" + hear(t, open, code, t0.Add(time.Second)) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + if len(ws) != 1 { + t.Fatalf("walks: %+v", ws) + } + w := ws[0] + if w.State != inventory.PlanDone || len(w.Tiers) != 0 || len(w.Modules) != 0 { + t.Fatalf("the walk reads %s, tiers %v, modules %v; want done with nothing to walk", w.State, w.Tiers, w.Modules) + } + if w.Delivery == nil || !strings.Contains(w.Delivery.Unchanged["one"], "its program is the same as running build") { + t.Fatalf("the walk does not say one matched what runs: %+v", w.Delivery) + } +} diff --git a/cmd/mesh-controller/same_program_test.go b/cmd/mesh-controller/same_program_test.go index 2d47d6bd..11afe41d 100644 --- a/cmd/mesh-controller/same_program_test.go +++ b/cmd/mesh-controller/same_program_test.go @@ -1,12 +1,9 @@ package main import ( - "context" "strings" "testing" - "time" - "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -96,46 +93,3 @@ func TestADependentOfABaseThatCameOutTheSameIsLeftOut(t *testing.T) { // Against the store, the path a real merge takes (novox/hq issues 721 and 611): a merge whose check built its one // module the same as the build the mesh holds is cut into a walk done with nothing to walk, whose record names the // module unchanged and why — the walk mesh-delivery reads as delivered, never stranded. -func TestAWalkWhoseEveryBuildCameOutTheSameIsDone(t *testing.T) { - open := windowed(t) - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - inv := open.inventory - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-one", Repository: "novox/one", Ref: "main", - Module: "one", Commit: "c0", On: "builder", Asked: t0.Add(-time.Hour), - Made: []inventory.Artifact{{Name: "one", Kind: "image", Reference: "store/one@sha256:aaaa"}}, - Sources: []inventory.BuildSource{{Paths: []string{"main.go", "go.mod"}}}}); err != nil { - t.Fatal(err) - } - // What runs: the module registered with that build's image (RegisteredHeld reads the module table). - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "one", Version: "1", Resources: []map[string]any{ - {"id": "app", "type": "container", "name": "one", "image": "store/one@sha256:aaaa"}}}, - inventory.Source{Repository: "novox/one", Ref: "main", BuiltFrom: "c0", Asked: time.Now()}); err != nil { - t.Fatal(err) - } - pull := link.PullRef{Owner: "novox", Repo: "one", Number: 7} - was := checkedBuildsOf - checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) { - return []link.CheckedBuild{{Pull: pull, Head: "headaaaaaaaa", Built: link.CheckBuilt{Module: "one", Key: "k", - Made: []link.MadeArtifact{{Name: "one", Kind: "image", Reference: "scratch/one@sha256:bbbb"}}, - Same: "unchanged: its program is the same as running build one aaaa", - SameAs: map[string]string{"one": "10.0.0.1:5000/one@sha256:aaaa"}}}}, nil - } - t.Cleanup(func() { checkedBuildsOf = was }) - code := repoMerge("one", "one1aaaaaaaa", t0) - code.Number, code.HeadSHA = 7, "headaaaaaaaa1111" - hear(t, open, code, t0.Add(time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - if len(ws) != 1 { - t.Fatalf("walks: %+v", ws) - } - w := ws[0] - if w.State != inventory.PlanDone || len(w.Tiers) != 0 || len(w.Modules) != 0 { - t.Fatalf("the walk reads %s, tiers %v, modules %v; want done with nothing to walk", w.State, w.Tiers, w.Modules) - } - if w.Delivery == nil || !strings.Contains(w.Delivery.Unchanged["one"], "its program is the same as running build") { - t.Fatalf("the walk does not say one matched what runs: %+v", w.Delivery) - } -} diff --git a/cmd/mesh-controller/same_source_test.go b/cmd/mesh-controller/same_source_test.go index 01abd454..695e2244 100644 --- a/cmd/mesh-controller/same_source_test.go +++ b/cmd/mesh-controller/same_source_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/sayable_integration_test.go b/cmd/mesh-controller/sayable_integration_test.go new file mode 100644 index 00000000..508cf740 --- /dev/null +++ b/cmd/mesh-controller/sayable_integration_test.go @@ -0,0 +1,150 @@ +//go:build integration + +package main + +import ( + "context" + "fmt" + "os" + "sort" + "strings" + "sync/atomic" + "testing" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/outward" +) + +func TestASingleLookIsHeldAndTheSecondInARowRaises(t *testing.T) { + var unanswered, broken atomic.Bool + held := conditions.Observation{Scope: conditions.ScopeSeat, ID: "mesh-dns-resolver.anchor", Token: "wrong", + Machine: "anchor", Severity: conditions.Urgent, Confirm: true, + Summary: "the mesh's resolver on anchor does not answer", Said: "no answer from 10.77.0.1"} + withProbes(t, + probe{ID: "P1", Asserts: "asks over the network", Kind: "resolver-wrong", Phase: 1, + run: func(context.Context, *doctor) ([]conditions.Observation, error) { + if unanswered.Load() { + return []conditions.Observation{held}, nil + } + return nil, nil + }}, + probe{ID: "P2", Asserts: "sometimes cannot run", Kind: "x", Phase: 1, + run: func(context.Context, *doctor) ([]conditions.Observation, error) { + if broken.Load() { + return nil, fmt.Errorf("a question timed out") + } + return nil, nil + }}, + ) + store := conditions.NewInMemory() + k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store}) + defer k.Close(context.Background()) + d := &doctor{keeper: k, host: "anchor"} + keys := func() []string { + open, err := k.Open(t.Context()) + if err != nil { + t.Fatal(err) + } + var out []string + for _, c := range open { + out = append(out, c.Key) + } + sort.Strings(out) + return out + } + + unanswered.Store(true) + broken.Store(true) + run := d.runOnce(t.Context(), "a test") + if open := keys(); len(open) != 0 { + t.Fatalf("one look raised %v", open) + } + if run.Probes[0].Verdict != verdictPass || len(run.Probes[0].Unconfirmed) != 1 || run.Probes[1].Verdict != verdictFailedToRun { + t.Fatalf("the first run's verdict hides what it saw: %+v", run.Probes) + } + + // Seen twice in a row: raised, and so is the probe that could not run twice. + run = d.runOnce(t.Context(), "a test") + if open := keys(); strings.Join(open, " ") != "probe.P2.failed seat.mesh-dns-resolver.anchor.wrong" { + t.Fatalf("two looks in a row left open %v", open) + } + if run.Probes[0].Verdict != verdictFail { + t.Fatalf("%+v", run.Probes[0]) + } + + // Open, and seen again: kept, never cleared and raised again. + d.runOnce(t.Context(), "a test") + if c, open, _ := k.Get(t.Context(), "seat.mesh-dns-resolver.anchor.wrong"); !open || c.Count != 1 || c.Observations != 2 { + t.Fatalf("the open condition was not kept as it was: %+v", c) + } + + // Answered again: cleared. Then one look alone raises nothing again. + unanswered.Store(false) + broken.Store(false) + d.runOnce(t.Context(), "a test") + if open := keys(); len(open) != 0 { + t.Fatalf("a passing run left open %v", open) + } + unanswered.Store(true) + d.runOnce(t.Context(), "a test") + if open := keys(); len(open) != 0 { + t.Fatalf("one look after a pass raised %v", open) + } +} + +// **Every condition this suite raises says itself in machine names and words** (novox/hq issue 277, +// ADR 0234 §6). Every observation a keeper takes in any test of this package — every signals row +// suppressed past its bound, every probe's finding, every event's condition — is held to the operator +// channel's content rule; one whose summary or key carries an address, a domain, a path or a secret's +// shape fails the suite, naming its source. The keeper would say such a summary in words at run time; +// this is where the producer is made to say it rightly in the first place. +func TestMain(m *testing.M) { + // **A command line this suite runs ends when it is done** (novox/hq issue 591). Many tests run this test + // binary as a verb's command line, and a binary built with the race detector sleeps a second before it + // exits (GORACE's atexit_sleep_ms), which was more than the rest of each such test. The children still + // run under the race detector and still fail on a race they find; they only stop waiting at their end. + // Set before any child starts, and only where nobody chose GORACE. + if _, chosen := os.LookupEnv("GORACE"); !chosen { + _ = os.Setenv("GORACE", "atexit_sleep_ms=0") + } + // The process a mesh-cli test runs as a command line: it says the verb and the caller it was given, and + // ends (meshcli_test.go). + // The process a mesh-cli test runs as `secret accept`: it reads its value exactly as `secret accept` does + // (valueFor) and says whether it is the one the test gave, never the value (meshcli_stdin_test.go). + if want := os.Getenv(secretAcceptWants); want != "" { + os.Exit(readAsSecretAccept(want, os.Args[1:])) + } + if os.Getenv(echoEnvironment) != "" { + fmt.Printf("verb=%q caller=%q terminal=%v\n", os.Getenv("MESH_VERB"), os.Getenv("MESH_CALLER"), startedAtTheTerminal()) + if os.Getenv(echoFails) != "" { + os.Exit(3) + } + os.Exit(0) + } + conditions.Unsayable = func(o conditions.Observation, field string, r outward.Refusal) { + unsaid.note(o, field, r) + } + conditions.Unworded = func(o conditions.Observation, why string) { + if o.Source != "test" { // a kind a test makes up to exercise the keeper + unworded.add(o, why) + } + } + code := m.Run() + if said := unworded.all(); len(said) > 0 { + fmt.Fprintf(os.Stderr, "FAIL: %d condition(s) raised in these tests have no plain words of their own, or "+ + "words that are not plain (novox/hq ADR 0253: a headline, an explanation and a resolved line the operator "+ + "reads at a glance — plain_words.go):\n %s\n", len(said), strings.Join(said, "\n ")) + if code == 0 { + code = 1 + } + } + if said := unsaid.all(); len(said) > 0 { + fmt.Fprintf(os.Stderr, "FAIL: %d condition(s) raised in these tests say what the operator's channel withholds "+ + "(an address, a domain, a path or a secret's shape belongs in the evidence, not the summary):\n %s\n", + len(said), strings.Join(said, "\n ")) + if code == 0 { + code = 1 + } + } + os.Exit(code) +} diff --git a/cmd/mesh-controller/sayable_test.go b/cmd/mesh-controller/sayable_test.go index 8e92dda6..1d51afa6 100644 --- a/cmd/mesh-controller/sayable_test.go +++ b/cmd/mesh-controller/sayable_test.go @@ -1,10 +1,8 @@ package main import ( - "context" "fmt" "net" - "os" "sort" "strings" "sync" @@ -19,63 +17,6 @@ import ( "github.com/novox/mesh-controller/internal/outward" ) -// **Every condition this suite raises says itself in machine names and words** (novox/hq issue 277, -// ADR 0234 §6). Every observation a keeper takes in any test of this package — every signals row -// suppressed past its bound, every probe's finding, every event's condition — is held to the operator -// channel's content rule; one whose summary or key carries an address, a domain, a path or a secret's -// shape fails the suite, naming its source. The keeper would say such a summary in words at run time; -// this is where the producer is made to say it rightly in the first place. -func TestMain(m *testing.M) { - // **A command line this suite runs ends when it is done** (novox/hq issue 591). Many tests run this test - // binary as a verb's command line, and a binary built with the race detector sleeps a second before it - // exits (GORACE's atexit_sleep_ms), which was more than the rest of each such test. The children still - // run under the race detector and still fail on a race they find; they only stop waiting at their end. - // Set before any child starts, and only where nobody chose GORACE. - if _, chosen := os.LookupEnv("GORACE"); !chosen { - _ = os.Setenv("GORACE", "atexit_sleep_ms=0") - } - // The process a mesh-cli test runs as a command line: it says the verb and the caller it was given, and - // ends (meshcli_test.go). - // The process a mesh-cli test runs as `secret accept`: it reads its value exactly as `secret accept` does - // (valueFor) and says whether it is the one the test gave, never the value (meshcli_stdin_test.go). - if want := os.Getenv(secretAcceptWants); want != "" { - os.Exit(readAsSecretAccept(want, os.Args[1:])) - } - if os.Getenv(echoEnvironment) != "" { - fmt.Printf("verb=%q caller=%q terminal=%v\n", os.Getenv("MESH_VERB"), os.Getenv("MESH_CALLER"), startedAtTheTerminal()) - if os.Getenv(echoFails) != "" { - os.Exit(3) - } - os.Exit(0) - } - conditions.Unsayable = func(o conditions.Observation, field string, r outward.Refusal) { - unsaid.note(o, field, r) - } - conditions.Unworded = func(o conditions.Observation, why string) { - if o.Source != "test" { // a kind a test makes up to exercise the keeper - unworded.add(o, why) - } - } - code := m.Run() - if said := unworded.all(); len(said) > 0 { - fmt.Fprintf(os.Stderr, "FAIL: %d condition(s) raised in these tests have no plain words of their own, or "+ - "words that are not plain (novox/hq ADR 0253: a headline, an explanation and a resolved line the operator "+ - "reads at a glance — plain_words.go):\n %s\n", len(said), strings.Join(said, "\n ")) - if code == 0 { - code = 1 - } - } - if said := unsaid.all(); len(said) > 0 { - fmt.Fprintf(os.Stderr, "FAIL: %d condition(s) raised in these tests say what the operator's channel withholds "+ - "(an address, a domain, a path or a secret's shape belongs in the evidence, not the summary):\n %s\n", - len(said), strings.Join(said, "\n ")) - if code == 0 { - code = 1 - } - } - os.Exit(code) -} - // unsaid collects, across the suite, every finding whose words the operator's channel would withhold. var unsaid unsayable @@ -284,79 +225,3 @@ func TestAResolverAnsweringWronglyIsSaidAtOnceInMachineNames(t *testing.T) { // **The self-check raises a finding one look can be wrong about on the second run in a row**, keeps it // open while it is seen, and clears it when it is not — and a probe that cannot run is said as a // condition only when the next run cannot run it either. Neither is ever a pass in the verdict. -func TestASingleLookIsHeldAndTheSecondInARowRaises(t *testing.T) { - var unanswered, broken atomic.Bool - held := conditions.Observation{Scope: conditions.ScopeSeat, ID: "mesh-dns-resolver.anchor", Token: "wrong", - Machine: "anchor", Severity: conditions.Urgent, Confirm: true, - Summary: "the mesh's resolver on anchor does not answer", Said: "no answer from 10.77.0.1"} - withProbes(t, - probe{ID: "P1", Asserts: "asks over the network", Kind: "resolver-wrong", Phase: 1, - run: func(context.Context, *doctor) ([]conditions.Observation, error) { - if unanswered.Load() { - return []conditions.Observation{held}, nil - } - return nil, nil - }}, - probe{ID: "P2", Asserts: "sometimes cannot run", Kind: "x", Phase: 1, - run: func(context.Context, *doctor) ([]conditions.Observation, error) { - if broken.Load() { - return nil, fmt.Errorf("a question timed out") - } - return nil, nil - }}, - ) - store := conditions.NewInMemory() - k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store}) - defer k.Close(context.Background()) - d := &doctor{keeper: k, host: "anchor"} - keys := func() []string { - open, err := k.Open(t.Context()) - if err != nil { - t.Fatal(err) - } - var out []string - for _, c := range open { - out = append(out, c.Key) - } - sort.Strings(out) - return out - } - - unanswered.Store(true) - broken.Store(true) - run := d.runOnce(t.Context(), "a test") - if open := keys(); len(open) != 0 { - t.Fatalf("one look raised %v", open) - } - if run.Probes[0].Verdict != verdictPass || len(run.Probes[0].Unconfirmed) != 1 || run.Probes[1].Verdict != verdictFailedToRun { - t.Fatalf("the first run's verdict hides what it saw: %+v", run.Probes) - } - - // Seen twice in a row: raised, and so is the probe that could not run twice. - run = d.runOnce(t.Context(), "a test") - if open := keys(); strings.Join(open, " ") != "probe.P2.failed seat.mesh-dns-resolver.anchor.wrong" { - t.Fatalf("two looks in a row left open %v", open) - } - if run.Probes[0].Verdict != verdictFail { - t.Fatalf("%+v", run.Probes[0]) - } - - // Open, and seen again: kept, never cleared and raised again. - d.runOnce(t.Context(), "a test") - if c, open, _ := k.Get(t.Context(), "seat.mesh-dns-resolver.anchor.wrong"); !open || c.Count != 1 || c.Observations != 2 { - t.Fatalf("the open condition was not kept as it was: %+v", c) - } - - // Answered again: cleared. Then one look alone raises nothing again. - unanswered.Store(false) - broken.Store(false) - d.runOnce(t.Context(), "a test") - if open := keys(); len(open) != 0 { - t.Fatalf("a passing run left open %v", open) - } - unanswered.Store(true) - d.runOnce(t.Context(), "a test") - if open := keys(); len(open) != 0 { - t.Fatalf("one look after a pass raised %v", open) - } -} diff --git a/cmd/mesh-controller/seat_dependencies_integration_test.go b/cmd/mesh-controller/seat_dependencies_integration_test.go new file mode 100644 index 00000000..c54f7c41 --- /dev/null +++ b/cmd/mesh-controller/seat_dependencies_integration_test.go @@ -0,0 +1,119 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestAnAssignmentWithoutItsHolderIsRefusedAndNotKept(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, serviceManagerHolder()) + register(t, open, packageManagerHolder()) + register(t, open, aDaemon()) + + _, err := assign(ctx, open, "laptop", "sshd") + if err == nil { + t.Fatal("sshd went onto a machine nothing holds the service manager of") + } + for _, want := range []string{catalogue.ServiceManagerSeat, "systemd", "ADR 0207"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal does not say %q:\n%v", want, err) + } + } + assigned, err := open.inventory.Assigned(ctx, "laptop") + if err != nil { + t.Fatal(err) + } + if contains(assigned, "sshd") { + t.Fatalf("a refused assignment was kept: %v", assigned) + } + + // The holders depend on each other, so neither goes on alone — and both go on in one act. + if _, err := assign(ctx, open, "laptop", "systemd"); err == nil { + t.Fatal("systemd went on alone though its package needs a package manager") + } + if said, err := assign(ctx, open, "laptop", "systemd", "pacman"); err != nil { + t.Fatalf("the two holders assigned together were refused: %v\n%s", err, said) + } + if said, err := assign(ctx, open, "laptop", "sshd"); err != nil { + t.Fatalf("sshd beside its holder was refused: %v\n%s", err, said) + } +} + +func TestUnassigningTheLastHolderUnderItsDependentsIsRefused(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, serviceManagerHolder()) + register(t, open, packageManagerHolder()) + register(t, open, aDaemon()) + if _, err := assign(ctx, open, "laptop", "systemd", "pacman", "sshd"); err != nil { + t.Fatal(err) + } + + _, err := unassign(ctx, open, "laptop", "systemd") + if err == nil { + t.Fatal("the service manager came off a machine still running services") + } + for _, want := range []string{catalogue.ServiceManagerSeat, "sshd", "pacman"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal does not name %q:\n%v", want, err) + } + } + assigned, _ := open.inventory.Assigned(ctx, "laptop") + if !contains(assigned, "systemd") { + t.Fatalf("a refused unassignment took the module off anyway: %v", assigned) + } + if _, err := unassign(ctx, open, "laptop", "sshd"); err != nil { + t.Fatalf("a dependent could not come off: %v", err) + } +} + +func TestStatusReportsAnUnheldDependencyWithoutRefusingTheMachine(t *testing.T) { + // The mesh as it ran before the switch (novox/hq ADR 0207 §4). + defer catalogue.EnforcingSeatDependencies(false)() + open := aMesh(t) + ctx := t.Context() + register(t, open, serviceManagerHolder()) + register(t, open, aDaemon()) + // Assigned straight into the store: a machine whose modules predate the rule, which is every + // machine on the day it ships. + if _, err := open.inventory.Assign(ctx, "laptop", "sshd"); err != nil { + t.Fatal(err) + } + + asked, err := theThreeQuestions(ctx, open) + if err != nil { + t.Fatal(err) + } + if _, refused := asked.refused["laptop"]; refused { + t.Fatalf("an unmet dependency refused the machine before the switch: %s", asked.refused["laptop"]) + } + if asked.well() { + t.Error("a mesh with an unheld dependency reads as all well") + } + got := printed(t, func() error { return printStatus(asked) }) + for _, want := range []string{"unheld", "laptop", "sshd", catalogue.ServiceManagerSeat, "systemd"} { + if !strings.Contains(got, want) { + t.Errorf("status does not say %q:\n%s", want, got) + } + } + body, err := statusAsJSON(asked) + if err != nil { + t.Fatal(err) + } + var doc struct { + Unheld []catalogue.Unheld `json:"unheld"` + } + if err := json.Unmarshal(body, &doc); err != nil { + t.Fatal(err) + } + if len(doc.Unheld) != 1 || doc.Unheld[0].Module != "sshd" || doc.Unheld[0].Seat != catalogue.ServiceManagerSeat { + t.Errorf("the document's unheld is %+v", doc.Unheld) + } +} diff --git a/cmd/mesh-controller/seat_dependencies_test.go b/cmd/mesh-controller/seat_dependencies_test.go index 9d47f4af..2a61cf56 100644 --- a/cmd/mesh-controller/seat_dependencies_test.go +++ b/cmd/mesh-controller/seat_dependencies_test.go @@ -1,7 +1,6 @@ package main import ( - "encoding/json" "strings" "testing" @@ -31,42 +30,6 @@ func aDaemon() catalogue.Manifest { Resources: []map[string]any{{"id": "sshd", "type": "service", "unit": "sshd.service"}}} } -func TestAnAssignmentWithoutItsHolderIsRefusedAndNotKept(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, serviceManagerHolder()) - register(t, open, packageManagerHolder()) - register(t, open, aDaemon()) - - _, err := assign(ctx, open, "laptop", "sshd") - if err == nil { - t.Fatal("sshd went onto a machine nothing holds the service manager of") - } - for _, want := range []string{catalogue.ServiceManagerSeat, "systemd", "ADR 0207"} { - if !strings.Contains(err.Error(), want) { - t.Errorf("the refusal does not say %q:\n%v", want, err) - } - } - assigned, err := open.inventory.Assigned(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if contains(assigned, "sshd") { - t.Fatalf("a refused assignment was kept: %v", assigned) - } - - // The holders depend on each other, so neither goes on alone — and both go on in one act. - if _, err := assign(ctx, open, "laptop", "systemd"); err == nil { - t.Fatal("systemd went on alone though its package needs a package manager") - } - if said, err := assign(ctx, open, "laptop", "systemd", "pacman"); err != nil { - t.Fatalf("the two holders assigned together were refused: %v\n%s", err, said) - } - if said, err := assign(ctx, open, "laptop", "sshd"); err != nil { - t.Fatalf("sshd beside its holder was refused: %v\n%s", err, said) - } -} - func TestTheControllerSeatsAssignTakesSeveralModulesAsOneAct(t *testing.T) { argv, err := argvFor("assign", map[string]any{"node": "laptop", "module": "systemd, pacman"}) if err != nil { @@ -76,75 +39,3 @@ func TestTheControllerSeatsAssignTakesSeveralModulesAsOneAct(t *testing.T) { t.Errorf("the seat's assign became %v", argv) } } - -func TestUnassigningTheLastHolderUnderItsDependentsIsRefused(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, serviceManagerHolder()) - register(t, open, packageManagerHolder()) - register(t, open, aDaemon()) - if _, err := assign(ctx, open, "laptop", "systemd", "pacman", "sshd"); err != nil { - t.Fatal(err) - } - - _, err := unassign(ctx, open, "laptop", "systemd") - if err == nil { - t.Fatal("the service manager came off a machine still running services") - } - for _, want := range []string{catalogue.ServiceManagerSeat, "sshd", "pacman"} { - if !strings.Contains(err.Error(), want) { - t.Errorf("the refusal does not name %q:\n%v", want, err) - } - } - assigned, _ := open.inventory.Assigned(ctx, "laptop") - if !contains(assigned, "systemd") { - t.Fatalf("a refused unassignment took the module off anyway: %v", assigned) - } - if _, err := unassign(ctx, open, "laptop", "sshd"); err != nil { - t.Fatalf("a dependent could not come off: %v", err) - } -} - -func TestStatusReportsAnUnheldDependencyWithoutRefusingTheMachine(t *testing.T) { - // The mesh as it ran before the switch (novox/hq ADR 0207 §4). - defer catalogue.EnforcingSeatDependencies(false)() - open := aMesh(t) - ctx := t.Context() - register(t, open, serviceManagerHolder()) - register(t, open, aDaemon()) - // Assigned straight into the store: a machine whose modules predate the rule, which is every - // machine on the day it ships. - if _, err := open.inventory.Assign(ctx, "laptop", "sshd"); err != nil { - t.Fatal(err) - } - - asked, err := theThreeQuestions(ctx, open) - if err != nil { - t.Fatal(err) - } - if _, refused := asked.refused["laptop"]; refused { - t.Fatalf("an unmet dependency refused the machine before the switch: %s", asked.refused["laptop"]) - } - if asked.well() { - t.Error("a mesh with an unheld dependency reads as all well") - } - got := printed(t, func() error { return printStatus(asked) }) - for _, want := range []string{"unheld", "laptop", "sshd", catalogue.ServiceManagerSeat, "systemd"} { - if !strings.Contains(got, want) { - t.Errorf("status does not say %q:\n%s", want, got) - } - } - body, err := statusAsJSON(asked) - if err != nil { - t.Fatal(err) - } - var doc struct { - Unheld []catalogue.Unheld `json:"unheld"` - } - if err := json.Unmarshal(body, &doc); err != nil { - t.Fatal(err) - } - if len(doc.Unheld) != 1 || doc.Unheld[0].Module != "sshd" || doc.Unheld[0].Seat != catalogue.ServiceManagerSeat { - t.Errorf("the document's unheld is %+v", doc.Unheld) - } -} diff --git a/cmd/mesh-controller/seatverbs_integration_test.go b/cmd/mesh-controller/seatverbs_integration_test.go new file mode 100644 index 00000000..84e0dea8 --- /dev/null +++ b/cmd/mesh-controller/seatverbs_integration_test.go @@ -0,0 +1,164 @@ +//go:build integration + +package main + +import ( + "context" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/link" +) + +func TestToolsAnswersTheSeatsRecords(t *testing.T) { + handlers, behind, err := seatToolHandlers() + if err != nil { + t.Fatal(err) + } + if len(behind) != 0 { + t.Fatalf("this build cannot run %v of its own seat's verbs", behind) + } + if len(handlers) != len(catalogue.ControllerVerbs) { + t.Fatalf("%d handlers for %d verbs", len(handlers), len(catalogue.ControllerVerbs)) + } + answer := seatTools() + seats, _ := answer["seats"].([]map[string]any) + var found bool + for _, s := range seats { + if s["seat"] == catalogue.ControllerSeatName { + found = true + tools, _ := s["tools"].([]map[string]any) + if len(tools) != len(catalogue.ControllerVerbs) || tools[0]["input"] == nil { + t.Fatalf("the controller seat's tools are not listed in full: %v", tools) + } + } + } + if !found { + t.Fatal("the mesh-controller seat is not in the listing") + } + // And what a verb replaces travels with it (novox/hq ADR 0245): the console and the agent's + // instructions read it from here. + var journal []string + for _, s := range seats { + if s["seat"] != catalogue.ServiceManagerSeat { + continue + } + tools, _ := s["tools"].([]map[string]any) + for _, tool := range tools { + if tool["name"] == "journal" { + journal, _ = tool["replaces"].([]string) + } + } + } + if len(journal) == 0 || journal[0] != "journalctl" { + t.Fatalf("the service manager's journal does not say it replaces journalctl: %v", journal) + } +} + +func TestAJSONVerbsAnswerIsItsStandardOutput(t *testing.T) { + jsonVerbs["-test.run"] = true + t.Cleanup(func() { delete(jsonVerbs, "-test.run") }) + answer, err := runVerb(t.Context(), []string{"-test.run", "TestAnswerEcho", "-test.v"}) + if err != nil { + t.Fatal(err) + } + if !answer.OK { + t.Fatalf("the command failed: %s", answer.Output) + } + if !strings.Contains(answer.Output, "PASS") { + t.Fatalf("stderr and stdout are both what the command said: %s", answer.Output) + } +} + +func TestARowAheadOfThisBuildIsServedAnyway(t *testing.T) { + seat, known := catalogue.SeatNamed(catalogue.ControllerSeatName) + if !known { + t.Fatal("no controller seat") + } + // The row as a newer control plane would have written it: every verb this build knows, and one + // it does not. + widened := seat + widened.Serves = append(append([]catalogue.Verb{}, seat.Serves...), + catalogue.Verb{Name: "teleport", Description: "a verb from a build that does not exist yet"}) + rows := catalogue.DefaultSeats() + for i := range rows { + if rows[i].Name == catalogue.ControllerSeatName { + rows[i] = widened + } + } + catalogue.UseSeats(rows) + t.Cleanup(func() { catalogue.UseSeats(catalogue.DefaultSeats()) }) + + handlers, behind, err := seatToolHandlers() + if err != nil { + t.Fatalf("a row with one unknown verb refused to serve at all: %v", err) + } + if len(behind) != 1 || behind[0] != "teleport" { + t.Fatalf("the verbs this build cannot run were reported as %v", behind) + } + if len(handlers) != len(widened.Serves) { + t.Fatalf("%d handlers for %d verbs in the row", len(handlers), len(widened.Serves)) + } + for _, known := range []string{"status", "nodes", "push"} { + if handlers[known] == nil { + t.Errorf("%s is not served although this build knows it", known) + } + } + _, err = handlers["teleport"](context.Background(), nil) + if err == nil { + t.Fatal("the unknown verb answered as though it had run") + } + for _, want := range []string{"teleport", "cannot run it", "behind"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the answer does not say %q: %v", want, err) + } + } +} + +func TestTheControllerAnnouncesTheVerbsItServes(t *testing.T) { + handlers, _, err := seatToolHandlers() + if err != nil { + t.Fatal(err) + } + info := seatAnnouncement(handlers) + if info.Name != catalogue.ControllerSeatName || info.ID == "" || info.Version == "" { + t.Fatalf("the service is not named for the seat: %+v", info.ServiceIdentity) + } + if len(info.Endpoints) != len(handlers) { + t.Fatalf("%d endpoints announced for %d verbs served", len(info.Endpoints), len(handlers)) + } + for _, e := range info.Endpoints { + verb := e.Metadata["tool"] + if _, served := handlers[verb]; !served || e.Name != catalogue.ControllerSeatName+"__"+verb { + t.Errorf("%s (%s) is announced and not served under that name", e.Name, verb) + } + if e.Metadata["kind"] != "seat" || e.Metadata["seat"] != catalogue.ControllerSeatName { + t.Errorf("%s is not announced as the seat's verb: %v", e.Name, e.Metadata) + } + if e.Subject != link.SeatToolSubject(catalogue.ControllerSeatName, verb) || e.QueueGroup != "seat."+catalogue.ControllerSeatName { + t.Errorf("%s is announced on %s/%s, not where it is served", e.Name, e.Subject, e.QueueGroup) + } + if e.Metadata["description"] == "" || e.Metadata["schema"] == "" || e.Metadata["scope"] != "mesh" { + t.Errorf("%s is announced without its description, schema or scope: %v", e.Name, e.Metadata) + } + } +} + +func TestNoVerbSetsTheOperatorsKeyOrRevealsASecret(t *testing.T) { + for _, argv := range [][]string{ + {"operator", "key", "set"}, {"operator", "issue"}, {"identity"}, {"token", "issue"}, {"broker", "users"}, + {"api"}, {"licence"}, {"secret", "export", "x"}, {"secret", "recover", "x"}, {"secret", "accept", "x"}, {"secret"}, + } { + if err := terminalOnly(argv); err == nil { + t.Errorf("%v passed", argv) + } + } + if err := terminalOnly([]string{"secret", "rotate", "n", "m", "s"}); err != nil { + t.Errorf("rotating seals to the machine that uses the secret, and stays a verb's: %v", err) + } + // A policy refusal is not a verb this binary is behind on: every verb is still served. + if _, behind, err := seatToolHandlers(); err != nil || len(behind) != 0 { + t.Fatalf("behind %v: %v", behind, err) + } +} diff --git a/cmd/mesh-controller/seatverbs_schema_integration_test.go b/cmd/mesh-controller/seatverbs_schema_integration_test.go new file mode 100644 index 00000000..88eb3882 --- /dev/null +++ b/cmd/mesh-controller/seatverbs_schema_integration_test.go @@ -0,0 +1,185 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "reflect" + "sort" + "strconv" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func servedSchemas(t *testing.T) map[string]catalogue.Verb { + t.Helper() + handlers, behind, err := seatToolHandlers() + if err != nil { + t.Fatal(err) + } + if len(behind) != 0 { + t.Fatalf("this build cannot run %v of its own seat's verbs", behind) + } + served := map[string]catalogue.Verb{} + for _, e := range seatAnnouncement(handlers).Endpoints { + var input map[string]any + if err := json.Unmarshal([]byte(e.Metadata["schema"]), &input); err != nil { + t.Fatalf("%s announces a schema that is not JSON: %v", e.Name, err) + } + served[e.Metadata["tool"]] = catalogue.Verb{Name: e.Metadata["tool"], Input: input} + } + if len(served) != len(catalogue.ControllerVerbs) { + t.Fatalf("%d verbs served for %d in the table", len(served), len(catalogue.ControllerVerbs)) + } + return served +} + +func TestNoArgumentAVerbIsGivenIsPassedOver(t *testing.T) { + for name, v := range servedSchemas(t) { + if inProcess[name] { + continue + } + names, switches := declaredArguments(v) + for _, subset := range subsetsOf(names) { + args := argumentsFor(subset, switches) + argv, err := argvFor(name, args) + if err != nil { + if strings.Contains(err.Error(), "does not declare") { + t.Errorf("%s %v: %v", name, args, err) + } + continue + } + for _, dropped := range subset { + fewer := map[string]any{} + for k, val := range args { + if k != dropped { + fewer[k] = val + } + } + without, err := argvFor(name, fewer) + if err == nil && reflect.DeepEqual(argv, without) && saidOutright[name] != dropped { + t.Errorf("%s ignores %q given with %v: %v either way", name, dropped, subset, argv) + } + } + } + } +} + +func TestAnArgumentAVerbDoesNotDeclareIsRefused(t *testing.T) { + for name, v := range servedSchemas(t) { + names, _ := declaredArguments(v) + strangers := []string{"no-such-argument"} + if !contains(names, "node") { + strangers = append(strangers, "node") + } + for _, stranger := range strangers { + args := sampleArguments(v) + args[stranger] = "x" + _, err := argvFor(name, args) + if inProcess[name] { + _, err = readArguments(name, args) + } + if err == nil || !strings.Contains(err.Error(), strconv.Quote(stranger)) { + t.Errorf("%s took %q, which it does not declare: %v", name, stranger, err) + } + } + } + if _, err := argvFor("clear", map[string]any{"dead": "yes"}); err == nil { + t.Error("a switch took a word that is neither true nor false, and would have read it as false") + } + if _, err := argvFor("node", map[string]any{"node": 7}); err == nil { + t.Error("a number was taken as a machine's name, or as no machine") + } + if argv, err := argvFor("clear", map[string]any{"dead": true}); err != nil || strings.Join(argv, " ") != "clear --dead" { + t.Errorf("a switch given as JSON true: %v %v", argv, err) + } +} + +func TestEveryFlagOfAVerbsCommandIsAnArgumentOrAccountedFor(t *testing.T) { + flags := commandFlags(t) + if len(flags["push"]) == 0 || len(flags["plan"]) == 0 { + t.Fatalf("reading the commands' flags found nothing for push or plan: %v", flags) + } + reached := map[string]map[string]bool{} // verb → flag sets its command lines reach + served := servedSchemas(t) + for name, v := range served { + if inProcess[name] || name == "command" { + continue + } + names, switches := declaredArguments(v) + for _, subset := range subsetsOf(names) { + argv, err := argvFor(name, argumentsFor(subset, switches)) + if err != nil { + continue + } + // The flag set is named by the longest run of leading words that names one. + var words []string + for _, w := range argv { + if strings.HasPrefix(w, "-") { + break + } + words = append(words, w) + } + for n := len(words); n > 0; n-- { + if _, has := flags[strings.Join(words[:n], " ")]; has { + if reached[name] == nil { + reached[name] = map[string]bool{} + } + reached[name][strings.Join(words[:n], " ")] = true + break + } + } + } + } + used := map[string]map[string]bool{} + verbs := make([]string, 0, len(reached)) + for verb := range reached { + verbs = append(verbs, verb) + } + sort.Strings(verbs) + for _, verb := range verbs { + declared, _ := declaredArguments(served[verb]) + for set := range reached[verb] { + if used[set] == nil { + used[set] = map[string]bool{} + } + for _, flagName := range flags[set] { + why, accounted := accountedFlags[set][flagName] + switch { + case accounted && strings.HasPrefix(why, "="): + used[set][flagName] = true + if !contains(declared, strings.TrimPrefix(why, "=")) { + t.Errorf("%s: --%s of `%s` is said to be carried by %q, which the schema does not declare", + verb, flagName, set, strings.TrimPrefix(why, "=")) + } + case accounted: + used[set][flagName] = true + case contains(declared, flagName): + default: + t.Errorf("%s runs `%s`, which takes --%s, and the verb's schema has no %q: declare it, "+ + "or say in accountedFlags why a caller does not reach it", verb, set, flagName, flagName) + } + } + } + } + for set, fs := range accountedFlags { + for flagName := range fs { + if !used[set][flagName] { + t.Errorf("accountedFlags names --%s of `%s`, which no verb's command takes any more", flagName, set) + } + } + } +} + +func TestEveryRequiredArgumentIsDescribed(t *testing.T) { + for name, v := range servedSchemas(t) { + names, _ := declaredArguments(v) + for k := range sampleArguments(v) { + if !contains(names, k) { + t.Errorf("%s requires %q and does not describe it", name, k) + } + } + } +} diff --git a/cmd/mesh-controller/seatverbs_schema_test.go b/cmd/mesh-controller/seatverbs_schema_test.go index a5c489dd..a14f1041 100644 --- a/cmd/mesh-controller/seatverbs_schema_test.go +++ b/cmd/mesh-controller/seatverbs_schema_test.go @@ -1,44 +1,17 @@ package main import ( - "encoding/json" "go/ast" "go/parser" "go/token" "path/filepath" - "reflect" - "sort" "strconv" "strings" "testing" - - "github.com/novox/mesh-controller/internal/catalogue" ) // The schemas the controller serves, verb by verb, as the console receives them: from the // announcement, not the table — what is checked is what a caller is shown (novox/hq issue 244). -func servedSchemas(t *testing.T) map[string]catalogue.Verb { - t.Helper() - handlers, behind, err := seatToolHandlers() - if err != nil { - t.Fatal(err) - } - if len(behind) != 0 { - t.Fatalf("this build cannot run %v of its own seat's verbs", behind) - } - served := map[string]catalogue.Verb{} - for _, e := range seatAnnouncement(handlers).Endpoints { - var input map[string]any - if err := json.Unmarshal([]byte(e.Metadata["schema"]), &input); err != nil { - t.Fatalf("%s announces a schema that is not JSON: %v", e.Name, err) - } - served[e.Metadata["tool"]] = catalogue.Verb{Name: e.Metadata["tool"], Input: input} - } - if len(served) != len(catalogue.ControllerVerbs) { - t.Fatalf("%d verbs served for %d in the table", len(served), len(catalogue.ControllerVerbs)) - } - return served -} // subsetsOf is every subset of the names, the empty one included. func subsetsOf(names []string) [][]string { @@ -78,69 +51,10 @@ var saidOutright = map[string]string{ // composes a command line that each given argument changed: taking any one away changes the line // or makes it refused. An argument the line is the same without is one the verb ignored — the // shape of the push that named a machine and pushed every machine behind. -func TestNoArgumentAVerbIsGivenIsPassedOver(t *testing.T) { - for name, v := range servedSchemas(t) { - if inProcess[name] { - continue - } - names, switches := declaredArguments(v) - for _, subset := range subsetsOf(names) { - args := argumentsFor(subset, switches) - argv, err := argvFor(name, args) - if err != nil { - if strings.Contains(err.Error(), "does not declare") { - t.Errorf("%s %v: %v", name, args, err) - } - continue - } - for _, dropped := range subset { - fewer := map[string]any{} - for k, val := range args { - if k != dropped { - fewer[k] = val - } - } - without, err := argvFor(name, fewer) - if err == nil && reflect.DeepEqual(argv, without) && saidOutright[name] != dropped { - t.Errorf("%s ignores %q given with %v: %v either way", name, dropped, subset, argv) - } - } - } - } -} // **An argument a verb does not declare is refused, naming it — never dropped.** Every verb, with // what it requires and one argument more; and `node` in particular, for every verb whose schema // does not take a machine. -func TestAnArgumentAVerbDoesNotDeclareIsRefused(t *testing.T) { - for name, v := range servedSchemas(t) { - names, _ := declaredArguments(v) - strangers := []string{"no-such-argument"} - if !contains(names, "node") { - strangers = append(strangers, "node") - } - for _, stranger := range strangers { - args := sampleArguments(v) - args[stranger] = "x" - _, err := argvFor(name, args) - if inProcess[name] { - _, err = readArguments(name, args) - } - if err == nil || !strings.Contains(err.Error(), strconv.Quote(stranger)) { - t.Errorf("%s took %q, which it does not declare: %v", name, stranger, err) - } - } - } - if _, err := argvFor("clear", map[string]any{"dead": "yes"}); err == nil { - t.Error("a switch took a word that is neither true nor false, and would have read it as false") - } - if _, err := argvFor("node", map[string]any{"node": 7}); err == nil { - t.Error("a number was taken as a machine's name, or as no machine") - } - if argv, err := argvFor("clear", map[string]any{"dead": true}); err != nil || strings.Join(argv, " ") != "clear --dead" { - t.Errorf("a switch given as JSON true: %v %v", argv, err) - } -} // The push that was the cause: a machine named is that machine; none named is the whole mesh, and // naming one beside behind is refused rather than one of the two guessed. @@ -297,91 +211,6 @@ var accountedFlags = map[string]map[string]string{ // fails this until somebody decides, in writing, how a caller reaches it (novox/hq issue 244). And // a flag accounted for that no longer exists fails too, so the table cannot rot into a list nobody // reads. -func TestEveryFlagOfAVerbsCommandIsAnArgumentOrAccountedFor(t *testing.T) { - flags := commandFlags(t) - if len(flags["push"]) == 0 || len(flags["plan"]) == 0 { - t.Fatalf("reading the commands' flags found nothing for push or plan: %v", flags) - } - reached := map[string]map[string]bool{} // verb → flag sets its command lines reach - served := servedSchemas(t) - for name, v := range served { - if inProcess[name] || name == "command" { - continue - } - names, switches := declaredArguments(v) - for _, subset := range subsetsOf(names) { - argv, err := argvFor(name, argumentsFor(subset, switches)) - if err != nil { - continue - } - // The flag set is named by the longest run of leading words that names one. - var words []string - for _, w := range argv { - if strings.HasPrefix(w, "-") { - break - } - words = append(words, w) - } - for n := len(words); n > 0; n-- { - if _, has := flags[strings.Join(words[:n], " ")]; has { - if reached[name] == nil { - reached[name] = map[string]bool{} - } - reached[name][strings.Join(words[:n], " ")] = true - break - } - } - } - } - used := map[string]map[string]bool{} - verbs := make([]string, 0, len(reached)) - for verb := range reached { - verbs = append(verbs, verb) - } - sort.Strings(verbs) - for _, verb := range verbs { - declared, _ := declaredArguments(served[verb]) - for set := range reached[verb] { - if used[set] == nil { - used[set] = map[string]bool{} - } - for _, flagName := range flags[set] { - why, accounted := accountedFlags[set][flagName] - switch { - case accounted && strings.HasPrefix(why, "="): - used[set][flagName] = true - if !contains(declared, strings.TrimPrefix(why, "=")) { - t.Errorf("%s: --%s of `%s` is said to be carried by %q, which the schema does not declare", - verb, flagName, set, strings.TrimPrefix(why, "=")) - } - case accounted: - used[set][flagName] = true - case contains(declared, flagName): - default: - t.Errorf("%s runs `%s`, which takes --%s, and the verb's schema has no %q: declare it, "+ - "or say in accountedFlags why a caller does not reach it", verb, set, flagName, flagName) - } - } - } - } - for set, fs := range accountedFlags { - for flagName := range fs { - if !used[set][flagName] { - t.Errorf("accountedFlags names --%s of `%s`, which no verb's command takes any more", flagName, set) - } - } - } -} // Every verb's required arguments are properties of its schema: a schema that requires what it // does not describe is the uncallable verb of issue 244 from the other side. -func TestEveryRequiredArgumentIsDescribed(t *testing.T) { - for name, v := range servedSchemas(t) { - names, _ := declaredArguments(v) - for k := range sampleArguments(v) { - if !contains(names, k) { - t.Errorf("%s requires %q and does not describe it", name, k) - } - } - } -} diff --git a/cmd/mesh-controller/seatverbs_test.go b/cmd/mesh-controller/seatverbs_test.go index c6bf7120..361fdc49 100644 --- a/cmd/mesh-controller/seatverbs_test.go +++ b/cmd/mesh-controller/seatverbs_test.go @@ -1,16 +1,12 @@ package main import ( - "context" "errors" "fmt" - "github.com/novox/mesh-controller/internal/link" "os" "path/filepath" "strings" "testing" - - "github.com/novox/mesh-controller/internal/catalogue" ) // `builds` given a build's id reads that build's log from the bus rather than listing builds @@ -182,68 +178,10 @@ func TestActsDoNotBlockTheCall(t *testing.T) { } // What `tools` answers is the seats' records, with each verb's schema. -func TestToolsAnswersTheSeatsRecords(t *testing.T) { - handlers, behind, err := seatToolHandlers() - if err != nil { - t.Fatal(err) - } - if len(behind) != 0 { - t.Fatalf("this build cannot run %v of its own seat's verbs", behind) - } - if len(handlers) != len(catalogue.ControllerVerbs) { - t.Fatalf("%d handlers for %d verbs", len(handlers), len(catalogue.ControllerVerbs)) - } - answer := seatTools() - seats, _ := answer["seats"].([]map[string]any) - var found bool - for _, s := range seats { - if s["seat"] == catalogue.ControllerSeatName { - found = true - tools, _ := s["tools"].([]map[string]any) - if len(tools) != len(catalogue.ControllerVerbs) || tools[0]["input"] == nil { - t.Fatalf("the controller seat's tools are not listed in full: %v", tools) - } - } - } - if !found { - t.Fatal("the mesh-controller seat is not in the listing") - } - // And what a verb replaces travels with it (novox/hq ADR 0245): the console and the agent's - // instructions read it from here. - var journal []string - for _, s := range seats { - if s["seat"] != catalogue.ServiceManagerSeat { - continue - } - tools, _ := s["tools"].([]map[string]any) - for _, tool := range tools { - if tool["name"] == "journal" { - journal, _ = tool["replaces"].([]string) - } - } - } - if len(journal) == 0 || journal[0] != "journalctl" { - t.Fatalf("the service manager's journal does not say it replaces journalctl: %v", journal) - } -} // A JSON verb's answer is parsed from what the command wrote to standard output alone; a warning it // printed beside the document does not take the document away. The test binary stands in for the // controller: `-test.run` with a name that matches nothing prints `ok` and a warning about no tests. -func TestAJSONVerbsAnswerIsItsStandardOutput(t *testing.T) { - jsonVerbs["-test.run"] = true - t.Cleanup(func() { delete(jsonVerbs, "-test.run") }) - answer, err := runVerb(t.Context(), []string{"-test.run", "TestAnswerEcho", "-test.v"}) - if err != nil { - t.Fatal(err) - } - if !answer.OK { - t.Fatalf("the command failed: %s", answer.Output) - } - if !strings.Contains(answer.Output, "PASS") { - t.Fatalf("stderr and stdout are both what the command said: %s", answer.Output) - } -} // `command` is the generic verb: the command line as given, split as a shell would, nothing added // (novox/hq ADR 0154, ADR 0175). It once carried an operator's `node account g14 jochen` too; a node's @@ -274,50 +212,6 @@ func TestCommandRunsTheLineAsGiven(t *testing.T) { // 0185). One unknown word cost the mesh ten minutes of silence on 2026-10-02, recoverable only by // a person running the binary by hand — the push that would have repaired it needs the control // plane that was down. -func TestARowAheadOfThisBuildIsServedAnyway(t *testing.T) { - seat, known := catalogue.SeatNamed(catalogue.ControllerSeatName) - if !known { - t.Fatal("no controller seat") - } - // The row as a newer control plane would have written it: every verb this build knows, and one - // it does not. - widened := seat - widened.Serves = append(append([]catalogue.Verb{}, seat.Serves...), - catalogue.Verb{Name: "teleport", Description: "a verb from a build that does not exist yet"}) - rows := catalogue.DefaultSeats() - for i := range rows { - if rows[i].Name == catalogue.ControllerSeatName { - rows[i] = widened - } - } - catalogue.UseSeats(rows) - t.Cleanup(func() { catalogue.UseSeats(catalogue.DefaultSeats()) }) - - handlers, behind, err := seatToolHandlers() - if err != nil { - t.Fatalf("a row with one unknown verb refused to serve at all: %v", err) - } - if len(behind) != 1 || behind[0] != "teleport" { - t.Fatalf("the verbs this build cannot run were reported as %v", behind) - } - if len(handlers) != len(widened.Serves) { - t.Fatalf("%d handlers for %d verbs in the row", len(handlers), len(widened.Serves)) - } - for _, known := range []string{"status", "nodes", "push"} { - if handlers[known] == nil { - t.Errorf("%s is not served although this build knows it", known) - } - } - _, err = handlers["teleport"](context.Background(), nil) - if err == nil { - t.Fatal("the unknown verb answered as though it had run") - } - for _, want := range []string{"teleport", "cannot run it", "behind"} { - if !strings.Contains(err.Error(), want) { - t.Errorf("the answer does not say %q: %v", want, err) - } - } -} // novox/hq ADR 0195: the console's discovery reads the machines and the modules; they answer as JSON, // as status and seats do, so nothing parses a printed column. @@ -335,34 +229,6 @@ func TestTheNodesAndModulesVerbsAnswerAsJSON(t *testing.T) { // novox/hq ADR 0197: the controller announces exactly the verbs it serves, each on the subject and // queue it serves it on, with the seat's own description and schema, in NATS's services format. -func TestTheControllerAnnouncesTheVerbsItServes(t *testing.T) { - handlers, _, err := seatToolHandlers() - if err != nil { - t.Fatal(err) - } - info := seatAnnouncement(handlers) - if info.Name != catalogue.ControllerSeatName || info.ID == "" || info.Version == "" { - t.Fatalf("the service is not named for the seat: %+v", info.ServiceIdentity) - } - if len(info.Endpoints) != len(handlers) { - t.Fatalf("%d endpoints announced for %d verbs served", len(info.Endpoints), len(handlers)) - } - for _, e := range info.Endpoints { - verb := e.Metadata["tool"] - if _, served := handlers[verb]; !served || e.Name != catalogue.ControllerSeatName+"__"+verb { - t.Errorf("%s (%s) is announced and not served under that name", e.Name, verb) - } - if e.Metadata["kind"] != "seat" || e.Metadata["seat"] != catalogue.ControllerSeatName { - t.Errorf("%s is not announced as the seat's verb: %v", e.Name, e.Metadata) - } - if e.Subject != link.SeatToolSubject(catalogue.ControllerSeatName, verb) || e.QueueGroup != "seat."+catalogue.ControllerSeatName { - t.Errorf("%s is announced on %s/%s, not where it is served", e.Name, e.Subject, e.QueueGroup) - } - if e.Metadata["description"] == "" || e.Metadata["schema"] == "" || e.Metadata["scope"] != "mesh" { - t.Errorf("%s is announced without its description, schema or scope: %v", e.Name, e.Metadata) - } - } -} // The generic verb only reads (novox/hq ADR 0266): an allow list of reading forms, and every line that // writes, issues, sets a key or reveals a secret refused — the chain a review found ran through it: set the @@ -402,23 +268,6 @@ func TestTheCommandVerbOnlyReads(t *testing.T) { } // What hands a caller a key, a credential or a secret is refused whichever verb composed it. -func TestNoVerbSetsTheOperatorsKeyOrRevealsASecret(t *testing.T) { - for _, argv := range [][]string{ - {"operator", "key", "set"}, {"operator", "issue"}, {"identity"}, {"token", "issue"}, {"broker", "users"}, - {"api"}, {"licence"}, {"secret", "export", "x"}, {"secret", "recover", "x"}, {"secret", "accept", "x"}, {"secret"}, - } { - if err := terminalOnly(argv); err == nil { - t.Errorf("%v passed", argv) - } - } - if err := terminalOnly([]string{"secret", "rotate", "n", "m", "s"}); err != nil { - t.Errorf("rotating seals to the machine that uses the secret, and stays a verb's: %v", err) - } - // A policy refusal is not a verb this binary is behind on: every verb is still served. - if _, behind, err := seatToolHandlers(); err != nil || len(behind) != 0 { - t.Fatalf("behind %v: %v", behind, err) - } -} // A value with a quote in it can be said on a `command` line, as in a shell, and a line whose quote // is left open is refused naming where it opened and how a quote is written, quoting none of it (novox/hq issue 294: an diff --git a/cmd/mesh-controller/secret_integration_test.go b/cmd/mesh-controller/secret_integration_test.go new file mode 100644 index 00000000..b3595be5 --- /dev/null +++ b/cmd/mesh-controller/secret_integration_test.go @@ -0,0 +1,33 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" +) + +func TestTheArgumentsAreRequired(t *testing.T) { + for _, args := range [][]string{ + {}, + {"accept"}, + {"accept", "anchor"}, + {"accept", "anchor", "umami"}, + {"give", "anchor", "umami", "database"}, + } { + if err := secretCommand(t.Context(), args); err == nil { + t.Errorf("%v was accepted", args) + } + } +} + +func TestTheCommandItselfSeesTheFlag(t *testing.T) { + err := secretCommand(t.Context(), + []string{"accept", "anchor", "umami", "database", "--from", "/nonexistent/nowhere"}) + if err == nil { + t.Fatal("a missing file was accepted") + } + if strings.Contains(err.Error(), "secret accept ") { + t.Fatalf("the command did not see its flag and complained about usage instead: %v", err) + } +} diff --git a/cmd/mesh-controller/secret_test.go b/cmd/mesh-controller/secret_test.go index 7753cf94..f90ec4e1 100644 --- a/cmd/mesh-controller/secret_test.go +++ b/cmd/mesh-controller/secret_test.go @@ -3,7 +3,6 @@ package main import ( "os" "path/filepath" - "strings" "testing" ) @@ -55,19 +54,6 @@ func TestAMissingFileIsRefused(t *testing.T) { } // The command refuses what it cannot act on, rather than acting on part of it. -func TestTheArgumentsAreRequired(t *testing.T) { - for _, args := range [][]string{ - {}, - {"accept"}, - {"accept", "anchor"}, - {"accept", "anchor", "umami"}, - {"give", "anchor", "umami", "database"}, - } { - if err := secretCommand(t.Context(), args); err == nil { - t.Errorf("%v was accepted", args) - } - } -} // The invocation that actually gets used, which the first version of these tests never tried. // @@ -92,16 +78,6 @@ func TestTheArgumentsAndTheFlagAreBothSeen(t *testing.T) { // Testing `split` alone left the command able to ignore it entirely — removing the call changed // no test. This reaches secretCommand: with a `--from` naming a file that is not there, the // complaint must be about the file. A complaint about usage would mean the flag was never seen. -func TestTheCommandItselfSeesTheFlag(t *testing.T) { - err := secretCommand(t.Context(), - []string{"accept", "anchor", "umami", "database", "--from", "/nonexistent/nowhere"}) - if err == nil { - t.Fatal("a missing file was accepted") - } - if strings.Contains(err.Error(), "secret accept ") { - t.Fatalf("the command did not see its flag and complained about usage instead: %v", err) - } -} func assert(t *testing.T, ok bool, format string, args ...any) { t.Helper() diff --git a/cmd/mesh-controller/selfexec_integration_test.go b/cmd/mesh-controller/selfexec_integration_test.go new file mode 100644 index 00000000..6b74d0d6 --- /dev/null +++ b/cmd/mesh-controller/selfexec_integration_test.go @@ -0,0 +1,138 @@ +//go:build integration + +package main + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/link" +) + +func TestSelfExecServer(t *testing.T) { + if os.Getenv(selfExecRole) != "server" { + t.Skip("run by TestAVerbRunsWhileItsBuildIsMovedOrDeleted") + } + line, _ := bufio.NewReader(os.Stdin).ReadString('\n') + if strings.TrimSpace(line) != "go" { + t.Fatalf("told %q", line) + } + if err := os.Setenv(selfExecRole, "verb"); err != nil { + t.Fatal(err) + } + answer, err := runVerb(t.Context(), []string{"-test.run", "^TestSelfExecVerb$", "-test.v"}) + said := map[string]any{"ok": answer.OK, "output": answer.Output} + if err != nil { + said["error"] = err.Error() + } + body, _ := json.Marshal(said) + fmt.Println("ANSWER " + string(body)) +} + +func TestAVerbRunsWhileItsBuildIsMovedOrDeleted(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("the image is named through /proc on Linux only") + } + self, err := os.Executable() + if err != nil { + t.Fatal(err) + } + for _, how := range []string{"moved into a closed directory", "deleted"} { + t.Run(how, func(t *testing.T) { + root := t.TempDir() + placed := filepath.Join(root, "mesh-controller", "mesh-controller") + if err := os.MkdirAll(filepath.Dir(placed), 0o755); err != nil { + t.Fatal(err) + } + copyFile(t, self, placed) + + server := exec.Command(placed, "-test.run", "^TestSelfExecServer$", "-test.v") + server.Env = append(os.Environ(), selfExecRole+"=server") + stdin, err := server.StdinPipe() + if err != nil { + t.Fatal(err) + } + stdout, err := server.StdoutPipe() + if err != nil { + t.Fatal(err) + } + server.Stderr = os.Stderr + if err := server.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = server.Process.Kill(); _ = server.Wait() }) + + // What the witness does to a running build, once the process runs. + switch how { + case "deleted": + if err := os.RemoveAll(filepath.Dir(placed)); err != nil { + t.Fatal(err) + } + default: + kept := filepath.Join(root, ".witness", "mesh-controller", "previous") + if err := os.MkdirAll(filepath.Dir(kept), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(filepath.Dir(placed), kept); err != nil { + t.Fatal(err) + } + if err := os.Chmod(filepath.Join(root, ".witness"), 0); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(filepath.Join(root, ".witness"), 0o700) }) + } + if _, err := io.WriteString(stdin, "go\n"); err != nil { + t.Fatal(err) + } + out, _ := io.ReadAll(stdout) + _ = server.Wait() + var said struct { + OK bool `json:"ok"` + Output string `json:"output"` + Error string `json:"error"` + } + found := false + for _, line := range strings.Split(string(out), "\n") { + if rest, ok := strings.CutPrefix(line, "ANSWER "); ok { + found = json.Unmarshal([]byte(rest), &said) == nil + } + } + if !found { + t.Fatalf("the controller standing in answered nothing:\n%s", out) + } + if said.Error != "" || !said.OK || !strings.Contains(said.Output, "the verb ran") { + t.Fatalf("the verb did not run from the controller's own image after its build was %s: %+v", how, said) + } + }) + } +} + +func TestAVerbThatCannotRunIsRefusedAsAHandover(t *testing.T) { + closed := filepath.Join(t.TempDir(), "closed") + if err := os.MkdirAll(closed, 0o700); err != nil { + t.Fatal(err) + } + was := ownImage + ownImage = func() string { return filepath.Join(closed, "gone", "mesh-controller") } + t.Cleanup(func() { ownImage = was }) + _, err := runVerb(t.Context(), []string{"status"}) + if !errors.Is(err, link.ErrHandingOver) { + t.Fatalf("a build that is not there was answered %v, not a handover", err) + } + + ownImage = was + handingOver.Store(true) + t.Cleanup(func() { handingOver.Store(false) }) + if _, err := runVerb(t.Context(), []string{"-test.run", "^$"}); !errors.Is(err, link.ErrHandingOver) { + t.Fatalf("a controller stopping ran a verb: %v", err) + } +} diff --git a/cmd/mesh-controller/selfexec_test.go b/cmd/mesh-controller/selfexec_test.go index 05d8102b..45bf4c28 100644 --- a/cmd/mesh-controller/selfexec_test.go +++ b/cmd/mesh-controller/selfexec_test.go @@ -1,19 +1,10 @@ package main import ( - "bufio" - "encoding/json" - "errors" "fmt" "io" "os" - "os/exec" - "path/filepath" - "runtime" - "strings" "testing" - - "github.com/novox/mesh-controller/internal/link" ) // The roles a copy of this test binary plays in TestAVerbRunsWhileItsBuildIsMovedOrDeleted. @@ -21,25 +12,6 @@ const selfExecRole = "MESH_CONTROLLER_SELFEXEC_ROLE" // TestSelfExecServer is a controller standing in, when run as one: it waits until its build has been // moved, then runs a verb as the seat runs one, and prints what came of it as JSON. -func TestSelfExecServer(t *testing.T) { - if os.Getenv(selfExecRole) != "server" { - t.Skip("run by TestAVerbRunsWhileItsBuildIsMovedOrDeleted") - } - line, _ := bufio.NewReader(os.Stdin).ReadString('\n') - if strings.TrimSpace(line) != "go" { - t.Fatalf("told %q", line) - } - if err := os.Setenv(selfExecRole, "verb"); err != nil { - t.Fatal(err) - } - answer, err := runVerb(t.Context(), []string{"-test.run", "^TestSelfExecVerb$", "-test.v"}) - said := map[string]any{"ok": answer.OK, "output": answer.Output} - if err != nil { - said["error"] = err.Error() - } - body, _ := json.Marshal(said) - fmt.Println("ANSWER " + string(body)) -} // TestSelfExecVerb is the verb, when run as one. func TestSelfExecVerb(t *testing.T) { @@ -56,106 +28,9 @@ func TestSelfExecVerb(t *testing.T) { // // A copy of this test binary is the controller: started from one place, moved into a directory closed // to everyone (or deleted), and only then asked to run a verb. -func TestAVerbRunsWhileItsBuildIsMovedOrDeleted(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("the image is named through /proc on Linux only") - } - self, err := os.Executable() - if err != nil { - t.Fatal(err) - } - for _, how := range []string{"moved into a closed directory", "deleted"} { - t.Run(how, func(t *testing.T) { - root := t.TempDir() - placed := filepath.Join(root, "mesh-controller", "mesh-controller") - if err := os.MkdirAll(filepath.Dir(placed), 0o755); err != nil { - t.Fatal(err) - } - copyFile(t, self, placed) - - server := exec.Command(placed, "-test.run", "^TestSelfExecServer$", "-test.v") - server.Env = append(os.Environ(), selfExecRole+"=server") - stdin, err := server.StdinPipe() - if err != nil { - t.Fatal(err) - } - stdout, err := server.StdoutPipe() - if err != nil { - t.Fatal(err) - } - server.Stderr = os.Stderr - if err := server.Start(); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = server.Process.Kill(); _ = server.Wait() }) - - // What the witness does to a running build, once the process runs. - switch how { - case "deleted": - if err := os.RemoveAll(filepath.Dir(placed)); err != nil { - t.Fatal(err) - } - default: - kept := filepath.Join(root, ".witness", "mesh-controller", "previous") - if err := os.MkdirAll(filepath.Dir(kept), 0o700); err != nil { - t.Fatal(err) - } - if err := os.Rename(filepath.Dir(placed), kept); err != nil { - t.Fatal(err) - } - if err := os.Chmod(filepath.Join(root, ".witness"), 0); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = os.Chmod(filepath.Join(root, ".witness"), 0o700) }) - } - if _, err := io.WriteString(stdin, "go\n"); err != nil { - t.Fatal(err) - } - out, _ := io.ReadAll(stdout) - _ = server.Wait() - var said struct { - OK bool `json:"ok"` - Output string `json:"output"` - Error string `json:"error"` - } - found := false - for _, line := range strings.Split(string(out), "\n") { - if rest, ok := strings.CutPrefix(line, "ANSWER "); ok { - found = json.Unmarshal([]byte(rest), &said) == nil - } - } - if !found { - t.Fatalf("the controller standing in answered nothing:\n%s", out) - } - if said.Error != "" || !said.OK || !strings.Contains(said.Output, "the verb ran") { - t.Fatalf("the verb did not run from the controller's own image after its build was %s: %+v", how, said) - } - }) - } -} // A verb the controller cannot start from its own build is refused as a handover — marked so its // caller asks again — never a permission error; and so is one arriving once the controller is stopping. -func TestAVerbThatCannotRunIsRefusedAsAHandover(t *testing.T) { - closed := filepath.Join(t.TempDir(), "closed") - if err := os.MkdirAll(closed, 0o700); err != nil { - t.Fatal(err) - } - was := ownImage - ownImage = func() string { return filepath.Join(closed, "gone", "mesh-controller") } - t.Cleanup(func() { ownImage = was }) - _, err := runVerb(t.Context(), []string{"status"}) - if !errors.Is(err, link.ErrHandingOver) { - t.Fatalf("a build that is not there was answered %v, not a handover", err) - } - - ownImage = was - handingOver.Store(true) - t.Cleanup(func() { handingOver.Store(false) }) - if _, err := runVerb(t.Context(), []string{"-test.run", "^$"}); !errors.Is(err, link.ErrHandingOver) { - t.Fatalf("a controller stopping ran a verb: %v", err) - } -} func copyFile(t *testing.T, from, to string) { t.Helper() diff --git a/cmd/mesh-controller/sendable_integration_test.go b/cmd/mesh-controller/sendable_integration_test.go new file mode 100644 index 00000000..e1817f27 --- /dev/null +++ b/cmd/mesh-controller/sendable_integration_test.go @@ -0,0 +1,354 @@ +//go:build integration + +package main + +import ( + "bytes" + "encoding/json" + "io" + "os" + "reflect" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/overlay" +) + +func TestAConvergedDeclarationIsByteForByteWhatItWas(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, helloWeb()) + if _, err := unassign(ctx, open, "laptop", overlay.Name); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "laptop", "hello-web"); err != nil { + t.Fatal(err) + } + declared := composed(t, open, "laptop") + if declared.Adoption != nil { + t.Fatal("a converged node was given an adoption envelope") + } + body, err := declared.Body() + if err != nil { + t.Fatal(err) + } + if string(body) != convergedBefore { + t.Fatalf("a converged declaration changed; an older host parses this strictly:\n%s\n%s", + body, convergedBefore) + } + if bytes.Contains(body, []byte(`"adoption"`)) { + t.Fatal("a converged declaration names adoption; an older host would refuse it") + } +} + +func TestAnAdoptedDeclarationCarriesItsModeAndWhatWasTaken(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, helloWeb()) + if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "anchor", "hello-web"); err != nil { + t.Fatal(err) + } + + declared := composed(t, open, "anchor") + if declared.Adoption == nil { + t.Fatal("an adopted node's declaration does not say it is adopted") + } + untaken := declared.Adoption.Untaken["hello-web"] + // Every kind — its directory too (novox/hq ADR 0103). + if !reflect.DeepEqual(untaken, []string{"hello-web.page", "hello-web.server", + "hello-web.served"}) { + t.Fatalf("hello-web's resources are not all named untaken: %v", declared.Adoption) + } + if len(declared.Adoption.Taken) != 0 { + t.Fatalf("nothing was taken, and the declaration says %v", declared.Adoption.Taken) + } + + body, err := declared.Body() + if err != nil { + t.Fatal(err) + } + var envelope map[string]any + if err := json.Unmarshal(body, &envelope); err != nil { + t.Fatal(err) + } + adoption, _ := envelope["adoption"].(map[string]any) + if _, ok := adoption["taken"].([]any); !ok { + t.Fatalf("taken is not a list on the wire, even empty: %s", body) + } + + // The digest the mesh compares is the digest of what is sent: status and push agree. + would, err := wouldSend(ctx, open, mustNodes(t, open)) + if err != nil { + t.Fatal(err) + } + if would["anchor"] != digestOf(body) { + t.Fatal("the digest the mesh compares is not of the declaration push sends") + } + + // plan --json prints that same envelope. + printed := stdoutOf(t, func() error { return planCommand(ctx, []string{"anchor", "--json"}) }) + var compact bytes.Buffer + if err := json.Compact(&compact, []byte(printed)); err != nil { + t.Fatalf("plan --json is not JSON: %v\n%s", err, printed) + } + if digestOf(compact.Bytes()) != digestOf(body) { + t.Fatalf("plan --json shows something other than what push sends:\n%s", printed) + } + + // Taking the module moves its resources out of untaken. + if err := open.inventory.Take(ctx, "anchor", "hello-web"); err != nil { + t.Fatal(err) + } + declared = composed(t, open, "anchor") + if _, still := declared.Adoption.Untaken["hello-web"]; still { + t.Fatalf("a taken module is still untaken: %v", declared.Adoption) + } + if !reflect.DeepEqual(declared.Adoption.Taken, []string{"hello-web"}) { + t.Fatalf("taken is %v", declared.Adoption.Taken) + } +} + +func stdoutOf(t *testing.T, run func() error) string { + t.Helper() + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + saved := os.Stdout + os.Stdout = w + done := make(chan string) + go func() { + all, _ := io.ReadAll(r) + done <- string(all) + }() + runErr := run() + os.Stdout = saved + w.Close() + out := <-done + if runErr != nil { + t.Fatalf("%v\n%s", runErr, out) + } + return out +} + +func TestConsumersAreToldTheGivenPort(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "store", Version: "1", + Provides: []catalogue.Offer{{Name: "database", Scope: catalogue.ScopeMesh}}, + Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}}, + Guards: []int{5432}, + Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store", + "ports": []any{"5432:5432"}, + "image": "registry.example/pg@sha256:" + strings.Repeat("b", 64)}}}) + register(t, open, catalogue.Manifest{Module: "app", Version: "1", Requires: []string{"database"}}) + if _, err := assign(ctx, open, "anchor", "store"); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "laptop", "app"); err != nil { + t.Fatal(err) + } + if err := open.inventory.SetSettings(ctx, "anchor", "store", + map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5433}}); err != nil { + t.Fatal(err) + } + + plan, _, err := planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + var told any + for _, n := range plan.Needs { + if n.Name == "database" { + told = n.Serves["port"] + } + } + if told != 5433 { + t.Fatalf("the consumer is told the store is on %v", told) + } + for _, r := range composed(t, open, "anchor").Resources { + if r["id"] == "store.server" && !reflect.DeepEqual(r["ports"], []any{"5433:5432"}) { + t.Fatalf("the store publishes %v", r["ports"]) + } + } + + // A port for the whole mesh is refused where it is set, not stored to refuse every node's + // declaration afterwards. + if err := open.inventory.SetSettings(ctx, "", "store", + map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5434}}); err == nil || + !strings.Contains(err.Error(), "per node") { + t.Fatalf("a port given for the whole mesh was not refused: %v", err) + } + plan, settings, err := planFor(ctx, open, "anchor") + if err != nil { + t.Fatal(err) + } + if _, err := declarationFor(ctx, open, "anchor", plan, settings); err != nil { + t.Fatalf("the refused mesh-wide layer was stored anyway: %v", err) + } +} + +func TestTheGuardIsSentForTakenModulesOnly(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "store", Version: "1", + Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}}, + Guards: []int{5432}, + Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store", + "ports": []any{"5432:5432"}, + "image": "registry.example/pg@sha256:" + strings.Repeat("b", 64)}}}) + if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "anchor", "store"); err != nil { + t.Fatal(err) + } + if hasID(composed(t, open, "anchor").Resources, catalogue.GuardID()) { + t.Fatal("an untaken store is guarded") + } + if err := open.inventory.Take(ctx, "anchor", "store"); err != nil { + t.Fatal(err) + } + for _, r := range composed(t, open, "anchor").Resources { + if r["id"] == catalogue.GuardID() { + if r["content"] != catalogue.AsGuard([]int{5432}) { + t.Fatalf("the taken store's guard is:\n%s", r["content"]) + } + return + } + } + t.Fatal("a taken store is not guarded") +} + +func TestTheMachineSideOfAMappingIsMovedEverywhereTheNumberIsUsed(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "forge", Version: "1", + Provides: []catalogue.Offer{{Name: "git-over-ssh", Scope: catalogue.ScopeMesh}}, + Serves: map[string]map[string]any{"git-over-ssh": {"port": 2222}}, + Listens: []catalogue.Listening{{Port: 2222, From: catalogue.FromMesh}}, + Resources: []map[string]any{{"id": "server", "type": "container", "name": "forge", + "ports": []any{"2222:22"}, + "image": "registry.example/forge@sha256:" + strings.Repeat("c", 64)}}}) + register(t, open, catalogue.Manifest{Module: "app", Version: "1", + Requires: []string{"git-over-ssh"}}) + if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "anchor", "forge"); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "laptop", "app"); err != nil { + t.Fatal(err) + } + if err := open.inventory.Take(ctx, "anchor", "forge"); err != nil { + t.Fatal(err) + } + if err := open.inventory.SetSettings(ctx, "anchor", "forge", + map[string]any{catalogue.PortsSetting: map[string]any{"2222": 222}}); err != nil { + t.Fatal(err) + } + + resources := composed(t, open, "anchor").Resources + var container, opening map[string]any + for _, r := range resources { + switch r["id"] { + case "forge.server": + container = r + case catalogue.OpeningID("tcp", 222, catalogue.PathForwarded): + opening = r + } + if id, _ := r["id"].(string); strings.HasPrefix(id, "adoption.opening-tcp-2222-") { + t.Errorf("the adopted node is told to open the port the forge was moved off: %s", id) + } + } + if container == nil || !reflect.DeepEqual(container["ports"], []any{"222:22"}) { + t.Fatalf("the forge's container publishes %v", container["ports"]) + } + if opening == nil || opening["to"] != 22 || opening["from"] != catalogue.OpeningFromMesh { + t.Fatalf("no opening for the port this node gave the forge: %v", opening) + } + var guard map[string]any + for _, r := range resources { + if r["id"] == catalogue.GuardID() { + guard = r + } + } + if guard == nil || guard["content"] != catalogue.AsGuard([]int{222}) { + t.Fatalf("the guard does not refuse the port the forge is on:\n%v", guard["content"]) + } + + // And the consumer on the other machine dials the same number. + plan, _, err := planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + var told any + for _, n := range plan.Needs { + if n.Name == "git-over-ssh" { + told = n.Serves["port"] + } + } + if told != 222 { + t.Fatalf("the consumer is told the forge answers on %v", told) + } +} + +func TestADefinitionMovingUnderAStoredSettingLeavesThatModuleOutNotTheMachine(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + web := helloWeb() + web.Resources[1]["ports"] = []any{"8080"} + register(t, open, web) + register(t, open, catalogue.Manifest{Module: "notes", Version: "1", + Resources: []map[string]any{{"id": "conf", "type": "file", "path": "/etc/notes.conf", "content": "x"}}}) + for _, m := range []string{"hello-web", "notes"} { + if _, err := assign(ctx, open, "laptop", m); err != nil { + t.Fatal(err) + } + } + // Judged where it is stored: a port the module does not publish is refused by name. + err := open.inventory.SetSettings(ctx, "laptop", "hello-web", + map[string]any{catalogue.PortsSetting: map[string]any{"9999": 10000}}) + if err == nil || !strings.Contains(err.Error(), "hello-web on laptop") || !strings.Contains(err.Error(), "9999") { + t.Fatalf("an impossible setting was stored: %v", err) + } + if err := open.inventory.SetSettings(ctx, "laptop", "hello-web", + map[string]any{catalogue.PortsSetting: map[string]any{"8080": 10000}}); err != nil { + t.Fatal(err) + } + if declared := composed(t, open, "laptop"); len(declared.LeftOut) != 0 { + t.Fatalf("a setting that composes left a module out: %v", declared.LeftOut) + } + + // The definition moves: the container publishes another port now. + web.Version = "2" + web.Resources[1]["ports"] = []any{"9090"} + register(t, open, web) + declared := composed(t, open, "laptop") + if len(declared.LeftOut) != 1 || declared.LeftOut[0] != "hello-web" { + t.Fatalf("hello-web is not left out: %v", declared.LeftOut) + } + if !strings.Contains(declared.leftOutWhy["hello-web"], "no container of its publishes 8080") { + t.Fatalf("why it was left out is not said: %v", declared.leftOutWhy) + } + if hasID(declared.Resources, "hello-web.server") || !hasID(declared.Resources, "notes.conf") { + t.Fatalf("the machine was not told everything else: %v", declared.Resources) + } + body, err := declared.Body() + if err != nil { + t.Fatal(err) + } + var env map[string]any + if err := json.Unmarshal(body, &env); err != nil { + t.Fatal(err) + } + left, _ := env["left_out"].([]any) + if len(left) != 1 || left[0] != "hello-web" { + t.Fatalf("the envelope does not say what was left out: %v", env) + } +} diff --git a/cmd/mesh-controller/sendable_test.go b/cmd/mesh-controller/sendable_test.go index 9b218695..47224eea 100644 --- a/cmd/mesh-controller/sendable_test.go +++ b/cmd/mesh-controller/sendable_test.go @@ -1,17 +1,12 @@ package main import ( - "bytes" "encoding/json" - "io" - "os" - "reflect" "strings" "testing" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/overlay" ) // novox/hq ADR 0100: every declaration an adopted node is sent says it is adopted and which modules @@ -54,103 +49,6 @@ func composed(t *testing.T, open *stores, node string) sendable { // this guard permits; a field it would refuse is the one it exists to catch. const convergedBefore = `{"declaration":1,"resources":[{"content":"hello","id":"hello-web.page","path":"/var/lib/hello-web/index.html","type":"file"},{"id":"hello-web.server","image":"registry.example/hello@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","name":"hello-web","type":"container"},{"id":"hello-web.served","path":"/var/lib/hello-web","type":"directory"}]}` -func TestAConvergedDeclarationIsByteForByteWhatItWas(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, helloWeb()) - if _, err := unassign(ctx, open, "laptop", overlay.Name); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "laptop", "hello-web"); err != nil { - t.Fatal(err) - } - declared := composed(t, open, "laptop") - if declared.Adoption != nil { - t.Fatal("a converged node was given an adoption envelope") - } - body, err := declared.Body() - if err != nil { - t.Fatal(err) - } - if string(body) != convergedBefore { - t.Fatalf("a converged declaration changed; an older host parses this strictly:\n%s\n%s", - body, convergedBefore) - } - if bytes.Contains(body, []byte(`"adoption"`)) { - t.Fatal("a converged declaration names adoption; an older host would refuse it") - } -} - -func TestAnAdoptedDeclarationCarriesItsModeAndWhatWasTaken(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, helloWeb()) - if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "anchor", "hello-web"); err != nil { - t.Fatal(err) - } - - declared := composed(t, open, "anchor") - if declared.Adoption == nil { - t.Fatal("an adopted node's declaration does not say it is adopted") - } - untaken := declared.Adoption.Untaken["hello-web"] - // Every kind — its directory too (novox/hq ADR 0103). - if !reflect.DeepEqual(untaken, []string{"hello-web.page", "hello-web.server", - "hello-web.served"}) { - t.Fatalf("hello-web's resources are not all named untaken: %v", declared.Adoption) - } - if len(declared.Adoption.Taken) != 0 { - t.Fatalf("nothing was taken, and the declaration says %v", declared.Adoption.Taken) - } - - body, err := declared.Body() - if err != nil { - t.Fatal(err) - } - var envelope map[string]any - if err := json.Unmarshal(body, &envelope); err != nil { - t.Fatal(err) - } - adoption, _ := envelope["adoption"].(map[string]any) - if _, ok := adoption["taken"].([]any); !ok { - t.Fatalf("taken is not a list on the wire, even empty: %s", body) - } - - // The digest the mesh compares is the digest of what is sent: status and push agree. - would, err := wouldSend(ctx, open, mustNodes(t, open)) - if err != nil { - t.Fatal(err) - } - if would["anchor"] != digestOf(body) { - t.Fatal("the digest the mesh compares is not of the declaration push sends") - } - - // plan --json prints that same envelope. - printed := stdoutOf(t, func() error { return planCommand(ctx, []string{"anchor", "--json"}) }) - var compact bytes.Buffer - if err := json.Compact(&compact, []byte(printed)); err != nil { - t.Fatalf("plan --json is not JSON: %v\n%s", err, printed) - } - if digestOf(compact.Bytes()) != digestOf(body) { - t.Fatalf("plan --json shows something other than what push sends:\n%s", printed) - } - - // Taking the module moves its resources out of untaken. - if err := open.inventory.Take(ctx, "anchor", "hello-web"); err != nil { - t.Fatal(err) - } - declared = composed(t, open, "anchor") - if _, still := declared.Adoption.Untaken["hello-web"]; still { - t.Fatalf("a taken module is still untaken: %v", declared.Adoption) - } - if !reflect.DeepEqual(declared.Adoption.Taken, []string{"hello-web"}) { - t.Fatalf("taken is %v", declared.Adoption.Taken) - } -} - func mustNodes(t *testing.T, open *stores) []inventory.Node { t.Helper() nodes, err := open.inventory.Nodes(t.Context()) @@ -161,122 +59,13 @@ func mustNodes(t *testing.T, open *stores) []inventory.Node { } // stdoutOf is what run printed. -func stdoutOf(t *testing.T, run func() error) string { - t.Helper() - r, w, err := os.Pipe() - if err != nil { - t.Fatal(err) - } - saved := os.Stdout - os.Stdout = w - done := make(chan string) - go func() { - all, _ := io.ReadAll(r) - done <- string(all) - }() - runErr := run() - os.Stdout = saved - w.Close() - out := <-done - if runErr != nil { - t.Fatalf("%v\n%s", runErr, out) - } - return out -} // novox/hq ADR 0100: a port a node was given for the store is where its consumers on other // machines are told to reach it, and a port given for the whole mesh is refused. -func TestConsumersAreToldTheGivenPort(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "store", Version: "1", - Provides: []catalogue.Offer{{Name: "database", Scope: catalogue.ScopeMesh}}, - Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}}, - Guards: []int{5432}, - Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store", - "ports": []any{"5432:5432"}, - "image": "registry.example/pg@sha256:" + strings.Repeat("b", 64)}}}) - register(t, open, catalogue.Manifest{Module: "app", Version: "1", Requires: []string{"database"}}) - if _, err := assign(ctx, open, "anchor", "store"); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "laptop", "app"); err != nil { - t.Fatal(err) - } - if err := open.inventory.SetSettings(ctx, "anchor", "store", - map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5433}}); err != nil { - t.Fatal(err) - } - - plan, _, err := planFor(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - var told any - for _, n := range plan.Needs { - if n.Name == "database" { - told = n.Serves["port"] - } - } - if told != 5433 { - t.Fatalf("the consumer is told the store is on %v", told) - } - for _, r := range composed(t, open, "anchor").Resources { - if r["id"] == "store.server" && !reflect.DeepEqual(r["ports"], []any{"5433:5432"}) { - t.Fatalf("the store publishes %v", r["ports"]) - } - } - - // A port for the whole mesh is refused where it is set, not stored to refuse every node's - // declaration afterwards. - if err := open.inventory.SetSettings(ctx, "", "store", - map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5434}}); err == nil || - !strings.Contains(err.Error(), "per node") { - t.Fatalf("a port given for the whole mesh was not refused: %v", err) - } - plan, settings, err := planFor(ctx, open, "anchor") - if err != nil { - t.Fatal(err) - } - if _, err := declarationFor(ctx, open, "anchor", plan, settings); err != nil { - t.Fatalf("the refused mesh-wide layer was stored anyway: %v", err) - } -} // novox/hq ADR 0103: the guard an adopted node is sent follows what was taken there. A store // assigned but not taken is not guarded — its port may still be the predecessor's — and taking it // guards it from the next declaration. -func TestTheGuardIsSentForTakenModulesOnly(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "store", Version: "1", - Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}}, - Guards: []int{5432}, - Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store", - "ports": []any{"5432:5432"}, - "image": "registry.example/pg@sha256:" + strings.Repeat("b", 64)}}}) - if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "anchor", "store"); err != nil { - t.Fatal(err) - } - if hasID(composed(t, open, "anchor").Resources, catalogue.GuardID()) { - t.Fatal("an untaken store is guarded") - } - if err := open.inventory.Take(ctx, "anchor", "store"); err != nil { - t.Fatal(err) - } - for _, r := range composed(t, open, "anchor").Resources { - if r["id"] == catalogue.GuardID() { - if r["content"] != catalogue.AsGuard([]int{5432}) { - t.Fatalf("the taken store's guard is:\n%s", r["content"]) - } - return - } - } - t.Fatal("a taken store is not guarded") -} // novox/hq ADR 0038 and 0100: a module may publish a port the long way — `2222:22`, because the // machine's own ssh daemon holds 22 — and say it listens on the machine side of that mapping, @@ -287,79 +76,6 @@ func TestTheGuardIsSentForTakenModulesOnly(t *testing.T) { // It reached none of them. The setting was refused outright for naming the machine side — so a // module's port could not be put back where the machine it replaces had it, and, worse, the // node's every push failed for as long as the setting existed. -func TestTheMachineSideOfAMappingIsMovedEverywhereTheNumberIsUsed(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "forge", Version: "1", - Provides: []catalogue.Offer{{Name: "git-over-ssh", Scope: catalogue.ScopeMesh}}, - Serves: map[string]map[string]any{"git-over-ssh": {"port": 2222}}, - Listens: []catalogue.Listening{{Port: 2222, From: catalogue.FromMesh}}, - Resources: []map[string]any{{"id": "server", "type": "container", "name": "forge", - "ports": []any{"2222:22"}, - "image": "registry.example/forge@sha256:" + strings.Repeat("c", 64)}}}) - register(t, open, catalogue.Manifest{Module: "app", Version: "1", - Requires: []string{"git-over-ssh"}}) - if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "anchor", "forge"); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "laptop", "app"); err != nil { - t.Fatal(err) - } - if err := open.inventory.Take(ctx, "anchor", "forge"); err != nil { - t.Fatal(err) - } - if err := open.inventory.SetSettings(ctx, "anchor", "forge", - map[string]any{catalogue.PortsSetting: map[string]any{"2222": 222}}); err != nil { - t.Fatal(err) - } - - resources := composed(t, open, "anchor").Resources - var container, opening map[string]any - for _, r := range resources { - switch r["id"] { - case "forge.server": - container = r - case catalogue.OpeningID("tcp", 222, catalogue.PathForwarded): - opening = r - } - if id, _ := r["id"].(string); strings.HasPrefix(id, "adoption.opening-tcp-2222-") { - t.Errorf("the adopted node is told to open the port the forge was moved off: %s", id) - } - } - if container == nil || !reflect.DeepEqual(container["ports"], []any{"222:22"}) { - t.Fatalf("the forge's container publishes %v", container["ports"]) - } - if opening == nil || opening["to"] != 22 || opening["from"] != catalogue.OpeningFromMesh { - t.Fatalf("no opening for the port this node gave the forge: %v", opening) - } - var guard map[string]any - for _, r := range resources { - if r["id"] == catalogue.GuardID() { - guard = r - } - } - if guard == nil || guard["content"] != catalogue.AsGuard([]int{222}) { - t.Fatalf("the guard does not refuse the port the forge is on:\n%v", guard["content"]) - } - - // And the consumer on the other machine dials the same number. - plan, _, err := planFor(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - var told any - for _, n := range plan.Needs { - if n.Name == "git-over-ssh" { - told = n.Serves["port"] - } - } - if told != 222 { - t.Fatalf("the consumer is told the forge answers on %v", told) - } -} func TestAnEmptyDeclarationSaysOwnsNothing(t *testing.T) { // The host refuses an empty body unless told the emptiness is meant (novox/hq issue 127). @@ -387,57 +103,3 @@ func TestAnEmptyDeclarationSaysOwnsNothing(t *testing.T) { // (novox/hq ADR 0163, rule 6): stored while it composed, a setting whose definition then moved from // under it leaves that module out of the declaration — said in the envelope, so the host keeps the // module's things — and the machine is told everything else. -func TestADefinitionMovingUnderAStoredSettingLeavesThatModuleOutNotTheMachine(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - web := helloWeb() - web.Resources[1]["ports"] = []any{"8080"} - register(t, open, web) - register(t, open, catalogue.Manifest{Module: "notes", Version: "1", - Resources: []map[string]any{{"id": "conf", "type": "file", "path": "/etc/notes.conf", "content": "x"}}}) - for _, m := range []string{"hello-web", "notes"} { - if _, err := assign(ctx, open, "laptop", m); err != nil { - t.Fatal(err) - } - } - // Judged where it is stored: a port the module does not publish is refused by name. - err := open.inventory.SetSettings(ctx, "laptop", "hello-web", - map[string]any{catalogue.PortsSetting: map[string]any{"9999": 10000}}) - if err == nil || !strings.Contains(err.Error(), "hello-web on laptop") || !strings.Contains(err.Error(), "9999") { - t.Fatalf("an impossible setting was stored: %v", err) - } - if err := open.inventory.SetSettings(ctx, "laptop", "hello-web", - map[string]any{catalogue.PortsSetting: map[string]any{"8080": 10000}}); err != nil { - t.Fatal(err) - } - if declared := composed(t, open, "laptop"); len(declared.LeftOut) != 0 { - t.Fatalf("a setting that composes left a module out: %v", declared.LeftOut) - } - - // The definition moves: the container publishes another port now. - web.Version = "2" - web.Resources[1]["ports"] = []any{"9090"} - register(t, open, web) - declared := composed(t, open, "laptop") - if len(declared.LeftOut) != 1 || declared.LeftOut[0] != "hello-web" { - t.Fatalf("hello-web is not left out: %v", declared.LeftOut) - } - if !strings.Contains(declared.leftOutWhy["hello-web"], "no container of its publishes 8080") { - t.Fatalf("why it was left out is not said: %v", declared.leftOutWhy) - } - if hasID(declared.Resources, "hello-web.server") || !hasID(declared.Resources, "notes.conf") { - t.Fatalf("the machine was not told everything else: %v", declared.Resources) - } - body, err := declared.Body() - if err != nil { - t.Fatal(err) - } - var env map[string]any - if err := json.Unmarshal(body, &env); err != nil { - t.Fatal(err) - } - left, _ := env["left_out"].([]any) - if len(left) != 1 || left[0] != "hello-web" { - t.Fatalf("the envelope does not say what was left out: %v", env) - } -} diff --git a/cmd/mesh-controller/sent_required_by_integration_test.go b/cmd/mesh-controller/sent_required_by_integration_test.go new file mode 100644 index 00000000..5cef69b8 --- /dev/null +++ b/cmd/mesh-controller/sent_required_by_integration_test.go @@ -0,0 +1,104 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestAModulePulledInIsHandedOnAMachineThatCanTakeItAndMirrored(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + client := aClientModule() + moved, kept := aBot("moved-bot", true), aBot("kept-bot", false) + moved.Provides = []catalogue.Offer{{Name: "moved-bot-api"}} + kept.Provides = []catalogue.Offer{{Name: "kept-bot-api"}} + desk := catalogue.Manifest{Module: "desk", Version: "1", Requires: []string{"moved-bot-api", "kept-bot-api"}} + for _, m := range []catalogue.Manifest{client, moved, kept, desk} { + register(t, open, m) + } + for _, m := range []string{client.Module, desk.Module} { + if _, err := assign(ctx, open, "laptop", m); err != nil { + t.Fatal(err) + } + } + if err := issueProvisionerClient(ctx, inv, client, "laptop", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222"); err != nil { + t.Fatal(err) + } + if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "laptop", Contract: link.HandedContract, + SaidAt: time.Now()}); err != nil { + t.Fatal(err) + } + + plan, settings, err := planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + if plan.RequiredBy("moved-bot") != "desk" || plan.RequiredBy("kept-bot") != "desk" { + t.Fatalf("the bots are on laptop because %v", plan.Because) + } + declared, err := declarationFor(ctx, open, "laptop", plan, settings) + if err != nil { + t.Fatal(err) + } + var movedToken, keptToken, movedConf map[string]any + for _, r := range declared.Resources { + switch r["id"] { + case "moved-bot.needs-token": + movedToken = r + case "kept-bot.needs-token": + keptToken = r + case "moved-bot.conf": + movedConf = r + } + } + if movedToken == nil || movedToken["sealed"] != nil || movedToken["owner-at-runtime"] != "owned" { + t.Fatalf("the pulled-in moved module's secret file is sent as %v: placed, with no value", movedToken) + } + if movedConf == nil || movedConf["content"] != nil || movedConf["owner-at-runtime"] != "owned" { + t.Fatalf("the pulled-in moved module's templated file is sent as %v: placed, with no text", movedConf) + } + if keptToken == nil || keptToken["sealed"] == nil { + t.Fatalf("the pulled-in module not moved has its secret sent as %v: sealed, as before", keptToken) + } + if _, held, err := inv.ModuleSecretIfIssued(ctx, "laptop", "moved-bot", "token"); err != nil || held { + t.Fatalf("the send made the pulled-in handed module's secret in the controller's store (held %v, %v)", held, err) + } + if _, has := declared.plans["kept-bot"]; has || len(declared.plans) != 1 { + t.Fatalf("the send hands %v", declared.plans) + } + + record, err := inv.NodeByName(ctx, "laptop") + if err != nil { + t.Fatal(err) + } + if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, inventory.Send{Digest: "first", + Provisioning: sentRecordOf(ctx, declared).plans}); err != nil { + t.Fatal(err) + } + inv.SetProvisioningPlans(inv.SentProvisioningPlans) + mirror, err := inv.AssignmentsMirror(ctx) + if err != nil { + t.Fatal(err) + } + var value inventory.AssignmentMirrored + if err := json.Unmarshal(mirror["laptop.moved-bot"], &value); err != nil { + t.Fatalf("no value for the pulled-in module: %s (%v)", mirror["laptop.moved-bot"], err) + } + if value.RequiredBy != "desk" || !value.Assigned.IsZero() || value.Provisioning == nil || + len(value.Provisioning.Files) != 1 || value.Provisioning.Files[0].Path != "/etc/moved-bot/bot.conf" || + len(value.Provisioning.Secrets) != 1 || value.Provisioning.Secrets[0].Name != "token" { + t.Fatalf("the pulled-in module is mirrored as %s", mirror["laptop.moved-bot"]) + } + if _, has := mirror["laptop.kept-bot"]; has { + t.Fatalf("a pulled-in module not handed is mirrored: %s", mirror["laptop.kept-bot"]) + } +} diff --git a/cmd/mesh-controller/sent_required_by_test.go b/cmd/mesh-controller/sent_required_by_test.go index 68553609..7f611ca2 100644 --- a/cmd/mesh-controller/sent_required_by_test.go +++ b/cmd/mesh-controller/sent_required_by_test.go @@ -1,15 +1,10 @@ package main import ( - "encoding/json" "strings" "testing" - "time" - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" ) // A send keeps, beside each handed module's plan, what requires a module only pulled in (issue 765), so the @@ -34,92 +29,3 @@ func TestASendKeepsWhatRequiresAPulledInModule(t *testing.T) { // node running a client, a moved module that nothing assigned there, required by one that is, has its files and its // own secret handed like an assigned one's, no secret of it made or sealed by the controller, and its plan mirrored // under its own key, saying what requires it. A module not moved, pulled in the same way, is sent as before. -func TestAModulePulledInIsHandedOnAMachineThatCanTakeItAndMirrored(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - client := aClientModule() - moved, kept := aBot("moved-bot", true), aBot("kept-bot", false) - moved.Provides = []catalogue.Offer{{Name: "moved-bot-api"}} - kept.Provides = []catalogue.Offer{{Name: "kept-bot-api"}} - desk := catalogue.Manifest{Module: "desk", Version: "1", Requires: []string{"moved-bot-api", "kept-bot-api"}} - for _, m := range []catalogue.Manifest{client, moved, kept, desk} { - register(t, open, m) - } - for _, m := range []string{client.Module, desk.Module} { - if _, err := assign(ctx, open, "laptop", m); err != nil { - t.Fatal(err) - } - } - if err := issueProvisionerClient(ctx, inv, client, "laptop", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222"); err != nil { - t.Fatal(err) - } - if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "laptop", Contract: link.HandedContract, - SaidAt: time.Now()}); err != nil { - t.Fatal(err) - } - - plan, settings, err := planFor(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - if plan.RequiredBy("moved-bot") != "desk" || plan.RequiredBy("kept-bot") != "desk" { - t.Fatalf("the bots are on laptop because %v", plan.Because) - } - declared, err := declarationFor(ctx, open, "laptop", plan, settings) - if err != nil { - t.Fatal(err) - } - var movedToken, keptToken, movedConf map[string]any - for _, r := range declared.Resources { - switch r["id"] { - case "moved-bot.needs-token": - movedToken = r - case "kept-bot.needs-token": - keptToken = r - case "moved-bot.conf": - movedConf = r - } - } - if movedToken == nil || movedToken["sealed"] != nil || movedToken["owner-at-runtime"] != "owned" { - t.Fatalf("the pulled-in moved module's secret file is sent as %v: placed, with no value", movedToken) - } - if movedConf == nil || movedConf["content"] != nil || movedConf["owner-at-runtime"] != "owned" { - t.Fatalf("the pulled-in moved module's templated file is sent as %v: placed, with no text", movedConf) - } - if keptToken == nil || keptToken["sealed"] == nil { - t.Fatalf("the pulled-in module not moved has its secret sent as %v: sealed, as before", keptToken) - } - if _, held, err := inv.ModuleSecretIfIssued(ctx, "laptop", "moved-bot", "token"); err != nil || held { - t.Fatalf("the send made the pulled-in handed module's secret in the controller's store (held %v, %v)", held, err) - } - if _, has := declared.plans["kept-bot"]; has || len(declared.plans) != 1 { - t.Fatalf("the send hands %v", declared.plans) - } - - record, err := inv.NodeByName(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, inventory.Send{Digest: "first", - Provisioning: sentRecordOf(ctx, declared).plans}); err != nil { - t.Fatal(err) - } - inv.SetProvisioningPlans(inv.SentProvisioningPlans) - mirror, err := inv.AssignmentsMirror(ctx) - if err != nil { - t.Fatal(err) - } - var value inventory.AssignmentMirrored - if err := json.Unmarshal(mirror["laptop.moved-bot"], &value); err != nil { - t.Fatalf("no value for the pulled-in module: %s (%v)", mirror["laptop.moved-bot"], err) - } - if value.RequiredBy != "desk" || !value.Assigned.IsZero() || value.Provisioning == nil || - len(value.Provisioning.Files) != 1 || value.Provisioning.Files[0].Path != "/etc/moved-bot/bot.conf" || - len(value.Provisioning.Secrets) != 1 || value.Provisioning.Secrets[0].Name != "token" { - t.Fatalf("the pulled-in module is mirrored as %s", mirror["laptop.moved-bot"]) - } - if _, has := mirror["laptop.kept-bot"]; has { - t.Fatalf("a pulled-in module not handed is mirrored: %s", mirror["laptop.kept-bot"]) - } -} diff --git a/cmd/mesh-controller/sequence_integration_test.go b/cmd/mesh-controller/sequence_integration_test.go new file mode 100644 index 00000000..85c5475f --- /dev/null +++ b/cmd/mesh-controller/sequence_integration_test.go @@ -0,0 +1,41 @@ +//go:build integration + +package main + +import "testing" + +func TestEachSendToANodeIsOneHigherAndReadable(t *testing.T) { + open := aMesh(t) + record, err := open.inventory.NodeByName(t.Context(), "anchor") + if err != nil { + t.Fatal(err) + } + // Sent nothing since numbering existed: what it would be sent is composed with zero, which is + // not on the wire, which is what it was actually sent. + if n, err := open.inventory.Sequence(t.Context(), record.ID); err != nil || n != 0 { + t.Fatalf("a fresh node reads sequence %d, %v", n, err) + } + first, err := open.inventory.NextSequence(t.Context(), record.ID) + if err != nil { + t.Fatal(err) + } + second, err := open.inventory.NextSequence(t.Context(), record.ID) + if err != nil { + t.Fatal(err) + } + if first != 1 || second != 2 { + t.Fatalf("two sends were numbered %d and %d", first, second) + } + // And the read path sees the last one taken, so the comparison composes what was sent. + if n, err := open.inventory.Sequence(t.Context(), record.ID); err != nil || n != 2 { + t.Fatalf("after two sends the node reads sequence %d, %v", n, err) + } + // Another node counts on its own. + other, err := open.inventory.NodeByName(t.Context(), "laptop") + if err != nil { + t.Fatal(err) + } + if n, err := open.inventory.NextSequence(t.Context(), other.ID); err != nil || n != 1 { + t.Fatalf("a second node's first send was numbered %d, %v", n, err) + } +} diff --git a/cmd/mesh-controller/sequence_test.go b/cmd/mesh-controller/sequence_test.go index 5d2dbd67..c86f9140 100644 --- a/cmd/mesh-controller/sequence_test.go +++ b/cmd/mesh-controller/sequence_test.go @@ -39,39 +39,3 @@ func TestAnUnnumberedSendIsByteForByteWhatItWasBefore(t *testing.T) { t.Fatalf("a send numbered zero put a sequence on the wire: %s", body) } } - -func TestEachSendToANodeIsOneHigherAndReadable(t *testing.T) { - open := aMesh(t) - record, err := open.inventory.NodeByName(t.Context(), "anchor") - if err != nil { - t.Fatal(err) - } - // Sent nothing since numbering existed: what it would be sent is composed with zero, which is - // not on the wire, which is what it was actually sent. - if n, err := open.inventory.Sequence(t.Context(), record.ID); err != nil || n != 0 { - t.Fatalf("a fresh node reads sequence %d, %v", n, err) - } - first, err := open.inventory.NextSequence(t.Context(), record.ID) - if err != nil { - t.Fatal(err) - } - second, err := open.inventory.NextSequence(t.Context(), record.ID) - if err != nil { - t.Fatal(err) - } - if first != 1 || second != 2 { - t.Fatalf("two sends were numbered %d and %d", first, second) - } - // And the read path sees the last one taken, so the comparison composes what was sent. - if n, err := open.inventory.Sequence(t.Context(), record.ID); err != nil || n != 2 { - t.Fatalf("after two sends the node reads sequence %d, %v", n, err) - } - // Another node counts on its own. - other, err := open.inventory.NodeByName(t.Context(), "laptop") - if err != nil { - t.Fatal(err) - } - if n, err := open.inventory.NextSequence(t.Context(), other.ID); err != nil || n != 1 { - t.Fatalf("a second node's first send was numbered %d, %v", n, err) - } -} diff --git a/cmd/mesh-controller/served_own_names_test.go b/cmd/mesh-controller/served_own_names_test.go index 3b7866ab..430d9f0f 100644 --- a/cmd/mesh-controller/served_own_names_test.go +++ b/cmd/mesh-controller/served_own_names_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/settings_effective_integration_test.go b/cmd/mesh-controller/settings_effective_integration_test.go new file mode 100644 index 00000000..e23f7a4a --- /dev/null +++ b/cmd/mesh-controller/settings_effective_integration_test.go @@ -0,0 +1,38 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestPreferencesListedFromTheStores(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "notes", Version: "1", + Settings: map[string]catalogue.SettingDeclaration{ + "font-size": {Kind: catalogue.KindPreference, Default: float64(10), Why: "readable at 100 DPI"}, + }, + Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644", + "content": "font = ${setting:font-size}\n"}}}) + if _, err := assign(ctx, open, "laptop", "notes"); err != nil { + t.Fatal(err) + } + if err := open.inventory.SetSettings(ctx, "laptop", "notes", map[string]any{"font-size": float64(16)}); err != nil { + t.Fatal(err) + } + all := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"preferences"}) }) + if !strings.Contains(all, "notes (on laptop)") || !strings.Contains(all, "laptop: 16 (the node)") || + !strings.Contains(all, "font-size, default 10: readable at 100 DPI") { + t.Fatalf("the listing:\n%s", all) + } + if got := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"preferences", "--node", "anchor"}) }); got != "no module on anchor declares a preference\n" { + t.Fatalf("a machine without the module:\n%s", got) + } + if err := settingsCommand(ctx, []string{"preferences", "--node", "nowhere"}); err == nil { + t.Fatal("a machine the mesh does not know was answered") + } +} diff --git a/cmd/mesh-controller/settings_effective_test.go b/cmd/mesh-controller/settings_effective_test.go index f248e2b4..23e20ab1 100644 --- a/cmd/mesh-controller/settings_effective_test.go +++ b/cmd/mesh-controller/settings_effective_test.go @@ -80,30 +80,3 @@ func TestPreferencesSayEachMachinesValueAndItsSource(t *testing.T) { // The listing over the real stores: each machine's value with its source; a machine names only the // modules on it; a machine the mesh does not know is refused (novox/hq ADR 0262). -func TestPreferencesListedFromTheStores(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "notes", Version: "1", - Settings: map[string]catalogue.SettingDeclaration{ - "font-size": {Kind: catalogue.KindPreference, Default: float64(10), Why: "readable at 100 DPI"}, - }, - Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644", - "content": "font = ${setting:font-size}\n"}}}) - if _, err := assign(ctx, open, "laptop", "notes"); err != nil { - t.Fatal(err) - } - if err := open.inventory.SetSettings(ctx, "laptop", "notes", map[string]any{"font-size": float64(16)}); err != nil { - t.Fatal(err) - } - all := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"preferences"}) }) - if !strings.Contains(all, "notes (on laptop)") || !strings.Contains(all, "laptop: 16 (the node)") || - !strings.Contains(all, "font-size, default 10: readable at 100 DPI") { - t.Fatalf("the listing:\n%s", all) - } - if got := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"preferences", "--node", "anchor"}) }); got != "no module on anchor declares a preference\n" { - t.Fatalf("a machine without the module:\n%s", got) - } - if err := settingsCommand(ctx, []string{"preferences", "--node", "nowhere"}); err == nil { - t.Fatal("a machine the mesh does not know was answered") - } -} diff --git a/cmd/mesh-controller/settings_walk_integration_test.go b/cmd/mesh-controller/settings_walk_integration_test.go new file mode 100644 index 00000000..90bfe9ef --- /dev/null +++ b/cmd/mesh-controller/settings_walk_integration_test.go @@ -0,0 +1,439 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" +) + +func TestAnApprovedLayerOpensTheWalkThatSendsIt(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "notes", Version: "1", + Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644", + "content": "x = ${setting:x}\n"}}}) + if _, err := assign(ctx, open, "laptop", "notes"); err != nil { + t.Fatal(err) + } + // A layer set by hand opens no walk: it still waits for a push (ADR 0344 §7). + if err := atTheTerminal(t, "settings", "set", "notes", `{"x":"0"}`, "--node", "laptop"); err != nil { + t.Fatal(err) + } + if plans, _ := open.inventory.OpenPlans(ctx); len(plans) != 0 { + t.Fatalf("a layer set at the terminal opened %d walk(s)", len(plans)) + } + set := setLayerIn(open) + now := time.Now() + values := map[string]any{"x": "1"} + p := settingsProposal{Module: "notes", Values: values, From: "g14/claude-code", At: now, + Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} + words, err := set(ctx, p, "s9", "approved by the operator (ask s9)") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(words, "+ x (ask s9); settings-") || !strings.Contains(words, "sends it to laptop, one machine at a time") { + t.Errorf("the warrant's words: %q", words) + } + plans, err := open.inventory.OpenPlans(ctx) + if err != nil || len(plans) != 1 { + t.Fatalf("open walks: %+v %v", plans, err) + } + w := plans[0] + if w.Release == nil || w.Release.Settings == nil || w.Release.Settings.Module != "notes" || w.Release.Settings.Ask != "s9" || + !reflect.DeepEqual(w.Release.Order, []string{"laptop"}) || w.Repository != "the settings of notes on the whole mesh" || + !reflect.DeepEqual(w.Tiers, [][]string{{"notes"}}) { + t.Fatalf("the settings walk: %+v %+v", w, w.Release) + } + + // A machine's layer where the module does not run: set, and nothing to send. + q := settingsProposal{Module: "notes", Node: "anchor", Values: values, From: "g14/claude-code", At: now, + Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} + words, err = set(ctx, q, "s10", "approved by the operator (ask s10)") + if err != nil || !strings.Contains(words, "no machine runs notes there, so there is nothing to send") { + t.Fatalf("a layer nobody runs: %q %v", words, err) + } + if plans, _ := open.inventory.OpenPlans(ctx); len(plans) != 1 { + t.Fatalf("a walk was opened for a machine that does not run the module: %d open", len(plans)) + } + + // A walk that cannot be opened is the act's failure, said with the push that sends it by hand. + was := openSettingsWalk + openSettingsWalk = func(context.Context, *stores, inventory.PlanSettings) (string, error) { + return "", fmt.Errorf("the plans are not to be had") + } + t.Cleanup(func() { openSettingsWalk = was }) + again := map[string]any{"x": "2"} + byHand := map[string]any{"x": "0"} + r := settingsProposal{Module: "notes", Node: "laptop", Values: again, Before: byHand, HadLayer: true, + From: "g14/claude-code", At: now, Digest: layerDigest(again), BeforeDigest: layerDigest(byHand)} + if _, err := set(ctx, r, "s11", "approved"); err == nil || !strings.Contains(err.Error(), "is set, and the walk that "+ + "sends it could not be opened: the plans are not to be had — `push laptop` sends it") { + t.Errorf("an unopened walk: %v", err) + } +} + +func aSettledMesh(t *testing.T) *walkMesh { + t.Helper() + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + w := &walkMesh{open: open, broken: map[string]bool{}, moves: map[string][]dataMove{}, + heard: map[string]bool{"anchor": true, "laptop": true}} + withConditionsInMemory(t) + was := doctorFrom + doctorFrom = nil + t.Cleanup(func() { doctorFrom = was }) + old := time.Now().Add(-2 * time.Hour) + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c1", Module: "app", Commit: "c1", + Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: old, At: old, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-1"}}}); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ + Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c1", Head: "c1", Asked: old}); err != nil { + t.Fatal(err) + } + for _, n := range []string{"anchor", "laptop"} { + if _, err := inv.Assign(ctx, n, "app"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, map[string]string{"app": "c1"}); err != nil { + t.Fatal(err) + } + } + wasMoves, wasHeard, wasSend, wasGather, wasData := machineMoves, releaseHeard, sendRollout, gatherGateFacts, settingsDataMoves + settingsDataMoves = func(_ context.Context, _ *stores, node string) ([]dataMove, error) { return w.moves[node], nil } + machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { + modules, err := open.inventory.Assigned(ctx, node) + if err != nil { + return nil, err + } + sent, known, err := open.inventory.SentBuilds(ctx, node) + if err != nil { + return nil, err + } + return f.moves(node, modules, sent, known, all), nil + } + releaseHeard = func(context.Context, *stores) (map[string]bool, error) { return w.heard, nil } + n := 0 + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + w.sent = append(w.sent, append([]string(nil), names...)) + w.kept = append(w.kept, len(names) == 1 && settingsKeptOn(ctx, names[0])) + current, err := open.inventory.CurrentBuilds(ctx) + if err != nil { + return nil, err + } + for _, node := range names { + modules, _ := open.inventory.Assigned(ctx, node) + carried := map[string]string{} + for _, m := range modules { + carried[m] = current[m].Commit + } + n++ + digest := fmt.Sprintf("d-%s-%d", node, n) + if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil { + return nil, err + } + if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, + Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { + return nil, err + } + } + return names, nil + } + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, + rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} + reports, err := open.inventory.LastReports(ctx) + if err != nil { + return f, err + } + for _, r := range reports { + if w.broken[r.Node] { + r.Outcome = inventory.OutcomeFailed + } + f.reports[r.Node] = r + } + return f, nil + } + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) + gateWithin, gateEvery = time.Hour, time.Hour + t.Cleanup(func() { + machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather + settingsDataMoves = wasData + gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound + }) + return w +} + +func TestASettingsWalkSendsOneMachineAtATimeEachJudged(t *testing.T) { + judgeLeavesAsTheCore(t) + w := aSettledMesh(t) + ctx := t.Context() + words, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s1", Change: "~ x"}) + if err != nil || !strings.Contains(words, "sends it to anchor, laptop") { + t.Fatalf("%q %v", words, err) + } + advancePlans(ctx, w.open) + if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) { + t.Fatalf("sent %v: the first machine alone, before it is judged", w.sent) + } + p := w.walk(t) + if p.Release.Gate == nil || p.Release.Next != 0 || !strings.Contains(p.Note, "judging app on anchor") { + t.Fatalf("the walk after its first send: %s %+v", p.Note, p.Release) + } + gateEvery = 0 + for i := 0; i < 6; i++ { + advancePlans(ctx, w.open) + } + if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}, {"laptop"}}) { + t.Fatalf("sent %v", w.sent) + } + p = w.walk(t) + if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, []string{"anchor", "laptop"}) || + p.Outcome() != inventory.WalkDelivered || !strings.Contains(p.Note, "delivered on anchor, laptop") { + t.Fatalf("the walk is %s: %s %+v", p.State, p.Note, p.Release) + } + // It said its start and its end, so the operator is told once (ADR 0308). + if p.Times == nil || p.Times.Started == nil || p.Times.Ended == nil { + t.Fatalf("the walk's start and end are not marked: %+v", p.Times) + } + // Nothing waits, and no walk of waiting builds opened beside it. + if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r != nil { + t.Fatalf("%+v %v", r, err) + } +} + +func TestAFailedSettingsWalkStopsAndHoldsNothingBack(t *testing.T) { + w := aSettledMesh(t) + ctx := t.Context() + inv := w.open.inventory + w.broken["anchor"] = true + if err := inv.KeepSettings(ctx, "", "app", map[string]any{"x": "1"}); err != nil { + t.Fatal(err) + } + if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s2"}); err != nil { + t.Fatal(err) + } + for i := 0; i < 4; i++ { + advancePlans(ctx, w.open) + } + p := w.walk(t) + if p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed || + !strings.Contains(p.Note, "the settings of app on the whole mesh failed its gate on anchor") || + !strings.Contains(p.Note, "the layer stays as the operator approved it") || !strings.Contains(p.Note, "not sent: laptop") { + t.Fatalf("the walk is %s: %s", p.State, p.Note) + } + if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) { + t.Fatalf("sent %v: anchor alone, nothing to laptop", w.sent) + } + if layer, has, err := inv.Layer(ctx, "", "app"); err != nil || !has || layer["x"] != "1" { + t.Fatalf("the layer after the failure: %v %v %v", layer, has, err) + } + // A build that waits for a gate is still walked: the failed settings walk holds nothing back. + w.broken["anchor"] = false + now := time.Now().Add(-time.Minute) + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", + Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ + Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil { + t.Fatal(err) + } + if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r == nil { + t.Fatalf("the waiting build is held back by a failed settings walk: %+v %v", r, err) + } +} + +func TestASettingsWalkSendsTheLayerAloneEveryBuildKept(t *testing.T) { + w := aSettledMesh(t) + ctx := t.Context() + inv := w.open.inventory + gateEvery = 0 + if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s3"}); err != nil { + t.Fatal(err) + } + for i := 0; i < 4; i++ { + advancePlans(ctx, w.open) + } + if !reflect.DeepEqual(w.kept, []bool{true, true}) || w.walk(t).State != inventory.PlanDone { + t.Fatalf("the sends kept every build: %v; the walk: %s", w.kept, w.walk(t).Note) + } + // What such a send keeps, on the store: a rolled-out build waiting there is kept at the one last sent. + now := time.Now().Add(-time.Minute) + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", + Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ + Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil { + t.Fatal(err) + } + kept, err := recordedKept(keepingBuildsForSettings(ctx, "anchor"), w.open, "anchor") + if err != nil || kept["app"] != "c1" { + t.Fatalf("a settings send to anchor keeps %v (%v), want app at c1", kept, err) + } + if kept, _ := recordedKept(ctx, w.open, "anchor"); kept["app"] != "" { + t.Fatalf("an ordinary send keeps a rolled-out build: %v", kept) + } + // And so the ungated build waiting on anchor is no move of the settings send's: nothing for a gate to see. + f, err := readMoveFacts(ctx, inv) + if err != nil { + t.Fatal(err) + } + waiting, _ := machineMoves(ctx, w.open, f, "anchor", false) + if len(waiting) != 1 || waiting[0].Module != "app" { + t.Fatalf("anchor waits for %+v, want app's c2", waiting) + } + if moved, err := withoutKept(keepingBuildsForSettings(ctx, "anchor"), w.open, "anchor", waiting); err != nil || len(moved) != 0 { + t.Fatalf("a settings send to anchor would move %+v (%v)", moved, err) + } +} + +func TestASettingsWalkStopsBeforeADataMoveAndPassesOverAMachineAway(t *testing.T) { + w := aSettledMesh(t) + ctx := t.Context() + gateEvery = 0 + w.moves["laptop"] = []dataMove{{Container: "app-server", Inside: "/data", From: "/srv/a", To: "/srv/b"}} + if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s4"}); err != nil { + t.Fatal(err) + } + for i := 0; i < 4; i++ { + advancePlans(ctx, w.open) + } + p := w.walk(t) + if p.State != inventory.PlanFailed || !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) || + !strings.Contains(p.Note, "not sent to laptop: a running module's data would move there") || + !strings.Contains(p.Note, "push laptop --move app") { + t.Fatalf("sent %v; the walk is %s: %s", w.sent, p.State, p.Note) + } + + v := aSettledMesh(t) + v.heard = map[string]bool{"laptop": true} + gateEvery = 0 + if _, err := openSettingsWalk(ctx, v.open, inventory.PlanSettings{Module: "app", Ask: "s5"}); err != nil { + t.Fatal(err) + } + for i := 0; i < 4; i++ { + advancePlans(ctx, v.open) + } + p = v.walk(t) + if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Skipped, []string{"anchor"}) || + !strings.Contains(p.Note, "delivered on laptop; not heard from, left as they were, their next send carries it: anchor") { + t.Fatalf("the walk with anchor away is %s: %s", p.State, p.Note) + } + + u := aSettledMesh(t) + u.heard = map[string]bool{} + if _, err := openSettingsWalk(ctx, u.open, inventory.PlanSettings{Module: "app", Ask: "s6"}); err != nil { + t.Fatal(err) + } + advancePlans(ctx, u.open) + if p := u.walk(t); p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed || + !strings.Contains(p.Note, "was sent to no machine") || len(u.sent) != 0 { + t.Fatalf("a walk that reached nobody is %s: %s", p.State, p.Note) + } +} + +func TestACrossingSendSupersedesASettingsGate(t *testing.T) { + judgeLeavesAsTheCore(t) + w := aSettledMesh(t) + ctx := t.Context() + inv := w.open.inventory + if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s7"}); err != nil { + t.Fatal(err) + } + advancePlans(ctx, w.open) + p := w.walk(t) + if g := p.Release.Gate; g == nil || g.To != "c1" || g.From != "c1" { + t.Fatalf("the settings gate judges no build: %+v", p.Release.Gate) + } + // Another walk sends anchor app's c2 while the settings gate judges there. + now := time.Now().Add(-time.Minute) + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", + Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, w.open, "anchor"), "d-merge", map[string]string{"app": "c2"}); err != nil { + t.Fatal(err) + } + if _, err := inv.RecordDoing(ctx, nodeID(t, w.open, "anchor"), inventory.Doing{Node: "anchor", + Outcome: inventory.OutcomeApplied, Declared: "d-merge", Applied: 1, At: time.Now()}); err != nil { + t.Fatal(err) + } + gateEvery = 0 + advancePlans(ctx, w.open) + p = w.walk(t) + if p.State != inventory.PlanSuperseded || p.Release.Gate == nil || p.Release.Gate.Verdict != inventory.GateSuperseded || + !strings.Contains(p.Note, "superseded on anchor") || len(p.Release.Done) != 0 { + t.Fatalf("the settings walk after a crossing send is %s: %s %+v", p.State, p.Note, p.Release) + } + if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) { + t.Fatalf("sent %v: laptop was sent after a superseded judging", w.sent) + } +} + +func TestASettingsWalkFailsOnARefusalThatCannotClear(t *testing.T) { + w := aSettledMesh(t) + ctx := t.Context() + inv := w.open.inventory + // What a send to a machine never sent can keep: nothing, and said as the refusal the walk fails on. + if _, err := inv.AddNode(ctx, "fresh"); err != nil { + t.Fatal(err) + } + if _, err := recordedKept(keepingBuildsForSettings(ctx, "fresh"), w.open, "fresh"); !errors.Is(err, errSettingsCannotKeep) { + t.Fatalf("a settings send to a machine never sent: %v", err) + } + w.heard = map[string]bool{"laptop": true} + settingsDataMoves = func(_ context.Context, _ *stores, node string) ([]dataMove, error) { + return nil, fmt.Errorf("%w: what %s was last sent is not known", errSettingsCannotKeep, node) + } + if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s8"}); err != nil { + t.Fatal(err) + } + advancePlans(ctx, w.open) + p := w.walk(t) + if p.State != inventory.PlanFailed || len(w.sent) != 0 || + !strings.Contains(p.Note, "not sent to laptop: the settings cannot be sent with every build kept: what laptop was last sent is not known") || + !strings.Contains(p.Note, "not heard from, left as they were: anchor") { + t.Fatalf("the walk is %s: %s", p.State, p.Note) + } + // It is not open, so a build waiting for a gate is walked. + now := time.Now().Add(-time.Minute) + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", + Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ + Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil { + t.Fatal(err) + } + if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r == nil { + t.Fatalf("the waiting build is held by the failed settings walk: %+v %v", r, err) + } +} diff --git a/cmd/mesh-controller/settings_walk_test.go b/cmd/mesh-controller/settings_walk_test.go index 45df338a..2d9e0c18 100644 --- a/cmd/mesh-controller/settings_walk_test.go +++ b/cmd/mesh-controller/settings_walk_test.go @@ -2,17 +2,11 @@ package main import ( "context" - "encoding/json" - "errors" - "fmt" "reflect" "strings" "testing" - "time" - "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" ) // novox/hq ADR 0344, issue 641: the operator's one Approve on a settings proposal is the act that makes it take @@ -108,71 +102,6 @@ func TestTheAskSaysAnApprovedLayerIsSentAtOnce(t *testing.T) { // The warrant sets the layer and opens the walk that sends it, naming the machines it reaches; a layer for a // machine that does not run the module opens none and says so. -func TestAnApprovedLayerOpensTheWalkThatSendsIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "notes", Version: "1", - Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644", - "content": "x = ${setting:x}\n"}}}) - if _, err := assign(ctx, open, "laptop", "notes"); err != nil { - t.Fatal(err) - } - // A layer set by hand opens no walk: it still waits for a push (ADR 0344 §7). - if err := atTheTerminal(t, "settings", "set", "notes", `{"x":"0"}`, "--node", "laptop"); err != nil { - t.Fatal(err) - } - if plans, _ := open.inventory.OpenPlans(ctx); len(plans) != 0 { - t.Fatalf("a layer set at the terminal opened %d walk(s)", len(plans)) - } - set := setLayerIn(open) - now := time.Now() - values := map[string]any{"x": "1"} - p := settingsProposal{Module: "notes", Values: values, From: "g14/claude-code", At: now, - Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} - words, err := set(ctx, p, "s9", "approved by the operator (ask s9)") - if err != nil { - t.Fatal(err) - } - if !strings.Contains(words, "+ x (ask s9); settings-") || !strings.Contains(words, "sends it to laptop, one machine at a time") { - t.Errorf("the warrant's words: %q", words) - } - plans, err := open.inventory.OpenPlans(ctx) - if err != nil || len(plans) != 1 { - t.Fatalf("open walks: %+v %v", plans, err) - } - w := plans[0] - if w.Release == nil || w.Release.Settings == nil || w.Release.Settings.Module != "notes" || w.Release.Settings.Ask != "s9" || - !reflect.DeepEqual(w.Release.Order, []string{"laptop"}) || w.Repository != "the settings of notes on the whole mesh" || - !reflect.DeepEqual(w.Tiers, [][]string{{"notes"}}) { - t.Fatalf("the settings walk: %+v %+v", w, w.Release) - } - - // A machine's layer where the module does not run: set, and nothing to send. - q := settingsProposal{Module: "notes", Node: "anchor", Values: values, From: "g14/claude-code", At: now, - Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} - words, err = set(ctx, q, "s10", "approved by the operator (ask s10)") - if err != nil || !strings.Contains(words, "no machine runs notes there, so there is nothing to send") { - t.Fatalf("a layer nobody runs: %q %v", words, err) - } - if plans, _ := open.inventory.OpenPlans(ctx); len(plans) != 1 { - t.Fatalf("a walk was opened for a machine that does not run the module: %d open", len(plans)) - } - - // A walk that cannot be opened is the act's failure, said with the push that sends it by hand. - was := openSettingsWalk - openSettingsWalk = func(context.Context, *stores, inventory.PlanSettings) (string, error) { - return "", fmt.Errorf("the plans are not to be had") - } - t.Cleanup(func() { openSettingsWalk = was }) - again := map[string]any{"x": "2"} - byHand := map[string]any{"x": "0"} - r := settingsProposal{Module: "notes", Node: "laptop", Values: again, Before: byHand, HadLayer: true, - From: "g14/claude-code", At: now, Digest: layerDigest(again), BeforeDigest: layerDigest(byHand)} - if _, err := set(ctx, r, "s11", "approved"); err == nil || !strings.Contains(err.Error(), "is set, and the walk that "+ - "sends it could not be opened: the plans are not to be had — `push laptop` sends it") { - t.Errorf("an unopened walk: %v", err) - } -} // walkMesh is a mesh where `app` runs c1 on anchor and laptop and nothing waits for a gate: every send applied at // once, recorded, and judged from the reports, with a machine named broken failing what it was sent. @@ -186,105 +115,6 @@ type walkMesh struct { heard map[string]bool } -func aSettledMesh(t *testing.T) *walkMesh { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - w := &walkMesh{open: open, broken: map[string]bool{}, moves: map[string][]dataMove{}, - heard: map[string]bool{"anchor": true, "laptop": true}} - withConditionsInMemory(t) - was := doctorFrom - doctorFrom = nil - t.Cleanup(func() { doctorFrom = was }) - old := time.Now().Add(-2 * time.Hour) - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c1", Module: "app", Commit: "c1", - Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: old, At: old, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-1"}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c1", Head: "c1", Asked: old}); err != nil { - t.Fatal(err) - } - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, "app"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, map[string]string{"app": "c1"}); err != nil { - t.Fatal(err) - } - } - wasMoves, wasHeard, wasSend, wasGather, wasData := machineMoves, releaseHeard, sendRollout, gatherGateFacts, settingsDataMoves - settingsDataMoves = func(_ context.Context, _ *stores, node string) ([]dataMove, error) { return w.moves[node], nil } - machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { - modules, err := open.inventory.Assigned(ctx, node) - if err != nil { - return nil, err - } - sent, known, err := open.inventory.SentBuilds(ctx, node) - if err != nil { - return nil, err - } - return f.moves(node, modules, sent, known, all), nil - } - releaseHeard = func(context.Context, *stores) (map[string]bool, error) { return w.heard, nil } - n := 0 - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - w.sent = append(w.sent, append([]string(nil), names...)) - w.kept = append(w.kept, len(names) == 1 && settingsKeptOn(ctx, names[0])) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - for _, node := range names { - modules, _ := open.inventory.Assigned(ctx, node) - carried := map[string]string{} - for _, m := range modules { - carried[m] = current[m].Commit - } - n++ - digest := fmt.Sprintf("d-%s-%d", node, n) - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil { - return nil, err - } - if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, - Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { - return nil, err - } - } - return names, nil - } - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, - rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} - reports, err := open.inventory.LastReports(ctx) - if err != nil { - return f, err - } - for _, r := range reports { - if w.broken[r.Node] { - r.Outcome = inventory.OutcomeFailed - } - f.reports[r.Node] = r - } - return f, nil - } - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - gateWithin, gateEvery = time.Hour, time.Hour - t.Cleanup(func() { - machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather - settingsDataMoves = wasData - gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound - }) - return w -} - func (w *walkMesh) walk(t *testing.T) inventory.Plan { t.Helper() plans, err := w.open.inventory.RecentPlans(t.Context(), 10) @@ -302,271 +132,21 @@ func (w *walkMesh) walk(t *testing.T) inventory.Plan { // The walk sends the first machine alone and judges the module there before it sends the next, and ends // delivered on both; nothing waited for a gate, so each send carried the layer alone. -func TestASettingsWalkSendsOneMachineAtATimeEachJudged(t *testing.T) { - judgeLeavesAsTheCore(t) - w := aSettledMesh(t) - ctx := t.Context() - words, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s1", Change: "~ x"}) - if err != nil || !strings.Contains(words, "sends it to anchor, laptop") { - t.Fatalf("%q %v", words, err) - } - advancePlans(ctx, w.open) - if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: the first machine alone, before it is judged", w.sent) - } - p := w.walk(t) - if p.Release.Gate == nil || p.Release.Next != 0 || !strings.Contains(p.Note, "judging app on anchor") { - t.Fatalf("the walk after its first send: %s %+v", p.Note, p.Release) - } - gateEvery = 0 - for i := 0; i < 6; i++ { - advancePlans(ctx, w.open) - } - if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}, {"laptop"}}) { - t.Fatalf("sent %v", w.sent) - } - p = w.walk(t) - if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, []string{"anchor", "laptop"}) || - p.Outcome() != inventory.WalkDelivered || !strings.Contains(p.Note, "delivered on anchor, laptop") { - t.Fatalf("the walk is %s: %s %+v", p.State, p.Note, p.Release) - } - // It said its start and its end, so the operator is told once (ADR 0308). - if p.Times == nil || p.Times.Started == nil || p.Times.Ended == nil { - t.Fatalf("the walk's start and end are not marked: %+v", p.Times) - } - // Nothing waits, and no walk of waiting builds opened beside it. - if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r != nil { - t.Fatalf("%+v %v", r, err) - } -} // A machine that fails what it was sent stops the walk there: the next is not sent, the layer stays, and the // failure holds back no walk of waiting builds. -func TestAFailedSettingsWalkStopsAndHoldsNothingBack(t *testing.T) { - w := aSettledMesh(t) - ctx := t.Context() - inv := w.open.inventory - w.broken["anchor"] = true - if err := inv.KeepSettings(ctx, "", "app", map[string]any{"x": "1"}); err != nil { - t.Fatal(err) - } - if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s2"}); err != nil { - t.Fatal(err) - } - for i := 0; i < 4; i++ { - advancePlans(ctx, w.open) - } - p := w.walk(t) - if p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed || - !strings.Contains(p.Note, "the settings of app on the whole mesh failed its gate on anchor") || - !strings.Contains(p.Note, "the layer stays as the operator approved it") || !strings.Contains(p.Note, "not sent: laptop") { - t.Fatalf("the walk is %s: %s", p.State, p.Note) - } - if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: anchor alone, nothing to laptop", w.sent) - } - if layer, has, err := inv.Layer(ctx, "", "app"); err != nil || !has || layer["x"] != "1" { - t.Fatalf("the layer after the failure: %v %v %v", layer, has, err) - } - // A build that waits for a gate is still walked: the failed settings walk holds nothing back. - w.broken["anchor"] = false - now := time.Now().Add(-time.Minute) - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", - Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil { - t.Fatal(err) - } - if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r == nil { - t.Fatalf("the waiting build is held back by a failed settings walk: %+v %v", r, err) - } -} // Each send of a settings walk keeps every build of its machine: the layer goes, and a build held for a person or // waiting for a gate stays where it is (ADR 0344 §3). -func TestASettingsWalkSendsTheLayerAloneEveryBuildKept(t *testing.T) { - w := aSettledMesh(t) - ctx := t.Context() - inv := w.open.inventory - gateEvery = 0 - if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s3"}); err != nil { - t.Fatal(err) - } - for i := 0; i < 4; i++ { - advancePlans(ctx, w.open) - } - if !reflect.DeepEqual(w.kept, []bool{true, true}) || w.walk(t).State != inventory.PlanDone { - t.Fatalf("the sends kept every build: %v; the walk: %s", w.kept, w.walk(t).Note) - } - // What such a send keeps, on the store: a rolled-out build waiting there is kept at the one last sent. - now := time.Now().Add(-time.Minute) - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", - Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil { - t.Fatal(err) - } - kept, err := recordedKept(keepingBuildsForSettings(ctx, "anchor"), w.open, "anchor") - if err != nil || kept["app"] != "c1" { - t.Fatalf("a settings send to anchor keeps %v (%v), want app at c1", kept, err) - } - if kept, _ := recordedKept(ctx, w.open, "anchor"); kept["app"] != "" { - t.Fatalf("an ordinary send keeps a rolled-out build: %v", kept) - } - // And so the ungated build waiting on anchor is no move of the settings send's: nothing for a gate to see. - f, err := readMoveFacts(ctx, inv) - if err != nil { - t.Fatal(err) - } - waiting, _ := machineMoves(ctx, w.open, f, "anchor", false) - if len(waiting) != 1 || waiting[0].Module != "app" { - t.Fatalf("anchor waits for %+v, want app's c2", waiting) - } - if moved, err := withoutKept(keepingBuildsForSettings(ctx, "anchor"), w.open, "anchor", waiting); err != nil || len(moved) != 0 { - t.Fatalf("a settings send to anchor would move %+v (%v)", moved, err) - } -} // A machine whose send would move a running module's data stops the walk there, failed and said with the push that // sends it on a person's word (ADR 0217 §3); one not heard from is passed over and named; and a walk that reached // no machine is not said delivered. -func TestASettingsWalkStopsBeforeADataMoveAndPassesOverAMachineAway(t *testing.T) { - w := aSettledMesh(t) - ctx := t.Context() - gateEvery = 0 - w.moves["laptop"] = []dataMove{{Container: "app-server", Inside: "/data", From: "/srv/a", To: "/srv/b"}} - if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s4"}); err != nil { - t.Fatal(err) - } - for i := 0; i < 4; i++ { - advancePlans(ctx, w.open) - } - p := w.walk(t) - if p.State != inventory.PlanFailed || !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) || - !strings.Contains(p.Note, "not sent to laptop: a running module's data would move there") || - !strings.Contains(p.Note, "push laptop --move app") { - t.Fatalf("sent %v; the walk is %s: %s", w.sent, p.State, p.Note) - } - - v := aSettledMesh(t) - v.heard = map[string]bool{"laptop": true} - gateEvery = 0 - if _, err := openSettingsWalk(ctx, v.open, inventory.PlanSettings{Module: "app", Ask: "s5"}); err != nil { - t.Fatal(err) - } - for i := 0; i < 4; i++ { - advancePlans(ctx, v.open) - } - p = v.walk(t) - if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Skipped, []string{"anchor"}) || - !strings.Contains(p.Note, "delivered on laptop; not heard from, left as they were, their next send carries it: anchor") { - t.Fatalf("the walk with anchor away is %s: %s", p.State, p.Note) - } - - u := aSettledMesh(t) - u.heard = map[string]bool{} - if _, err := openSettingsWalk(ctx, u.open, inventory.PlanSettings{Module: "app", Ask: "s6"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, u.open) - if p := u.walk(t); p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed || - !strings.Contains(p.Note, "was sent to no machine") || len(u.sent) != 0 { - t.Fatalf("a walk that reached nobody is %s: %s", p.State, p.Note) - } -} // The gate keeps the module's build its send kept, so a crossing send — a merge's walk sending the machine a new // build of the module while the settings gate judges there — supersedes it, and its report is never read as the // settings send's (issue 641 review, finding 1). -func TestACrossingSendSupersedesASettingsGate(t *testing.T) { - judgeLeavesAsTheCore(t) - w := aSettledMesh(t) - ctx := t.Context() - inv := w.open.inventory - if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s7"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, w.open) - p := w.walk(t) - if g := p.Release.Gate; g == nil || g.To != "c1" || g.From != "c1" { - t.Fatalf("the settings gate judges no build: %+v", p.Release.Gate) - } - // Another walk sends anchor app's c2 while the settings gate judges there. - now := time.Now().Add(-time.Minute) - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", - Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, w.open, "anchor"), "d-merge", map[string]string{"app": "c2"}); err != nil { - t.Fatal(err) - } - if _, err := inv.RecordDoing(ctx, nodeID(t, w.open, "anchor"), inventory.Doing{Node: "anchor", - Outcome: inventory.OutcomeApplied, Declared: "d-merge", Applied: 1, At: time.Now()}); err != nil { - t.Fatal(err) - } - gateEvery = 0 - advancePlans(ctx, w.open) - p = w.walk(t) - if p.State != inventory.PlanSuperseded || p.Release.Gate == nil || p.Release.Gate.Verdict != inventory.GateSuperseded || - !strings.Contains(p.Note, "superseded on anchor") || len(p.Release.Done) != 0 { - t.Fatalf("the settings walk after a crossing send is %s: %s %+v", p.State, p.Note, p.Release) - } - if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: laptop was sent after a superseded judging", w.sent) - } -} // A send refusal that cannot clear by itself — what the machine was last sent is not known, or a build it runs is // gone — fails the walk with why, rather than keeping it open and holding the walk of waiting builds; and a // failure names the machines passed over as away (issue 641 review, findings 2 and 3). -func TestASettingsWalkFailsOnARefusalThatCannotClear(t *testing.T) { - w := aSettledMesh(t) - ctx := t.Context() - inv := w.open.inventory - // What a send to a machine never sent can keep: nothing, and said as the refusal the walk fails on. - if _, err := inv.AddNode(ctx, "fresh"); err != nil { - t.Fatal(err) - } - if _, err := recordedKept(keepingBuildsForSettings(ctx, "fresh"), w.open, "fresh"); !errors.Is(err, errSettingsCannotKeep) { - t.Fatalf("a settings send to a machine never sent: %v", err) - } - w.heard = map[string]bool{"laptop": true} - settingsDataMoves = func(_ context.Context, _ *stores, node string) ([]dataMove, error) { - return nil, fmt.Errorf("%w: what %s was last sent is not known", errSettingsCannotKeep, node) - } - if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s8"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, w.open) - p := w.walk(t) - if p.State != inventory.PlanFailed || len(w.sent) != 0 || - !strings.Contains(p.Note, "not sent to laptop: the settings cannot be sent with every build kept: what laptop was last sent is not known") || - !strings.Contains(p.Note, "not heard from, left as they were: anchor") { - t.Fatalf("the walk is %s: %s", p.State, p.Note) - } - // It is not open, so a build waiting for a gate is walked. - now := time.Now().Add(-time.Minute) - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", - Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil { - t.Fatal(err) - } - if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r == nil { - t.Fatalf("the waiting build is held by the failed settings walk: %+v %v", r, err) - } -} diff --git a/cmd/mesh-controller/standing_integration_test.go b/cmd/mesh-controller/standing_integration_test.go new file mode 100644 index 00000000..625f0f15 --- /dev/null +++ b/cmd/mesh-controller/standing_integration_test.go @@ -0,0 +1,105 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/link" +) + +func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "idp", Version: "1", + Receives: map[string]string{"oidc-client": "/var/lib/mesh/idp/mesh.json"}}) + if _, err := assign(ctx, open, "anchor", "idp"); err != nil { + t.Fatal(err) + } + kept := standings{keeper: func() *conditions.Keeper { return conditionsFrom }} + since := time.Now().Add(-23 * time.Hour) + failing := link.Standing{Module: "idp", Failing: true, Provider: "oidc-client", ProviderNode: "anchor", + Consumer: "mesh_laptop_dashboard", Node: "laptop", Class: "credentials-rejected", + Error: `Keycloak token request failed: 401 {"error":"invalid_grant"}`, Since: since, Attempts: 31000} + if _, err := kept.Stood(ctx, failing); err != nil { + t.Fatal(err) + } + + asked, err := theThreeQuestions(ctx, open) + if err != nil { + t.Fatal(err) + } + if asked.well() { + t.Fatal("a mesh whose identity provider fails a consumer reads as well") + } + said := printed(t, func() error { return printStatus(asked) }) + for _, want := range []string{"1 open condition(s)", "provider.idp.anchor.mesh_laptop_dashboard.failing", + "idp on anchor keeps failing mesh_laptop_dashboard on laptop", "credentials-rejected", "31000 attempt(s)"} { + if !strings.Contains(said, want) { + t.Fatalf("status does not say %q:\n%s", want, said) + } + } + if strings.Contains(said, "all doing what they were told") { + t.Fatalf("status said all well beside a failing provider:\n%s", said) + } + if !strings.HasPrefix(said, "1 open condition(s)") { + t.Fatalf("status does not lead with what is open:\n%s", said) + } + body, err := statusAsJSON(asked) + if err != nil { + t.Fatal(err) + } + var doc struct { + Conditions []conditions.Condition `json:"conditions"` + Failing []conditions.Condition `json:"failing"` + } + if err := json.Unmarshal(body, &doc); err != nil || len(doc.Failing) != 1 || len(doc.Conditions) != 1 || + !strings.Contains(doc.Failing[0].Evidence[0].Said, "invalid_grant") { + t.Fatalf("the document does not carry it: %v\n%s", err, body) + } + // Both machines' `node show` name it: where the provider runs, and where the consumer is. + for _, node := range []string{"anchor", "laptop"} { + shown := printed(t, func() error { return showNode(ctx, open, node) }) + if !strings.Contains(shown, "open condition(s) about this machine") || !strings.Contains(shown, "mesh_laptop_dashboard") { + t.Fatalf("node show %s does not name it:\n%s", node, shown) + } + } + + // Recovered: gone, and the mesh may be well again as far as this is concerned. + failing.Failing = false + if cleared, err := kept.Stood(ctx, failing); err != nil || !cleared { + t.Fatalf("%v %v", cleared, err) + } + asked, err = theThreeQuestions(ctx, open) + if err != nil { + t.Fatal(err) + } + if len(asked.conditions) != 0 { + t.Fatalf("a recovered consumer is still named: %+v", asked.conditions) + } +} + +func TestAnUnassignedProvidersLastWordIsCleared(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + kept := standings{keeper: func() *conditions.Keeper { return conditionsFrom }} + if _, err := kept.Stood(ctx, link.Standing{Module: "gone", Failing: true, + ProviderNode: "anchor", Consumer: "x", Since: time.Now()}); err != nil { + t.Fatal(err) + } + all, err := conditionsFrom.Open(ctx) + if err != nil || len(all) != 1 { + t.Fatalf("%+v %v", all, err) + } + if err := unassignedProviders(ctx, open.inventory, conditionsFrom, all); err != nil { + t.Fatal(err) + } + if all, _ := conditionsFrom.Open(ctx); len(all) != 0 { + t.Fatalf("%+v", all) + } +} diff --git a/cmd/mesh-controller/standing_test.go b/cmd/mesh-controller/standing_test.go index d174dd0c..88d69600 100644 --- a/cmd/mesh-controller/standing_test.go +++ b/cmd/mesh-controller/standing_test.go @@ -1,12 +1,9 @@ package main import ( - "encoding/json" "strings" "testing" - "time" - "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/link" ) @@ -15,98 +12,9 @@ import ( // the condition store's first kind (to-be 45 §2). On 2026-10-05 the identity provider refused every // consumer for a day and status called the mesh well (04-ISSUES/179): this is that day, told to the // controller the way the provider now tells it. -func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "idp", Version: "1", - Receives: map[string]string{"oidc-client": "/var/lib/mesh/idp/mesh.json"}}) - if _, err := assign(ctx, open, "anchor", "idp"); err != nil { - t.Fatal(err) - } - kept := standings{keeper: func() *conditions.Keeper { return conditionsFrom }} - since := time.Now().Add(-23 * time.Hour) - failing := link.Standing{Module: "idp", Failing: true, Provider: "oidc-client", ProviderNode: "anchor", - Consumer: "mesh_laptop_dashboard", Node: "laptop", Class: "credentials-rejected", - Error: `Keycloak token request failed: 401 {"error":"invalid_grant"}`, Since: since, Attempts: 31000} - if _, err := kept.Stood(ctx, failing); err != nil { - t.Fatal(err) - } - - asked, err := theThreeQuestions(ctx, open) - if err != nil { - t.Fatal(err) - } - if asked.well() { - t.Fatal("a mesh whose identity provider fails a consumer reads as well") - } - said := printed(t, func() error { return printStatus(asked) }) - for _, want := range []string{"1 open condition(s)", "provider.idp.anchor.mesh_laptop_dashboard.failing", - "idp on anchor keeps failing mesh_laptop_dashboard on laptop", "credentials-rejected", "31000 attempt(s)"} { - if !strings.Contains(said, want) { - t.Fatalf("status does not say %q:\n%s", want, said) - } - } - if strings.Contains(said, "all doing what they were told") { - t.Fatalf("status said all well beside a failing provider:\n%s", said) - } - if !strings.HasPrefix(said, "1 open condition(s)") { - t.Fatalf("status does not lead with what is open:\n%s", said) - } - body, err := statusAsJSON(asked) - if err != nil { - t.Fatal(err) - } - var doc struct { - Conditions []conditions.Condition `json:"conditions"` - Failing []conditions.Condition `json:"failing"` - } - if err := json.Unmarshal(body, &doc); err != nil || len(doc.Failing) != 1 || len(doc.Conditions) != 1 || - !strings.Contains(doc.Failing[0].Evidence[0].Said, "invalid_grant") { - t.Fatalf("the document does not carry it: %v\n%s", err, body) - } - // Both machines' `node show` name it: where the provider runs, and where the consumer is. - for _, node := range []string{"anchor", "laptop"} { - shown := printed(t, func() error { return showNode(ctx, open, node) }) - if !strings.Contains(shown, "open condition(s) about this machine") || !strings.Contains(shown, "mesh_laptop_dashboard") { - t.Fatalf("node show %s does not name it:\n%s", node, shown) - } - } - - // Recovered: gone, and the mesh may be well again as far as this is concerned. - failing.Failing = false - if cleared, err := kept.Stood(ctx, failing); err != nil || !cleared { - t.Fatalf("%v %v", cleared, err) - } - asked, err = theThreeQuestions(ctx, open) - if err != nil { - t.Fatal(err) - } - if len(asked.conditions) != 0 { - t.Fatalf("a recovered consumer is still named: %+v", asked.conditions) - } -} // A provider no longer assigned where it ran has nothing running to fail anybody: its last word is // cleared on the next look, said as resolved by the assignment. -func TestAnUnassignedProvidersLastWordIsCleared(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - kept := standings{keeper: func() *conditions.Keeper { return conditionsFrom }} - if _, err := kept.Stood(ctx, link.Standing{Module: "gone", Failing: true, - ProviderNode: "anchor", Consumer: "x", Since: time.Now()}); err != nil { - t.Fatal(err) - } - all, err := conditionsFrom.Open(ctx) - if err != nil || len(all) != 1 { - t.Fatalf("%+v %v", all, err) - } - if err := unassignedProviders(ctx, open.inventory, conditionsFrom, all); err != nil { - t.Fatal(err) - } - if all, _ := conditionsFrom.Open(ctx); len(all) != 0 { - t.Fatalf("%+v", all) - } -} // **The store away holds a recovery** (ADR 0224 §3): a standing that cannot be kept is asked again. func TestAStandingTheStoreCannotKeepIsAskedAgain(t *testing.T) { diff --git a/cmd/mesh-controller/status_summary_test.go b/cmd/mesh-controller/status_summary_test.go index 6f8a6984..8b342cf3 100644 --- a/cmd/mesh-controller/status_summary_test.go +++ b/cmd/mesh-controller/status_summary_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/status_test.go b/cmd/mesh-controller/status_test.go index db5102ce..2272a6d7 100644 --- a/cmd/mesh-controller/status_test.go +++ b/cmd/mesh-controller/status_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/template_integration_test.go b/cmd/mesh-controller/template_integration_test.go new file mode 100644 index 00000000..89338cd5 --- /dev/null +++ b/cmd/mesh-controller/template_integration_test.go @@ -0,0 +1,148 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + snapshot "github.com/novox/mesh-controller/internal/facts" + "github.com/novox/mesh-controller/internal/inventory" +) + +func templatedMesh(t *testing.T) (snapshot.Facts, map[string]string) { + t.Helper() + open := aMesh(t) + ctx := t.Context() + manifests := map[string]string{ + "lemurs": `{"module":"lemurs","version":"1","capabilities":["systemd"], + "resources":[{"id":"service","type":"service","unit":"lemurs.service","state":"running","boot":"enabled"}]}`, + "notifier": inlineNotifier, + } + for name, raw := range manifests { + m, err := catalogue.ParseManifest([]byte(raw)) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", + Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "laptop", name); err != nil { + t.Fatalf("assign %s: %v", name, err) + } + } + f, err := gatherFacts(ctx, open, "2.11.17") + if err != nil { + t.Fatal(err) + } + return f, manifests +} + +func TestTheMergeGateJudgesAChangeToATemplateAlone(t *testing.T) { + f, manifests := templatedMesh(t) + + // The module moved to its template, the same text: nothing changes on any machine. + v := gateJudged(t, f, aTemplatedTree(t, manifests, "volume = 3\n", true), "modules/notifier/module.json", + notifierTemplate) + if v.Verdict == "fail" || v.Modules["notifier"] != "" { + t.Fatalf("the module moved to its template with the same text reads as a change (%v):\n%s", v.Modules, v.Report()) + } + + // The template alone changed: its module changed, and composes. + v = gateJudged(t, f, aTemplatedTree(t, manifests, "volume = 4\n", true), notifierTemplate) + if v.Verdict == "fail" || v.Modules["notifier"] != "changed" { + t.Fatalf("a change to the template alone is not a change to its module (%v):\n%s", v.Modules, v.Report()) + } + if v.Width == nil || strings.Join(v.Width.Modules, ",") != "notifier" { + t.Fatalf("a change to the template rebuilds %+v, not its module", v.Width) + } + + // The template alone changed to text the controller refuses: the pull request fails, naming the module. + v = gateJudged(t, f, aTemplatedTree(t, manifests, "store = ${dir:nowhere}\n", true), notifierTemplate) + if v.Verdict != "fail" || !strings.Contains(v.Report(), "notifier") { + t.Fatalf("a template the controller refuses passed the gate:\n%s", v.Report()) + } + + // A symlinked files/ fails: nothing is read from outside the tree. + tree := aTree(t, withEdit(manifests, "notifier", "")) + elsewhere := t.TempDir() + if err := os.WriteFile(filepath.Join(elsewhere, "notifier.conf"), []byte("volume = 3\n"), 0o644); err != nil { + t.Fatal(err) + } + aTemplatedModule(t, filepath.Join(tree, "modules", "notifier"), templatedNotifier, nil) + if err := os.Symlink(elsewhere, filepath.Join(tree, "modules", "notifier", "files")); err != nil { + t.Fatal(err) + } + v = gateJudged(t, f, tree, notifierTemplate) + if v.Verdict != "fail" || !strings.Contains(v.Report(), "files/ is a symlink") { + t.Fatalf("a symlinked files/ passed the gate:\n%s", v.Report()) + } + + // And the template removed from the tree fails, naming it. + v = gateJudged(t, f, aTemplatedTree(t, manifests, "", false), notifierTemplate) + if v.Verdict != "fail" || !strings.Contains(v.Report(), "files/notifier.conf") { + t.Fatalf("a missing template passed the gate:\n%s", v.Report()) + } +} + +func TestTheMergeGateResolvesATemplatedModuleToItsInlineTwin(t *testing.T) { + dir := t.TempDir() + path := aTemplatedModule(t, dir, templatedNotifier, map[string]string{"files/notifier.conf": "volume = 3\n"}) + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + m, err := catalogue.ParseManifest(raw) + if err != nil { + t.Fatal(err) + } + texts, err := catalogue.ReadTemplates(m, dir) + if err != nil { + t.Fatal(err) + } + resolved, err := standInBuild(m, texts) + if err != nil { + t.Fatal(err) + } + inline, err := catalogue.ParseManifest([]byte(inlineNotifier)) + if err != nil { + t.Fatal(err) + } + if builtAlike(resolved) != builtAlike(inline) { + got, _ := json.Marshal(resolved) + t.Fatalf("the templated module resolves to %s, not its inline twin", got) + } +} + +func TestReplay876(t *testing.T) { + moved := strings.Replace(templatedNotifier, `"version":"1",`, `"version":"1","moved":true,`, 1) + judge := func(manifest string, files map[string]string) (map[string]catalogue.Manifest, []string) { + tree := t.TempDir() + aTemplatedModule(t, filepath.Join(tree, "modules", "notifier"), manifest, files) + out, failures, err := shelfWithChange(map[string]catalogue.Manifest{}, nil, snapshot.Facts{}, + "novox/mesh-catalog", tree, map[string]string{}, nil) + if err != nil { + t.Fatal(err) + } + return out, failures + } + out, failures := judge(moved, map[string]string{"files/notifier.conf": "volume = 3\n"}) + if len(failures) > 0 { + t.Fatalf("a moved module naming its templates was refused: %v", failures) + } + if h := out["notifier"].Handed; h == nil || h.Templates["conf"] != "notifier.conf" { + t.Fatalf("the gate did not hand the moved module its templates: %+v", h) + } + + // A moved module naming no template is still refused. + none := strings.Replace(strings.Replace(moved, `"template":"files/notifier.conf"`, + `"content":"volume = 3\n"`, 1), `"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},`, "", 1) + if _, failures := judge(none, nil); len(failures) != 1 || !strings.Contains(failures[0], "names no template") { + t.Fatalf("a moved module naming no template was not refused: %v", failures) + } +} diff --git a/cmd/mesh-controller/template_test.go b/cmd/mesh-controller/template_test.go index 2a14e3e4..2ece11f5 100644 --- a/cmd/mesh-controller/template_test.go +++ b/cmd/mesh-controller/template_test.go @@ -2,15 +2,10 @@ package main import ( "bytes" - "encoding/json" "os" "path/filepath" "strings" "testing" - - "github.com/novox/mesh-controller/internal/catalogue" - snapshot "github.com/novox/mesh-controller/internal/facts" - "github.com/novox/mesh-controller/internal/inventory" ) // A file's text kept as a template file in its module's repository (novox/hq ADR 0349 §2, design 57 package T1, @@ -83,34 +78,6 @@ func TestModuleCheckReadsTemplatesFromTheTreeBesideTheManifest(t *testing.T) { } // templatedMesh is catalogueMesh with the notifier on the laptop, registered as built: its text inline. -func templatedMesh(t *testing.T) (snapshot.Facts, map[string]string) { - t.Helper() - open := aMesh(t) - ctx := t.Context() - manifests := map[string]string{ - "lemurs": `{"module":"lemurs","version":"1","capabilities":["systemd"], - "resources":[{"id":"service","type":"service","unit":"lemurs.service","state":"running","boot":"enabled"}]}`, - "notifier": inlineNotifier, - } - for name, raw := range manifests { - m, err := catalogue.ParseManifest([]byte(raw)) - if err != nil { - t.Fatalf("%s: %v", name, err) - } - if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", - Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "laptop", name); err != nil { - t.Fatalf("assign %s: %v", name, err) - } - } - f, err := gatherFacts(ctx, open, "2.11.17") - if err != nil { - t.Fatal(err) - } - return f, manifests -} // aTemplatedTree is the catalogue's checkout with the notifier's text in files/. func aTemplatedTree(t *testing.T, manifests map[string]string, text string, has bool) string { @@ -126,82 +93,7 @@ func aTemplatedTree(t *testing.T, manifests map[string]string, text string, has const notifierTemplate = "modules/notifier/files/notifier.conf" -func TestTheMergeGateJudgesAChangeToATemplateAlone(t *testing.T) { - f, manifests := templatedMesh(t) - - // The module moved to its template, the same text: nothing changes on any machine. - v := gateJudged(t, f, aTemplatedTree(t, manifests, "volume = 3\n", true), "modules/notifier/module.json", - notifierTemplate) - if v.Verdict == "fail" || v.Modules["notifier"] != "" { - t.Fatalf("the module moved to its template with the same text reads as a change (%v):\n%s", v.Modules, v.Report()) - } - - // The template alone changed: its module changed, and composes. - v = gateJudged(t, f, aTemplatedTree(t, manifests, "volume = 4\n", true), notifierTemplate) - if v.Verdict == "fail" || v.Modules["notifier"] != "changed" { - t.Fatalf("a change to the template alone is not a change to its module (%v):\n%s", v.Modules, v.Report()) - } - if v.Width == nil || strings.Join(v.Width.Modules, ",") != "notifier" { - t.Fatalf("a change to the template rebuilds %+v, not its module", v.Width) - } - - // The template alone changed to text the controller refuses: the pull request fails, naming the module. - v = gateJudged(t, f, aTemplatedTree(t, manifests, "store = ${dir:nowhere}\n", true), notifierTemplate) - if v.Verdict != "fail" || !strings.Contains(v.Report(), "notifier") { - t.Fatalf("a template the controller refuses passed the gate:\n%s", v.Report()) - } - - // A symlinked files/ fails: nothing is read from outside the tree. - tree := aTree(t, withEdit(manifests, "notifier", "")) - elsewhere := t.TempDir() - if err := os.WriteFile(filepath.Join(elsewhere, "notifier.conf"), []byte("volume = 3\n"), 0o644); err != nil { - t.Fatal(err) - } - aTemplatedModule(t, filepath.Join(tree, "modules", "notifier"), templatedNotifier, nil) - if err := os.Symlink(elsewhere, filepath.Join(tree, "modules", "notifier", "files")); err != nil { - t.Fatal(err) - } - v = gateJudged(t, f, tree, notifierTemplate) - if v.Verdict != "fail" || !strings.Contains(v.Report(), "files/ is a symlink") { - t.Fatalf("a symlinked files/ passed the gate:\n%s", v.Report()) - } - - // And the template removed from the tree fails, naming it. - v = gateJudged(t, f, aTemplatedTree(t, manifests, "", false), notifierTemplate) - if v.Verdict != "fail" || !strings.Contains(v.Report(), "files/notifier.conf") { - t.Fatalf("a missing template passed the gate:\n%s", v.Report()) - } -} - // What the gate composes for the module in its template form is what the mesh composes for it inline. -func TestTheMergeGateResolvesATemplatedModuleToItsInlineTwin(t *testing.T) { - dir := t.TempDir() - path := aTemplatedModule(t, dir, templatedNotifier, map[string]string{"files/notifier.conf": "volume = 3\n"}) - raw, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - m, err := catalogue.ParseManifest(raw) - if err != nil { - t.Fatal(err) - } - texts, err := catalogue.ReadTemplates(m, dir) - if err != nil { - t.Fatal(err) - } - resolved, err := standInBuild(m, texts) - if err != nil { - t.Fatal(err) - } - inline, err := catalogue.ParseManifest([]byte(inlineNotifier)) - if err != nil { - t.Fatal(err) - } - if builtAlike(resolved) != builtAlike(inline) { - got, _ := json.Marshal(resolved) - t.Fatalf("the templated module resolves to %s, not its inline twin", got) - } -} // A module whose files/ is unpacked onto the machine names its own templates directory (issue 731): `module check` // reads its templates from there and says of it exactly what it says of the same module with its text inline. @@ -243,30 +135,3 @@ func TestModuleCheckReadsTemplatesFromTheTemplatesDirectoryTheManifestNames(t *t // Issue 876: the gate judges a moved module that names its templates as registration reads it after a real build, // handed its templates; it refused every such module as "marked moved and names no template". This calls the gate's // own shelf (shelfWithChange), which needs no database. -func TestReplay876(t *testing.T) { - moved := strings.Replace(templatedNotifier, `"version":"1",`, `"version":"1","moved":true,`, 1) - judge := func(manifest string, files map[string]string) (map[string]catalogue.Manifest, []string) { - tree := t.TempDir() - aTemplatedModule(t, filepath.Join(tree, "modules", "notifier"), manifest, files) - out, failures, err := shelfWithChange(map[string]catalogue.Manifest{}, nil, snapshot.Facts{}, - "novox/mesh-catalog", tree, map[string]string{}, nil) - if err != nil { - t.Fatal(err) - } - return out, failures - } - out, failures := judge(moved, map[string]string{"files/notifier.conf": "volume = 3\n"}) - if len(failures) > 0 { - t.Fatalf("a moved module naming its templates was refused: %v", failures) - } - if h := out["notifier"].Handed; h == nil || h.Templates["conf"] != "notifier.conf" { - t.Fatalf("the gate did not hand the moved module its templates: %+v", h) - } - - // A moved module naming no template is still refused. - none := strings.Replace(strings.Replace(moved, `"template":"files/notifier.conf"`, - `"content":"volume = 3\n"`, 1), `"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},`, "", 1) - if _, failures := judge(none, nil); len(failures) != 1 || !strings.Contains(failures[0], "names no template") { - t.Fatalf("a moved module naming no template was not refused: %v", failures) - } -} diff --git a/cmd/mesh-controller/terminal_settings_test.go b/cmd/mesh-controller/terminal_settings_test.go index 783ca251..62dd3954 100644 --- a/cmd/mesh-controller/terminal_settings_test.go +++ b/cmd/mesh-controller/terminal_settings_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/tier_send_integration_test.go b/cmd/mesh-controller/tier_send_integration_test.go new file mode 100644 index 00000000..d05dc9ca --- /dev/null +++ b/cmd/mesh-controller/tier_send_integration_test.go @@ -0,0 +1,410 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" +) + +func aTierMesh(t *testing.T, modules ...string) *tierMesh { + t.Helper() + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + tm := &tierMesh{open: open, modules: modules} + tm.keeper, _ = withConditionsInMemory(t) + was := doctorFrom + doctorFrom = &doctor{open: open, keeper: tm.keeper, teller: &conditions.Told{}} + t.Cleanup(func() { doctorFrom = was }) + + register := func(m, commit string, asked time.Time) { + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m, BuiltFrom: commit, + Head: commit, Asked: asked}); err != nil { + t.Fatal(err) + } + } + first := map[string]string{} + for _, m := range modules { + for i, commit := range []string{"c1", "c2"} { + at := time.Now().Add(-2 * time.Hour) + if commit == "c2" { + at = time.Now().Add(-time.Minute) + } + b := inventory.Build{ID: fmt.Sprintf("build-%s-%d", m, i+1), Module: m, Commit: commit, + Repository: "novox/mesh-catalog", Path: "modules/" + m, Asked: at, At: at} + b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit}) + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + } + register(m, "c1", time.Now().Add(-2*time.Hour)) + first[m] = "c1" + for _, n := range []string{"anchor", "laptop"} { + if _, err := inv.Assign(ctx, n, m); err != nil { + t.Fatal(err) + } + } + } + for _, n := range []string{"anchor", "laptop"} { + if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", first); err != nil { + t.Fatal(err) + } + } + for _, m := range modules { + register(m, "c2", time.Now().Add(-time.Minute)) + } + + n := 0 + wasSend := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + tm.sent = append(tm.sent, append([]string(nil), names...)) + current, err := open.inventory.CurrentBuilds(ctx) + if err != nil { + return nil, err + } + builds := map[string]string{} + for _, m := range tm.modules { + builds[m] = current[m].Commit + } + for _, node := range names { + n++ + digest := fmt.Sprintf("d-%s-%d", node, n) + if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, builds); err != nil { + return nil, err + } + if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, + Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { + return nil, err + } + } + return names, nil + } + t.Cleanup(func() { sendRollout = wasSend }) + + wasGather := gatherGateFacts + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, + rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}, judged: true, + open: append([]conditions.Condition(nil), tm.conds...)} + reports, err := open.inventory.LastReports(ctx) + if err != nil { + return f, err + } + for _, r := range reports { + f.reports[r.Node] = r + f.served[r.Node] = served{runtime: true, tools: map[string]bool{}} + } + return f, nil + } + t.Cleanup(func() { gatherGateFacts = wasGather }) + + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) + // One judging per advance until a test says otherwise: a pass waits gateEvery for the next. + gateWithin, gateEvery = time.Hour, time.Hour + t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) + + built := time.Now().UTC() + states := map[string]*inventory.PlanModule{} + for _, m := range modules { + states[m] = &inventory.PlanModule{State: "built", BuiltAt: &built, Commit: "c2", Build: "build-" + m + "-2"} + } + plan := inventory.Plan{ID: "plan-tier", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", + Created: built, State: inventory.PlanBuilding, Tiers: [][]string{modules}, Modules: states} + if err := inv.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + return tm +} + +func TestATierOfManyModulesIsSentToEachMachineOnce(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + judgeLeavesAsTheCore(t) + var modules []string + for i := 1; i <= 12; i++ { + modules = append(modules, fmt.Sprintf("m%02d", i)) + } + tm := aTierMesh(t, modules...) + ctx := t.Context() + + advancePlans(ctx, tm.open) + if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) { + t.Fatalf("sent %v: the first machine once, for the whole tier", tm.sent) + } + p := tm.plan(t) + lead := p.Modules["m01"] + if lead.Gate == nil || len(lead.Gate.Carried) != len(modules) || lead.GatedBy != "" { + t.Fatalf("the gate on the first module does not judge the whole send: %+v", lead.Gate) + } + for _, m := range modules { + s := p.Modules[m] + if s.FirstAt == nil || !reflect.DeepEqual(s.First, []string{"anchor"}) || s.Previous != "c1" { + t.Fatalf("%s's first send is not recorded, or what anchor ran before is not c1: %+v", m, s) + } + if m != "m01" && s.GatedBy != "m01" { + t.Fatalf("%s is not gated by the send's gate: %+v", m, s) + } + } + + gateEvery = 0 + for i := 0; i < 6; i++ { + advancePlans(ctx, tm.open) + } + p = tm.plan(t) + if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) { + t.Fatalf("sent %v: one send per machine for the tier", tm.sent) + } + if p.State != inventory.PlanDone { + t.Fatalf("the plan is %s: %s", p.State, p.Note) + } + for _, m := range modules { + if g := p.Modules[m].Gate; g == nil || g.Verdict != inventory.GatePassed { + t.Fatalf("%s's gate: %+v", m, g) + } + if v, found, err := tm.open.inventory.GateOf(ctx, "build-"+m+"-2"); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("%s's pass was not kept: %+v %v %v", m, v, found, err) + } + } +} + +func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + tm := aTierMesh(t, "app1", "app2", broker.RuntimeModule) + ctx := t.Context() + inv := tm.open.inventory + // The node tools are held inside their witness window, which the node-engine keeps while they settle: their + // one reading does not count yet, so the core behind raised meanwhile is theirs (novox/hq ADR 0351 decision 4). + gateWitnessWindow = time.Hour + + advancePlans(ctx, tm.open) + if len(tm.sent) != 1 { + t.Fatalf("sent %v", tm.sent) + } + tm.raise("anchor", kindCoreBehind, "anchor runs the node tools it was last sent, not yet reported applied, "+ + "and no plan is rolling them out") + gateEvery = 0 + advancePlans(ctx, tm.open) + g := tm.plan(t).Modules["app1"].Gate + if g.Verdict != "" || !reflect.DeepEqual(g.Failing, []string{broker.RuntimeModule}) { + t.Fatalf("inside the settle window the gate is %+v: only the node tools wait on it", g) + } + + gateBound = -time.Second + advancePlans(ctx, tm.open) + p := tm.plan(t) + g = p.Modules["app1"].Gate + if p.State != inventory.PlanFailed || g.Verdict != inventory.GateFailed || + !reflect.DeepEqual(g.Failing, []string{broker.RuntimeModule}) { + t.Fatalf("the plan is %s (%s), the gate %+v", p.State, p.Note, g) + } + current, _ := inv.CurrentBuilds(ctx) + if current[broker.RuntimeModule].Commit != "c1" { + t.Fatalf("the node tools were not put back: %s", current[broker.RuntimeModule].Commit) + } + for _, m := range []string{"app1", "app2"} { + if current[m].Commit != "c2" { + t.Fatalf("%s was put back for the node tools' condition: %s", m, current[m].Commit) + } + if failed, _ := inv.GateFailed(ctx, "build-"+m+"-2"); failed { + t.Fatalf("%s's build was marked failed for the node tools' condition", m) + } + } + // Healthy for the passes asked, the module the gate is kept on keeps its own pass (novox/hq issue 318 + // review): it is not blamed, and not left without a verdict. + if !strings.Contains(p.Note, "app1 passed on its own and is kept") { + t.Fatalf("the plan blames the module its gate was kept on, or leaves it unjudged: %s", p.Note) + } + if v, found, _ := inv.GateOf(ctx, "build-app1-2"); !found || v.Verdict != inventory.GatePassed { + t.Fatalf("app1's own pass was not kept: %+v", v) + } +} + +func TestAMachineUnhealthyAsAWholeFailsTheWholeSendTogether(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + judgeLeavesAsTheCore(t) + tm := aTierMesh(t, "app1", "app2", "app3") + ctx := t.Context() + inv := tm.open.inventory + advancePlans(ctx, tm.open) + tm.raise("anchor", "disk-full", "anchor's root file system is full") + gateEvery, gateBound = 0, -time.Second + advancePlans(ctx, tm.open) + p := tm.plan(t) + g := p.Modules["app1"].Gate + if p.State != inventory.PlanFailed || !strings.Contains(g.Why, "anchor as a whole") || + !reflect.DeepEqual(g.Failing, []string{"app1", "app2", "app3"}) { + t.Fatalf("the plan is %s (%s), the gate %+v", p.State, p.Note, g) + } + current, _ := inv.CurrentBuilds(ctx) + for _, m := range []string{"app1", "app2", "app3"} { + if current[m].Commit != "c1" { + t.Fatalf("%s was not put back with the send: %s", m, current[m].Commit) + } + } + if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"anchor"}}) { + t.Fatalf("sent %v: the first send and one send putting all of it back, and nothing to laptop", tm.sent) + } + if !strings.Contains(p.Note, "put back app1 to c1, app2 to c1, app3 to c1 on anchor, in one send") { + t.Fatalf("the plan does not say what was put back: %s", p.Note) + } +} + +func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + judgeLeavesAsTheCore(t) + g := aGateMesh(t) + ctx := t.Context() + inv := g.open.inventory + // An earlier send carried c2 to anchor before the plan's own. + if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-early", map[string]string{"app": "c2"}); err != nil { + t.Fatal(err) + } + advancePlans(ctx, g.open) + if p := g.plan(t); p.Modules["app"].Previous != "c2" { + t.Fatalf("what anchor ran before the send: %+v", p.Modules["app"]) + } + g.health["anchor"] = healthNotYet + gateEvery, gateBound = 0, -time.Second + advancePlans(ctx, g.open) + + p := g.plan(t) + gate := p.Modules["app"].Gate + if p.State != inventory.PlanFailed || gate.Verdict != inventory.GateFailed || gate.Rollback != gateUnchanged || + !strings.Contains(p.Note, "app was left as it was") { + t.Fatalf("the plan is %s (%s), the gate %+v", p.State, p.Note, gate) + } + if failed, _ := inv.GateFailed(ctx, "build-2"); failed { + t.Fatal("a build its send changed nothing of was marked failed") + } + if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { + t.Fatalf("the module was moved: %s", current["app"].Commit) + } + open, _ := g.keeper.Open(ctx) + for _, c := range open { + if c.Kind == kindRollbackFailed || c.Kind == kindRolledBack { + t.Fatalf("said as a rollback: %+v", c) + } + } + + was := askABuild + askABuild = func(context.Context, buildSource, string, string) (string, error) { return "build-3", nil } + t.Cleanup(func() { askABuild = was }) + g.health["anchor"] = healthGood + said, err := retryPlan(ctx, g.open, "plan-gate") + if err != nil { + t.Fatalf("a plan stopped at a gate that judged no build was not retried: %v", err) + } + p = g.plan(t) + s := p.Modules["app"] + if p.State != inventory.PlanBuilding || s.State != "asked" || s.Build != "build-3" || s.FirstAt != nil || s.Gate != nil { + t.Fatalf("retried as %q: the plan is %s, app %+v", said, p.State, s) + } +} + +func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + judgeLeavesAsTheCore(t) + for _, order := range [][]string{{broker.RuntimeModule, "app1"}, {"app1", broker.RuntimeModule}} { + t.Run("gate kept on "+order[0], func(t *testing.T) { + tm := aTierMesh(t, order...) + ctx := t.Context() + inv := tm.open.inventory + tierFacts := gatherGateFacts + var seen []string + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + f, err := tierFacts(ctx, open, component) + current, _ := open.inventory.CurrentBuilds(ctx) + seen = append(seen, current[broker.RuntimeModule].Commit) + if current[broker.RuntimeModule].Commit == "c2" { + f.rolledBack["anchor"] = []lease.Rollback{{Component: lease.ComponentNodeTools, + Outcome: lease.OutcomeRolledBack, From: "c2", To: "c1", At: time.Now(), Why: "the node tools did not answer"}} + } + return f, err + } + // app1, judged as the core, counts a reading at most every 100 ms: it cannot have its three in the + // judgings that find the node tools broken, and is judged to its own pass after them (the hold the + // settle time gave before novox/hq ADR 0352 counted readings from the send). + gateEvery, gateWithin, gateBound = 100*time.Millisecond, time.Hour, 10*time.Second + for i := 0; i < 20 && len(seen) < 2; i++ { + advancePlans(ctx, tm.open) + } + if len(seen) < 2 || seen[0] != "c2" || seen[1] != "c1" { + t.Fatalf("the node tools' registered build at each judging: %v; want c2, then c1 at once", seen) + } + if failed, _ := inv.GateFailed(ctx, "build-"+broker.RuntimeModule+"-2"); !failed { + t.Fatal("the node tools' build is not marked failed at once") + } + if p := tm.plan(t); p.State == inventory.PlanFailed { + t.Fatalf("the plan failed at once: %s; want it judging app1 first", p.Note) + } + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); { + advancePlans(ctx, tm.open) + if tm.plan(t).State == inventory.PlanFailed { + break + } + time.Sleep(20 * time.Millisecond) + } + p := tm.plan(t) + if p.State != inventory.PlanFailed { + t.Fatalf("the plan is %s: %s; want failed, for the node tools", p.State, p.Note) + } + if v, found, _ := inv.GateOf(ctx, "build-app1-2"); !found || v.Verdict != inventory.GatePassed { + t.Fatalf("app1's verdict: %+v; want its own pass (plan: %s)", v, p.Note) + } + if current, _ := inv.CurrentBuilds(ctx); current["app1"].Commit != "c2" || current[broker.RuntimeModule].Commit != "c1" { + t.Fatalf("registered app1 %s, node tools %s; want c2 and c1", current["app1"].Commit, + current[broker.RuntimeModule].Commit) + } + }) + } +} + +func TestASendCarryingTheToolRunnerWaitsForItsWitnessWindow(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + tm := aTierMesh(t, broker.RuntimeModule, "app1") + ctx := t.Context() + // One reading counted per advance, as every test with a gate left open does: a gate judged with no time + // between readings would count a pass at every look and never let an advance rest. + gateEvery, gateWitnessWindow = time.Hour, time.Hour + for i := 0; i < 6; i++ { + advancePlans(ctx, tm.open) + } + p := tm.plan(t) + lead := p.Modules[broker.RuntimeModule].Gate + if lead == nil { + lead = p.Modules["app1"].Gate + } + if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != 1 { + t.Fatalf("passed inside the witness window: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead) + } + gateEvery, gateWitnessWindow = 0, 0 + for i := 0; i < 6; i++ { + advancePlans(ctx, tm.open) + } + p = tm.plan(t) + if p.State != inventory.PlanDone || !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) { + t.Fatalf("the plan is %s (%s), sent %v", p.State, p.Note, tm.sent) + } + for _, m := range []string{broker.RuntimeModule, "app1"} { + if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < 1 || + !strings.Contains(g.Why, "the tool runner's rule")) { + t.Fatalf("%s's gate: %+v; want passed on one reading by the tool runner's rule", m, g) + } + } +} diff --git a/cmd/mesh-controller/tier_send_test.go b/cmd/mesh-controller/tier_send_test.go index 590900ad..188ec090 100644 --- a/cmd/mesh-controller/tier_send_test.go +++ b/cmd/mesh-controller/tier_send_test.go @@ -1,19 +1,13 @@ package main import ( - "context" - "encoding/json" - "fmt" - "reflect" "strings" "testing" "time" "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" ) // A plan's tier is sent to each machine once, and judged there by one gate (novox/hq issue 281). @@ -37,121 +31,6 @@ type tierMesh struct { keeper *conditions.Keeper } -func aTierMesh(t *testing.T, modules ...string) *tierMesh { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - tm := &tierMesh{open: open, modules: modules} - tm.keeper, _ = withConditionsInMemory(t) - was := doctorFrom - doctorFrom = &doctor{open: open, keeper: tm.keeper, teller: &conditions.Told{}} - t.Cleanup(func() { doctorFrom = was }) - - register := func(m, commit string, asked time.Time) { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m, BuiltFrom: commit, - Head: commit, Asked: asked}); err != nil { - t.Fatal(err) - } - } - first := map[string]string{} - for _, m := range modules { - for i, commit := range []string{"c1", "c2"} { - at := time.Now().Add(-2 * time.Hour) - if commit == "c2" { - at = time.Now().Add(-time.Minute) - } - b := inventory.Build{ID: fmt.Sprintf("build-%s-%d", m, i+1), Module: m, Commit: commit, - Repository: "novox/mesh-catalog", Path: "modules/" + m, Asked: at, At: at} - b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit}) - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - register(m, "c1", time.Now().Add(-2*time.Hour)) - first[m] = "c1" - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, m); err != nil { - t.Fatal(err) - } - } - } - for _, n := range []string{"anchor", "laptop"} { - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", first); err != nil { - t.Fatal(err) - } - } - for _, m := range modules { - register(m, "c2", time.Now().Add(-time.Minute)) - } - - n := 0 - wasSend := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - tm.sent = append(tm.sent, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - builds := map[string]string{} - for _, m := range tm.modules { - builds[m] = current[m].Commit - } - for _, node := range names { - n++ - digest := fmt.Sprintf("d-%s-%d", node, n) - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, builds); err != nil { - return nil, err - } - if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, - Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { - return nil, err - } - } - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - - wasGather := gatherGateFacts - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, - rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}, judged: true, - open: append([]conditions.Condition(nil), tm.conds...)} - reports, err := open.inventory.LastReports(ctx) - if err != nil { - return f, err - } - for _, r := range reports { - f.reports[r.Node] = r - f.served[r.Node] = served{runtime: true, tools: map[string]bool{}} - } - return f, nil - } - t.Cleanup(func() { gatherGateFacts = wasGather }) - - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - // One judging per advance until a test says otherwise: a pass waits gateEvery for the next. - gateWithin, gateEvery = time.Hour, time.Hour - t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) - - built := time.Now().UTC() - states := map[string]*inventory.PlanModule{} - for _, m := range modules { - states[m] = &inventory.PlanModule{State: "built", BuiltAt: &built, Commit: "c2", Build: "build-" + m + "-2"} - } - plan := inventory.Plan{ID: "plan-tier", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", - Created: built, State: inventory.PlanBuilding, Tiers: [][]string{modules}, Modules: states} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - return tm -} - func (tm *tierMesh) plan(t *testing.T) inventory.Plan { t.Helper() p, err := tm.open.inventory.PlanByID(t.Context(), "plan-tier") @@ -170,142 +49,13 @@ func (tm *tierMesh) raise(machine, kind, summary string) { // The incident's tier, small: a dozen modules whose first machine is the same are sent there in one // send, judged by one gate, and the rest of their machines are sent once — two sends, not two dozen. -func TestATierOfManyModulesIsSentToEachMachineOnce(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - var modules []string - for i := 1; i <= 12; i++ { - modules = append(modules, fmt.Sprintf("m%02d", i)) - } - tm := aTierMesh(t, modules...) - ctx := t.Context() - - advancePlans(ctx, tm.open) - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: the first machine once, for the whole tier", tm.sent) - } - p := tm.plan(t) - lead := p.Modules["m01"] - if lead.Gate == nil || len(lead.Gate.Carried) != len(modules) || lead.GatedBy != "" { - t.Fatalf("the gate on the first module does not judge the whole send: %+v", lead.Gate) - } - for _, m := range modules { - s := p.Modules[m] - if s.FirstAt == nil || !reflect.DeepEqual(s.First, []string{"anchor"}) || s.Previous != "c1" { - t.Fatalf("%s's first send is not recorded, or what anchor ran before is not c1: %+v", m, s) - } - if m != "m01" && s.GatedBy != "m01" { - t.Fatalf("%s is not gated by the send's gate: %+v", m, s) - } - } - - gateEvery = 0 - for i := 0; i < 6; i++ { - advancePlans(ctx, tm.open) - } - p = tm.plan(t) - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) { - t.Fatalf("sent %v: one send per machine for the tier", tm.sent) - } - if p.State != inventory.PlanDone { - t.Fatalf("the plan is %s: %s", p.State, p.Note) - } - for _, m := range modules { - if g := p.Modules[m].Gate; g == nil || g.Verdict != inventory.GatePassed { - t.Fatalf("%s's gate: %+v", m, g) - } - if v, found, err := tm.open.inventory.GateOf(ctx, "build-"+m+"-2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("%s's pass was not kept: %+v %v %v", m, v, found, err) - } - } -} // The core behind about the node tools, while the node tools are themselves among what was sent and // settling, is the node tools' — inside their settle window, the gate's bound — and no failure of the // modules sent with them: at the bound only the node tools are put back. -func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - tm := aTierMesh(t, "app1", "app2", broker.RuntimeModule) - ctx := t.Context() - inv := tm.open.inventory - // The node tools are held inside their witness window, which the node-engine keeps while they settle: their - // one reading does not count yet, so the core behind raised meanwhile is theirs (novox/hq ADR 0351 decision 4). - gateWitnessWindow = time.Hour - - advancePlans(ctx, tm.open) - if len(tm.sent) != 1 { - t.Fatalf("sent %v", tm.sent) - } - tm.raise("anchor", kindCoreBehind, "anchor runs the node tools it was last sent, not yet reported applied, "+ - "and no plan is rolling them out") - gateEvery = 0 - advancePlans(ctx, tm.open) - g := tm.plan(t).Modules["app1"].Gate - if g.Verdict != "" || !reflect.DeepEqual(g.Failing, []string{broker.RuntimeModule}) { - t.Fatalf("inside the settle window the gate is %+v: only the node tools wait on it", g) - } - - gateBound = -time.Second - advancePlans(ctx, tm.open) - p := tm.plan(t) - g = p.Modules["app1"].Gate - if p.State != inventory.PlanFailed || g.Verdict != inventory.GateFailed || - !reflect.DeepEqual(g.Failing, []string{broker.RuntimeModule}) { - t.Fatalf("the plan is %s (%s), the gate %+v", p.State, p.Note, g) - } - current, _ := inv.CurrentBuilds(ctx) - if current[broker.RuntimeModule].Commit != "c1" { - t.Fatalf("the node tools were not put back: %s", current[broker.RuntimeModule].Commit) - } - for _, m := range []string{"app1", "app2"} { - if current[m].Commit != "c2" { - t.Fatalf("%s was put back for the node tools' condition: %s", m, current[m].Commit) - } - if failed, _ := inv.GateFailed(ctx, "build-"+m+"-2"); failed { - t.Fatalf("%s's build was marked failed for the node tools' condition", m) - } - } - // Healthy for the passes asked, the module the gate is kept on keeps its own pass (novox/hq issue 318 - // review): it is not blamed, and not left without a verdict. - if !strings.Contains(p.Note, "app1 passed on its own and is kept") { - t.Fatalf("the plan blames the module its gate was kept on, or leaves it unjudged: %s", p.Note) - } - if v, found, _ := inv.GateOf(ctx, "build-app1-2"); !found || v.Verdict != inventory.GatePassed { - t.Fatalf("app1's own pass was not kept: %+v", v) - } -} // A machine-level condition about anything else holds the machine back as a whole: everything the send // moved there fails together, once, with that reason — not one module for it. -func TestAMachineUnhealthyAsAWholeFailsTheWholeSendTogether(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - tm := aTierMesh(t, "app1", "app2", "app3") - ctx := t.Context() - inv := tm.open.inventory - advancePlans(ctx, tm.open) - tm.raise("anchor", "disk-full", "anchor's root file system is full") - gateEvery, gateBound = 0, -time.Second - advancePlans(ctx, tm.open) - p := tm.plan(t) - g := p.Modules["app1"].Gate - if p.State != inventory.PlanFailed || !strings.Contains(g.Why, "anchor as a whole") || - !reflect.DeepEqual(g.Failing, []string{"app1", "app2", "app3"}) { - t.Fatalf("the plan is %s (%s), the gate %+v", p.State, p.Note, g) - } - current, _ := inv.CurrentBuilds(ctx) - for _, m := range []string{"app1", "app2", "app3"} { - if current[m].Commit != "c1" { - t.Fatalf("%s was not put back with the send: %s", m, current[m].Commit) - } - } - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"anchor"}}) { - t.Fatalf("sent %v: the first send and one send putting all of it back, and nothing to laptop", tm.sent) - } - if !strings.Contains(p.Note, "put back app1 to c1, app2 to c1, app3 to c1 on anchor, in one send") { - t.Fatalf("the plan does not say what was put back: %s", p.Note) - } -} // The machine-level conditions, sorted: one naming a module is that module's; the core behind is the // core's when the send moved it and nobody's when it did not; anything else is the machine's as a whole. @@ -342,153 +92,12 @@ func TestTheConditionsAboutAMachineItself(t *testing.T) { // fails a gate as no verdict on its build: left as it was, never marked failed, nothing said as a // rollback that failed; and `plans retry` asks it again rather than refusing a build that was never // judged (the incident's "NOT put back: no build kept"). -func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - g := aGateMesh(t) - ctx := t.Context() - inv := g.open.inventory - // An earlier send carried c2 to anchor before the plan's own. - if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-early", map[string]string{"app": "c2"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, g.open) - if p := g.plan(t); p.Modules["app"].Previous != "c2" { - t.Fatalf("what anchor ran before the send: %+v", p.Modules["app"]) - } - g.health["anchor"] = healthNotYet - gateEvery, gateBound = 0, -time.Second - advancePlans(ctx, g.open) - - p := g.plan(t) - gate := p.Modules["app"].Gate - if p.State != inventory.PlanFailed || gate.Verdict != inventory.GateFailed || gate.Rollback != gateUnchanged || - !strings.Contains(p.Note, "app was left as it was") { - t.Fatalf("the plan is %s (%s), the gate %+v", p.State, p.Note, gate) - } - if failed, _ := inv.GateFailed(ctx, "build-2"); failed { - t.Fatal("a build its send changed nothing of was marked failed") - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { - t.Fatalf("the module was moved: %s", current["app"].Commit) - } - open, _ := g.keeper.Open(ctx) - for _, c := range open { - if c.Kind == kindRollbackFailed || c.Kind == kindRolledBack { - t.Fatalf("said as a rollback: %+v", c) - } - } - - was := askABuild - askABuild = func(context.Context, buildSource, string, string) (string, error) { return "build-3", nil } - t.Cleanup(func() { askABuild = was }) - g.health["anchor"] = healthGood - said, err := retryPlan(ctx, g.open, "plan-gate") - if err != nil { - t.Fatalf("a plan stopped at a gate that judged no build was not retried: %v", err) - } - p = g.plan(t) - s := p.Modules["app"] - if p.State != inventory.PlanBuilding || s.State != "asked" || s.Build != "build-3" || s.FirstAt != nil || s.Gate != nil { - t.Fatalf("retried as %q: the plan is %s, app %+v", said, p.State, s) - } -} // **In a plan's tier path, a broken module is put back at once too** (novox/hq issue 318 review): the node // tools' witness reverts them on the first machine. Whether the gate is kept on them or on the module beside // them, they are registered back and marked in the judging that finds them broken, the plan goes on judging // the other module to its own pass, and only then fails. -func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - for _, order := range [][]string{{broker.RuntimeModule, "app1"}, {"app1", broker.RuntimeModule}} { - t.Run("gate kept on "+order[0], func(t *testing.T) { - tm := aTierMesh(t, order...) - ctx := t.Context() - inv := tm.open.inventory - tierFacts := gatherGateFacts - var seen []string - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f, err := tierFacts(ctx, open, component) - current, _ := open.inventory.CurrentBuilds(ctx) - seen = append(seen, current[broker.RuntimeModule].Commit) - if current[broker.RuntimeModule].Commit == "c2" { - f.rolledBack["anchor"] = []lease.Rollback{{Component: lease.ComponentNodeTools, - Outcome: lease.OutcomeRolledBack, From: "c2", To: "c1", At: time.Now(), Why: "the node tools did not answer"}} - } - return f, err - } - // app1, judged as the core, counts a reading at most every 100 ms: it cannot have its three in the - // judgings that find the node tools broken, and is judged to its own pass after them (the hold the - // settle time gave before novox/hq ADR 0352 counted readings from the send). - gateEvery, gateWithin, gateBound = 100*time.Millisecond, time.Hour, 10*time.Second - for i := 0; i < 20 && len(seen) < 2; i++ { - advancePlans(ctx, tm.open) - } - if len(seen) < 2 || seen[0] != "c2" || seen[1] != "c1" { - t.Fatalf("the node tools' registered build at each judging: %v; want c2, then c1 at once", seen) - } - if failed, _ := inv.GateFailed(ctx, "build-"+broker.RuntimeModule+"-2"); !failed { - t.Fatal("the node tools' build is not marked failed at once") - } - if p := tm.plan(t); p.State == inventory.PlanFailed { - t.Fatalf("the plan failed at once: %s; want it judging app1 first", p.Note) - } - for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); { - advancePlans(ctx, tm.open) - if tm.plan(t).State == inventory.PlanFailed { - break - } - time.Sleep(20 * time.Millisecond) - } - p := tm.plan(t) - if p.State != inventory.PlanFailed { - t.Fatalf("the plan is %s: %s; want failed, for the node tools", p.State, p.Note) - } - if v, found, _ := inv.GateOf(ctx, "build-app1-2"); !found || v.Verdict != inventory.GatePassed { - t.Fatalf("app1's verdict: %+v; want its own pass (plan: %s)", v, p.Note) - } - if current, _ := inv.CurrentBuilds(ctx); current["app1"].Commit != "c2" || current[broker.RuntimeModule].Commit != "c1" { - t.Fatalf("registered app1 %s, node tools %s; want c2 and c1", current["app1"].Commit, - current[broker.RuntimeModule].Commit) - } - }) - } -} // **A send carrying the tool runner waits for its witness window** (issue 524 review, novox/hq ADR 0351 // decision 4): the leaf beside the node tools has its one reading at once, but the send is judged by the tool // runner's rule: one reading, not before the node-engine's witness window for it has ended. -func TestASendCarryingTheToolRunnerWaitsForItsWitnessWindow(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - tm := aTierMesh(t, broker.RuntimeModule, "app1") - ctx := t.Context() - // One reading counted per advance, as every test with a gate left open does: a gate judged with no time - // between readings would count a pass at every look and never let an advance rest. - gateEvery, gateWitnessWindow = time.Hour, time.Hour - for i := 0; i < 6; i++ { - advancePlans(ctx, tm.open) - } - p := tm.plan(t) - lead := p.Modules[broker.RuntimeModule].Gate - if lead == nil { - lead = p.Modules["app1"].Gate - } - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != 1 { - t.Fatalf("passed inside the witness window: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead) - } - gateEvery, gateWitnessWindow = 0, 0 - for i := 0; i < 6; i++ { - advancePlans(ctx, tm.open) - } - p = tm.plan(t) - if p.State != inventory.PlanDone || !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) { - t.Fatalf("the plan is %s (%s), sent %v", p.State, p.Note, tm.sent) - } - for _, m := range []string{broker.RuntimeModule, "app1"} { - if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < 1 || - !strings.Contains(g.Why, "the tool runner's rule")) { - t.Fatalf("%s's gate: %+v; want passed on one reading by the tool runner's rule", m, g) - } - } -} diff --git a/cmd/mesh-controller/try_integration_test.go b/cmd/mesh-controller/try_integration_test.go new file mode 100644 index 00000000..c3b930e8 --- /dev/null +++ b/cmd/mesh-controller/try_integration_test.go @@ -0,0 +1,39 @@ +//go:build integration + +package main + +import ( + "context" + "testing" + + "github.com/novox/mesh-controller/internal/link" +) + +func TestATrysOutcomeSetsNoStatusAndRegistersNothing(t *testing.T) { + said := &saidOnTheBus{} + checkEvents = said + defer func() { checkEvents = nil }() + for name, result := range map[string]link.BuildResult{ + "a try": {ID: "build-1", Ref: "c0ffee", Checked: &link.CheckRequest{Owner: "novox", Repo: "a", Try: true}, + Check: &link.CheckOutcome{Verdict: "pass", RepoCheck: &link.CheckLayer{Verdict: "fail", Summary: "failed"}}}, + "a try that could not run": {ID: "build-2", Checked: &link.CheckRequest{Owner: "novox", Repo: "a", Try: true}, + Failed: "cannot clone"}, + // A build seat from before the mark echoes the ask without it: a check of no pull request and no + // group is still nobody's status. + "a try echoed by an older build seat": {ID: "build-3", Checked: &link.CheckRequest{Owner: "novox", Repo: "a"}, + Check: &link.CheckOutcome{Verdict: "pass"}}, + } { + if err := (builds{}).Built(context.Background(), result); err != nil { + t.Fatalf("%s: %v", name, err) + } + } + if len(said.events) != 0 { + t.Fatalf("a try's outcome was said for the forge: %v", said.events) + } + // A pull request's check is still said. + _ = (builds{}).Built(context.Background(), link.BuildResult{ID: "build-4", Checked: &link.CheckRequest{ + Owner: "novox", Repo: "a", Number: 7}, Check: &link.CheckOutcome{Verdict: "pass"}}) + if len(said.events) != 1 { + t.Fatalf("a pull request's check was said %d time(s)", len(said.events)) + } +} diff --git a/cmd/mesh-controller/try_test.go b/cmd/mesh-controller/try_test.go index 5f72bcda..f76953c1 100644 --- a/cmd/mesh-controller/try_test.go +++ b/cmd/mesh-controller/try_test.go @@ -40,34 +40,6 @@ func TestATryAsksACheckOfTheRepositorysOwnScriptAlone(t *testing.T) { // A try's outcome is heard and left there: no `checked` is said, so the forge's holder sets no status, and // nothing is recorded or registered (the handler holds no store here, and would fail reaching for one). -func TestATrysOutcomeSetsNoStatusAndRegistersNothing(t *testing.T) { - said := &saidOnTheBus{} - checkEvents = said - defer func() { checkEvents = nil }() - for name, result := range map[string]link.BuildResult{ - "a try": {ID: "build-1", Ref: "c0ffee", Checked: &link.CheckRequest{Owner: "novox", Repo: "a", Try: true}, - Check: &link.CheckOutcome{Verdict: "pass", RepoCheck: &link.CheckLayer{Verdict: "fail", Summary: "failed"}}}, - "a try that could not run": {ID: "build-2", Checked: &link.CheckRequest{Owner: "novox", Repo: "a", Try: true}, - Failed: "cannot clone"}, - // A build seat from before the mark echoes the ask without it: a check of no pull request and no - // group is still nobody's status. - "a try echoed by an older build seat": {ID: "build-3", Checked: &link.CheckRequest{Owner: "novox", Repo: "a"}, - Check: &link.CheckOutcome{Verdict: "pass"}}, - } { - if err := (builds{}).Built(context.Background(), result); err != nil { - t.Fatalf("%s: %v", name, err) - } - } - if len(said.events) != 0 { - t.Fatalf("a try's outcome was said for the forge: %v", said.events) - } - // A pull request's check is still said. - _ = (builds{}).Built(context.Background(), link.BuildResult{ID: "build-4", Checked: &link.CheckRequest{ - Owner: "novox", Repo: "a", Number: 7}, Check: &link.CheckOutcome{Verdict: "pass"}}) - if len(said.events) != 1 { - t.Fatalf("a pull request's check was said %d time(s)", len(said.events)) - } -} // saidOnTheBus keeps the seat events said, and nothing else. type saidOnTheBus struct { diff --git a/cmd/mesh-controller/unbound_test.go b/cmd/mesh-controller/unbound_test.go index 8bbc28a7..554a744f 100644 --- a/cmd/mesh-controller/unbound_test.go +++ b/cmd/mesh-controller/unbound_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/unheld_report_integration_test.go b/cmd/mesh-controller/unheld_report_integration_test.go new file mode 100644 index 00000000..ec1bdd51 --- /dev/null +++ b/cmd/mesh-controller/unheld_report_integration_test.go @@ -0,0 +1,36 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestAnAssignmentSaysOnlyWhatItChangedOnItsOwnNode(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, aContainer("web")) + register(t, open, aContainer("db")) + // anchor already lacks a runtime for db: true, and not this act's to say. + if _, err := open.inventory.Assign(ctx, "anchor", "db"); err != nil { + t.Fatal(err) + } + if _, err := open.inventory.Assign(ctx, "laptop", "db"); err != nil { + t.Fatal(err) + } + said, err := assign(ctx, open, "laptop", "web") + if err != nil { + t.Fatalf("%v\n%s", err, said) + } + if !strings.Contains(said, "web on laptop depends on "+catalogue.ContainerRuntimeSeat) { + t.Errorf("the assignment does not say what the module it assigned depends on:\n%s", said) + } + for _, not := range []string{"db on laptop", "db on anchor", "anchor:"} { + if strings.Contains(said, not) { + t.Errorf("the assignment says %q, which it did not change:\n%s", not, said) + } + } +} diff --git a/cmd/mesh-controller/unheld_report_test.go b/cmd/mesh-controller/unheld_report_test.go index 9430d671..6d7d2e61 100644 --- a/cmd/mesh-controller/unheld_report_test.go +++ b/cmd/mesh-controller/unheld_report_test.go @@ -17,32 +17,6 @@ func aContainer(name string) catalogue.Manifest { Resources: []map[string]any{{"id": name, "type": "container", "image": name}}} } -func TestAnAssignmentSaysOnlyWhatItChangedOnItsOwnNode(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, aContainer("web")) - register(t, open, aContainer("db")) - // anchor already lacks a runtime for db: true, and not this act's to say. - if _, err := open.inventory.Assign(ctx, "anchor", "db"); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.Assign(ctx, "laptop", "db"); err != nil { - t.Fatal(err) - } - said, err := assign(ctx, open, "laptop", "web") - if err != nil { - t.Fatalf("%v\n%s", err, said) - } - if !strings.Contains(said, "web on laptop depends on "+catalogue.ContainerRuntimeSeat) { - t.Errorf("the assignment does not say what the module it assigned depends on:\n%s", said) - } - for _, not := range []string{"db on laptop", "db on anchor", "anchor:"} { - if strings.Contains(said, not) { - t.Errorf("the assignment says %q, which it did not change:\n%s", not, said) - } - } -} - func TestAnAssignmentThatMeetsADependencySaysSo(t *testing.T) { shelf := map[string]catalogue.Manifest{ "web": aContainer("web"), diff --git a/cmd/mesh-controller/unseen_integration_test.go b/cmd/mesh-controller/unseen_integration_test.go new file mode 100644 index 00000000..e15a2acb --- /dev/null +++ b/cmd/mesh-controller/unseen_integration_test.go @@ -0,0 +1,19 @@ +//go:build integration + +package main + +import ( + "context" + "strings" + "testing" +) + +func TestAPushNamingNoMachineIsRefusedUnlessItSaysAll(t *testing.T) { + err := pushCommand(context.Background(), nil) + if err == nil || !strings.Contains(err.Error(), "--all") { + t.Fatalf("a bare push was not refused: %v", err) + } + if err := pushCommand(context.Background(), []string{"--all", "--behind"}); err == nil { + t.Fatal("--all with --behind was accepted") + } +} diff --git a/cmd/mesh-controller/unseen_test.go b/cmd/mesh-controller/unseen_test.go index b675766b..6d8ad915 100644 --- a/cmd/mesh-controller/unseen_test.go +++ b/cmd/mesh-controller/unseen_test.go @@ -2,7 +2,6 @@ package main import ( "bytes" - "context" "reflect" "strings" "testing" @@ -140,16 +139,6 @@ func TestARunningContainersDataMovingIsAMoveAndAnAddedMountIsNot(t *testing.T) { } } -func TestAPushNamingNoMachineIsRefusedUnlessItSaysAll(t *testing.T) { - err := pushCommand(context.Background(), nil) - if err == nil || !strings.Contains(err.Error(), "--all") { - t.Fatalf("a bare push was not refused: %v", err) - } - if err := pushCommand(context.Background(), []string{"--all", "--behind"}); err == nil { - t.Fatal("--all with --behind was accepted") - } -} - func TestTheVerbsCarryReadReplaceAndMove(t *testing.T) { for _, c := range []struct { verb string diff --git a/cmd/mesh-controller/untaken_integration_test.go b/cmd/mesh-controller/untaken_integration_test.go new file mode 100644 index 00000000..1eb62216 --- /dev/null +++ b/cmd/mesh-controller/untaken_integration_test.go @@ -0,0 +1,80 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestStatusNamesAModuleHeldBecauseNothingTookIt(t *testing.T) { + open := aMesh(t) + heldOn(t, open, "anchor", "route-proxy", "ca", "certs", "server") + + asked, err := theThreeQuestions(t.Context(), open) + if err != nil { + t.Fatal(err) + } + if got := asked.untaken["anchor"]["route-proxy"]; got != 3 { + t.Fatalf("status counted %d resources held for route-proxy, wanted 3", got) + } +} + +func TestAHeldModuleIsFoundFromWhatTheMachineReported(t *testing.T) { + // End to end through the store, so the condition above is reached by real data and not only by + // a constructed value: the machine reports, the mesh records, status asks. + open := aMesh(t) + heldOn(t, open, "anchor", "route-proxy", "ca", "server") + + asked, err := theThreeQuestions(t.Context(), open) + if err != nil { + t.Fatal(err) + } + if len(asked.untaken) == 0 { + t.Fatal("what the machine reported holding did not reach status") + } + if asked.well() { + t.Fatal("a mesh whose machine reported holds reads as well") + } +} + +func TestTheJSONStatusCarriesWhatIsHeldAndForWhichModule(t *testing.T) { + open := aMesh(t) + heldOn(t, open, "anchor", "route-proxy", "ca", "certs") + + asked, err := theThreeQuestions(t.Context(), open) + if err != nil { + t.Fatal(err) + } + body, err := statusAsJSON(asked) + if err != nil { + t.Fatal(err) + } + var doc struct { + Untaken []struct { + Node string `json:"node"` + Module string `json:"module"` + Held int `json:"held"` + } `json:"untaken"` + } + if err := json.Unmarshal(body, &doc); err != nil { + t.Fatal(err) + } + if len(doc.Untaken) != 1 { + t.Fatalf("the document carries %d untaken rows, wanted 1: %s", len(doc.Untaken), body) + } + row := doc.Untaken[0] + if row.Node != "anchor" || row.Module != "route-proxy" || row.Held != 2 { + t.Fatalf("the row is %+v, wanted anchor/route-proxy/2", row) + } + // Absent rather than empty when nothing is held, so a well mesh's document does not carry a + // field a reader has to interpret. + clean, err := statusAsJSON(answers{}) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(clean), "untaken") { + t.Fatalf("a mesh holding nothing still names untaken: %s", clean) + } +} diff --git a/cmd/mesh-controller/untaken_test.go b/cmd/mesh-controller/untaken_test.go index cbd47674..8b768b36 100644 --- a/cmd/mesh-controller/untaken_test.go +++ b/cmd/mesh-controller/untaken_test.go @@ -1,8 +1,6 @@ package main import ( - "encoding/json" - "strings" "testing" "github.com/novox/mesh-controller/internal/inventory" @@ -30,19 +28,6 @@ func heldOn(t *testing.T, open *stores, node, module string, ids ...string) { } } -func TestStatusNamesAModuleHeldBecauseNothingTookIt(t *testing.T) { - open := aMesh(t) - heldOn(t, open, "anchor", "route-proxy", "ca", "certs", "server") - - asked, err := theThreeQuestions(t.Context(), open) - if err != nil { - t.Fatal(err) - } - if got := asked.untaken["anchor"]["route-proxy"]; got != 3 { - t.Fatalf("status counted %d resources held for route-proxy, wanted 3", got) - } -} - func TestAHeldModuleStopsTheMeshReadingAsWell(t *testing.T) { // The whole of the fault. "all doing what they were told" was true throughout the outage, and // true is not the same as safe to act on: the machine was doing what it was told, and what it @@ -62,61 +47,3 @@ func TestAHeldModuleStopsTheMeshReadingAsWell(t *testing.T) { t.Fatal("the well condition is not stable") } } - -func TestAHeldModuleIsFoundFromWhatTheMachineReported(t *testing.T) { - // End to end through the store, so the condition above is reached by real data and not only by - // a constructed value: the machine reports, the mesh records, status asks. - open := aMesh(t) - heldOn(t, open, "anchor", "route-proxy", "ca", "server") - - asked, err := theThreeQuestions(t.Context(), open) - if err != nil { - t.Fatal(err) - } - if len(asked.untaken) == 0 { - t.Fatal("what the machine reported holding did not reach status") - } - if asked.well() { - t.Fatal("a mesh whose machine reported holds reads as well") - } -} - -func TestTheJSONStatusCarriesWhatIsHeldAndForWhichModule(t *testing.T) { - open := aMesh(t) - heldOn(t, open, "anchor", "route-proxy", "ca", "certs") - - asked, err := theThreeQuestions(t.Context(), open) - if err != nil { - t.Fatal(err) - } - body, err := statusAsJSON(asked) - if err != nil { - t.Fatal(err) - } - var doc struct { - Untaken []struct { - Node string `json:"node"` - Module string `json:"module"` - Held int `json:"held"` - } `json:"untaken"` - } - if err := json.Unmarshal(body, &doc); err != nil { - t.Fatal(err) - } - if len(doc.Untaken) != 1 { - t.Fatalf("the document carries %d untaken rows, wanted 1: %s", len(doc.Untaken), body) - } - row := doc.Untaken[0] - if row.Node != "anchor" || row.Module != "route-proxy" || row.Held != 2 { - t.Fatalf("the row is %+v, wanted anchor/route-proxy/2", row) - } - // Absent rather than empty when nothing is held, so a well mesh's document does not carry a - // field a reader has to interpret. - clean, err := statusAsJSON(answers{}) - if err != nil { - t.Fatal(err) - } - if strings.Contains(string(clean), "untaken") { - t.Fatalf("a mesh holding nothing still names untaken: %s", clean) - } -} diff --git a/cmd/mesh-controller/urgent_unhealthy_test.go b/cmd/mesh-controller/urgent_unhealthy_test.go index 661ff392..3ad82512 100644 --- a/cmd/mesh-controller/urgent_unhealthy_test.go +++ b/cmd/mesh-controller/urgent_unhealthy_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/vault_handed_test.go b/cmd/mesh-controller/vault_handed_test.go index 51349b3e..8e9ca26b 100644 --- a/cmd/mesh-controller/vault_handed_test.go +++ b/cmd/mesh-controller/vault_handed_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/vault_secrets_integration_test.go b/cmd/mesh-controller/vault_secrets_integration_test.go new file mode 100644 index 00000000..989d17f1 --- /dev/null +++ b/cmd/mesh-controller/vault_secrets_integration_test.go @@ -0,0 +1,229 @@ +//go:build integration + +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "log" + "strings" + "sync" + "testing" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestAModulesSecretsAreTheVaultsOnceItIsMarkedMoved(t *testing.T) { + open := aMesh(t) + register(t, open, aBot("moved-bot", true)) + register(t, open, aBot("kept-bot", false)) + for module, want := range map[string]bool{"moved-bot": true, "kept-bot": false, "nowhere": false} { + got, err := secretsOfTheVault(module) + if err != nil || got != want { + t.Errorf("%s: the vault's %v (%v), want %v", module, got, err, want) + } + } +} + +func TestAnAcceptedValueCrossesTheBusOnlySealedToTheVault(t *testing.T) { + conn, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(conn.Close) + before := handActConn + handActConn = conn + t.Cleanup(func() { handActConn = before }) + + var tapped bytes.Buffer + var tapMu sync.Mutex + tap, err := conn.Subscribe(">", func(m *nats.Msg) { + tapMu.Lock() + defer tapMu.Unlock() + tapped.WriteString(m.Subject + " " + string(m.Data) + "\n") + }) + if err != nil { + t.Fatal(err) + } + defer tap.Unsubscribe() + + vault := newStandInVault(t) + var subjects []string + holder, err := conn.Subscribe(broker.CalledPattern(link.SeatToolSubject("mesh-vault", "accept")), func(m *nats.Msg) { + subjects = append(subjects, m.Subject) + var args map[string]any + _ = json.Unmarshal(m.Data, &args) + result, err := vault.ask("accept", args) + answer := link.Answer{Result: result} + if err != nil { + answer = link.Answer{Error: err.Error()} + } + raw, _ := json.Marshal(answer) + _ = m.Respond(raw) + }) + if err != nil { + t.Fatal(err) + } + defer holder.Unsubscribe() + + var logged bytes.Buffer + was := log.Writer() + log.SetOutput(&logged) + t.Cleanup(func() { log.SetOutput(was) }) + + const value = "pw-of-last-year-that-must-not-show" + words, err := acceptIntoVault(theVault(context.Background()), provisionArgs("novox", "postgres"), value, "jochen") + if err != nil { + t.Fatal(err) + } + if vault.taken["accept"] != value { + t.Fatalf("the vault opened %q", vault.taken["accept"]) + } + if err := conn.Flush(); err != nil { + t.Fatal(err) + } + tapMu.Lock() + crossed := tapped.String() + tapMu.Unlock() + if !strings.Contains(crossed, "mesh.seat.mesh-vault.call.accept.controller") { + t.Fatalf("the tap heard no call to the vault: %s", crossed) + } + for _, s := range subjects { + if !strings.HasSuffix(s, ".controller") { + t.Errorf("the vault was asked on %s, not in the controller's name", s) + } + } + for where, text := range map[string]string{"the bus": crossed, "the log": logged.String(), "the answer": words} { + if strings.Contains(text, value) || strings.Contains(text, "must-not-show") { + t.Errorf("the value crossed %s in the clear", where) + } + } + if !strings.Contains(words, "sealed at this terminal") { + t.Errorf("the answer: %q", words) + } + // A vault that refuses the first step: nothing is sealed or sent. + vault.refuse = errors.New("the vault has no operator key") + if _, err := acceptIntoVault(vault.ask, ownSecretArgs("g14", "dunst", "token"), value, "jochen"); err == nil || + !strings.Contains(err.Error(), "nothing was read or sealed") { + t.Errorf("a refused call said %v", err) + } +} + +func TestAnIssuedValueForAProviderFilledTheNewWayGoesToTheVault(t *testing.T) { + var r issuedRecorder + vault := newStandInVault(t) + taker := r.taker(t, func(issuedProvider, map[string]any) (link.Answer, error) { + t.Fatal("the controller asked the provider's tool itself") + return link.Answer{}, nil + }) + taker.keep = func(string, string, string) (inventory.IssuedKept, error) { + t.Fatal("today's store was asked to keep it") + return inventory.IssuedKept{}, nil + } + taker.ofTheVault = switchOf(func(m string) bool { return m == "plex" }) + taker.vault = vault.ask + words, err := taker.take("plex-api") + if err != nil { + t.Fatal(err) + } + if len(r.sent) != 0 { + t.Errorf("machines were sent: %v", r.sent) + } + if len(r.acts) != 1 || r.acts[0].Verb != handActIssued || !strings.Contains(r.acts[0].Why, "kept in the vault") { + t.Errorf("the act: %+v", r.acts) + } + said := vault.said() + for _, want := range []string{`"verb":"issued"`, `"provider":"ace"`, `"module":"plex"`, `"tool":"plex_issued_credential"`} { + if !strings.Contains(said, want) { + t.Errorf("the vault was told %s, without %s", said, want) + } + } + if strings.Contains(said, "seal_to") || strings.Contains(said, "sealed") || strings.Contains(said, `"call"`) { + t.Errorf("the controller carried a key or a sealed value: %s", said) + } + if !strings.Contains(words, "provision/ace/plex-api") { + t.Errorf("said %q", words) + } + // The same value again: the vault writes nothing, and nothing is recorded. + again := newStandInVault(t) + again.again = true + r2 := issuedRecorder{} + t2 := r2.taker(t, issuedSealed(t, "the-servers-own-token")) + t2.ofTheVault, t2.vault = switchOf(func(string) bool { return true }), again.ask + if words, err := t2.take("plex-api"); err != nil || len(r2.acts) != 0 || !strings.Contains(words, "nothing was written") { + t.Errorf("%q %v %+v", words, err, r2.acts) + } + // The vault refuses (its tool refused it): said, nothing recorded. + refused := newStandInVault(t) + refused.refuse = errors.New("its tool plex_issued_credential refused: this tool answers the vault alone") + r4 := issuedRecorder{} + t4 := r4.taker(t, issuedSealed(t, "x")) + t4.ofTheVault, t4.vault = switchOf(func(string) bool { return true }), refused.ask + if _, err := t4.take("plex-api"); err == nil || !strings.Contains(err.Error(), "answers the vault alone") || len(r4.acts) != 0 { + t.Errorf("a refusal: %v, acts %d", err, len(r4.acts)) + } + + // Filled the old way: today's path. + var r3 issuedRecorder + t3 := r3.taker(t, issuedSealed(t, "the-servers-own-token")) + t3.ofTheVault = switchOf(func(string) bool { return false }) + t3.vault = func(string, map[string]any) (json.RawMessage, error) { + t.Fatal("the vault was asked for a provider filled the old way") + return nil, nil + } + if _, err := t3.take("plex-api"); err != nil || len(r3.kept) != 1 { + t.Errorf("today's path: %v, kept %d", err, len(r3.kept)) + } +} + +func TestAnIssuedValuePerHolderNamesTheHoldersToTheVault(t *testing.T) { + var r issuedRecorder + vault := newStandInVault(t) + vault.perHolder = true + taker := r.taker(t, func(issuedProvider, map[string]any) (link.Answer, error) { + t.Fatal("the controller asked the provider's tool itself") + return link.Answer{}, nil + }) + taker.providers = func(string) ([]issuedProvider, bool, error) { + return []issuedProvider{{Node: "novox", Module: "gitea", Tool: "gitea_agent_token", Holders: []issuedHolder{ + {Node: "anchor", Module: "claude-code"}, {Node: "laptop", Module: "claude-code"}, + {Node: "shanks", Module: "claude-code"}}}}, true, nil + } + taker.ofTheVault = switchOf(func(m string) bool { return m == "gitea" }) + taker.vault = vault.ask + words, err := taker.take("gitea-read") + if err != nil { + t.Fatal(err) + } + said := vault.said() + if !strings.Contains(said, `"holders":[{"node":"anchor","module":"claude-code"},{"node":"laptop","module":"claude-code"},{"node":"shanks","module":"claude-code"}]`) { + t.Errorf("the vault was told %s", said) + } + for _, want := range []string{"provision/novox/gitea-read/anchor/claude-code", "provision/novox/gitea-read/laptop/claude-code", + "gave nothing for claude-code on shanks"} { + if !strings.Contains(words, want) { + t.Errorf("said %q, without %q", words, want) + } + } + if len(r.acts) != 1 || !strings.Contains(r.acts[0].Why, "claude-code on anchor") || !strings.Contains(r.acts[0].Why, "claude-code on laptop") { + t.Errorf("the act: %+v", r.acts) + } + if len(r.sent) != 0 || len(r.kept) != 0 { + t.Error("the controller kept or sent something") + } + // The same values again: nothing written, nothing recorded. + again := newStandInVault(t) + again.perHolder, again.again = true, true + r2 := issuedRecorder{} + t2 := r2.taker(t, nil) + t2.providers, t2.ofTheVault, t2.vault = taker.providers, taker.ofTheVault, again.ask + if words, err := t2.take("gitea-read"); err != nil || len(r2.acts) != 0 || !strings.Contains(words, "nothing was written") { + t.Errorf("again: %q %v %+v", words, err, r2.acts) + } +} diff --git a/cmd/mesh-controller/vault_secrets_test.go b/cmd/mesh-controller/vault_secrets_test.go index 6ded75a4..e31041b7 100644 --- a/cmd/mesh-controller/vault_secrets_test.go +++ b/cmd/mesh-controller/vault_secrets_test.go @@ -1,24 +1,16 @@ package main import ( - "bytes" - "context" "encoding/json" "errors" "fmt" - "log" "strings" "sync" "testing" - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" "github.com/novox/mesh-controller/internal/secrets" - "github.com/novox/mesh-controller/internal/testbus" ) // The controller's secret verbs reach the vault only for a module filled the new way (design 57's V2, issue 675, @@ -128,17 +120,6 @@ func (v *standInVault) said() string { // A module's secrets are the vault's once the catalogue marks it moved (design 57 V4, issue 647): its machine's // client asks the vault for them. Every module not marked, and one the catalogue does not hold, keeps today's path. -func TestAModulesSecretsAreTheVaultsOnceItIsMarkedMoved(t *testing.T) { - open := aMesh(t) - register(t, open, aBot("moved-bot", true)) - register(t, open, aBot("kept-bot", false)) - for module, want := range map[string]bool{"moved-bot": true, "kept-bot": false, "nowhere": false} { - got, err := secretsOfTheVault(module) - if err != nil || got != want { - t.Errorf("%s: the vault's %v (%v), want %v", module, got, err, want) - } - } -} // A switch that cannot be read does nothing, and says why: never today's path by default. func TestASwitchThatCannotBeReadDoesNothing(t *testing.T) { @@ -175,89 +156,6 @@ func TestTheSwitchRunsOnePathByTheModulesForm(t *testing.T) { // `secret accept` for a module filled the new way: the value read at the terminal is sealed there to the key the // vault made for this one call; no clear value crosses the bus or reaches the controller's log. Over a real bus, // with a tap on every subject. -func TestAnAcceptedValueCrossesTheBusOnlySealedToTheVault(t *testing.T) { - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - before := handActConn - handActConn = conn - t.Cleanup(func() { handActConn = before }) - - var tapped bytes.Buffer - var tapMu sync.Mutex - tap, err := conn.Subscribe(">", func(m *nats.Msg) { - tapMu.Lock() - defer tapMu.Unlock() - tapped.WriteString(m.Subject + " " + string(m.Data) + "\n") - }) - if err != nil { - t.Fatal(err) - } - defer tap.Unsubscribe() - - vault := newStandInVault(t) - var subjects []string - holder, err := conn.Subscribe(broker.CalledPattern(link.SeatToolSubject("mesh-vault", "accept")), func(m *nats.Msg) { - subjects = append(subjects, m.Subject) - var args map[string]any - _ = json.Unmarshal(m.Data, &args) - result, err := vault.ask("accept", args) - answer := link.Answer{Result: result} - if err != nil { - answer = link.Answer{Error: err.Error()} - } - raw, _ := json.Marshal(answer) - _ = m.Respond(raw) - }) - if err != nil { - t.Fatal(err) - } - defer holder.Unsubscribe() - - var logged bytes.Buffer - was := log.Writer() - log.SetOutput(&logged) - t.Cleanup(func() { log.SetOutput(was) }) - - const value = "pw-of-last-year-that-must-not-show" - words, err := acceptIntoVault(theVault(context.Background()), provisionArgs("novox", "postgres"), value, "jochen") - if err != nil { - t.Fatal(err) - } - if vault.taken["accept"] != value { - t.Fatalf("the vault opened %q", vault.taken["accept"]) - } - if err := conn.Flush(); err != nil { - t.Fatal(err) - } - tapMu.Lock() - crossed := tapped.String() - tapMu.Unlock() - if !strings.Contains(crossed, "mesh.seat.mesh-vault.call.accept.controller") { - t.Fatalf("the tap heard no call to the vault: %s", crossed) - } - for _, s := range subjects { - if !strings.HasSuffix(s, ".controller") { - t.Errorf("the vault was asked on %s, not in the controller's name", s) - } - } - for where, text := range map[string]string{"the bus": crossed, "the log": logged.String(), "the answer": words} { - if strings.Contains(text, value) || strings.Contains(text, "must-not-show") { - t.Errorf("the value crossed %s in the clear", where) - } - } - if !strings.Contains(words, "sealed at this terminal") { - t.Errorf("the answer: %q", words) - } - // A vault that refuses the first step: nothing is sealed or sent. - vault.refuse = errors.New("the vault has no operator key") - if _, err := acceptIntoVault(vault.ask, ownSecretArgs("g14", "dunst", "token"), value, "jochen"); err == nil || - !strings.Contains(err.Error(), "nothing was read or sealed") { - t.Errorf("a refused call said %v", err) - } -} // The desk's path for a module filled the new way (issue 723): the controller asks the vault once; the vault // records the ask, holds its bounds and asks the desk's prompt itself, so the controller never reaches the desk and @@ -341,122 +239,11 @@ func TestAnAskForAModuleFilledTheNewWayIsTheVaults(t *testing.T) { // provider's machine, module and tool; the vault asks the tool itself; the controller never reaches the tool and // nothing is kept, sealed or sent here. A provider filled the old way keeps today's path and the vault is never // asked. -func TestAnIssuedValueForAProviderFilledTheNewWayGoesToTheVault(t *testing.T) { - var r issuedRecorder - vault := newStandInVault(t) - taker := r.taker(t, func(issuedProvider, map[string]any) (link.Answer, error) { - t.Fatal("the controller asked the provider's tool itself") - return link.Answer{}, nil - }) - taker.keep = func(string, string, string) (inventory.IssuedKept, error) { - t.Fatal("today's store was asked to keep it") - return inventory.IssuedKept{}, nil - } - taker.ofTheVault = switchOf(func(m string) bool { return m == "plex" }) - taker.vault = vault.ask - words, err := taker.take("plex-api") - if err != nil { - t.Fatal(err) - } - if len(r.sent) != 0 { - t.Errorf("machines were sent: %v", r.sent) - } - if len(r.acts) != 1 || r.acts[0].Verb != handActIssued || !strings.Contains(r.acts[0].Why, "kept in the vault") { - t.Errorf("the act: %+v", r.acts) - } - said := vault.said() - for _, want := range []string{`"verb":"issued"`, `"provider":"ace"`, `"module":"plex"`, `"tool":"plex_issued_credential"`} { - if !strings.Contains(said, want) { - t.Errorf("the vault was told %s, without %s", said, want) - } - } - if strings.Contains(said, "seal_to") || strings.Contains(said, "sealed") || strings.Contains(said, `"call"`) { - t.Errorf("the controller carried a key or a sealed value: %s", said) - } - if !strings.Contains(words, "provision/ace/plex-api") { - t.Errorf("said %q", words) - } - // The same value again: the vault writes nothing, and nothing is recorded. - again := newStandInVault(t) - again.again = true - r2 := issuedRecorder{} - t2 := r2.taker(t, issuedSealed(t, "the-servers-own-token")) - t2.ofTheVault, t2.vault = switchOf(func(string) bool { return true }), again.ask - if words, err := t2.take("plex-api"); err != nil || len(r2.acts) != 0 || !strings.Contains(words, "nothing was written") { - t.Errorf("%q %v %+v", words, err, r2.acts) - } - // The vault refuses (its tool refused it): said, nothing recorded. - refused := newStandInVault(t) - refused.refuse = errors.New("its tool plex_issued_credential refused: this tool answers the vault alone") - r4 := issuedRecorder{} - t4 := r4.taker(t, issuedSealed(t, "x")) - t4.ofTheVault, t4.vault = switchOf(func(string) bool { return true }), refused.ask - if _, err := t4.take("plex-api"); err == nil || !strings.Contains(err.Error(), "answers the vault alone") || len(r4.acts) != 0 { - t.Errorf("a refusal: %v, acts %d", err, len(r4.acts)) - } - - // Filled the old way: today's path. - var r3 issuedRecorder - t3 := r3.taker(t, issuedSealed(t, "the-servers-own-token")) - t3.ofTheVault = switchOf(func(string) bool { return false }) - t3.vault = func(string, map[string]any) (json.RawMessage, error) { - t.Fatal("the vault was asked for a provider filled the old way") - return nil, nil - } - if _, err := t3.take("plex-api"); err != nil || len(r3.kept) != 1 { - t.Errorf("today's path: %v, kept %d", err, len(r3.kept)) - } -} // `secret issued` names to the vault the provision's holders from that provider (issue 749): a tool may answer a // value for each, which the vault keeps as that holder's own; the controller says what was kept for whom and which // holder the tool gave nothing for, never a value. A provider answering one value is told the holders all the same // and keeps the provision's one credential. -func TestAnIssuedValuePerHolderNamesTheHoldersToTheVault(t *testing.T) { - var r issuedRecorder - vault := newStandInVault(t) - vault.perHolder = true - taker := r.taker(t, func(issuedProvider, map[string]any) (link.Answer, error) { - t.Fatal("the controller asked the provider's tool itself") - return link.Answer{}, nil - }) - taker.providers = func(string) ([]issuedProvider, bool, error) { - return []issuedProvider{{Node: "novox", Module: "gitea", Tool: "gitea_agent_token", Holders: []issuedHolder{ - {Node: "anchor", Module: "claude-code"}, {Node: "laptop", Module: "claude-code"}, - {Node: "shanks", Module: "claude-code"}}}}, true, nil - } - taker.ofTheVault = switchOf(func(m string) bool { return m == "gitea" }) - taker.vault = vault.ask - words, err := taker.take("gitea-read") - if err != nil { - t.Fatal(err) - } - said := vault.said() - if !strings.Contains(said, `"holders":[{"node":"anchor","module":"claude-code"},{"node":"laptop","module":"claude-code"},{"node":"shanks","module":"claude-code"}]`) { - t.Errorf("the vault was told %s", said) - } - for _, want := range []string{"provision/novox/gitea-read/anchor/claude-code", "provision/novox/gitea-read/laptop/claude-code", - "gave nothing for claude-code on shanks"} { - if !strings.Contains(words, want) { - t.Errorf("said %q, without %q", words, want) - } - } - if len(r.acts) != 1 || !strings.Contains(r.acts[0].Why, "claude-code on anchor") || !strings.Contains(r.acts[0].Why, "claude-code on laptop") { - t.Errorf("the act: %+v", r.acts) - } - if len(r.sent) != 0 || len(r.kept) != 0 { - t.Error("the controller kept or sent something") - } - // The same values again: nothing written, nothing recorded. - again := newStandInVault(t) - again.perHolder, again.again = true, true - r2 := issuedRecorder{} - t2 := r2.taker(t, nil) - t2.providers, t2.ofTheVault, t2.vault = taker.providers, taker.ofTheVault, again.ask - if words, err := t2.take("gitea-read"); err != nil || len(r2.acts) != 0 || !strings.Contains(words, "nothing was written") { - t.Errorf("again: %q %v %+v", words, err, r2.acts) - } -} // The holders named are the provision's holders from that provider, each consuming module on each machine once. func TestTheHoldersNamedAreThoseOfThatProvider(t *testing.T) { diff --git a/cmd/mesh-controller/vault_switch_test.go b/cmd/mesh-controller/vault_switch_test.go index 6cf12f65..98c9f189 100644 --- a/cmd/mesh-controller/vault_switch_test.go +++ b/cmd/mesh-controller/vault_switch_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/verb_looks_test.go b/cmd/mesh-controller/verb_looks_test.go index d16e190a..ced2f5c3 100644 --- a/cmd/mesh-controller/verb_looks_test.go +++ b/cmd/mesh-controller/verb_looks_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/waiting_provider_test.go b/cmd/mesh-controller/waiting_provider_test.go index e220363f..d324ad83 100644 --- a/cmd/mesh-controller/waiting_provider_test.go +++ b/cmd/mesh-controller/waiting_provider_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/walk_said_integration_test.go b/cmd/mesh-controller/walk_said_integration_test.go new file mode 100644 index 00000000..eec2119c --- /dev/null +++ b/cmd/mesh-controller/walk_said_integration_test.go @@ -0,0 +1,266 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "os" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestAWalkLetGoAndStoppedByVerbsIsSaidStartedAndEnded(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + next := planMovedHeard(t) + register(t, open, catalogue.Manifest{Module: "app", Version: "1"}) + if _, err := assign(ctx, open, "anchor", "app"); err != nil { + t.Fatal(err) + } + cut := time.Now().UTC() + waiting := inventory.Plan{ID: "plan-308-live", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c8c8c8c8", + Created: cut, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, + Modules: map[string]*inventory.PlanModule{"app": {}}, + Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}, + Times: &inventory.PlanTimes{Cut: &cut, Class: inventory.ClassLeaf}} + if err := open.inventory.SavePlan(ctx, &waiting); err != nil { + t.Fatal(err) + } + if err := deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}); err != nil { + t.Fatal(err) + } + started := next(inventory.WalkStarted) + touches, _ := json.Marshal(started["touches"]) + if started["walk"] != waiting.ID || string(touches) != `[{"machine":"anchor","modules":["app"]}]` || + started["budget_class"] != inventory.ClassLeaf || + started["bound_seconds"] != float64(2*deliveryBudgets[inventory.ClassLeaf]/time.Second) { + t.Fatalf("the start: %v", started) + } + if err := deliveryCommand(ctx, []string{"stop", waiting.ID, "--why", "a test", "--by", "a person"}); err != nil { + t.Fatal(err) + } + ended := next(inventory.WalkEnded) + touches, _ = json.Marshal(ended["touches"]) + if ended["walk"] != waiting.ID || ended["outcome"] != inventory.WalkStopped || string(touches) != `[]` { + t.Fatalf("the end, which sent nothing: %v", ended) + } +} + +func planMovedHeard(t *testing.T) func(phase string) map[string]any { + t.Helper() + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + before := handActConn + handActConn = js.Conn() + t.Cleanup(func() { handActConn = before }) + heard := make(chan *nats.Msg, 64) + sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), heard) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + return func(phase string) map[string]any { + t.Helper() + deadline := time.After(5 * time.Second) + for { + select { + case m := <-heard: + var said map[string]any + if err := json.Unmarshal(m.Data, &said); err != nil { + t.Fatal(err) + } + if said["phase"] == phase { + return said + } + case <-deadline: + t.Fatalf("no walk was said %s", phase) + } + } + } +} + +func TestAWalkStoppedOrClosedByAPersonIsSaidEndedStopped(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + next := planMovedHeard(t) + register(t, open, catalogue.Manifest{Module: "app", Version: "1"}) + if _, err := assign(ctx, open, "anchor", "app"); err != nil { + t.Fatal(err) + } + for _, verb := range []string{"stop", "close"} { + cut := time.Now().UTC() + walk := inventory.Plan{ID: "plan-308-" + verb, Repository: "novox/mesh-catalog", Branch: "main", + Commit: "c8c8c8c8", Created: cut, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, + Modules: map[string]*inventory.PlanModule{"app": {}}, + Times: &inventory.PlanTimes{Cut: &cut, Started: &cut, Class: inventory.ClassLeaf}} + if err := open.inventory.SavePlan(ctx, &walk); err != nil { + t.Fatal(err) + } + if err := plansCommand(ctx, []string{verb, walk.ID, "--why", "a test"}); err != nil { + t.Fatal(err) + } + ended := next(inventory.WalkEnded) + if ended["walk"] != walk.ID || ended["outcome"] != inventory.WalkStopped { + t.Fatalf("plans %s: the end is %v", verb, ended) + } + p, err := open.inventory.PlanByID(ctx, walk.ID) + if err != nil { + t.Fatal(err) + } + if p.Outcome() != inventory.WalkStopped || p.Delivery == nil || p.Delivery.StoppedWhy != "a test" { + t.Fatalf("plans %s kept the walk %s, %+v", verb, p.Outcome(), p.Delivery) + } + } +} + +func TestAWalkRetriedByAPersonIsSaidStartedAgain(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + asksRecorded(t) + next := planMovedHeard(t) + twoTiers(t, open) + before := time.Now().UTC().Add(-time.Hour) + ended := before.Add(time.Minute) + failed := inventory.Plan{ID: "plan-308-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee", + Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}}, + Note: "a failed to build in tier 0", + Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", + Why: link.KilledByHand}}, + Times: &inventory.PlanTimes{Cut: &before, Started: &before, Ended: &ended, Class: inventory.ClassLeaf}} + if err := open.inventory.SavePlan(ctx, &failed); err != nil { + t.Fatal(err) + } + if err := plansCommand(ctx, []string{"retry", failed.ID}); err != nil { + t.Fatal(err) + } + started := next(inventory.WalkStarted) + if started["walk"] != failed.ID || started["budget_class"] != inventory.ClassLeaf { + t.Fatalf("the retried start: %v", started) + } +} + +func TestABacklogReleasedByAPersonIsSaidStarted(t *testing.T) { + b := aBacklog(t) + next := planMovedHeard(t) + if err := backlogCommand(t.Context(), "release-backlog", []string{"--why", "a test"}); err != nil { + t.Fatal(err) + } + started := next(inventory.WalkStarted) + if p := b.release(t); started["walk"] != p.ID { + t.Fatalf("the release's start: %v", started) + } +} + +func asTyped(t *testing.T, args ...string) error { + t.Helper() + was := os.Args + os.Args = append([]string{"mesh-controller"}, args...) + t.Cleanup(func() { os.Args = was }) + err := run() + // run gives back the lease as the process would end; the test's process goes on, and takes it again. + theLease.mu.Lock() + theLease.held, theLease.kv, theLease.noBus = nil, nil, false + theLease.mu.Unlock() + return err +} + +func TestAWalkEndedByCancelOrClearIsSaidEnded(t *testing.T) { + _, ask := aBuildQueue(t) + open := aMesh(t) + next := planMovedHeard(t) + twoTiers(t, open) + for _, verb := range [][]string{{"cancel"}, {"clear"}} { + id := link.NewBuildID(time.Now()) + ask(id, "https://forge.example/novox/a.git") + asked, _ := link.BuildAskedAt(id) + walk := aStartedWalkAsking(t, open, "plan-308-"+verb[0], id, asked) + args := verb + if verb[0] == "cancel" { + args = append(args, id) + } + if err := asTyped(t, args...); err != nil { + t.Fatal(err) + } + ended := next(inventory.WalkEnded) + if ended["walk"] != walk.ID || ended["outcome"] != inventory.WalkFailed { + t.Fatalf("%s: the end is %v", verb[0], ended) + } + // Ended, and so the next walk may be cut on the same repository. + p, _ := open.inventory.PlanByID(t.Context(), walk.ID) + if p.State != inventory.PlanFailed { + t.Fatalf("%s left the walk %s", verb[0], p.State) + } + } +} + +func TestAWalkRestartedByARebuildIsSaidStarted(t *testing.T) { + open := aMesh(t) + asksRecorded(t) + next := planMovedHeard(t) + twoTiers(t, open) + before := time.Now().UTC().Add(-time.Hour) + ended := before.Add(time.Minute) + failed := inventory.Plan{ID: "plan-308-rebuild", Repository: "novox/a", Branch: "main", Commit: "c0ffee", + Created: before, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}}, + Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", + Why: link.CancelledByHand}}, + Times: &inventory.PlanTimes{Cut: &before, Started: &before, Ended: &ended, Class: inventory.ClassLeaf}} + if err := open.inventory.SavePlan(t.Context(), &failed); err != nil { + t.Fatal(err) + } + if err := asTyped(t, "rebuild", "a"); err != nil { + t.Fatal(err) + } + if started := next(inventory.WalkStarted); started["walk"] != failed.ID { + t.Fatalf("the rebuilt walk's start: %v", started) + } +} + +func TestAWalkStoppedByHandAndRetriedEndsFailedWhenItFailsAgain(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + asksRecorded(t) + twoTiers(t, open) + for _, again := range [][]string{{"plans", "retry"}, {"rebuild", "a"}} { + before := time.Now().UTC().Add(-time.Hour) + walk := aStartedWalkAsking(t, open, "plan-308-again-"+again[0], "build-1", before) + if err := asTyped(t, "plans", "stop", walk.ID, "--why", "a test"); err != nil { + t.Fatal(err) + } + // The build asked before the stop fails after it. + p, _ := open.inventory.PlanByID(ctx, walk.ID) + p.Modules["a"].State, p.Modules["a"].Why = "failed", "it broke" + if err := open.inventory.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + args := again + if again[0] == "plans" { + args = append(append([]string(nil), again...), walk.ID) + } + if err := asTyped(t, args...); err != nil { + t.Fatal(err) + } + p, _ = open.inventory.PlanByID(ctx, walk.ID) + if p.State != inventory.PlanBuilding || (p.Delivery != nil && p.Delivery.Stopped != "") { + t.Fatalf("%s: the walk is %s and keeps its stop: %+v", again[0], p.State, p.Delivery) + } + asking, _ := link.BuildAskedAt(p.Modules["a"].Build) + planBuilt(ctx, open, "a", "c0ffee", "it broke again", asking, p.Modules["a"].Build) + if p, _ = open.inventory.PlanByID(ctx, walk.ID); p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed { + t.Fatalf("%s: failing again, the walk is %s and ends %s", again[0], p.State, p.Outcome()) + } + } +} diff --git a/cmd/mesh-controller/walk_said_test.go b/cmd/mesh-controller/walk_said_test.go index 22ef9a15..aac59370 100644 --- a/cmd/mesh-controller/walk_said_test.go +++ b/cmd/mesh-controller/walk_said_test.go @@ -3,18 +3,13 @@ package main import ( "context" "encoding/json" - "os" "reflect" "testing" "time" - "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" ) // fakeWalkFacts is the inventory as far as a walk's touches read it. @@ -234,176 +229,21 @@ func TestPlanMovedCarriesTheWalkWholeAndOnItsStartWhatItTouches(t *testing.T) { // Live, with the store and a bus: a walk let go by the delivery's verb is said started with what it touches and // its bound, and stopped by the verb is said ended, stopped (novox/hq ADR 0308 How it is checked §1). -func TestAWalkLetGoAndStoppedByVerbsIsSaidStartedAndEnded(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - next := planMovedHeard(t) - register(t, open, catalogue.Manifest{Module: "app", Version: "1"}) - if _, err := assign(ctx, open, "anchor", "app"); err != nil { - t.Fatal(err) - } - cut := time.Now().UTC() - waiting := inventory.Plan{ID: "plan-308-live", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c8c8c8c8", - Created: cut, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {}}, - Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}, - Times: &inventory.PlanTimes{Cut: &cut, Class: inventory.ClassLeaf}} - if err := open.inventory.SavePlan(ctx, &waiting); err != nil { - t.Fatal(err) - } - if err := deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}); err != nil { - t.Fatal(err) - } - started := next(inventory.WalkStarted) - touches, _ := json.Marshal(started["touches"]) - if started["walk"] != waiting.ID || string(touches) != `[{"machine":"anchor","modules":["app"]}]` || - started["budget_class"] != inventory.ClassLeaf || - started["bound_seconds"] != float64(2*deliveryBudgets[inventory.ClassLeaf]/time.Second) { - t.Fatalf("the start: %v", started) - } - if err := deliveryCommand(ctx, []string{"stop", waiting.ID, "--why", "a test", "--by", "a person"}); err != nil { - t.Fatal(err) - } - ended := next(inventory.WalkEnded) - touches, _ = json.Marshal(ended["touches"]) - if ended["walk"] != waiting.ID || ended["outcome"] != inventory.WalkStopped || string(touches) != `[]` { - t.Fatalf("the end, which sent nothing: %v", ended) - } -} // planMovedHeard raises a bus of the test's own, points the verbs' hand-act connection at it, and returns what // waits for the next plan-moved said with the given phase, failing the test when none comes within 5 s. -func planMovedHeard(t *testing.T) func(phase string) map[string]any { - t.Helper() - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - before := handActConn - handActConn = js.Conn() - t.Cleanup(func() { handActConn = before }) - heard := make(chan *nats.Msg, 64) - sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), heard) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = sub.Unsubscribe() }) - return func(phase string) map[string]any { - t.Helper() - deadline := time.After(5 * time.Second) - for { - select { - case m := <-heard: - var said map[string]any - if err := json.Unmarshal(m.Data, &said); err != nil { - t.Fatal(err) - } - if said["phase"] == phase { - return said - } - case <-deadline: - t.Fatalf("no walk was said %s", phase) - } - } - } -} // Live, with the store and a bus: a walk a person stops with `plans stop` is said ended, outcome stopped, by the // verb itself, and is kept stopped by that person; `plans close` ends one the same way (novox/hq ADR 0308 // decision 1, review 15 on mesh-controller#247). -func TestAWalkStoppedOrClosedByAPersonIsSaidEndedStopped(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - next := planMovedHeard(t) - register(t, open, catalogue.Manifest{Module: "app", Version: "1"}) - if _, err := assign(ctx, open, "anchor", "app"); err != nil { - t.Fatal(err) - } - for _, verb := range []string{"stop", "close"} { - cut := time.Now().UTC() - walk := inventory.Plan{ID: "plan-308-" + verb, Repository: "novox/mesh-catalog", Branch: "main", - Commit: "c8c8c8c8", Created: cut, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {}}, - Times: &inventory.PlanTimes{Cut: &cut, Started: &cut, Class: inventory.ClassLeaf}} - if err := open.inventory.SavePlan(ctx, &walk); err != nil { - t.Fatal(err) - } - if err := plansCommand(ctx, []string{verb, walk.ID, "--why", "a test"}); err != nil { - t.Fatal(err) - } - ended := next(inventory.WalkEnded) - if ended["walk"] != walk.ID || ended["outcome"] != inventory.WalkStopped { - t.Fatalf("plans %s: the end is %v", verb, ended) - } - p, err := open.inventory.PlanByID(ctx, walk.ID) - if err != nil { - t.Fatal(err) - } - if p.Outcome() != inventory.WalkStopped || p.Delivery == nil || p.Delivery.StoppedWhy != "a test" { - t.Fatalf("plans %s kept the walk %s, %+v", verb, p.Outcome(), p.Delivery) - } - } -} // Live, with the store and a bus: a failed walk retried with `plans retry` starts again, and the verb says its // start (novox/hq ADR 0308 decision 1, review 15 on mesh-controller#247). -func TestAWalkRetriedByAPersonIsSaidStartedAgain(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - asksRecorded(t) - next := planMovedHeard(t) - twoTiers(t, open) - before := time.Now().UTC().Add(-time.Hour) - ended := before.Add(time.Minute) - failed := inventory.Plan{ID: "plan-308-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee", - Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}}, - Note: "a failed to build in tier 0", - Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", - Why: link.KilledByHand}}, - Times: &inventory.PlanTimes{Cut: &before, Started: &before, Ended: &ended, Class: inventory.ClassLeaf}} - if err := open.inventory.SavePlan(ctx, &failed); err != nil { - t.Fatal(err) - } - if err := plansCommand(ctx, []string{"retry", failed.ID}); err != nil { - t.Fatal(err) - } - started := next(inventory.WalkStarted) - if started["walk"] != failed.ID || started["budget_class"] != inventory.ClassLeaf { - t.Fatalf("the retried start: %v", started) - } -} // Live, with the store and a bus: the release plan `upgrade release-backlog` opens is said started by the verb // (novox/hq ADR 0308 decision 1, review 15 on mesh-controller#247). -func TestABacklogReleasedByAPersonIsSaidStarted(t *testing.T) { - b := aBacklog(t) - next := planMovedHeard(t) - if err := backlogCommand(t.Context(), "release-backlog", []string{"--why", "a test"}); err != nil { - t.Fatal(err) - } - started := next(inventory.WalkStarted) - if p := b.release(t); started["walk"] != p.ID { - t.Fatalf("the release's start: %v", started) - } -} // asTyped runs a verb as the controller's command line runs it, through run, as a person's terminal would. -func asTyped(t *testing.T, args ...string) error { - t.Helper() - was := os.Args - os.Args = append([]string{"mesh-controller"}, args...) - t.Cleanup(func() { os.Args = was }) - err := run() - // run gives back the lease as the process would end; the test's process goes on, and takes it again. - theLease.mu.Lock() - theLease.held, theLease.kv, theLease.noBus = nil, nil, false - theLease.mu.Unlock() - return err -} // aStartedWalkAsking keeps a live walk, said started, whose first tier asked a as build. func aStartedWalkAsking(t *testing.T, open *stores, id, build string, asked time.Time) inventory.Plan { @@ -420,95 +260,10 @@ func aStartedWalkAsking(t *testing.T, open *stores, id, build string, asked time // Live, with the store and a bus: `cancel ` and `clear` fail the walk that asked the build, and the verb // says its end, failed (novox/hq ADR 0308 decision 1, review 21 on mesh-controller#247). -func TestAWalkEndedByCancelOrClearIsSaidEnded(t *testing.T) { - _, ask := aBuildQueue(t) - open := aMesh(t) - next := planMovedHeard(t) - twoTiers(t, open) - for _, verb := range [][]string{{"cancel"}, {"clear"}} { - id := link.NewBuildID(time.Now()) - ask(id, "https://forge.example/novox/a.git") - asked, _ := link.BuildAskedAt(id) - walk := aStartedWalkAsking(t, open, "plan-308-"+verb[0], id, asked) - args := verb - if verb[0] == "cancel" { - args = append(args, id) - } - if err := asTyped(t, args...); err != nil { - t.Fatal(err) - } - ended := next(inventory.WalkEnded) - if ended["walk"] != walk.ID || ended["outcome"] != inventory.WalkFailed { - t.Fatalf("%s: the end is %v", verb[0], ended) - } - // Ended, and so the next walk may be cut on the same repository. - p, _ := open.inventory.PlanByID(t.Context(), walk.ID) - if p.State != inventory.PlanFailed { - t.Fatalf("%s left the walk %s", verb[0], p.State) - } - } -} // Live, with the store and a bus: `rebuild ` joining a failed walk sets it building again, and the verb // says its start (novox/hq ADR 0308 decision 1, review 21 on mesh-controller#247). -func TestAWalkRestartedByARebuildIsSaidStarted(t *testing.T) { - open := aMesh(t) - asksRecorded(t) - next := planMovedHeard(t) - twoTiers(t, open) - before := time.Now().UTC().Add(-time.Hour) - ended := before.Add(time.Minute) - failed := inventory.Plan{ID: "plan-308-rebuild", Repository: "novox/a", Branch: "main", Commit: "c0ffee", - Created: before, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}}, - Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", - Why: link.CancelledByHand}}, - Times: &inventory.PlanTimes{Cut: &before, Started: &before, Ended: &ended, Class: inventory.ClassLeaf}} - if err := open.inventory.SavePlan(t.Context(), &failed); err != nil { - t.Fatal(err) - } - if err := asTyped(t, "rebuild", "a"); err != nil { - t.Fatal(err) - } - if started := next(inventory.WalkStarted); started["walk"] != failed.ID { - t.Fatalf("the rebuilt walk's start: %v", started) - } -} // Live, with the store: a walk a person stopped and then retried, by `plans retry` or by `rebuild`, walks again // without the stop, so when it fails again it ends failed, not stopped, and the search for the merge that // brought the failure is not kept from it (novox/hq ADR 0308 decision 1, review 21 on mesh-controller#247). -func TestAWalkStoppedByHandAndRetriedEndsFailedWhenItFailsAgain(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - asksRecorded(t) - twoTiers(t, open) - for _, again := range [][]string{{"plans", "retry"}, {"rebuild", "a"}} { - before := time.Now().UTC().Add(-time.Hour) - walk := aStartedWalkAsking(t, open, "plan-308-again-"+again[0], "build-1", before) - if err := asTyped(t, "plans", "stop", walk.ID, "--why", "a test"); err != nil { - t.Fatal(err) - } - // The build asked before the stop fails after it. - p, _ := open.inventory.PlanByID(ctx, walk.ID) - p.Modules["a"].State, p.Modules["a"].Why = "failed", "it broke" - if err := open.inventory.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - args := again - if again[0] == "plans" { - args = append(append([]string(nil), again...), walk.ID) - } - if err := asTyped(t, args...); err != nil { - t.Fatal(err) - } - p, _ = open.inventory.PlanByID(ctx, walk.ID) - if p.State != inventory.PlanBuilding || (p.Delivery != nil && p.Delivery.Stopped != "") { - t.Fatalf("%s: the walk is %s and keeps its stop: %+v", again[0], p.State, p.Delivery) - } - asking, _ := link.BuildAskedAt(p.Modules["a"].Build) - planBuilt(ctx, open, "a", "c0ffee", "it broke again", asking, p.Modules["a"].Build) - if p, _ = open.inventory.PlanByID(ctx, walk.ID); p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed { - t.Fatalf("%s: failing again, the walk is %s and ends %s", again[0], p.State, p.Outcome()) - } - } -} diff --git a/cmd/mesh-controller/walk_wait_rebuild_754_integration_test.go b/cmd/mesh-controller/walk_wait_rebuild_754_integration_test.go new file mode 100644 index 00000000..ae027544 --- /dev/null +++ b/cmd/mesh-controller/walk_wait_rebuild_754_integration_test.go @@ -0,0 +1,63 @@ +//go:build integration + +package main + +import ( + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestARebuildInTheWaitIsNotTheWalksBuild(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + asked := asksRecorded(t) + twoTiers(t, open) + created := time.Now().UTC().Add(-10 * time.Minute) + plan := inventory.Plan{ID: "plan-754", Repository: "novox/a", Branch: "main", Commit: "c0ffee", Created: created, + State: inventory.PlanBuilding, Tiers: [][]string{{"a", "b"}}, Modules: map[string]*inventory.PlanModule{}, + Delivery: &inventory.PlanDelivery{Awaits: "mesh-delivery"}} + if err := open.inventory.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + + // A rebuild of a, beside the waiting walk: not joined, and its outcome not taken. + joined, said, err := joinAPlan(ctx, open, "a") + if err != nil || joined || said == "" { + t.Fatalf("a rebuild joined a waiting walk: %v %q %v", joined, said, err) + } + planBuilt(ctx, open, "a", "c0ffee", "", time.Now().UTC(), "build-rebuild") + p, _ := open.inventory.PlanByID(ctx, plan.ID) + if s := p.Modules["a"]; s != nil && s.State != "" { + t.Fatalf("the waiting walk took a rebuild as its build: %+v", s) + } + if len(*asked) != 0 { + t.Fatalf("a waiting walk asked %v", *asked) + } + + // Let go: both are asked, and their outcomes complete the tier. + if _, err := letGo(ctx, open.inventory, plan.ID, "operator", "test"); err != nil { + t.Fatal(err) + } + advancePlans(ctx, open) + p, _ = open.inventory.PlanByID(ctx, plan.ID) + for _, m := range []string{"a", "b"} { + if s := p.Modules[m]; s == nil || s.State != "asked" || s.Build == "" { + t.Fatalf("let go, %s is %+v", m, s) + } + } + if len(*asked) != 2 { + t.Fatalf("asked %v", *asked) + } + for _, m := range []string{"a", "b"} { + id := p.Modules[m].Build + at, _ := link.BuildAskedAt(id) + planBuilt(ctx, open, m, "c0ffee", "", at, id) + } + advancePlans(ctx, open) + if p, _ = open.inventory.PlanByID(ctx, plan.ID); p.Open() && p.Tier == 0 { + t.Fatalf("the walk did not get past its tier: %s at %d, %q", p.State, p.Tier, p.Note) + } +} diff --git a/cmd/mesh-controller/walk_wait_rebuild_754_test.go b/cmd/mesh-controller/walk_wait_rebuild_754_test.go index 962fcebd..ee9616b3 100644 --- a/cmd/mesh-controller/walk_wait_rebuild_754_test.go +++ b/cmd/mesh-controller/walk_wait_rebuild_754_test.go @@ -6,7 +6,6 @@ import ( "time" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" ) // novox/hq issue 754: a walk waiting for its delivery's word takes no build's outcome, nor does a walk take one for @@ -69,54 +68,3 @@ func TestATierPartlyAskedAsksTheRest(t *testing.T) { // novox/hq issue 754, as seen on 2026-10-14: a rebuild lands while the walk waits for its word; the walk is let go; // it asks every module of its tier, and completes it once those builds come in. -func TestARebuildInTheWaitIsNotTheWalksBuild(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - asked := asksRecorded(t) - twoTiers(t, open) - created := time.Now().UTC().Add(-10 * time.Minute) - plan := inventory.Plan{ID: "plan-754", Repository: "novox/a", Branch: "main", Commit: "c0ffee", Created: created, - State: inventory.PlanBuilding, Tiers: [][]string{{"a", "b"}}, Modules: map[string]*inventory.PlanModule{}, - Delivery: &inventory.PlanDelivery{Awaits: "mesh-delivery"}} - if err := open.inventory.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - - // A rebuild of a, beside the waiting walk: not joined, and its outcome not taken. - joined, said, err := joinAPlan(ctx, open, "a") - if err != nil || joined || said == "" { - t.Fatalf("a rebuild joined a waiting walk: %v %q %v", joined, said, err) - } - planBuilt(ctx, open, "a", "c0ffee", "", time.Now().UTC(), "build-rebuild") - p, _ := open.inventory.PlanByID(ctx, plan.ID) - if s := p.Modules["a"]; s != nil && s.State != "" { - t.Fatalf("the waiting walk took a rebuild as its build: %+v", s) - } - if len(*asked) != 0 { - t.Fatalf("a waiting walk asked %v", *asked) - } - - // Let go: both are asked, and their outcomes complete the tier. - if _, err := letGo(ctx, open.inventory, plan.ID, "operator", "test"); err != nil { - t.Fatal(err) - } - advancePlans(ctx, open) - p, _ = open.inventory.PlanByID(ctx, plan.ID) - for _, m := range []string{"a", "b"} { - if s := p.Modules[m]; s == nil || s.State != "asked" || s.Build == "" { - t.Fatalf("let go, %s is %+v", m, s) - } - } - if len(*asked) != 2 { - t.Fatalf("asked %v", *asked) - } - for _, m := range []string{"a", "b"} { - id := p.Modules[m].Build - at, _ := link.BuildAskedAt(id) - planBuilt(ctx, open, m, "c0ffee", "", at, id) - } - advancePlans(ctx, open) - if p, _ = open.inventory.PlanByID(ctx, plan.ID); p.Open() && p.Tier == 0 { - t.Fatalf("the walk did not get past its tier: %s at %d, %q", p.State, p.Tier, p.Note) - } -} diff --git a/internal/builder/build_source_test.go b/internal/builder/build_source_test.go index 5c1fdc48..e356ccd7 100644 --- a/internal/builder/build_source_test.go +++ b/internal/builder/build_source_test.go @@ -1,3 +1,5 @@ +//go:build integration + package builder import ( diff --git a/internal/builder/builder_integration_test.go b/internal/builder/builder_integration_test.go new file mode 100644 index 00000000..56eac672 --- /dev/null +++ b/internal/builder/builder_integration_test.go @@ -0,0 +1,362 @@ +//go:build integration + +package builder + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestABuildProducesAManifestThePinsAreIn(t *testing.T) { + r, workspace := aRepository(t, withBoth, map[string]string{ + "Dockerfile": "FROM scratch", "files/theme.conf": "dark", + }) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if got.Commit != "c0ffeec0ffeec0ffeec0ffeec0ffeec0ffeec0ff" { + t.Fatalf("the commit was not recorded: %q", got.Commit) + } + if got.Manifest.Resources[0]["image"] == nil { + t.Fatalf("the image was not pinned: %v", got.Manifest.Resources[0]) + } + digest, _ := got.Manifest.Resources[1]["digest"].(string) + if !strings.HasPrefix(digest, "sha256:") { + t.Fatalf("the archive was not pinned: %v", got.Manifest.Resources[1]) + } +} + +func TestTwoBuildsOfOneCommitProduceOneDigest(t *testing.T) { + // Or nothing downstream can tell "this changed" from "this was built again", and every + // rebuild looks like a change to every machine holding it. + var digests []string + for i := 0; i < 2; i++ { + r, workspace := aRepository(t, withBoth, map[string]string{ + "Dockerfile": "FROM scratch", "files/a.conf": "one", "files/b.conf": "two", + }) + // A year apart, so a packer carrying timestamps cannot accidentally agree. + r.stamped = time.Date(2020+i, time.March, 3, 4, 5, 6, 0, time.UTC) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + for _, b := range got.Built { + if b.Kind == catalogue.ArtifactArchive { + digests = append(digests, b.Digest) + } + } + } + if digests[0] != digests[1] { + t.Fatalf("two builds of one commit produced %s and %s", digests[0], digests[1]) + } +} + +func TestNothingIsPublishedUntilEverythingIsBuilt(t *testing.T) { + // Half a module in the store under a digest the mesh never records is reachable, + // unreferenced, and indistinguishable from something in use. + r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch"}) + // `files` is missing, so packing the archive fails — after the image would have been pushed. + _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err == nil { + t.Fatal("a build with a missing input succeeded") + } + if len(r.archives) != 0 { + t.Fatalf("an archive was published by a failed build: %v", r.archives) + } +} + +func TestARepositoryWithNoManifestSaysSo(t *testing.T) { + workspace := t.TempDir() + r := &recorded{contents: map[string]string{"README.md": "nothing to see"}} + _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err == nil { + t.Fatal("a repository with nothing saying what it is was built") + } + if !strings.Contains(err.Error(), ManifestName) { + t.Fatalf("the failure does not name what is missing: %v", err) + } +} + +func TestAModuleThatBuildsNothingStillProducesAManifest(t *testing.T) { + // Most of what a person installs is configuration. + r, workspace := aRepository(t, `{"module":"shell","version":"1","resources":[ + {"id":"rc","type":"file","path":"/etc/zsh/zshrc","content":"setopt"}]}`, nil) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/shell.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if len(got.Built) != 0 { + t.Fatalf("something was built: %v", got.Built) + } + if got.Manifest.Module != "shell" || len(got.Manifest.Resources) != 1 { + t.Fatalf("got %+v", got.Manifest) + } + for _, line := range r.ran { + if strings.HasPrefix(line, "docker") { + t.Fatalf("docker was run for a module that builds nothing: %q", line) + } + } +} + +func TestTheTreeIsFreshEveryTime(t *testing.T) { + // A build that reuses a working tree can succeed because of something a previous build left + // behind, and that is a build nobody can reproduce. + r, workspace := aRepository(t, withBoth, map[string]string{ + "Dockerfile": "FROM scratch", "files/a": "b", + }) + leftover := filepath.Join(workspace, "source", "files", "from-last-time") + if err := os.MkdirAll(filepath.Dir(leftover), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(leftover, []byte("stale"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(leftover); err == nil { + t.Fatal("a previous build's file survived into this one") + } +} + +func TestABuildThatCannotPushFails(t *testing.T) { + r, workspace := aRepository(t, withBoth, map[string]string{ + "Dockerfile": "FROM scratch", "files/a": "b", + }) + r.failPush = true + if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil); err == nil { + t.Fatal("a build that could publish nothing reported success") + } +} + +func TestAnUpstreamImageIsMirroredRatherThanBuilt(t *testing.T) { + // A module usually runs software it did not write. Naming the upstream reference directly + // would need every machine to reach a public registry, and would pin to a tag somebody else + // can move. + const mirrors = `{"module":"postgres","version":"1", + "build":{"artifacts":[{"name":"store","kind":"upstream","from":"postgres:17-alpine"}]}, + "resources":[{"id":"db","type":"container","name":"mesh-postgres","artifact":"store"}]}` + + r, workspace := aRepository(t, mirrors, nil) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/postgres.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + // Pulled, not built. + var pulled, built bool + for _, line := range r.ran { + if strings.HasPrefix(line, "docker pull postgres:17-alpine") { + pulled = true + } + if strings.HasPrefix(line, "docker build") { + built = true + } + } + if !pulled { + t.Fatalf("the upstream image was not fetched: %v", r.ran) + } + if built { + t.Fatalf("something was built for an image that is mirrored: %v", r.ran) + } + // And the resource names what this registry serves, pinned by the digest it assigned. + image, _ := got.Manifest.Resources[0]["image"].(string) + if !strings.Contains(image, "@sha256:") { + t.Fatalf("the mirrored image is not pinned by digest: %q", image) + } + if strings.Contains(image, "17-alpine") { + t.Fatalf("the resource still names the upstream tag: %q", image) + } +} + +func TestAModuleIsBuiltFromItsPathWithinTheRepository(t *testing.T) { + r := &recorded{contents: map[string]string{ + "README.md": "this repository holds several modules", + "modules/shell/" + ManifestName: withBoth, + "modules/shell/Dockerfile": "FROM scratch", + "modules/shell/files/theme.conf": "dark", + // A second module beside it, so what is built is chosen by the path rather than by + // happening to be the only manifest in the clone. + "modules/other/" + ManifestName: `{"module":"other","version":"1"}`, + }} + got, err := Build(context.Background(), r.run, r, + "https://forge.invalid/catalogue.git", "modules/shell", "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if got.Manifest.Module != "meshboard" { + t.Fatalf("built %q, which is not the module at the path asked for", got.Manifest.Module) + } + if got.Manifest.Resources[0]["image"] == nil { + t.Fatalf("the image was not pinned: %v", got.Manifest.Resources[0]) + } +} + +func TestAPathThatLeavesTheRepositoryIsRefused(t *testing.T) { + for _, escaping := range []string{"../../etc", "/etc"} { + r := &recorded{contents: map[string]string{ManifestName: withBoth}} + _, err := Build(context.Background(), r.run, r, + "https://forge.invalid/x.git", escaping, "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil) + if err == nil { + t.Fatalf("%q was accepted as a module's path", escaping) + } + if !strings.Contains(err.Error(), "leaves the repository") && + !strings.Contains(err.Error(), "absolute path") { + t.Fatalf("the refusal of %q does not say why: %v", escaping, err) + } + } +} + +func TestAnArtifactWithItsOwnContextIsBuiltFromThere(t *testing.T) { + const withContext = `{"module":"route-proxy","version":"1", + "build":{"artifacts":[ + {"name":"server","kind":"image","from":"Dockerfile", + "context":{"repository":"https://forge.invalid/source.git","ref":"main"}}]}}` + r := &recorded{ + contents: map[string]string{ + ManifestName: withContext, + // The recipe lives with the packaging, not the source — read from here regardless of + // where the build context comes from. FROM scratch declares no base, so what is under + // test — where the context comes from — is not entangled with ADR 0097's own checks. + "Dockerfile": "FROM scratch\nCOPY go.mod ./\n", + }, + secondary: map[string]map[string]string{ + // go.mod exists only in the second repository. A build context taken from the wrong + // place would never find it, which a real docker build would refuse on — the fake + // does not read files, so what is checked below is that the build was even pointed + // at the right place, not that COPY would have succeeded. + "https://forge.invalid/source.git": {"go.mod": "module route-proxy\n"}, + }, + } + _, err := Build(context.Background(), r.run, r, + "https://forge.invalid/catalogue.git", "", "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + + var clonedSource bool + for _, line := range r.ran { + if strings.HasPrefix(line, "git clone") && strings.Contains(line, "https://forge.invalid/source.git") { + clonedSource = true + } + } + if !clonedSource { + t.Fatalf("the artifact's own context was never cloned: %v", r.ran) + } + + buildIndex := -1 + for i, line := range r.ran { + if strings.HasPrefix(line, "docker build ") { + buildIndex = i + } + } + if buildIndex == -1 { + t.Fatal("no docker build was run") + } + build := r.ran[buildIndex] + buildDir := r.dirs[buildIndex] + + if !strings.Contains(buildDir, "context-server") { + t.Errorf("docker build ran from %q, not the artifact's own cloned context", buildDir) + } + recipe := strings.SplitN(strings.SplitN(build, "-f ", 2)[1], " ", 2)[0] + if !filepath.IsAbs(recipe) { + t.Errorf("the recipe %q is not an absolute path, so it is read relative to whatever "+ + "directory the build context moved to rather than where it actually is", recipe) + } + if !strings.HasSuffix(recipe, string(filepath.Separator)+"Dockerfile") { + t.Errorf("the recipe is not the module's own Dockerfile: %q", recipe) + } + if !strings.HasSuffix(build, " .") { + t.Errorf("the build was not given a context: %s", build) + } +} + +func TestABuildOffersTheForgesCredentialThroughGitsOwnStore(t *testing.T) { + r, workspace := aRepository(t, withBoth, map[string]string{ + "Dockerfile": "FROM scratch", "files/theme.conf": "dark", + }) + _, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", + workspace, nil, Npmrc{}, + GitCredential{URL: "http://mesh_novox_builder:sw0rdfi5h@forge.invalid:20000"}, nil) + if err != nil { + t.Fatal(err) + } + clone := r.ran[0] + stored := storeNamedIn(t, clone) + for _, line := range r.ran { + if strings.Contains(line, "sw0rdfi5h") { + t.Fatalf("the secret is in a command line, readable by anything that can list processes: %s", line) + } + } + // Outside the workspace a check's container mounts, and gone once the build is (novox/hq issue 462); what + // it held while git read it is checked with the check's clones (TestACheckContainerSeesNoForgeCredential). + if withinDir(workspace, stored) { + t.Fatalf("the credential store %s is inside the workspace %s", stored, workspace) + } + if _, err := os.Stat(stored); !os.IsNotExist(err) { + t.Fatalf("the credential store outlives the build: %v", err) + } + if _, err := os.Stat(filepath.Join(workspace, "git-credentials")); !os.IsNotExist(err) { + t.Fatal("a credential file is left in the workspace") + } +} + +func TestABuildWithNoCredentialClonesExactlyAsBefore(t *testing.T) { + r, workspace := aRepository(t, withBoth, map[string]string{ + "Dockerfile": "FROM scratch", "files/theme.conf": "dark", + }) + _, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", + workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(r.ran[0], "git clone --quiet ") { + t.Fatalf("a credential-less clone grew flags: %s", r.ran[0]) + } + if _, err := os.Stat(filepath.Join(workspace, "git-credentials")); !os.IsNotExist(err) { + t.Fatal("a credential file was written with no credential to put in it") + } +} + +func TestAContextCloneCarriesTheSameCredentialStore(t *testing.T) { + const withContext = `{"module":"route-proxy","version":"1", + "build":{"artifacts":[ + {"name":"server","kind":"image","from":"Dockerfile", + "context":{"repository":"https://forge.invalid/source.git","ref":"main"}}]}}` + r := &recorded{ + contents: map[string]string{ + ManifestName: withContext, + "Dockerfile": "FROM scratch\nCOPY go.mod ./\n", + }, + secondary: map[string]map[string]string{ + "https://forge.invalid/source.git": {"go.mod": "module route-proxy\n"}, + }, + } + workspace := t.TempDir() + _, err := Build(context.Background(), r.run, r, + "https://forge.invalid/catalogue.git", "", "", workspace, nil, Npmrc{}, + GitCredential{URL: "https://builder:s3cret@forge.invalid"}, nil) + if err != nil { + t.Fatal(err) + } + stored := storeNamedIn(t, r.ran[0]) + var contextClone string + for _, line := range r.ran { + if strings.Contains(line, "clone") && strings.Contains(line, "source.git") { + contextClone = line + } + } + if contextClone == "" { + t.Fatalf("the context was never cloned: %v", r.ran) + } + if !strings.Contains(contextClone, "credential.helper=store --file="+stored) { + t.Fatalf("the context clone does not name the credential store: %s", contextClone) + } +} diff --git a/internal/builder/builder_test.go b/internal/builder/builder_test.go index 4bfe447f..1cc165ac 100644 --- a/internal/builder/builder_test.go +++ b/internal/builder/builder_test.go @@ -130,168 +130,6 @@ const withBoth = `{"module":"meshboard","version":"1", {"id":"svc","type":"container","name":"meshboard","artifact":"server"}, {"id":"theme","type":"archive","path":"/opt/meshboard","artifact":"look"}]}` -func TestABuildProducesAManifestThePinsAreIn(t *testing.T) { - r, workspace := aRepository(t, withBoth, map[string]string{ - "Dockerfile": "FROM scratch", "files/theme.conf": "dark", - }) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if got.Commit != "c0ffeec0ffeec0ffeec0ffeec0ffeec0ffeec0ff" { - t.Fatalf("the commit was not recorded: %q", got.Commit) - } - if got.Manifest.Resources[0]["image"] == nil { - t.Fatalf("the image was not pinned: %v", got.Manifest.Resources[0]) - } - digest, _ := got.Manifest.Resources[1]["digest"].(string) - if !strings.HasPrefix(digest, "sha256:") { - t.Fatalf("the archive was not pinned: %v", got.Manifest.Resources[1]) - } -} - -func TestTwoBuildsOfOneCommitProduceOneDigest(t *testing.T) { - // Or nothing downstream can tell "this changed" from "this was built again", and every - // rebuild looks like a change to every machine holding it. - var digests []string - for i := 0; i < 2; i++ { - r, workspace := aRepository(t, withBoth, map[string]string{ - "Dockerfile": "FROM scratch", "files/a.conf": "one", "files/b.conf": "two", - }) - // A year apart, so a packer carrying timestamps cannot accidentally agree. - r.stamped = time.Date(2020+i, time.March, 3, 4, 5, 6, 0, time.UTC) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - for _, b := range got.Built { - if b.Kind == catalogue.ArtifactArchive { - digests = append(digests, b.Digest) - } - } - } - if digests[0] != digests[1] { - t.Fatalf("two builds of one commit produced %s and %s", digests[0], digests[1]) - } -} - -func TestNothingIsPublishedUntilEverythingIsBuilt(t *testing.T) { - // Half a module in the store under a digest the mesh never records is reachable, - // unreferenced, and indistinguishable from something in use. - r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch"}) - // `files` is missing, so packing the archive fails — after the image would have been pushed. - _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err == nil { - t.Fatal("a build with a missing input succeeded") - } - if len(r.archives) != 0 { - t.Fatalf("an archive was published by a failed build: %v", r.archives) - } -} - -func TestARepositoryWithNoManifestSaysSo(t *testing.T) { - workspace := t.TempDir() - r := &recorded{contents: map[string]string{"README.md": "nothing to see"}} - _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err == nil { - t.Fatal("a repository with nothing saying what it is was built") - } - if !strings.Contains(err.Error(), ManifestName) { - t.Fatalf("the failure does not name what is missing: %v", err) - } -} - -func TestAModuleThatBuildsNothingStillProducesAManifest(t *testing.T) { - // Most of what a person installs is configuration. - r, workspace := aRepository(t, `{"module":"shell","version":"1","resources":[ - {"id":"rc","type":"file","path":"/etc/zsh/zshrc","content":"setopt"}]}`, nil) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/shell.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if len(got.Built) != 0 { - t.Fatalf("something was built: %v", got.Built) - } - if got.Manifest.Module != "shell" || len(got.Manifest.Resources) != 1 { - t.Fatalf("got %+v", got.Manifest) - } - for _, line := range r.ran { - if strings.HasPrefix(line, "docker") { - t.Fatalf("docker was run for a module that builds nothing: %q", line) - } - } -} - -func TestTheTreeIsFreshEveryTime(t *testing.T) { - // A build that reuses a working tree can succeed because of something a previous build left - // behind, and that is a build nobody can reproduce. - r, workspace := aRepository(t, withBoth, map[string]string{ - "Dockerfile": "FROM scratch", "files/a": "b", - }) - leftover := filepath.Join(workspace, "source", "files", "from-last-time") - if err := os.MkdirAll(filepath.Dir(leftover), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(leftover, []byte("stale"), 0o644); err != nil { - t.Fatal(err) - } - if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil); err != nil { - t.Fatal(err) - } - if _, err := os.Stat(leftover); err == nil { - t.Fatal("a previous build's file survived into this one") - } -} - -func TestABuildThatCannotPushFails(t *testing.T) { - r, workspace := aRepository(t, withBoth, map[string]string{ - "Dockerfile": "FROM scratch", "files/a": "b", - }) - r.failPush = true - if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil); err == nil { - t.Fatal("a build that could publish nothing reported success") - } -} - -func TestAnUpstreamImageIsMirroredRatherThanBuilt(t *testing.T) { - // A module usually runs software it did not write. Naming the upstream reference directly - // would need every machine to reach a public registry, and would pin to a tag somebody else - // can move. - const mirrors = `{"module":"postgres","version":"1", - "build":{"artifacts":[{"name":"store","kind":"upstream","from":"postgres:17-alpine"}]}, - "resources":[{"id":"db","type":"container","name":"mesh-postgres","artifact":"store"}]}` - - r, workspace := aRepository(t, mirrors, nil) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/postgres.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - // Pulled, not built. - var pulled, built bool - for _, line := range r.ran { - if strings.HasPrefix(line, "docker pull postgres:17-alpine") { - pulled = true - } - if strings.HasPrefix(line, "docker build") { - built = true - } - } - if !pulled { - t.Fatalf("the upstream image was not fetched: %v", r.ran) - } - if built { - t.Fatalf("something was built for an image that is mirrored: %v", r.ran) - } - // And the resource names what this registry serves, pinned by the digest it assigned. - image, _ := got.Manifest.Resources[0]["image"].(string) - if !strings.Contains(image, "@sha256:") { - t.Fatalf("the mirrored image is not pinned by digest: %q", image) - } - if strings.Contains(image, "17-alpine") { - t.Fatalf("the resource still names the upstream tag: %q", image) - } -} - func TestAnUpstreamImageWithNoTagIsRefused(t *testing.T) { // What gets mirrored would be whatever `latest` means today, and a module pinned to that is // not pinned. @@ -317,148 +155,19 @@ func TestAnUpstreamReferenceIsNotAPathInTheRepository(t *testing.T) { // **A module is a repository and a path within it** (novox/hq ADR 0069). The catalogue holds its // modules one to a directory and the system this replaces has always built one that way, so a // builder that could only read a repository's root could build none of what exists. -func TestAModuleIsBuiltFromItsPathWithinTheRepository(t *testing.T) { - r := &recorded{contents: map[string]string{ - "README.md": "this repository holds several modules", - "modules/shell/" + ManifestName: withBoth, - "modules/shell/Dockerfile": "FROM scratch", - "modules/shell/files/theme.conf": "dark", - // A second module beside it, so what is built is chosen by the path rather than by - // happening to be the only manifest in the clone. - "modules/other/" + ManifestName: `{"module":"other","version":"1"}`, - }} - got, err := Build(context.Background(), r.run, r, - "https://forge.invalid/catalogue.git", "modules/shell", "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if got.Manifest.Module != "meshboard" { - t.Fatalf("built %q, which is not the module at the path asked for", got.Manifest.Module) - } - if got.Manifest.Resources[0]["image"] == nil { - t.Fatalf("the image was not pinned: %v", got.Manifest.Resources[0]) - } -} // A build reads only its own tree. A path climbing out of the clone would otherwise let a build // read — and an archive artifact publish — whatever the build machine happens to hold, which is // the one thing a machine that builds other people's repositories must not do. -func TestAPathThatLeavesTheRepositoryIsRefused(t *testing.T) { - for _, escaping := range []string{"../../etc", "/etc"} { - r := &recorded{contents: map[string]string{ManifestName: withBoth}} - _, err := Build(context.Background(), r.run, r, - "https://forge.invalid/x.git", escaping, "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil) - if err == nil { - t.Fatalf("%q was accepted as a module's path", escaping) - } - if !strings.Contains(err.Error(), "leaves the repository") && - !strings.Contains(err.Error(), "absolute path") { - t.Fatalf("the refusal of %q does not say why: %v", escaping, err) - } - } -} // **Packaging and source are allowed to live apart** — a module that ships only the recipe for // source that lives in a second repository (the reference route-proxy, packaged in the catalogue // but built from mesh-controller's own repository) names where that source actually is, rather // than vendoring a second copy the two could drift from. -func TestAnArtifactWithItsOwnContextIsBuiltFromThere(t *testing.T) { - const withContext = `{"module":"route-proxy","version":"1", - "build":{"artifacts":[ - {"name":"server","kind":"image","from":"Dockerfile", - "context":{"repository":"https://forge.invalid/source.git","ref":"main"}}]}}` - r := &recorded{ - contents: map[string]string{ - ManifestName: withContext, - // The recipe lives with the packaging, not the source — read from here regardless of - // where the build context comes from. FROM scratch declares no base, so what is under - // test — where the context comes from — is not entangled with ADR 0097's own checks. - "Dockerfile": "FROM scratch\nCOPY go.mod ./\n", - }, - secondary: map[string]map[string]string{ - // go.mod exists only in the second repository. A build context taken from the wrong - // place would never find it, which a real docker build would refuse on — the fake - // does not read files, so what is checked below is that the build was even pointed - // at the right place, not that COPY would have succeeded. - "https://forge.invalid/source.git": {"go.mod": "module route-proxy\n"}, - }, - } - _, err := Build(context.Background(), r.run, r, - "https://forge.invalid/catalogue.git", "", "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - - var clonedSource bool - for _, line := range r.ran { - if strings.HasPrefix(line, "git clone") && strings.Contains(line, "https://forge.invalid/source.git") { - clonedSource = true - } - } - if !clonedSource { - t.Fatalf("the artifact's own context was never cloned: %v", r.ran) - } - - buildIndex := -1 - for i, line := range r.ran { - if strings.HasPrefix(line, "docker build ") { - buildIndex = i - } - } - if buildIndex == -1 { - t.Fatal("no docker build was run") - } - build := r.ran[buildIndex] - buildDir := r.dirs[buildIndex] - - if !strings.Contains(buildDir, "context-server") { - t.Errorf("docker build ran from %q, not the artifact's own cloned context", buildDir) - } - recipe := strings.SplitN(strings.SplitN(build, "-f ", 2)[1], " ", 2)[0] - if !filepath.IsAbs(recipe) { - t.Errorf("the recipe %q is not an absolute path, so it is read relative to whatever "+ - "directory the build context moved to rather than where it actually is", recipe) - } - if !strings.HasSuffix(recipe, string(filepath.Separator)+"Dockerfile") { - t.Errorf("the recipe is not the module's own Dockerfile: %q", recipe) - } - if !strings.HasSuffix(build, " .") { - t.Errorf("the build was not given a context: %s", build) - } -} // The forge credential is offered through git's own credential store — a file, never argv — and // git decides when it applies. What is checked: the clone names the store, the secret never // appears in a command line, and the file holds exactly the URL at 0600. -func TestABuildOffersTheForgesCredentialThroughGitsOwnStore(t *testing.T) { - r, workspace := aRepository(t, withBoth, map[string]string{ - "Dockerfile": "FROM scratch", "files/theme.conf": "dark", - }) - _, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", - workspace, nil, Npmrc{}, - GitCredential{URL: "http://mesh_novox_builder:sw0rdfi5h@forge.invalid:20000"}, nil) - if err != nil { - t.Fatal(err) - } - clone := r.ran[0] - stored := storeNamedIn(t, clone) - for _, line := range r.ran { - if strings.Contains(line, "sw0rdfi5h") { - t.Fatalf("the secret is in a command line, readable by anything that can list processes: %s", line) - } - } - // Outside the workspace a check's container mounts, and gone once the build is (novox/hq issue 462); what - // it held while git read it is checked with the check's clones (TestACheckContainerSeesNoForgeCredential). - if withinDir(workspace, stored) { - t.Fatalf("the credential store %s is inside the workspace %s", stored, workspace) - } - if _, err := os.Stat(stored); !os.IsNotExist(err) { - t.Fatalf("the credential store outlives the build: %v", err) - } - if _, err := os.Stat(filepath.Join(workspace, "git-credentials")); !os.IsNotExist(err) { - t.Fatal("a credential file is left in the workspace") - } -} // storeNamedIn is the credential store a git command line offers. func storeNamedIn(t *testing.T, line string) string { @@ -472,57 +181,6 @@ func storeNamedIn(t *testing.T, line string) string { // Without a credential, a clone is exactly the invocation it always was, and no credential file // appears — the builder a mesh of public repositories runs is unchanged. -func TestABuildWithNoCredentialClonesExactlyAsBefore(t *testing.T) { - r, workspace := aRepository(t, withBoth, map[string]string{ - "Dockerfile": "FROM scratch", "files/theme.conf": "dark", - }) - _, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", - workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if !strings.HasPrefix(r.ran[0], "git clone --quiet ") { - t.Fatalf("a credential-less clone grew flags: %s", r.ran[0]) - } - if _, err := os.Stat(filepath.Join(workspace, "git-credentials")); !os.IsNotExist(err) { - t.Fatal("a credential file was written with no credential to put in it") - } -} // An artifact's own context is cloned with the same offer: a private module whose context is a // second private repository on the same forge builds, and the secret still never reaches argv. -func TestAContextCloneCarriesTheSameCredentialStore(t *testing.T) { - const withContext = `{"module":"route-proxy","version":"1", - "build":{"artifacts":[ - {"name":"server","kind":"image","from":"Dockerfile", - "context":{"repository":"https://forge.invalid/source.git","ref":"main"}}]}}` - r := &recorded{ - contents: map[string]string{ - ManifestName: withContext, - "Dockerfile": "FROM scratch\nCOPY go.mod ./\n", - }, - secondary: map[string]map[string]string{ - "https://forge.invalid/source.git": {"go.mod": "module route-proxy\n"}, - }, - } - workspace := t.TempDir() - _, err := Build(context.Background(), r.run, r, - "https://forge.invalid/catalogue.git", "", "", workspace, nil, Npmrc{}, - GitCredential{URL: "https://builder:s3cret@forge.invalid"}, nil) - if err != nil { - t.Fatal(err) - } - stored := storeNamedIn(t, r.ran[0]) - var contextClone string - for _, line := range r.ran { - if strings.Contains(line, "clone") && strings.Contains(line, "source.git") { - contextClone = line - } - } - if contextClone == "" { - t.Fatalf("the context was never cloned: %v", r.ran) - } - if !strings.Contains(contextClone, "credential.helper=store --file="+stored) { - t.Fatalf("the context clone does not name the credential store: %s", contextClone) - } -} diff --git a/internal/builder/bundle_integration_test.go b/internal/builder/bundle_integration_test.go new file mode 100644 index 00000000..80a69e81 --- /dev/null +++ b/internal/builder/bundle_integration_test.go @@ -0,0 +1,166 @@ +//go:build integration + +package builder + +import ( + "context" + "strings" + "testing" +) + +func TestABundleIsCompiledAndPackedWithNoDockerfile(t *testing.T) { + r, workspace := aRepository(t, aBundle, map[string]string{"index.ts": "console.log(1)"}) + held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} + + got, err := Build(context.Background(), compiling{r}.run, r, + "https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatalf("a module with a language and no Dockerfile did not build: %v", err) + } + + // Compiled in the toolchain the mesh chose, not in one the module named. + var compiled string + for _, line := range r.ran { + if strings.HasPrefix(line, "docker run") { + compiled = line + } + } + if compiled == "" { + t.Fatalf("nothing was compiled:\n%s", strings.Join(r.ran, "\n")) + } + if !strings.Contains(compiled, "mesh-tools/build@sha256:") { + t.Fatalf("the compile did not run in the mesh's own toolchain: %s", compiled) + } + if strings.Contains(strings.Join(r.ran, "\n"), "docker build") { + t.Fatalf("a bundle invoked a Dockerfile build, which is the thing it exists to avoid:\n%s", + strings.Join(r.ran, "\n")) + } + + // And pinned by a digest of what came out, like any other artifact. + digest, _ := got.Manifest.Resources[0]["digest"].(string) + if !strings.HasPrefix(digest, "sha256:") { + t.Fatalf("the bundle was not pinned: %v", got.Manifest.Resources[0]) + } + + // **One file per entrypoint and launcher, in the toolchain's bundler** (novox/hq ADR 0193). A + // second run in the same toolchain image bundles each into the artifact's bundled output, the SDK + // inlined, refusing by name in an image that predates the bundler; and the toolchain's + // node_modules is no longer copied into a bundle that keeps nothing external. + var bundling []string + for _, line := range r.ran { + if strings.HasPrefix(line, "docker run") && strings.Contains(line, "esbuild") { + bundling = append(bundling, line) + } + } + if len(bundling) != 2 { + t.Fatalf("want one bundling run for the entrypoints and one for the launchers:\n%s", strings.Join(r.ran, "\n")) + } + for _, want := range []string{"mesh-tools/build@sha256:", "predates one-file bundles", "--bundle", "--format=esm", + "--platform=node", "--outdir=" + Out("code") + ".bundled", Out("code") + "/index.js"} { + if !strings.Contains(bundling[0], want) { + t.Errorf("the entrypoints' bundling lacks %q: %s", want, bundling[0]) + } + } + if !strings.Contains(bundling[1], Out("code")+"/index.serve.mjs") || !strings.Contains(bundling[1], "--out-extension:.js=.mjs") { + t.Errorf("the launcher is not bundled under its own name: %s", bundling[1]) + } + if strings.Contains(strings.Join(r.ran, "\n"), "/app/runtime") { + t.Errorf("the toolchain's node_modules was copied into a bundle that keeps nothing external:\n%s", strings.Join(r.ran, "\n")) + } + if strings.Index(strings.Join(r.ran, "\n"), "--outDir") > strings.Index(strings.Join(r.ran, "\n"), "esbuild") { + t.Fatal("the bundler ran before the compile wrote its output") + } +} + +func TestABundleKeepingAPackageExternalCarriesTheToolchainsModules(t *testing.T) { + manifest := strings.Replace(aBundle, `"entrypoints":["index.js"]`, `"entrypoints":["index.js"],"external":["sharp"]`, 1) + r, workspace := aRepository(t, manifest, map[string]string{"index.ts": "console.log(1)"}) + held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} + if _, err := Build(context.Background(), compiling{r}.run, r, + "https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, GitCredential{}, nil); err != nil { + t.Fatal(err) + } + all := strings.Join(r.ran, "\n") + if !strings.Contains(all, "--external:sharp") || !strings.Contains(all, "/app/runtime") { + t.Errorf("an external package was not kept as an import with the toolchain's modules beside it:\n%s", all) + } +} + +func TestABundleWhoseToolchainIsNotHeldIsRefusedFirst(t *testing.T) { + r, workspace := aRepository(t, aBundle, map[string]string{"index.ts": "console.log(1)"}) + + _, err := Build(context.Background(), compiling{r}.run, r, + "https://forge.invalid/greeter.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err == nil { + t.Fatal("a bundle was built with no toolchain to compile it in") + } + if !strings.Contains(err.Error(), "mesh-tools") { + t.Fatalf("the refusal does not name what has to be built first: %v", err) + } + for _, line := range r.ran { + if strings.HasPrefix(line, "docker run") { + t.Fatalf("a compile was attempted before the refusal: %s", line) + } + } +} + +func TestABundleInAnUnknownLanguageIsRefused(t *testing.T) { + manifest := strings.Replace(aBundle, `"language":"typescript"`, `"language":"cobol"`, 1) + r, workspace := aRepository(t, manifest, map[string]string{"index.ts": "x"}) + + _, err := Build(context.Background(), compiling{r}.run, r, + "https://forge.invalid/greeter.git", "", "", workspace, + map[string]string{"mesh-tools/build": "registry.invalid/x@sha256:" + strings.Repeat("c", 64)}, Npmrc{}, GitCredential{}, nil) + if err == nil { + t.Fatal("a language nothing can compile was accepted") + } + if !strings.Contains(err.Error(), "typescript") { + t.Fatalf("the refusal does not say what would have worked: %v", err) + } +} + +func TestTwoBundlesInOneModuleArePackedSeparately(t *testing.T) { + const two = `{"module":"greeter","version":"1", + "build":{"artifacts":[ + {"name":"daemon","kind":"bundle","language":"typescript","entrypoints":["index.js"]}, + {"name":"tools","kind":"bundle","language":"typescript","entrypoints":["index.js"]}]}, + "resources":[ + {"id":"a","type":"archive","path":"/opt/greeter/daemon","artifact":"daemon"}, + {"id":"b","type":"archive","path":"/opt/greeter/tools","artifact":"tools"}]}` + + r, workspace := aRepository(t, two, map[string]string{"index.ts": "console.log(1)"}) + held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} + + got, err := Build(context.Background(), compiling{r}.run, r, + "https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatalf("a module with two bundles did not build: %v", err) + } + + // Compiled into two different places. Only the compile lines: the copy of each bundle's + // dependencies names the same directory again, deliberately. + var outputs []string + for _, line := range r.ran { + if !strings.Contains(line, "--outDir") { + continue + } + for _, part := range strings.Fields(line) { + if strings.HasPrefix(part, ".mesh-build/") { + outputs = append(outputs, part) + } + } + } + if len(outputs) != 2 || outputs[0] == outputs[1] { + t.Fatalf("two bundles did not get their own output directories: %v", outputs) + } + + // And published as two artifacts, each with its own digest. + if len(r.archives) != 2 { + t.Fatalf("expected two archives published, got %v", r.archives) + } + first, _ := got.Manifest.Resources[0]["digest"].(string) + second, _ := got.Manifest.Resources[1]["digest"].(string) + if first == "" || second == "" { + t.Fatalf("a bundle was not pinned: %v", got.Manifest.Resources) + } +} diff --git a/internal/builder/bundle_test.go b/internal/builder/bundle_test.go index a29225e6..ccabe300 100644 --- a/internal/builder/bundle_test.go +++ b/internal/builder/bundle_test.go @@ -4,7 +4,6 @@ import ( "context" "os" "path/filepath" - "strings" "testing" ) @@ -49,85 +48,9 @@ func (c compiling) run(ctx context.Context, dir, name string, args ...string) (s // **A module says what it is written in, and needs no Dockerfile.** This is the whole point of the // bundle recipe: the same module previously needed a hand-written recipe repeating an incantation // that is easy to get wrong in ways that fail somewhere else. -func TestABundleIsCompiledAndPackedWithNoDockerfile(t *testing.T) { - r, workspace := aRepository(t, aBundle, map[string]string{"index.ts": "console.log(1)"}) - held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} - - got, err := Build(context.Background(), compiling{r}.run, r, - "https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatalf("a module with a language and no Dockerfile did not build: %v", err) - } - - // Compiled in the toolchain the mesh chose, not in one the module named. - var compiled string - for _, line := range r.ran { - if strings.HasPrefix(line, "docker run") { - compiled = line - } - } - if compiled == "" { - t.Fatalf("nothing was compiled:\n%s", strings.Join(r.ran, "\n")) - } - if !strings.Contains(compiled, "mesh-tools/build@sha256:") { - t.Fatalf("the compile did not run in the mesh's own toolchain: %s", compiled) - } - if strings.Contains(strings.Join(r.ran, "\n"), "docker build") { - t.Fatalf("a bundle invoked a Dockerfile build, which is the thing it exists to avoid:\n%s", - strings.Join(r.ran, "\n")) - } - - // And pinned by a digest of what came out, like any other artifact. - digest, _ := got.Manifest.Resources[0]["digest"].(string) - if !strings.HasPrefix(digest, "sha256:") { - t.Fatalf("the bundle was not pinned: %v", got.Manifest.Resources[0]) - } - - // **One file per entrypoint and launcher, in the toolchain's bundler** (novox/hq ADR 0193). A - // second run in the same toolchain image bundles each into the artifact's bundled output, the SDK - // inlined, refusing by name in an image that predates the bundler; and the toolchain's - // node_modules is no longer copied into a bundle that keeps nothing external. - var bundling []string - for _, line := range r.ran { - if strings.HasPrefix(line, "docker run") && strings.Contains(line, "esbuild") { - bundling = append(bundling, line) - } - } - if len(bundling) != 2 { - t.Fatalf("want one bundling run for the entrypoints and one for the launchers:\n%s", strings.Join(r.ran, "\n")) - } - for _, want := range []string{"mesh-tools/build@sha256:", "predates one-file bundles", "--bundle", "--format=esm", - "--platform=node", "--outdir=" + Out("code") + ".bundled", Out("code") + "/index.js"} { - if !strings.Contains(bundling[0], want) { - t.Errorf("the entrypoints' bundling lacks %q: %s", want, bundling[0]) - } - } - if !strings.Contains(bundling[1], Out("code")+"/index.serve.mjs") || !strings.Contains(bundling[1], "--out-extension:.js=.mjs") { - t.Errorf("the launcher is not bundled under its own name: %s", bundling[1]) - } - if strings.Contains(strings.Join(r.ran, "\n"), "/app/runtime") { - t.Errorf("the toolchain's node_modules was copied into a bundle that keeps nothing external:\n%s", strings.Join(r.ran, "\n")) - } - if strings.Index(strings.Join(r.ran, "\n"), "--outDir") > strings.Index(strings.Join(r.ran, "\n"), "esbuild") { - t.Fatal("the bundler ran before the compile wrote its output") - } -} // A bundle naming packages it keeps external is bundled with them as imports, and carries the // toolchain's node_modules for them — the one case it still does. -func TestABundleKeepingAPackageExternalCarriesTheToolchainsModules(t *testing.T) { - manifest := strings.Replace(aBundle, `"entrypoints":["index.js"]`, `"entrypoints":["index.js"],"external":["sharp"]`, 1) - r, workspace := aRepository(t, manifest, map[string]string{"index.ts": "console.log(1)"}) - held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} - if _, err := Build(context.Background(), compiling{r}.run, r, - "https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, GitCredential{}, nil); err != nil { - t.Fatal(err) - } - all := strings.Join(r.ran, "\n") - if !strings.Contains(all, "--external:sharp") || !strings.Contains(all, "/app/runtime") { - t.Errorf("an external package was not kept as an import with the toolchain's modules beside it:\n%s", all) - } -} // A language whose bundle carries its own dependencies copies nothing in: a Go binary is static. func TestOnlyALanguageWithARuntimeDirectoryCopiesDependenciesIn(t *testing.T) { @@ -146,39 +69,8 @@ func TestOnlyALanguageWithARuntimeDirectoryCopiesDependenciesIn(t *testing.T) { // **Refused before anything is built, naming what to build first.** A base the mesh has not built // is not a compile that fails on its first line — it is a question somebody can answer, and saying // it early is the difference between a fixable message and one about a missing image. -func TestABundleWhoseToolchainIsNotHeldIsRefusedFirst(t *testing.T) { - r, workspace := aRepository(t, aBundle, map[string]string{"index.ts": "console.log(1)"}) - - _, err := Build(context.Background(), compiling{r}.run, r, - "https://forge.invalid/greeter.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err == nil { - t.Fatal("a bundle was built with no toolchain to compile it in") - } - if !strings.Contains(err.Error(), "mesh-tools") { - t.Fatalf("the refusal does not name what has to be built first: %v", err) - } - for _, line := range r.ran { - if strings.HasPrefix(line, "docker run") { - t.Fatalf("a compile was attempted before the refusal: %s", line) - } - } -} // A language the mesh does not build is refused the same way, and names what it can build. -func TestABundleInAnUnknownLanguageIsRefused(t *testing.T) { - manifest := strings.Replace(aBundle, `"language":"typescript"`, `"language":"cobol"`, 1) - r, workspace := aRepository(t, manifest, map[string]string{"index.ts": "x"}) - - _, err := Build(context.Background(), compiling{r}.run, r, - "https://forge.invalid/greeter.git", "", "", workspace, - map[string]string{"mesh-tools/build": "registry.invalid/x@sha256:" + strings.Repeat("c", 64)}, Npmrc{}, GitCredential{}, nil) - if err == nil { - t.Fatal("a language nothing can compile was accepted") - } - if !strings.Contains(err.Error(), "typescript") { - t.Fatalf("the refusal does not say what would have worked: %v", err) - } -} // **One module, two bundles, and neither packs the other.** // @@ -186,48 +78,3 @@ func TestABundleInAnUnknownLanguageIsRefused(t *testing.T) { // daemon in one language and tools in another (ADR 0040). An earlier version of this compiled // every bundle into the toolchain's single output directory, so two of them would overwrite each // other and then be packed together — one artifact containing both, twice. -func TestTwoBundlesInOneModuleArePackedSeparately(t *testing.T) { - const two = `{"module":"greeter","version":"1", - "build":{"artifacts":[ - {"name":"daemon","kind":"bundle","language":"typescript","entrypoints":["index.js"]}, - {"name":"tools","kind":"bundle","language":"typescript","entrypoints":["index.js"]}]}, - "resources":[ - {"id":"a","type":"archive","path":"/opt/greeter/daemon","artifact":"daemon"}, - {"id":"b","type":"archive","path":"/opt/greeter/tools","artifact":"tools"}]}` - - r, workspace := aRepository(t, two, map[string]string{"index.ts": "console.log(1)"}) - held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} - - got, err := Build(context.Background(), compiling{r}.run, r, - "https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatalf("a module with two bundles did not build: %v", err) - } - - // Compiled into two different places. Only the compile lines: the copy of each bundle's - // dependencies names the same directory again, deliberately. - var outputs []string - for _, line := range r.ran { - if !strings.Contains(line, "--outDir") { - continue - } - for _, part := range strings.Fields(line) { - if strings.HasPrefix(part, ".mesh-build/") { - outputs = append(outputs, part) - } - } - } - if len(outputs) != 2 || outputs[0] == outputs[1] { - t.Fatalf("two bundles did not get their own output directories: %v", outputs) - } - - // And published as two artifacts, each with its own digest. - if len(r.archives) != 2 { - t.Fatalf("expected two archives published, got %v", r.archives) - } - first, _ := got.Manifest.Resources[0]["digest"].(string) - second, _ := got.Manifest.Resources[1]["digest"].(string) - if first == "" || second == "" { - t.Fatalf("a bundle was not pinned: %v", got.Manifest.Resources) - } -} diff --git a/internal/builder/check_base_test.go b/internal/builder/check_base_test.go index 16aa5484..e1887137 100644 --- a/internal/builder/check_base_test.go +++ b/internal/builder/check_base_test.go @@ -1,3 +1,5 @@ +//go:build integration + package builder import ( diff --git a/internal/builder/check_credential_integration_test.go b/internal/builder/check_credential_integration_test.go new file mode 100644 index 00000000..6e56036b --- /dev/null +++ b/internal/builder/check_credential_integration_test.go @@ -0,0 +1,180 @@ +//go:build integration + +package builder + +import ( + "context" + "errors" + "io/fs" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestACheckContainerSeesNoForgeCredential(t *testing.T) { + repo, head := aCheckedRepository(t, map[string]string{CheckScript: "echo checked\n"}) + besideRepo, besideHead := aCheckedRepository(t, map[string]string{"README": "beside"}) + // A clone source that carries userinfo, as a forge's clone URL may: git records it as given in the + // clone's .git/config, which the container reads. + besideURL := "file://beside-user:" + besideSecret + "@" + besideRepo + workspace := t.TempDir() + + var stores []string + reached := false + var leaks []string + run := func(ctx context.Context, dir, name string, args ...string) (string, error) { + switch name { + case "git": + for _, a := range args { + if f, ok := strings.CutPrefix(a, "credential.helper=store --file="); ok { + stores = append(stores, f) + raw, err := os.ReadFile(f) + if err != nil || strings.TrimSpace(string(raw)) != forgeURL { + t.Errorf("git is offered a store that does not hold the credential as given: %q, %v", raw, err) + } + if info, err := os.Stat(f); err == nil && info.Mode().Perm() != 0o600 { + t.Errorf("the credential store is readable beyond its owner: %v", info.Mode()) + } + } + } + if len(args) >= 2 && args[len(args)-3] == "--quiet" && hasString(args, "clone") { + source := args[len(args)-2] + if u, err := url.Parse(source); err == nil && u.User != nil { + // Git cannot reach a file:// URL with userinfo; clone it without, then record it as git + // would have: as given. + clone := append(append([]string{}, args[:len(args)-2]...), bareURL(t, source), args[len(args)-1]) + if out, err := Command(ctx, dir, "git", clone...); err != nil { + return out, err + } + return Command(ctx, filepath.Join(dir, args[len(args)-1]), "git", "remote", "set-url", "origin", source) + } + } + return Command(ctx, dir, name, args...) + case "docker": + if !reached { + reached = true + // The first container: everything the workspace holds is what the toolchain container sees. + filepath.WalkDir(workspace, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + raw, _ := os.ReadFile(path) + s := string(raw) + if strings.Contains(s, forgeSecret) || strings.Contains(s, besideSecret) || + strings.Contains(s, npmSecret) || d.Name() == "git-credentials" || d.Name() == ".npmrc" || + strings.Contains(s, "credential.helper") { + leaks = append(leaks, path) + } + return nil + }) + for _, f := range stores { + if _, err := os.Stat(f); !errors.Is(err, os.ErrNotExist) { + leaks = append(leaks, f+" (still there when the first container runs)") + } + if rel, err := filepath.Rel(workspace, f); err == nil && !strings.HasPrefix(rel, "..") { + leaks = append(leaks, f+" (inside the workspace the container mounts)") + } + } + } + if len(args) > 0 && args[0] == "ps" { + return "", nil + } + return "", errors.New("no container runtime in this test") + } + return "", errors.New("unexpected command " + name) + } + // A credential an older builder left in the workspace is removed too: the forge's, and the .npmrc a + // build wrote into the tree it cloned. + if err := os.WriteFile(filepath.Join(workspace, "git-credentials"), []byte(forgeURL+"\n"), 0o600); err != nil { + t.Fatal(err) + } + for _, dir := range []string{"source/x", "context-server"} { + if err := os.MkdirAll(filepath.Join(workspace, dir), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(workspace, dir, ".npmrc"), + []byte("//forge.invalid/api/packages/novox/npm/:_authToken="+npmSecret+"\n"), 0o600); err != nil { + t.Fatal(err) + } + } + _, err := Check(t.Context(), run, CheckSpec{ID: "check-462", Repository: repo, Ref: head, Owner: "novox", + Repo: "mesh-controller", Number: 1, Toolchain: "golang", Beside: map[string]Beside{ + "mesh-catalog": {Repository: besideURL, Ref: besideHead}}}, workspace, aFactsRegistry(t), + GitCredential{URL: forgeURL}, nil) + if err == nil { + t.Fatal("the check ran past its first container in a test with none") + } + if !reached { + t.Fatalf("the check never reached its first container: %v", err) + } + if len(stores) == 0 { + t.Fatal("no clone was offered the forge credential") + } + if len(leaks) > 0 { + t.Fatalf("the check's container sees the credential:\n%s", strings.Join(leaks, "\n")) + } +} + +func TestACheckLinePublishedToTheLogIsRedacted(t *testing.T) { + var said []string + say := func(step, format string, args ...any) { + if step == "output" && len(args) > 0 { + said = append(said, args[0].(string)) + } + } + var out tail + layer := ownCheck(t.Context(), CheckSpec{Toolchain: "golang"}, []ScriptPart{{Toolchain: "go", Script: CheckScript}}, + t.TempDir(), &out, func() bool { return false }, func(string, string) *exec.Cmd { + return exec.CommandContext(t.Context(), "sh", "-c", "echo cloning http://mesh_builder:t0ps3cret-forge@forge.invalid/novox/x.git; "+ + "echo token ghp_abcdefghijklmnopqrstuvwxyz0123456789") + }, say) + if layer == nil || layer.Verdict != "pass" { + t.Fatalf("the check answered %+v\n%s", layer, out.String()) + } + joined := strings.Join(said, "\n") + if strings.Contains(joined, "t0ps3cret-forge") || strings.Contains(joined, "ghp_abcdef") { + t.Fatalf("a secret the check printed is published to the build's log:\n%s", joined) + } + if !strings.Contains(joined, "http://mesh_builder:[redacted: a password in a URI]@forge.invalid/novox/x.git") { + t.Fatalf("the line is not said with what was there named:\n%s", joined) + } +} + +func TestABuildsCloneRecordsNoUserinfo(t *testing.T) { + repo, _ := aCheckedRepository(t, map[string]string{ManifestName: `{"module":"plain","version":"1"}`}) + source := "file://build-user:" + besideSecret + "@" + repo + workspace := t.TempDir() + tree := filepath.Join(workspace, "source") + var configs []string + run := func(ctx context.Context, dir, name string, args ...string) (string, error) { + if name == "git" && hasString(args, "clone") && args[len(args)-2] == source { + clone := append(append([]string{}, args[:len(args)-2]...), bareURL(t, source), args[len(args)-1]) + if out, err := Command(ctx, dir, "git", clone...); err != nil { + return out, err + } + return Command(ctx, args[len(args)-1], "git", "remote", "set-url", "origin", source) + } + if name == "git" && len(args) > 0 && args[0] == "rev-parse" { + raw, _ := os.ReadFile(filepath.Join(tree, ".git", "config")) + configs = append(configs, string(raw)) + } + return Command(ctx, dir, name, args...) + } + if _, err := Build(t.Context(), run, &recorded{}, source, "", "", workspace, nil, Npmrc{}, GitCredential{}, nil); err != nil { + t.Fatal(err) + } + if len(configs) == 0 { + t.Fatal("the build never read its clone") + } + for _, c := range configs { + if strings.Contains(c, besideSecret) || strings.Contains(c, "build-user") { + t.Fatalf("the build's clone records the credential it was cloned with:\n%s", c) + } + } + if _, err := os.Stat(tree); !os.IsNotExist(err) { + t.Fatalf("the build's tree outlives the build: %v", err) + } +} diff --git a/internal/builder/check_credential_test.go b/internal/builder/check_credential_test.go index 7fc52aa9..f0c63f1c 100644 --- a/internal/builder/check_credential_test.go +++ b/internal/builder/check_credential_test.go @@ -1,18 +1,12 @@ package builder import ( - "context" "crypto/sha256" "encoding/hex" "encoding/json" - "errors" - "io/fs" "net/http" "net/http/httptest" "net/url" - "os" - "os/exec" - "path/filepath" "strings" "testing" "time" @@ -67,135 +61,7 @@ func bareURL(t *testing.T, raw string) string { return u.String() } -func TestACheckContainerSeesNoForgeCredential(t *testing.T) { - repo, head := aCheckedRepository(t, map[string]string{CheckScript: "echo checked\n"}) - besideRepo, besideHead := aCheckedRepository(t, map[string]string{"README": "beside"}) - // A clone source that carries userinfo, as a forge's clone URL may: git records it as given in the - // clone's .git/config, which the container reads. - besideURL := "file://beside-user:" + besideSecret + "@" + besideRepo - workspace := t.TempDir() - - var stores []string - reached := false - var leaks []string - run := func(ctx context.Context, dir, name string, args ...string) (string, error) { - switch name { - case "git": - for _, a := range args { - if f, ok := strings.CutPrefix(a, "credential.helper=store --file="); ok { - stores = append(stores, f) - raw, err := os.ReadFile(f) - if err != nil || strings.TrimSpace(string(raw)) != forgeURL { - t.Errorf("git is offered a store that does not hold the credential as given: %q, %v", raw, err) - } - if info, err := os.Stat(f); err == nil && info.Mode().Perm() != 0o600 { - t.Errorf("the credential store is readable beyond its owner: %v", info.Mode()) - } - } - } - if len(args) >= 2 && args[len(args)-3] == "--quiet" && hasString(args, "clone") { - source := args[len(args)-2] - if u, err := url.Parse(source); err == nil && u.User != nil { - // Git cannot reach a file:// URL with userinfo; clone it without, then record it as git - // would have: as given. - clone := append(append([]string{}, args[:len(args)-2]...), bareURL(t, source), args[len(args)-1]) - if out, err := Command(ctx, dir, "git", clone...); err != nil { - return out, err - } - return Command(ctx, filepath.Join(dir, args[len(args)-1]), "git", "remote", "set-url", "origin", source) - } - } - return Command(ctx, dir, name, args...) - case "docker": - if !reached { - reached = true - // The first container: everything the workspace holds is what the toolchain container sees. - filepath.WalkDir(workspace, func(path string, d fs.DirEntry, err error) error { - if err != nil || d.IsDir() { - return nil - } - raw, _ := os.ReadFile(path) - s := string(raw) - if strings.Contains(s, forgeSecret) || strings.Contains(s, besideSecret) || - strings.Contains(s, npmSecret) || d.Name() == "git-credentials" || d.Name() == ".npmrc" || - strings.Contains(s, "credential.helper") { - leaks = append(leaks, path) - } - return nil - }) - for _, f := range stores { - if _, err := os.Stat(f); !errors.Is(err, os.ErrNotExist) { - leaks = append(leaks, f+" (still there when the first container runs)") - } - if rel, err := filepath.Rel(workspace, f); err == nil && !strings.HasPrefix(rel, "..") { - leaks = append(leaks, f+" (inside the workspace the container mounts)") - } - } - } - if len(args) > 0 && args[0] == "ps" { - return "", nil - } - return "", errors.New("no container runtime in this test") - } - return "", errors.New("unexpected command " + name) - } - // A credential an older builder left in the workspace is removed too: the forge's, and the .npmrc a - // build wrote into the tree it cloned. - if err := os.WriteFile(filepath.Join(workspace, "git-credentials"), []byte(forgeURL+"\n"), 0o600); err != nil { - t.Fatal(err) - } - for _, dir := range []string{"source/x", "context-server"} { - if err := os.MkdirAll(filepath.Join(workspace, dir), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(workspace, dir, ".npmrc"), - []byte("//forge.invalid/api/packages/novox/npm/:_authToken="+npmSecret+"\n"), 0o600); err != nil { - t.Fatal(err) - } - } - _, err := Check(t.Context(), run, CheckSpec{ID: "check-462", Repository: repo, Ref: head, Owner: "novox", - Repo: "mesh-controller", Number: 1, Toolchain: "golang", Beside: map[string]Beside{ - "mesh-catalog": {Repository: besideURL, Ref: besideHead}}}, workspace, aFactsRegistry(t), - GitCredential{URL: forgeURL}, nil) - if err == nil { - t.Fatal("the check ran past its first container in a test with none") - } - if !reached { - t.Fatalf("the check never reached its first container: %v", err) - } - if len(stores) == 0 { - t.Fatal("no clone was offered the forge credential") - } - if len(leaks) > 0 { - t.Fatalf("the check's container sees the credential:\n%s", strings.Join(leaks, "\n")) - } -} - // Every line a repository's own check prints is published to the build's log redacted. -func TestACheckLinePublishedToTheLogIsRedacted(t *testing.T) { - var said []string - say := func(step, format string, args ...any) { - if step == "output" && len(args) > 0 { - said = append(said, args[0].(string)) - } - } - var out tail - layer := ownCheck(t.Context(), CheckSpec{Toolchain: "golang"}, []ScriptPart{{Toolchain: "go", Script: CheckScript}}, - t.TempDir(), &out, func() bool { return false }, func(string, string) *exec.Cmd { - return exec.CommandContext(t.Context(), "sh", "-c", "echo cloning http://mesh_builder:t0ps3cret-forge@forge.invalid/novox/x.git; "+ - "echo token ghp_abcdefghijklmnopqrstuvwxyz0123456789") - }, say) - if layer == nil || layer.Verdict != "pass" { - t.Fatalf("the check answered %+v\n%s", layer, out.String()) - } - joined := strings.Join(said, "\n") - if strings.Contains(joined, "t0ps3cret-forge") || strings.Contains(joined, "ghp_abcdef") { - t.Fatalf("a secret the check printed is published to the build's log:\n%s", joined) - } - if !strings.Contains(joined, "http://mesh_builder:[redacted: a password in a URI]@forge.invalid/novox/x.git") { - t.Fatalf("the line is not said with what was there named:\n%s", joined) - } -} func TestTheRedactorHidesTheForgeCredentialAndShapes(t *testing.T) { r := redactorFor(GitCredential{URL: forgeURL}) @@ -214,38 +80,3 @@ func TestTheRedactorHidesTheForgeCredentialAndShapes(t *testing.T) { // A build's clone of a URL carrying userinfo records it without, so no build leaves a credential in // workspace/source/.git/config while it runs either (novox/hq issue 462); the tree itself is gone when the // build ends. -func TestABuildsCloneRecordsNoUserinfo(t *testing.T) { - repo, _ := aCheckedRepository(t, map[string]string{ManifestName: `{"module":"plain","version":"1"}`}) - source := "file://build-user:" + besideSecret + "@" + repo - workspace := t.TempDir() - tree := filepath.Join(workspace, "source") - var configs []string - run := func(ctx context.Context, dir, name string, args ...string) (string, error) { - if name == "git" && hasString(args, "clone") && args[len(args)-2] == source { - clone := append(append([]string{}, args[:len(args)-2]...), bareURL(t, source), args[len(args)-1]) - if out, err := Command(ctx, dir, "git", clone...); err != nil { - return out, err - } - return Command(ctx, args[len(args)-1], "git", "remote", "set-url", "origin", source) - } - if name == "git" && len(args) > 0 && args[0] == "rev-parse" { - raw, _ := os.ReadFile(filepath.Join(tree, ".git", "config")) - configs = append(configs, string(raw)) - } - return Command(ctx, dir, name, args...) - } - if _, err := Build(t.Context(), run, &recorded{}, source, "", "", workspace, nil, Npmrc{}, GitCredential{}, nil); err != nil { - t.Fatal(err) - } - if len(configs) == 0 { - t.Fatal("the build never read its clone") - } - for _, c := range configs { - if strings.Contains(c, besideSecret) || strings.Contains(c, "build-user") { - t.Fatalf("the build's clone records the credential it was cloned with:\n%s", c) - } - } - if _, err := os.Stat(tree); !os.IsNotExist(err) { - t.Fatalf("the build's tree outlives the build: %v", err) - } -} diff --git a/internal/builder/check_integration_test.go b/internal/builder/check_integration_test.go new file mode 100644 index 00000000..6e9f9992 --- /dev/null +++ b/internal/builder/check_integration_test.go @@ -0,0 +1,399 @@ +//go:build integration + +package builder + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestTheThrowawayStoreFlushesNothing(t *testing.T) { + said := strings.Join(StoreSettings, " ") + for _, want := range []string{"-c fsync=off", "-c synchronous_commit=off", "-c full_page_writes=off"} { + if !strings.Contains(said, want) { + t.Errorf("the store is raised with %q, not %q", said, want) + } + } + if os.Getenv("MESH_TEST_DOCKER") != "1" { + t.Skip("MESH_TEST_DOCKER=1: the settings are proven on a raised store") + } + id := fmt.Sprintf("check-store-%d", time.Now().UnixNano()) + labelledRun := Labelled(Command, id) + t.Cleanup(func() { _, _ = RemoveContainersOf(context.Background(), Command, id) }) + if _, err := throwaway(t.Context(), labelledRun, Command, id+"-store", StoreImage("17.11"), 5432, + []string{"-e", "POSTGRES_PASSWORD=check"}, StoreSettings); err != nil { + t.Fatal(err) + } + if err := waitFor(t.Context(), Command, id+"-store", []string{"pg_isready", "-U", "postgres"}); err != nil { + t.Fatal(err) + } + for setting, want := range map[string]string{"fsync": "off", "synchronous_commit": "off", "full_page_writes": "off"} { + var got string + for try := 0; try < 10; try++ { + out, err := exec.Command("docker", "exec", id+"-store", "psql", "-U", "postgres", "-tAc", "SHOW "+setting).CombinedOutput() + if got = strings.TrimSpace(string(out)); err == nil { + break + } + time.Sleep(500 * time.Millisecond) + } + if got != want { + t.Errorf("%s is %q on the raised store", setting, got) + } + } +} + +func aCheckedRepository(t *testing.T, files map[string]string) (string, string) { + t.Helper() + dir := t.TempDir() + git := func(args ...string) string { + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org", + "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + git("init", "--quiet", "-b", "main") + for name, body := range files { + if err := os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o755); err != nil { + t.Fatal(err) + } + } + git("add", "-A") + git("commit", "--quiet", "-m", "x") + return dir, git("rev-parse", "HEAD") +} + +func labelled(id string) []string { + out, _ := exec.Command("docker", "ps", "-aq", "--filter", "label="+BuildLabel+"="+id).Output() + return strings.Fields(string(out)) +} + +func TestACheckRunsTheRepositorysOwnScriptBesideTheMeshsVersionsAndLeavesNothing(t *testing.T) { + registry := checkEnvironment(t) + // The script proves what it was given: the facts, a store that answers, a bus that answers — and no + // container runtime socket. + script := `set -e +test -s "$MESH_FACTS" +grep -q '"bus": "2.11.17"' "$MESH_FACTS" || grep -q '"bus":"2.11.17"' "$MESH_FACTS" +case "$MESH_TEST_POSTGRES" in postgres://*127.0.0.1:*) ;; *) echo "no store: $MESH_TEST_POSTGRES"; exit 1;; esac +case "$MESH_TEST_NATS" in nats://127.0.0.1:*) ;; *) echo "no bus: $MESH_TEST_NATS"; exit 1;; esac +test "$MESH_CHECK_REPOSITORY" = "novox/hq" +test "$MESH_CHECK_CHANGED" = "a.go,b.go" +test ! -S /var/run/docker.sock || { echo "the check holds the container runtime's socket"; exit 1; } +echo checked +` + repo, head := aCheckedRepository(t, map[string]string{CheckScript: script}) + id := fmt.Sprintf("check-test-%d", time.Now().UnixNano()) + v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: head, Owner: "novox", + Repo: "hq", Number: 7, Paths: []string{"a.go", "b.go"}, Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + // Nothing of the graph touched: the gate a pass that says so, the repository's own check run. + if v.Gate == nil || v.Gate.Verdict != "pass" || v.Gate.Summary != noModule { + t.Errorf("the gate answered %+v", v.Gate) + } + if v.Repo == nil || v.Repo.Verdict != "pass" || !strings.Contains(v.Report, "checked") { + t.Fatalf("the repository's check answered %+v\n%s", v.Repo, v.Report) + } + if left := labelled(id); len(left) > 0 { + t.Errorf("the check left %d container(s) behind", len(left)) + } +} + +func TestAFailingCheckFailsAndOneThatCannotRunIsNeverAPass(t *testing.T) { + registry := checkEnvironment(t) + repo, head := aCheckedRepository(t, map[string]string{CheckScript: "echo 'resource \"x.service\": refused'; exit 3\n"}) + v, err := Check(t.Context(), Command, CheckSpec{ID: fmt.Sprintf("check-fail-%d", time.Now().UnixNano()), + Repository: repo, Ref: head, Owner: "novox", Repo: "hq", Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if v.Repo == nil || v.Repo.Verdict != "fail" || !strings.Contains(v.Repo.Summary, "refused") { + t.Fatalf("a failing script answered %+v", v.Repo) + } + + // A script in a toolchain the mesh does not hold: an error, never a pass. + repo, head = aCheckedRepository(t, map[string]string{CheckScript: "# mesh-check-toolchain: cobol\nexit 0\n"}) + v, err = Check(t.Context(), Command, CheckSpec{ID: fmt.Sprintf("check-cobol-%d", time.Now().UnixNano()), + Repository: repo, Ref: head, Owner: "novox", Repo: "hq", Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) + if err != nil || v.Repo == nil || v.Repo.Verdict != "error" { + t.Fatalf("a script in a toolchain nobody holds answered %+v, %v", v.Repo, err) + } + + // Past its bound: an error, not a pass. + was := CheckTimeout + CheckTimeout = 25 * time.Second + t.Cleanup(func() { CheckTimeout = was }) + repo, head = aCheckedRepository(t, map[string]string{CheckScript: "sleep 120\n"}) + v, err = Check(context.Background(), Command, CheckSpec{ID: fmt.Sprintf("check-slow-%d", time.Now().UnixNano()), + Repository: repo, Ref: head, Owner: "novox", Repo: "hq", Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) + if err == nil && (v.Repo == nil || v.Repo.Verdict != "error") { + t.Fatalf("a check past its bound answered %+v", v.Repo) + } + + // No facts: it cannot run, and says so. + repo, head = aCheckedRepository(t, map[string]string{CheckScript: "exit 0\n"}) + _, err = Check(t.Context(), Command, CheckSpec{ID: "check-nofacts", Repository: repo, Ref: head, Owner: "novox", + Repo: "hq", Toolchain: goToolchain}, t.TempDir(), "127.0.0.1:1", GitCredential{}, nil) + if err == nil || !strings.Contains(err.Error(), "facts snapshot") { + t.Fatalf("a check with no facts said %v", err) + } +} + +func TestACheckWithNothingToRunSaysSo(t *testing.T) { + repo, head := aCheckedRepository(t, map[string]string{"README.md": "x"}) + v, err := Check(t.Context(), Command, CheckSpec{ID: "check-nothing", Repository: repo, Ref: head, Owner: "novox", + Repo: "hq"}, t.TempDir(), "", GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if v.Gate == nil || v.Gate.Verdict != "pass" || v.Repo == nil || v.Repo.Verdict != "warning" || + !strings.Contains(v.Repo.Summary, CheckScript) { + t.Fatalf("a check with nothing to run said %+v / %+v", v.Gate, v.Repo) + } + // A gated change never takes that path: with no store to read the facts from, it cannot run. + _, err = Check(t.Context(), Command, CheckSpec{ID: "check-gated", Repository: repo, Ref: head, Owner: "novox", + Repo: "mesh-catalog", Modules: []string{"gitea"}}, t.TempDir(), "", GitCredential{}, nil) + if err == nil { + t.Fatal("a change touching a module ran nothing and was not refused") + } +} + +func TestTheGateRunsWhenTheGraphIsTouchedBesideTheRepositorysOwnCheck(t *testing.T) { + registry := checkEnvironment(t) + judgeRepo, judgeHead := aCheckedRepository(t, aJudge) + beside := map[string]Beside{"mesh-controller": {Repository: judgeRepo, Ref: judgeHead}} + check := func(files map[string]string, judge string) CheckVerdict { + t.Helper() + repo, head := aCheckedRepository(t, files) + id := fmt.Sprintf("check-gate-%d", time.Now().UnixNano()) + v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: head, Owner: "novox", + Repo: "mesh-catalog", Number: 9, Paths: []string{"modules/gitea/index.ts"}, Beside: beside, + Modules: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}, Judge: judge, + Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if left := labelled(id); len(left) > 0 { + t.Errorf("the check left %d container(s) behind", len(left)) + } + return v + } + v := check(map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`, CheckScript: "echo own; exit 0\n"}, "") + if v.Gate == nil || v.Gate.Verdict != "warning" || v.Gate.Summary != "a merge rebuilds 14 module(s)" || + strings.Join(v.Gate.Modules, ",") != "gitea" || v.Verdict != "warning" { + t.Fatalf("the gate answered %+v\n%s", v.Gate, v.Report) + } + if v.Repo == nil || v.Repo.Verdict != "pass" { + t.Fatalf("its own check answered %+v", v.Repo) + } + + v = check(map[string]string{"modules/gitea/module.json": `{"module":"gitea","broken":true}`}, "") + if v.Gate.Verdict != "fail" || !strings.Contains(v.Gate.Summary, "module check") || v.Repo.Verdict != "warning" { + t.Fatalf("a manifest the judge refuses answered %+v / %+v\n%s", v.Gate, v.Repo, v.Report) + } + + // A fault the base branch already had is said and fails nothing; one the change brings fails. + repo, _ := aCheckedRepository(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea","broken":true}`}) + git := func(args ...string) string { + cmd := exec.Command("git", args...) + cmd.Dir = repo + cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org", + "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + git("checkout", "--quiet", "-b", "change") + if err := os.WriteFile(filepath.Join(repo, "modules/gitea/index.ts"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + git("add", "-A") + git("commit", "--quiet", "-m", "y") + id := fmt.Sprintf("check-base-%d", time.Now().UnixNano()) + v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: git("rev-parse", "HEAD"), Owner: "novox", + Repo: "mesh-catalog", Base: "main", Paths: []string{"modules/gitea/index.ts"}, Beside: beside, + Modules: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}, Toolchain: goToolchain}, + t.TempDir(), registry, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if v.Gate.Verdict != "warning" || !strings.Contains(v.Report, "already") { + t.Fatalf("a fault the base already had failed the change: %+v\n%s", v.Gate, v.Report) + } + + // A delivery group (novox/hq ADR 0239): the other heads cloned beside it at their heads and handed to the + // gate to compose with this one; the repository's own check is each pull request's, not the group's. + app, appHead := aCheckedRepository(t, map[string]string{"module.json": `{"module":"app"}`}) + repo2, head2 := aCheckedRepository(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`, + CheckScript: "exit 1\n"}) + gid := fmt.Sprintf("check-group-%d", time.Now().UnixNano()) + v, err = Check(t.Context(), Command, CheckSpec{ID: gid, Repository: repo2, Ref: head2, Owner: "novox", + Repo: "mesh-catalog", Paths: []string{"modules/gitea/module.json"}, Beside: beside, Modules: []string{"gitea"}, + Manifests: []string{"modules/gitea/module.json"}, Toolchain: goToolchain, + Group: []GroupHead{{Owner: "novox", Repo: "app", Repository: app, Ref: appHead, Paths: []string{"module.json"}}}}, + t.TempDir(), registry, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if v.Gate.Verdict != "pass" || v.Gate.Summary != "composed with novox/app" { + t.Fatalf("the group's other head was not composed: %+v\n%s", v.Gate, v.Report) + } + if v.Repo == nil || v.Repo.Verdict != "pass" || !strings.Contains(v.Repo.Summary, "group") { + t.Fatalf("a group's check ran a member's own script: %+v", v.Repo) + } + + // A change to the controller judges itself: one that does not build fails its own gate. + v = check(map[string]string{"go.mod": "module x\n\ngo 1.22\n", "cmd/mesh-controller/main.go": "package main\nfunc main() { nope }\n", + "modules/gitea/module.json": "{}"}, "self") + if v.Gate.Verdict != "fail" || !strings.Contains(v.Gate.Summary, "does not build") { + t.Fatalf("a controller that does not build judged itself %+v", v.Gate) + } +} + +func TestABuildSaysWhetherItsCommitIsOnTheTrunk(t *testing.T) { + repo, onMain := aCheckedRepository(t, map[string]string{"module.json": `{"module":"x","version":"1"}`}) + git := func(dir string, args ...string) string { + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org", + "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + git(repo, "checkout", "--quiet", "-b", "feature") + if err := os.WriteFile(filepath.Join(repo, "x"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + git(repo, "add", "-A") + git(repo, "commit", "--quiet", "-m", "off the trunk") + offMain := git(repo, "rev-parse", "HEAD") + git(repo, "checkout", "--quiet", "main") + + clone := filepath.Join(t.TempDir(), "clone") + git(filepath.Dir(clone), "clone", "--quiet", repo, clone) + if trunk, on := trunkOf(t.Context(), Command, clone, onMain); trunk != "main" || !on { + t.Errorf("a commit on main reads as on %q: %v", trunk, on) + } + if trunk, on := trunkOf(t.Context(), Command, clone, offMain); trunk != "main" || on { + t.Errorf("a feature branch's commit reads as on %q: %v", trunk, on) + } + if got := strings.Join(branchesHolding(t.Context(), Command, clone, offMain), ","); got != "feature" { + t.Errorf("the feature branch's commit is said to be on %q", got) + } + if got := strings.Join(branchesHolding(t.Context(), Command, clone, onMain), ","); got != "feature,main" { + t.Errorf("main's commit is said to be on %q", got) + } + // A tree that says no trunk is not known — never read as on it. + if trunk, on := trunkOf(t.Context(), Command, repo, onMain); trunk != "" || on { + t.Errorf("a repository with no origin reads as trunk %q, on %v", trunk, on) + } +} + +func TestARedeliveredCheckRemovesWhatItsEarlierDeliveryLeft(t *testing.T) { + registry := checkEnvironment(t) + repo, head := aCheckedRepository(t, map[string]string{CheckScript: "echo checked\n"}) + id := fmt.Sprintf("check-again-%d", time.Now().UnixNano()) + if out, err := exec.Command("docker", "run", "-d", "--rm", "--label", BuildLabel+"="+id, "--name", id+"-store", + "postgres:17-alpine", "sleep", "300").CombinedOutput(); err != nil { + t.Skipf("no container for the earlier delivery: %v %s", err, out) + } + v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: head, Owner: "novox", + Repo: "hq", Number: 7, Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) + if err != nil { + t.Fatalf("a redelivered check could not run: %v", err) + } + if v.Repo == nil || v.Repo.Verdict != "pass" { + t.Errorf("the repository's check answered %+v", v.Repo) + } + if left := labelled(id); len(left) > 0 { + t.Errorf("the check left %d container(s) behind", len(left)) + } +} + +func TestARepositoryInTwoLanguagesIsCheckedInBoth(t *testing.T) { + script := "#!/bin/sh\n# mesh-check-toolchain: typescript\n# mesh-check-also: go replays/merge-check.sh\nnpm test\n" + parts := ScriptParts([]byte(script)) + if len(parts) != 2 || parts[0] != (ScriptPart{"typescript", CheckScript}) || + parts[1] != (ScriptPart{"go", "replays/merge-check.sh"}) { + t.Fatalf("the parts read as %+v", parts) + } + if got := ScriptParts([]byte("#!/bin/sh\nset -eu\n")); len(got) != 1 || got[0] != (ScriptPart{"go", CheckScript}) { + t.Fatalf("a script declaring nothing reads as %+v", got) + } + + // Each part run where its toolchain is: here, the image the part was given is what the command says. + held := CheckSpec{Toolchain: "go-image", Toolchains: map[string]string{"typescript": "ts-image", "go": "go-image"}} + run := func(t *testing.T, spec CheckSpec, files map[string]string, parts []ScriptPart) (*Layer, []string) { + t.Helper() + tree := t.TempDir() + for name, body := range files { + if err := os.MkdirAll(filepath.Dir(filepath.Join(tree, name)), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tree, name), []byte(body), 0o755); err != nil { + t.Fatal(err) + } + } + var images []string + var out tail + layer := ownCheck(t.Context(), spec, parts, tree, &out, func() bool { return false }, + func(image, script string) *exec.Cmd { + images = append(images, image+" "+script) + cmd := exec.CommandContext(t.Context(), "sh", script) + cmd.Dir = tree + return cmd + }, func(string, string, ...any) {}) + return layer, images + } + twoParts := []ScriptPart{{"typescript", CheckScript}, {"go", "replays/merge-check.sh"}} + + layer, images := run(t, held, map[string]string{CheckScript: "exit 0\n", "replays/merge-check.sh": "exit 0\n"}, twoParts) + if layer.Verdict != "pass" || !strings.Contains(layer.Summary, "replays/merge-check.sh (go)") || + strings.Join(images, ", ") != "ts-image merge-check.sh, go-image replays/merge-check.sh" { + t.Errorf("both parts passing answered %+v, ran %v", layer, images) + } + layer, _ = run(t, held, map[string]string{CheckScript: "exit 0\n", + "replays/merge-check.sh": "echo 'not gofmt'\"'\"'d:'; echo register.go; exit 1\n"}, twoParts) + if layer.Verdict != "fail" || !strings.Contains(layer.Summary, "replays/merge-check.sh failed") || + !strings.Contains(layer.Summary, "register.go") || !strings.Contains(layer.Summary, "merge-check.sh (typescript) passed") { + t.Errorf("a failing Go part answered %+v", layer) + } + layer, images = run(t, held, map[string]string{CheckScript: "exit 2\n", "replays/merge-check.sh": "exit 0\n"}, twoParts) + if layer.Verdict != "fail" || len(images) != 1 { + t.Errorf("a failing first part answered %+v and ran %v", layer, images) + } + layer, _ = run(t, held, map[string]string{CheckScript: "exit 0\n"}, twoParts) + if layer.Verdict != "fail" || !strings.Contains(layer.Summary, "does not hold") { + t.Errorf("a declared part the repository lacks answered %+v", layer) + } + layer, _ = run(t, held, map[string]string{CheckScript: "exit 0\n"}, []ScriptPart{{"typescript", CheckScript}, + {"go", "../elsewhere.sh"}}) + if layer.Verdict != "fail" || !strings.Contains(layer.Summary, "not a path inside") { + t.Errorf("a part outside the repository answered %+v", layer) + } + layer, _ = run(t, CheckSpec{Toolchains: map[string]string{"typescript": "ts-image"}}, + map[string]string{CheckScript: "exit 0\n", "replays/merge-check.sh": "exit 0\n"}, twoParts) + if layer.Verdict != "error" || !strings.Contains(layer.Summary, "go toolchain") { + t.Errorf("a part in a toolchain the mesh does not hold answered %+v — never a pass", layer) + } +} diff --git a/internal/builder/check_merge_test.go b/internal/builder/check_merge_test.go index 86ad2da0..3bc42bc7 100644 --- a/internal/builder/check_merge_test.go +++ b/internal/builder/check_merge_test.go @@ -1,3 +1,5 @@ +//go:build integration + package builder import ( diff --git a/internal/builder/check_test.go b/internal/builder/check_test.go index 1eb5890a..b712b25d 100644 --- a/internal/builder/check_test.go +++ b/internal/builder/check_test.go @@ -1,11 +1,8 @@ package builder import ( - "context" "encoding/json" - "fmt" "os" - "os/exec" "path/filepath" "strings" "testing" @@ -35,69 +32,8 @@ func TestTheStoreAndBusAChecksStandsOnAreTheOnesTheMeshRuns(t *testing.T) { // **Issue 306**: the throwaway store is raised without durability — it lives for one check — so a suite // that makes a database per test does not wait on the disk of a busy machine for every flush. -func TestTheThrowawayStoreFlushesNothing(t *testing.T) { - said := strings.Join(StoreSettings, " ") - for _, want := range []string{"-c fsync=off", "-c synchronous_commit=off", "-c full_page_writes=off"} { - if !strings.Contains(said, want) { - t.Errorf("the store is raised with %q, not %q", said, want) - } - } - if os.Getenv("MESH_TEST_DOCKER") != "1" { - t.Skip("MESH_TEST_DOCKER=1: the settings are proven on a raised store") - } - id := fmt.Sprintf("check-store-%d", time.Now().UnixNano()) - labelledRun := Labelled(Command, id) - t.Cleanup(func() { _, _ = RemoveContainersOf(context.Background(), Command, id) }) - if _, err := throwaway(t.Context(), labelledRun, Command, id+"-store", StoreImage("17.11"), 5432, - []string{"-e", "POSTGRES_PASSWORD=check"}, StoreSettings); err != nil { - t.Fatal(err) - } - if err := waitFor(t.Context(), Command, id+"-store", []string{"pg_isready", "-U", "postgres"}); err != nil { - t.Fatal(err) - } - for setting, want := range map[string]string{"fsync": "off", "synchronous_commit": "off", "full_page_writes": "off"} { - var got string - for try := 0; try < 10; try++ { - out, err := exec.Command("docker", "exec", id+"-store", "psql", "-U", "postgres", "-tAc", "SHOW "+setting).CombinedOutput() - if got = strings.TrimSpace(string(out)); err == nil { - break - } - time.Sleep(500 * time.Millisecond) - } - if got != want { - t.Errorf("%s is %q on the raised store", setting, got) - } - } -} // aRepository is a git repository holding these files, committed, and its head. -func aCheckedRepository(t *testing.T, files map[string]string) (string, string) { - t.Helper() - dir := t.TempDir() - git := func(args ...string) string { - cmd := exec.Command("git", args...) - cmd.Dir = dir - cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org", - "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org") - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("git %v: %v\n%s", args, err, out) - } - return strings.TrimSpace(string(out)) - } - git("init", "--quiet", "-b", "main") - for name, body := range files { - if err := os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o755); err != nil { - t.Fatal(err) - } - } - git("add", "-A") - git("commit", "--quiet", "-m", "x") - return dir, git("rev-parse", "HEAD") -} func checkEnvironment(t *testing.T) string { t.Helper() @@ -121,104 +57,8 @@ func checkEnvironment(t *testing.T) string { // goToolchain is the Go image a check's script runs in here: the base the controller's own image is built on. const goToolchain = "golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c" -func labelled(id string) []string { - out, _ := exec.Command("docker", "ps", "-aq", "--filter", "label="+BuildLabel+"="+id).Output() - return strings.Fields(string(out)) -} - -func TestACheckRunsTheRepositorysOwnScriptBesideTheMeshsVersionsAndLeavesNothing(t *testing.T) { - registry := checkEnvironment(t) - // The script proves what it was given: the facts, a store that answers, a bus that answers — and no - // container runtime socket. - script := `set -e -test -s "$MESH_FACTS" -grep -q '"bus": "2.11.17"' "$MESH_FACTS" || grep -q '"bus":"2.11.17"' "$MESH_FACTS" -case "$MESH_TEST_POSTGRES" in postgres://*127.0.0.1:*) ;; *) echo "no store: $MESH_TEST_POSTGRES"; exit 1;; esac -case "$MESH_TEST_NATS" in nats://127.0.0.1:*) ;; *) echo "no bus: $MESH_TEST_NATS"; exit 1;; esac -test "$MESH_CHECK_REPOSITORY" = "novox/hq" -test "$MESH_CHECK_CHANGED" = "a.go,b.go" -test ! -S /var/run/docker.sock || { echo "the check holds the container runtime's socket"; exit 1; } -echo checked -` - repo, head := aCheckedRepository(t, map[string]string{CheckScript: script}) - id := fmt.Sprintf("check-test-%d", time.Now().UnixNano()) - v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: head, Owner: "novox", - Repo: "hq", Number: 7, Paths: []string{"a.go", "b.go"}, Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - // Nothing of the graph touched: the gate a pass that says so, the repository's own check run. - if v.Gate == nil || v.Gate.Verdict != "pass" || v.Gate.Summary != noModule { - t.Errorf("the gate answered %+v", v.Gate) - } - if v.Repo == nil || v.Repo.Verdict != "pass" || !strings.Contains(v.Report, "checked") { - t.Fatalf("the repository's check answered %+v\n%s", v.Repo, v.Report) - } - if left := labelled(id); len(left) > 0 { - t.Errorf("the check left %d container(s) behind", len(left)) - } -} - -func TestAFailingCheckFailsAndOneThatCannotRunIsNeverAPass(t *testing.T) { - registry := checkEnvironment(t) - repo, head := aCheckedRepository(t, map[string]string{CheckScript: "echo 'resource \"x.service\": refused'; exit 3\n"}) - v, err := Check(t.Context(), Command, CheckSpec{ID: fmt.Sprintf("check-fail-%d", time.Now().UnixNano()), - Repository: repo, Ref: head, Owner: "novox", Repo: "hq", Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if v.Repo == nil || v.Repo.Verdict != "fail" || !strings.Contains(v.Repo.Summary, "refused") { - t.Fatalf("a failing script answered %+v", v.Repo) - } - - // A script in a toolchain the mesh does not hold: an error, never a pass. - repo, head = aCheckedRepository(t, map[string]string{CheckScript: "# mesh-check-toolchain: cobol\nexit 0\n"}) - v, err = Check(t.Context(), Command, CheckSpec{ID: fmt.Sprintf("check-cobol-%d", time.Now().UnixNano()), - Repository: repo, Ref: head, Owner: "novox", Repo: "hq", Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) - if err != nil || v.Repo == nil || v.Repo.Verdict != "error" { - t.Fatalf("a script in a toolchain nobody holds answered %+v, %v", v.Repo, err) - } - - // Past its bound: an error, not a pass. - was := CheckTimeout - CheckTimeout = 25 * time.Second - t.Cleanup(func() { CheckTimeout = was }) - repo, head = aCheckedRepository(t, map[string]string{CheckScript: "sleep 120\n"}) - v, err = Check(context.Background(), Command, CheckSpec{ID: fmt.Sprintf("check-slow-%d", time.Now().UnixNano()), - Repository: repo, Ref: head, Owner: "novox", Repo: "hq", Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) - if err == nil && (v.Repo == nil || v.Repo.Verdict != "error") { - t.Fatalf("a check past its bound answered %+v", v.Repo) - } - - // No facts: it cannot run, and says so. - repo, head = aCheckedRepository(t, map[string]string{CheckScript: "exit 0\n"}) - _, err = Check(t.Context(), Command, CheckSpec{ID: "check-nofacts", Repository: repo, Ref: head, Owner: "novox", - Repo: "hq", Toolchain: goToolchain}, t.TempDir(), "127.0.0.1:1", GitCredential{}, nil) - if err == nil || !strings.Contains(err.Error(), "facts snapshot") { - t.Fatalf("a check with no facts said %v", err) - } -} - // A repository that touches nothing of the graph and has no script of its own runs nothing, and says // both: the gate a pass that names why, the repository a warning — never silent. -func TestACheckWithNothingToRunSaysSo(t *testing.T) { - repo, head := aCheckedRepository(t, map[string]string{"README.md": "x"}) - v, err := Check(t.Context(), Command, CheckSpec{ID: "check-nothing", Repository: repo, Ref: head, Owner: "novox", - Repo: "hq"}, t.TempDir(), "", GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if v.Gate == nil || v.Gate.Verdict != "pass" || v.Repo == nil || v.Repo.Verdict != "warning" || - !strings.Contains(v.Repo.Summary, CheckScript) { - t.Fatalf("a check with nothing to run said %+v / %+v", v.Gate, v.Repo) - } - // A gated change never takes that path: with no store to read the facts from, it cannot run. - _, err = Check(t.Context(), Command, CheckSpec{ID: "check-gated", Repository: repo, Ref: head, Owner: "novox", - Repo: "mesh-catalog", Modules: []string{"gitea"}}, t.TempDir(), "", GitCredential{}, nil) - if err == nil { - t.Fatal("a change touching a module ran nothing and was not refused") - } -} // A script declares its toolchain among its first lines; go when it declares none. func TestAScriptDeclaresItsToolchain(t *testing.T) { @@ -314,143 +154,8 @@ func main() { `, } -func TestTheGateRunsWhenTheGraphIsTouchedBesideTheRepositorysOwnCheck(t *testing.T) { - registry := checkEnvironment(t) - judgeRepo, judgeHead := aCheckedRepository(t, aJudge) - beside := map[string]Beside{"mesh-controller": {Repository: judgeRepo, Ref: judgeHead}} - check := func(files map[string]string, judge string) CheckVerdict { - t.Helper() - repo, head := aCheckedRepository(t, files) - id := fmt.Sprintf("check-gate-%d", time.Now().UnixNano()) - v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: head, Owner: "novox", - Repo: "mesh-catalog", Number: 9, Paths: []string{"modules/gitea/index.ts"}, Beside: beside, - Modules: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}, Judge: judge, - Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if left := labelled(id); len(left) > 0 { - t.Errorf("the check left %d container(s) behind", len(left)) - } - return v - } - v := check(map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`, CheckScript: "echo own; exit 0\n"}, "") - if v.Gate == nil || v.Gate.Verdict != "warning" || v.Gate.Summary != "a merge rebuilds 14 module(s)" || - strings.Join(v.Gate.Modules, ",") != "gitea" || v.Verdict != "warning" { - t.Fatalf("the gate answered %+v\n%s", v.Gate, v.Report) - } - if v.Repo == nil || v.Repo.Verdict != "pass" { - t.Fatalf("its own check answered %+v", v.Repo) - } - - v = check(map[string]string{"modules/gitea/module.json": `{"module":"gitea","broken":true}`}, "") - if v.Gate.Verdict != "fail" || !strings.Contains(v.Gate.Summary, "module check") || v.Repo.Verdict != "warning" { - t.Fatalf("a manifest the judge refuses answered %+v / %+v\n%s", v.Gate, v.Repo, v.Report) - } - - // A fault the base branch already had is said and fails nothing; one the change brings fails. - repo, _ := aCheckedRepository(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea","broken":true}`}) - git := func(args ...string) string { - cmd := exec.Command("git", args...) - cmd.Dir = repo - cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org", - "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org") - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("git %v: %v\n%s", args, err, out) - } - return strings.TrimSpace(string(out)) - } - git("checkout", "--quiet", "-b", "change") - if err := os.WriteFile(filepath.Join(repo, "modules/gitea/index.ts"), []byte("x"), 0o644); err != nil { - t.Fatal(err) - } - git("add", "-A") - git("commit", "--quiet", "-m", "y") - id := fmt.Sprintf("check-base-%d", time.Now().UnixNano()) - v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: git("rev-parse", "HEAD"), Owner: "novox", - Repo: "mesh-catalog", Base: "main", Paths: []string{"modules/gitea/index.ts"}, Beside: beside, - Modules: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}, Toolchain: goToolchain}, - t.TempDir(), registry, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if v.Gate.Verdict != "warning" || !strings.Contains(v.Report, "already") { - t.Fatalf("a fault the base already had failed the change: %+v\n%s", v.Gate, v.Report) - } - - // A delivery group (novox/hq ADR 0239): the other heads cloned beside it at their heads and handed to the - // gate to compose with this one; the repository's own check is each pull request's, not the group's. - app, appHead := aCheckedRepository(t, map[string]string{"module.json": `{"module":"app"}`}) - repo2, head2 := aCheckedRepository(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`, - CheckScript: "exit 1\n"}) - gid := fmt.Sprintf("check-group-%d", time.Now().UnixNano()) - v, err = Check(t.Context(), Command, CheckSpec{ID: gid, Repository: repo2, Ref: head2, Owner: "novox", - Repo: "mesh-catalog", Paths: []string{"modules/gitea/module.json"}, Beside: beside, Modules: []string{"gitea"}, - Manifests: []string{"modules/gitea/module.json"}, Toolchain: goToolchain, - Group: []GroupHead{{Owner: "novox", Repo: "app", Repository: app, Ref: appHead, Paths: []string{"module.json"}}}}, - t.TempDir(), registry, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if v.Gate.Verdict != "pass" || v.Gate.Summary != "composed with novox/app" { - t.Fatalf("the group's other head was not composed: %+v\n%s", v.Gate, v.Report) - } - if v.Repo == nil || v.Repo.Verdict != "pass" || !strings.Contains(v.Repo.Summary, "group") { - t.Fatalf("a group's check ran a member's own script: %+v", v.Repo) - } - - // A change to the controller judges itself: one that does not build fails its own gate. - v = check(map[string]string{"go.mod": "module x\n\ngo 1.22\n", "cmd/mesh-controller/main.go": "package main\nfunc main() { nope }\n", - "modules/gitea/module.json": "{}"}, "self") - if v.Gate.Verdict != "fail" || !strings.Contains(v.Gate.Summary, "does not build") { - t.Fatalf("a controller that does not build judged itself %+v", v.Gate) - } -} - // **Only a commit on the trunk is published** (novox/hq ADR 0238): the build seat reads, from the clone it // just made, the branch the forge names its default and whether the commit built is on it. -func TestABuildSaysWhetherItsCommitIsOnTheTrunk(t *testing.T) { - repo, onMain := aCheckedRepository(t, map[string]string{"module.json": `{"module":"x","version":"1"}`}) - git := func(dir string, args ...string) string { - cmd := exec.Command("git", args...) - cmd.Dir = dir - cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org", - "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org") - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("git %v: %v\n%s", args, err, out) - } - return strings.TrimSpace(string(out)) - } - git(repo, "checkout", "--quiet", "-b", "feature") - if err := os.WriteFile(filepath.Join(repo, "x"), []byte("x"), 0o644); err != nil { - t.Fatal(err) - } - git(repo, "add", "-A") - git(repo, "commit", "--quiet", "-m", "off the trunk") - offMain := git(repo, "rev-parse", "HEAD") - git(repo, "checkout", "--quiet", "main") - - clone := filepath.Join(t.TempDir(), "clone") - git(filepath.Dir(clone), "clone", "--quiet", repo, clone) - if trunk, on := trunkOf(t.Context(), Command, clone, onMain); trunk != "main" || !on { - t.Errorf("a commit on main reads as on %q: %v", trunk, on) - } - if trunk, on := trunkOf(t.Context(), Command, clone, offMain); trunk != "main" || on { - t.Errorf("a feature branch's commit reads as on %q: %v", trunk, on) - } - if got := strings.Join(branchesHolding(t.Context(), Command, clone, offMain), ","); got != "feature" { - t.Errorf("the feature branch's commit is said to be on %q", got) - } - if got := strings.Join(branchesHolding(t.Context(), Command, clone, onMain), ","); got != "feature,main" { - t.Errorf("main's commit is said to be on %q", got) - } - // A tree that says no trunk is not known — never read as on it. - if trunk, on := trunkOf(t.Context(), Command, repo, onMain); trunk != "" || on { - t.Errorf("a repository with no origin reads as trunk %q, on %v", trunk, on) - } -} // **Issue 286**: a failed merge-check.sh was said by its last line — a bare "FAIL", or the name of a file // gofmt listed — which named nothing a reader could act on. The status says what failed. @@ -497,99 +202,12 @@ func TestTheSeatCallsAGateThatRaisedNoMachineAnError(t *testing.T) { // **Issue 285**: an ask redelivered after its holder stopped mid-check found its own throwaway store still // there under its name, and the check said it could not run. What an earlier delivery left goes first. -func TestARedeliveredCheckRemovesWhatItsEarlierDeliveryLeft(t *testing.T) { - registry := checkEnvironment(t) - repo, head := aCheckedRepository(t, map[string]string{CheckScript: "echo checked\n"}) - id := fmt.Sprintf("check-again-%d", time.Now().UnixNano()) - if out, err := exec.Command("docker", "run", "-d", "--rm", "--label", BuildLabel+"="+id, "--name", id+"-store", - "postgres:17-alpine", "sleep", "300").CombinedOutput(); err != nil { - t.Skipf("no container for the earlier delivery: %v %s", err, out) - } - v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: head, Owner: "novox", - Repo: "hq", Number: 7, Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) - if err != nil { - t.Fatalf("a redelivered check could not run: %v", err) - } - if v.Repo == nil || v.Repo.Verdict != "pass" { - t.Errorf("the repository's check answered %+v", v.Repo) - } - if left := labelled(id); len(left) > 0 { - t.Errorf("the check left %d container(s) behind", len(left)) - } -} // **A repository in two languages checks both, each in its own toolchain** (novox/hq issue 302): the lab's // merge-check.sh runs in the TypeScript toolchain, and its Go replays were said "NOT CHECKED HERE" on every // pull request, so a register that did not compile would have merged green. Its script declares the Go // part; the check runs it in the Go toolchain's container after the script, and the layer passes only // when both do. -func TestARepositoryInTwoLanguagesIsCheckedInBoth(t *testing.T) { - script := "#!/bin/sh\n# mesh-check-toolchain: typescript\n# mesh-check-also: go replays/merge-check.sh\nnpm test\n" - parts := ScriptParts([]byte(script)) - if len(parts) != 2 || parts[0] != (ScriptPart{"typescript", CheckScript}) || - parts[1] != (ScriptPart{"go", "replays/merge-check.sh"}) { - t.Fatalf("the parts read as %+v", parts) - } - if got := ScriptParts([]byte("#!/bin/sh\nset -eu\n")); len(got) != 1 || got[0] != (ScriptPart{"go", CheckScript}) { - t.Fatalf("a script declaring nothing reads as %+v", got) - } - - // Each part run where its toolchain is: here, the image the part was given is what the command says. - held := CheckSpec{Toolchain: "go-image", Toolchains: map[string]string{"typescript": "ts-image", "go": "go-image"}} - run := func(t *testing.T, spec CheckSpec, files map[string]string, parts []ScriptPart) (*Layer, []string) { - t.Helper() - tree := t.TempDir() - for name, body := range files { - if err := os.MkdirAll(filepath.Dir(filepath.Join(tree, name)), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(tree, name), []byte(body), 0o755); err != nil { - t.Fatal(err) - } - } - var images []string - var out tail - layer := ownCheck(t.Context(), spec, parts, tree, &out, func() bool { return false }, - func(image, script string) *exec.Cmd { - images = append(images, image+" "+script) - cmd := exec.CommandContext(t.Context(), "sh", script) - cmd.Dir = tree - return cmd - }, func(string, string, ...any) {}) - return layer, images - } - twoParts := []ScriptPart{{"typescript", CheckScript}, {"go", "replays/merge-check.sh"}} - - layer, images := run(t, held, map[string]string{CheckScript: "exit 0\n", "replays/merge-check.sh": "exit 0\n"}, twoParts) - if layer.Verdict != "pass" || !strings.Contains(layer.Summary, "replays/merge-check.sh (go)") || - strings.Join(images, ", ") != "ts-image merge-check.sh, go-image replays/merge-check.sh" { - t.Errorf("both parts passing answered %+v, ran %v", layer, images) - } - layer, _ = run(t, held, map[string]string{CheckScript: "exit 0\n", - "replays/merge-check.sh": "echo 'not gofmt'\"'\"'d:'; echo register.go; exit 1\n"}, twoParts) - if layer.Verdict != "fail" || !strings.Contains(layer.Summary, "replays/merge-check.sh failed") || - !strings.Contains(layer.Summary, "register.go") || !strings.Contains(layer.Summary, "merge-check.sh (typescript) passed") { - t.Errorf("a failing Go part answered %+v", layer) - } - layer, images = run(t, held, map[string]string{CheckScript: "exit 2\n", "replays/merge-check.sh": "exit 0\n"}, twoParts) - if layer.Verdict != "fail" || len(images) != 1 { - t.Errorf("a failing first part answered %+v and ran %v", layer, images) - } - layer, _ = run(t, held, map[string]string{CheckScript: "exit 0\n"}, twoParts) - if layer.Verdict != "fail" || !strings.Contains(layer.Summary, "does not hold") { - t.Errorf("a declared part the repository lacks answered %+v", layer) - } - layer, _ = run(t, held, map[string]string{CheckScript: "exit 0\n"}, []ScriptPart{{"typescript", CheckScript}, - {"go", "../elsewhere.sh"}}) - if layer.Verdict != "fail" || !strings.Contains(layer.Summary, "not a path inside") { - t.Errorf("a part outside the repository answered %+v", layer) - } - layer, _ = run(t, CheckSpec{Toolchains: map[string]string{"typescript": "ts-image"}}, - map[string]string{CheckScript: "exit 0\n", "replays/merge-check.sh": "exit 0\n"}, twoParts) - if layer.Verdict != "error" || !strings.Contains(layer.Summary, "go toolchain") { - t.Errorf("a part in a toolchain the mesh does not hold answered %+v — never a pass", layer) - } -} // A branch tried before its pull request may narrow what the repository's script runs (novox/hq issue 686); a // pull request's check is handed nothing, and runs whole. diff --git a/internal/builder/dependencies_test.go b/internal/builder/dependencies_test.go index fbcade29..c70c77bc 100644 --- a/internal/builder/dependencies_test.go +++ b/internal/builder/dependencies_test.go @@ -1,3 +1,5 @@ +//go:build integration + package builder import ( diff --git a/internal/builder/failures_integration_test.go b/internal/builder/failures_integration_test.go new file mode 100644 index 00000000..b714f1bb --- /dev/null +++ b/internal/builder/failures_integration_test.go @@ -0,0 +1,152 @@ +//go:build integration + +package builder + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func ownCheckOf(t *testing.T, printed string) (*Layer, []string) { + t.Helper() + tree := t.TempDir() + if err := os.WriteFile(filepath.Join(tree, "printed.txt"), []byte(printed), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tree, CheckScript), []byte("cat printed.txt\nexit 1\n"), 0o755); err != nil { + t.Fatal(err) + } + var out tail + var logged []string + layer := ownCheck(t.Context(), CheckSpec{Toolchain: "go-image"}, []ScriptPart{{"go", CheckScript}}, tree, &out, + func() bool { return false }, + func(_, script string) *exec.Cmd { + cmd := exec.CommandContext(t.Context(), "sh", script) + cmd.Dir = tree + return cmd + }, func(step, format string, args ...any) { + if step == "output" { + logged = append(logged, fmt.Sprintf(format, args...)) + } + }) + if layer == nil || layer.Verdict != "fail" { + t.Fatalf("a failing check answered %+v", layer) + } + return layer, logged +} + +func TestAFailureEarlyInALongRunIsStillNamed(t *testing.T) { + layer, _ := ownCheckOf(t, captured(t, "a-long-run-failing-early.txt")) + if !strings.Contains(layer.Summary, "failed: --- FAIL: TestTheEnvelopeIsPinned (0.00s)") { + t.Errorf("a failure 600 lines from the end is said as %q", layer.Summary) + } + if !strings.Contains(layer.Summary, "TestSubtests/the_hub") { + t.Errorf("the second failing test is not named: %q", layer.Summary) + } +} + +func TestARaceRunOfEveryPackageNamesItsFirstFailure(t *testing.T) { + layer, _ := ownCheckOf(t, captured(t, "a-race-run-of-every-package.txt")) + if !strings.Contains(layer.Summary, "failed: --- FAIL: TestTheEnvelopeIsPinned (0.00s)") || + !strings.Contains(layer.Summary, "TestAMapIsNil") || !strings.Contains(layer.Summary, "TestACounterIsShared") { + t.Errorf("a run failing in four packages is said as %q", layer.Summary) + } +} + +func TestABuildErrorIsNamedByTheError(t *testing.T) { + layer, _ := ownCheckOf(t, captured(t, "a-build-error.txt")) + if !strings.HasSuffix(layer.Summary, "failed: broken/broken.go:3:28: undefined: undefinedThing") { + t.Errorf("a build error is said as %q", layer.Summary) + } +} + +func TestTheWholeOutputIsInTheBuildsLog(t *testing.T) { + printed := captured(t, "a-long-run-failing-early.txt") + _, logged := ownCheckOf(t, printed) + want := strings.Split(strings.TrimRight(printed, "\n"), "\n") + if len(logged) != len(want) { + t.Fatalf("the build's log holds %d line(s) of the %d printed", len(logged), len(want)) + } + for i := range want { + if logged[i] != want[i] { + t.Fatalf("line %d is logged as %q, printed as %q", i+1, logged[i], want[i]) + } + } +} + +func TestTheVerdictKeepsWhatFailedWhole(t *testing.T) { + layer, _ := ownCheckOf(t, captured(t, "a-long-run-failing-early.txt")) + want := "--- FAIL: TestTheEnvelopeIsPinned (0.00s)\n" + + " early_test.go:9: the envelope's subject is \"mesh.a\", not \"mesh.b\"\n" + + " early_test.go:10: a second line of the same failure\n" + + "--- FAIL: TestSubtests (0.00s)\n" + + " --- FAIL: TestSubtests/the_hub (0.00s)\n" + + " early_test.go:14: the hub does not compose\n" + + "FAIL\texample.com/cap/early\t" + if !strings.HasPrefix(layer.Failed, want) || strings.Contains(layer.Failed, "kept what home-server says") { + t.Errorf("the early failure is kept as:\n%s", layer.Failed) + } + + layer, _ = ownCheckOf(t, captured(t, "a-panic.txt")) + for _, line := range []string{"--- FAIL: TestAMapIsNil (0.00s)", "panic: assignment to entry in nil map", + "[running]:", "example.com/cap/panics.TestAMapIsNil(", "/src/cap/panics/panics_test.go:7", + "FAIL\texample.com/cap/panics\t"} { + if !strings.Contains(layer.Failed, line) { + t.Errorf("a panic is kept without %q:\n%s", line, layer.Failed) + } + } + + layer, _ = ownCheckOf(t, captured(t, "a-data-race.txt")) + race := captured(t, "a-data-race.txt") + report := race[:strings.Index(race, "--- FAIL:")] + if !strings.HasPrefix(layer.Failed, report) || + !strings.Contains(layer.Failed, "--- FAIL: TestACounterIsShared (0.00s)\n testing.go:1865: race detected") { + t.Errorf("a data race is kept as:\n%s", layer.Failed) + } + + layer, _ = ownCheckOf(t, captured(t, "a-build-error.txt")) + if layer.Failed != "# example.com/cap/broken\nbroken/broken.go:3:28: undefined: undefinedThing\n"+ + "FAIL\texample.com/cap/broken [build failed]" { + t.Errorf("a build error is kept as:\n%s", layer.Failed) + } +} + +func TestWhatFailedIsBoundedAndSurvivesTheReportsCut(t *testing.T) { + var b strings.Builder + for i := range 3000 { + fmt.Fprintf(&b, "--- FAIL: TestNumber%d (0.00s)\n x_test.go:1: %s\n", i, strings.Repeat("why ", 600)) + fmt.Fprintf(&b, "2026/10/11 01:30:03 a log line %d\n", i) + } + b.WriteString("FAIL\tx/everything\t1s\nFAIL\n") + layer, logged := ownCheckOf(t, b.String()) + if len(layer.Failed) > failureBytes+2*failureLineBytes || !strings.Contains(layer.Failed, "more line(s) naming what failed") { + t.Errorf("what failed in a run that fails everywhere is %d bytes, ending %q", len(layer.Failed), + layer.Failed[max(0, len(layer.Failed)-200):]) + } + if !strings.Contains(layer.Summary, "--- FAIL: TestNumber0 (0.00s); and TestNumber1, ") || + !strings.Contains(layer.Summary, "and 2994 more") { + t.Errorf("a run failing 3000 tests is said as %q", layer.Summary) + } + + var out tail + out.Write([]byte(b.String())) + report := withWhatFailed(out.String(), layer) + const maxCheckReport = 60 << 10 // as the controller cuts it, from the front + if len(report) > maxCheckReport { + report = report[len(report)-maxCheckReport:] + } + if !strings.Contains(report, "--- what failed") || !strings.Contains(report, "--- FAIL: TestNumber0 (0.00s)") { + t.Error("what failed did not survive the controller's cut of the report") + } + + // The build's log keeps its first logLines lines, says how many it left out, then what failed. + if len(logged) < logLines+2 || logged[logLines] != "… 1002 more line(s) of its output are not in this log, which keeps its first 8000" || + logged[logLines+1] != "--- what failed, picked from the whole of its output" || + logged[logLines+2] != "--- FAIL: TestNumber0 (0.00s)" { + t.Errorf("the log past its bound says %q", logged[logLines:min(len(logged), logLines+3)]) + } +} diff --git a/internal/builder/failures_test.go b/internal/builder/failures_test.go index e6278dfa..65a81beb 100644 --- a/internal/builder/failures_test.go +++ b/internal/builder/failures_test.go @@ -1,43 +1,13 @@ package builder import ( - "fmt" "os" - "os/exec" "path/filepath" - "strings" "testing" ) // ownCheckOf runs a repository's own check whose script prints a captured output and fails, as the build // seat runs it, and answers the layer and every line it said in the build's log. -func ownCheckOf(t *testing.T, printed string) (*Layer, []string) { - t.Helper() - tree := t.TempDir() - if err := os.WriteFile(filepath.Join(tree, "printed.txt"), []byte(printed), 0o644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(tree, CheckScript), []byte("cat printed.txt\nexit 1\n"), 0o755); err != nil { - t.Fatal(err) - } - var out tail - var logged []string - layer := ownCheck(t.Context(), CheckSpec{Toolchain: "go-image"}, []ScriptPart{{"go", CheckScript}}, tree, &out, - func() bool { return false }, - func(_, script string) *exec.Cmd { - cmd := exec.CommandContext(t.Context(), "sh", script) - cmd.Dir = tree - return cmd - }, func(step, format string, args ...any) { - if step == "output" { - logged = append(logged, fmt.Sprintf(format, args...)) - } - }) - if layer == nil || layer.Verdict != "fail" { - t.Fatalf("a failing check answered %+v", layer) - } - return layer, logged -} func captured(t *testing.T, name string) string { t.Helper() @@ -52,124 +22,18 @@ func captured(t *testing.T, name string) string { // name: the package's own logs, printed after its `--- FAIL:` lines, pushed them out of the last 200 lines, // which was all the summary was named from. The first failing test is named, and the others after it, // however much came after them. -func TestAFailureEarlyInALongRunIsStillNamed(t *testing.T) { - layer, _ := ownCheckOf(t, captured(t, "a-long-run-failing-early.txt")) - if !strings.Contains(layer.Summary, "failed: --- FAIL: TestTheEnvelopeIsPinned (0.00s)") { - t.Errorf("a failure 600 lines from the end is said as %q", layer.Summary) - } - if !strings.Contains(layer.Summary, "TestSubtests/the_hub") { - t.Errorf("the second failing test is not named: %q", layer.Summary) - } -} // One -race run of every package: the first failure is the earliest, not the last within the tail. -func TestARaceRunOfEveryPackageNamesItsFirstFailure(t *testing.T) { - layer, _ := ownCheckOf(t, captured(t, "a-race-run-of-every-package.txt")) - if !strings.Contains(layer.Summary, "failed: --- FAIL: TestTheEnvelopeIsPinned (0.00s)") || - !strings.Contains(layer.Summary, "TestAMapIsNil") || !strings.Contains(layer.Summary, "TestACounterIsShared") { - t.Errorf("a run failing in four packages is said as %q", layer.Summary) - } -} // A package that does not build is named by its error, not by its bare "[build failed]". -func TestABuildErrorIsNamedByTheError(t *testing.T) { - layer, _ := ownCheckOf(t, captured(t, "a-build-error.txt")) - if !strings.HasSuffix(layer.Summary, "failed: broken/broken.go:3:28: undefined: undefinedThing") { - t.Errorf("a build error is said as %q", layer.Summary) - } -} // The whole of what the check printed is in the build's log, line by line — where before none of it was. -func TestTheWholeOutputIsInTheBuildsLog(t *testing.T) { - printed := captured(t, "a-long-run-failing-early.txt") - _, logged := ownCheckOf(t, printed) - want := strings.Split(strings.TrimRight(printed, "\n"), "\n") - if len(logged) != len(want) { - t.Fatalf("the build's log holds %d line(s) of the %d printed", len(logged), len(want)) - } - for i := range want { - if logged[i] != want[i] { - t.Fatalf("line %d is logged as %q, printed as %q", i+1, logged[i], want[i]) - } - } -} // What failed is kept whole in the verdict: each --- FAIL with its messages, a panic with its first frames, // a data race report from rule to rule — and none of the log lines around them. -func TestTheVerdictKeepsWhatFailedWhole(t *testing.T) { - layer, _ := ownCheckOf(t, captured(t, "a-long-run-failing-early.txt")) - want := "--- FAIL: TestTheEnvelopeIsPinned (0.00s)\n" + - " early_test.go:9: the envelope's subject is \"mesh.a\", not \"mesh.b\"\n" + - " early_test.go:10: a second line of the same failure\n" + - "--- FAIL: TestSubtests (0.00s)\n" + - " --- FAIL: TestSubtests/the_hub (0.00s)\n" + - " early_test.go:14: the hub does not compose\n" + - "FAIL\texample.com/cap/early\t" - if !strings.HasPrefix(layer.Failed, want) || strings.Contains(layer.Failed, "kept what home-server says") { - t.Errorf("the early failure is kept as:\n%s", layer.Failed) - } - - layer, _ = ownCheckOf(t, captured(t, "a-panic.txt")) - for _, line := range []string{"--- FAIL: TestAMapIsNil (0.00s)", "panic: assignment to entry in nil map", - "[running]:", "example.com/cap/panics.TestAMapIsNil(", "/src/cap/panics/panics_test.go:7", - "FAIL\texample.com/cap/panics\t"} { - if !strings.Contains(layer.Failed, line) { - t.Errorf("a panic is kept without %q:\n%s", line, layer.Failed) - } - } - - layer, _ = ownCheckOf(t, captured(t, "a-data-race.txt")) - race := captured(t, "a-data-race.txt") - report := race[:strings.Index(race, "--- FAIL:")] - if !strings.HasPrefix(layer.Failed, report) || - !strings.Contains(layer.Failed, "--- FAIL: TestACounterIsShared (0.00s)\n testing.go:1865: race detected") { - t.Errorf("a data race is kept as:\n%s", layer.Failed) - } - - layer, _ = ownCheckOf(t, captured(t, "a-build-error.txt")) - if layer.Failed != "# example.com/cap/broken\nbroken/broken.go:3:28: undefined: undefinedThing\n"+ - "FAIL\texample.com/cap/broken [build failed]" { - t.Errorf("a build error is kept as:\n%s", layer.Failed) - } -} // A run that fails everywhere is bounded in what it keeps, and what it keeps survives the controller's cut // of a report to its last 60 KiB, said at the report's end. -func TestWhatFailedIsBoundedAndSurvivesTheReportsCut(t *testing.T) { - var b strings.Builder - for i := range 3000 { - fmt.Fprintf(&b, "--- FAIL: TestNumber%d (0.00s)\n x_test.go:1: %s\n", i, strings.Repeat("why ", 600)) - fmt.Fprintf(&b, "2026/10/11 01:30:03 a log line %d\n", i) - } - b.WriteString("FAIL\tx/everything\t1s\nFAIL\n") - layer, logged := ownCheckOf(t, b.String()) - if len(layer.Failed) > failureBytes+2*failureLineBytes || !strings.Contains(layer.Failed, "more line(s) naming what failed") { - t.Errorf("what failed in a run that fails everywhere is %d bytes, ending %q", len(layer.Failed), - layer.Failed[max(0, len(layer.Failed)-200):]) - } - if !strings.Contains(layer.Summary, "--- FAIL: TestNumber0 (0.00s); and TestNumber1, ") || - !strings.Contains(layer.Summary, "and 2994 more") { - t.Errorf("a run failing 3000 tests is said as %q", layer.Summary) - } - - var out tail - out.Write([]byte(b.String())) - report := withWhatFailed(out.String(), layer) - const maxCheckReport = 60 << 10 // as the controller cuts it, from the front - if len(report) > maxCheckReport { - report = report[len(report)-maxCheckReport:] - } - if !strings.Contains(report, "--- what failed") || !strings.Contains(report, "--- FAIL: TestNumber0 (0.00s)") { - t.Error("what failed did not survive the controller's cut of the report") - } - - // The build's log keeps its first logLines lines, says how many it left out, then what failed. - if len(logged) < logLines+2 || logged[logLines] != "… 1002 more line(s) of its output are not in this log, which keeps its first 8000" || - logged[logLines+1] != "--- what failed, picked from the whole of its output" || - logged[logLines+2] != "--- FAIL: TestNumber0 (0.00s)" { - t.Errorf("the log past its bound says %q", logged[logLines:min(len(logged), logLines+3)]) - } -} // A passing check says nothing of what failed. func TestAPassingCheckReportsAsBefore(t *testing.T) { diff --git a/internal/builder/gosource_integration_test.go b/internal/builder/gosource_integration_test.go new file mode 100644 index 00000000..ae57922d --- /dev/null +++ b/internal/builder/gosource_integration_test.go @@ -0,0 +1,84 @@ +//go:build integration + +package builder + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestAGoProgramsBuildSourceIsItsImportClosure(t *testing.T) { + root := aGoTree(t, aProgram) + got, err := GoBuildSource(root, "cmd/prog") + if err != nil { + t.Fatal(err) + } + for _, c := range []struct { + file string + held bool + why string + }{ + {"cmd/prog/main.go", true, "the program itself"}, + {"cmd/prog/helper.go", true, "a file added to the program's package"}, + {"lib/lib.go", true, "a package it imports"}, + {"emb/emb_amd64.s", true, "assembly beside a package's Go"}, + {"lib/lib_plan9.go", true, "a file one system builds"}, + {"plan9only/p.go", true, "what a file one system builds imports"}, + {"emb/static/index.html", true, "an embedded file"}, + {"emb/static/deep/style.css", true, "an embedded file below the pattern's directory"}, + {"emb/a b.txt", true, "an embed named outright, quoted"}, + {"vendor/example.org/dep/dep.go", true, "a vendored package it imports"}, + {"vendor/example.org/dep/sub/sub.go", true, "what a vendored package imports"}, + {"vendor/example.net/local/pkg/p.go", true, "a replaced module, as vendored"}, + {"third_party/local/pkg/p.go", true, "a replaced module, at its directory"}, + {"third_party/local/go.mod", true, "a replaced module's requirements"}, + {"go.mod", true, "the module's requirements"}, + {"go.sum", true, "the module's sums"}, + {"vendor/modules.txt", true, "the vendored modules"}, + {"go.work", true, "a workspace, were one made"}, + {"lib/lib_test.go", false, "a test is not built"}, + {"testonly/t.go", false, "a package only a test imports"}, + {"lib/sub/sub.go", false, "a package below an imported one, not imported"}, + {"unrelated/u.go", false, "a package nothing imports"}, + {"vendor/example.org/other/other.go", false, "a vendored package nothing imports"}, + {"README.md", false, "a file no build reads"}, + } { + if SourceHolds(got, c.file) != c.held { + t.Errorf("%s (%s): held %v, wanted %v\n %v", c.file, c.why, !c.held, c.held, got) + } + } + + // **Never narrower than go list -deps**: every package go names, and every file it compiles or embeds, + // is held. Where no go command is at hand, said, not passed. + goCmd, err := exec.LookPath("go") + if err != nil { + t.Skip("NOT COMPARED: no go command to list the closure with") + } + list := exec.Command(goCmd, "list", "-deps", "-f", + `{{if not .Standard}}{{$d := .Dir}}{{range .GoFiles}}{{$d}}/{{.}} +{{end}}{{range .SFiles}}{{$d}}/{{.}} +{{end}}{{range .EmbedFiles}}{{$d}}/{{.}} +{{end}}{{end}}`, "./cmd/prog") + list.Dir = root + list.Env = append(os.Environ(), "GOFLAGS=-mod=vendor", "GOPROXY=off", "GOWORK=off", "GOOS=linux", "GOARCH=amd64") + out, err := list.CombinedOutput() + if err != nil { + t.Fatalf("go list: %v\n%s", err, out) + } + real, _ := filepath.EvalSymlinks(root) + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + if line == "" { + continue + } + rel, err := filepath.Rel(real, line) + if err != nil || strings.HasPrefix(rel, "..") { + rel, _ = filepath.Rel(root, line) + } + if !SourceHolds(got, filepath.ToSlash(rel)) { + t.Errorf("go list builds %s, and the build source does not hold it", rel) + } + } +} diff --git a/internal/builder/gosource_test.go b/internal/builder/gosource_test.go index fd5aa2e6..80fbea1d 100644 --- a/internal/builder/gosource_test.go +++ b/internal/builder/gosource_test.go @@ -2,7 +2,6 @@ package builder import ( "os" - "os/exec" "path/filepath" "strings" "testing" @@ -60,79 +59,6 @@ var aProgram = map[string]string{ "README.md": "x", } -func TestAGoProgramsBuildSourceIsItsImportClosure(t *testing.T) { - root := aGoTree(t, aProgram) - got, err := GoBuildSource(root, "cmd/prog") - if err != nil { - t.Fatal(err) - } - for _, c := range []struct { - file string - held bool - why string - }{ - {"cmd/prog/main.go", true, "the program itself"}, - {"cmd/prog/helper.go", true, "a file added to the program's package"}, - {"lib/lib.go", true, "a package it imports"}, - {"emb/emb_amd64.s", true, "assembly beside a package's Go"}, - {"lib/lib_plan9.go", true, "a file one system builds"}, - {"plan9only/p.go", true, "what a file one system builds imports"}, - {"emb/static/index.html", true, "an embedded file"}, - {"emb/static/deep/style.css", true, "an embedded file below the pattern's directory"}, - {"emb/a b.txt", true, "an embed named outright, quoted"}, - {"vendor/example.org/dep/dep.go", true, "a vendored package it imports"}, - {"vendor/example.org/dep/sub/sub.go", true, "what a vendored package imports"}, - {"vendor/example.net/local/pkg/p.go", true, "a replaced module, as vendored"}, - {"third_party/local/pkg/p.go", true, "a replaced module, at its directory"}, - {"third_party/local/go.mod", true, "a replaced module's requirements"}, - {"go.mod", true, "the module's requirements"}, - {"go.sum", true, "the module's sums"}, - {"vendor/modules.txt", true, "the vendored modules"}, - {"go.work", true, "a workspace, were one made"}, - {"lib/lib_test.go", false, "a test is not built"}, - {"testonly/t.go", false, "a package only a test imports"}, - {"lib/sub/sub.go", false, "a package below an imported one, not imported"}, - {"unrelated/u.go", false, "a package nothing imports"}, - {"vendor/example.org/other/other.go", false, "a vendored package nothing imports"}, - {"README.md", false, "a file no build reads"}, - } { - if SourceHolds(got, c.file) != c.held { - t.Errorf("%s (%s): held %v, wanted %v\n %v", c.file, c.why, !c.held, c.held, got) - } - } - - // **Never narrower than go list -deps**: every package go names, and every file it compiles or embeds, - // is held. Where no go command is at hand, said, not passed. - goCmd, err := exec.LookPath("go") - if err != nil { - t.Skip("NOT COMPARED: no go command to list the closure with") - } - list := exec.Command(goCmd, "list", "-deps", "-f", - `{{if not .Standard}}{{$d := .Dir}}{{range .GoFiles}}{{$d}}/{{.}} -{{end}}{{range .SFiles}}{{$d}}/{{.}} -{{end}}{{range .EmbedFiles}}{{$d}}/{{.}} -{{end}}{{end}}`, "./cmd/prog") - list.Dir = root - list.Env = append(os.Environ(), "GOFLAGS=-mod=vendor", "GOPROXY=off", "GOWORK=off", "GOOS=linux", "GOARCH=amd64") - out, err := list.CombinedOutput() - if err != nil { - t.Fatalf("go list: %v\n%s", err, out) - } - real, _ := filepath.EvalSymlinks(root) - for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { - if line == "" { - continue - } - rel, err := filepath.Rel(real, line) - if err != nil || strings.HasPrefix(rel, "..") { - rel, _ = filepath.Rel(root, line) - } - if !SourceHolds(got, filepath.ToSlash(rel)) { - t.Errorf("go list builds %s, and the build source does not hold it", rel) - } - } -} - // A file added to the program's import closure moves its build source with it: the closure read again // grows by the package. func TestTheBuildSourceGrowsWithAnImport(t *testing.T) { diff --git a/internal/builder/judge_build_test.go b/internal/builder/judge_build_test.go index f4998c2e..63a3c896 100644 --- a/internal/builder/judge_build_test.go +++ b/internal/builder/judge_build_test.go @@ -1,3 +1,5 @@ +//go:build integration + package builder import ( diff --git a/internal/builder/kill_integration_test.go b/internal/builder/kill_integration_test.go new file mode 100644 index 00000000..ee2fd00a --- /dev/null +++ b/internal/builder/kill_integration_test.go @@ -0,0 +1,58 @@ +//go:build integration + +package builder + +import ( + "context" + "errors" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +func TestAKilledBuildEndsItsWholeProcessGroup(t *testing.T) { + dir := t.TempDir() + child := filepath.Join(dir, "child") + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + began := time.Now() + go func() { + // A shell that starts a grandchild and waits on it: killing the shell alone would leave the + // grandchild running, holding the output open. + _, err := Command(ctx, dir, "sh", "-c", `sleep 60 & echo $! > child; wait`) + done <- err + }() + var pid int + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) { + if raw, err := os.ReadFile(child); err == nil && strings.TrimSpace(string(raw)) != "" { + pid, _ = strconv.Atoi(strings.TrimSpace(string(raw))) + break + } + } + if pid == 0 { + t.Fatal("the command never started its child") + } + cancel() + select { + case err := <-done: + if err == nil { + t.Fatal("a killed command reported success") + } + case <-time.After(KillWait + 5*time.Second): + t.Fatal("the killed command never returned") + } + if took := time.Since(began); took > KillWait { + t.Errorf("ending the command took %s: the group was not killed, the wait ran out", took) + } + for deadline := time.Now().Add(2 * time.Second); time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) { + if err := syscall.Kill(pid, 0); errors.Is(err, syscall.ESRCH) { + return + } + } + _ = syscall.Kill(pid, syscall.SIGKILL) + t.Fatalf("the grandchild %d outlived the kill", pid) +} diff --git a/internal/builder/kill_test.go b/internal/builder/kill_test.go index 30b63c9b..b3779f65 100644 --- a/internal/builder/kill_test.go +++ b/internal/builder/kill_test.go @@ -2,61 +2,13 @@ package builder import ( "context" - "errors" - "os" - "path/filepath" "reflect" - "strconv" "strings" - "syscall" "testing" - "time" ) // A build killed by hand (novox/hq ADR 0219) ends every command it started: the context's end kills // the command's whole process group, not the one process the context knows about. -func TestAKilledBuildEndsItsWholeProcessGroup(t *testing.T) { - dir := t.TempDir() - child := filepath.Join(dir, "child") - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan error, 1) - began := time.Now() - go func() { - // A shell that starts a grandchild and waits on it: killing the shell alone would leave the - // grandchild running, holding the output open. - _, err := Command(ctx, dir, "sh", "-c", `sleep 60 & echo $! > child; wait`) - done <- err - }() - var pid int - for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) { - if raw, err := os.ReadFile(child); err == nil && strings.TrimSpace(string(raw)) != "" { - pid, _ = strconv.Atoi(strings.TrimSpace(string(raw))) - break - } - } - if pid == 0 { - t.Fatal("the command never started its child") - } - cancel() - select { - case err := <-done: - if err == nil { - t.Fatal("a killed command reported success") - } - case <-time.After(KillWait + 5*time.Second): - t.Fatal("the killed command never returned") - } - if took := time.Since(began); took > KillWait { - t.Errorf("ending the command took %s: the group was not killed, the wait ran out", took) - } - for deadline := time.Now().Add(2 * time.Second); time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) { - if err := syscall.Kill(pid, 0); errors.Is(err, syscall.ESRCH) { - return - } - } - _ = syscall.Kill(pid, syscall.SIGKILL) - t.Fatalf("the grandchild %d outlived the kill", pid) -} // Every `docker run` a build starts carries its id as a label; nothing else is touched. func TestEveryContainerABuildStartsCarriesItsID(t *testing.T) { diff --git a/internal/builder/lend_integration_test.go b/internal/builder/lend_integration_test.go new file mode 100644 index 00000000..234a8419 --- /dev/null +++ b/internal/builder/lend_integration_test.go @@ -0,0 +1,26 @@ +//go:build integration + +package builder + +import ( + "context" + "strings" + "testing" +) + +func TestAnImageIsPushedOnlyAfterItsBasesAreLent(t *testing.T) { + r, workspace := aRepository(t, anImage, map[string]string{"modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}"}) + r.contents["modules/bus/module.json"] = anImage + l := &lending{mirroring: &mirroring{recorded: r, at: "127.0.0.1:5000"}} + if _, err := Build(context.Background(), r.run, l, "https://forge.invalid/catalogue.git", "modules/bus", "", workspace, + nil, Npmrc{}, GitCredential{}, nil); err != nil { + t.Fatal(err) + } + want := []string{ + "lend bus/server 127.0.0.1:5000/upstream/docker.io/vendor/server@sha256:" + strings.Repeat("1", 64), + "push bus/server", + } + if strings.Join(l.order, "\n") != strings.Join(want, "\n") { + t.Fatalf("the build did:\n%s\nwant:\n%s", strings.Join(l.order, "\n"), strings.Join(want, "\n")) + } +} diff --git a/internal/builder/lend_test.go b/internal/builder/lend_test.go index ff92a998..833c29ec 100644 --- a/internal/builder/lend_test.go +++ b/internal/builder/lend_test.go @@ -121,20 +121,3 @@ func (l *lending) PublishImage(ctx context.Context, localTag, repository string) l.order = append(l.order, "push "+repository) return l.mirroring.PublishImage(ctx, localTag, repository) } - -func TestAnImageIsPushedOnlyAfterItsBasesAreLent(t *testing.T) { - r, workspace := aRepository(t, anImage, map[string]string{"modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}"}) - r.contents["modules/bus/module.json"] = anImage - l := &lending{mirroring: &mirroring{recorded: r, at: "127.0.0.1:5000"}} - if _, err := Build(context.Background(), r.run, l, "https://forge.invalid/catalogue.git", "modules/bus", "", workspace, - nil, Npmrc{}, GitCredential{}, nil); err != nil { - t.Fatal(err) - } - want := []string{ - "lend bus/server 127.0.0.1:5000/upstream/docker.io/vendor/server@sha256:" + strings.Repeat("1", 64), - "push bus/server", - } - if strings.Join(l.order, "\n") != strings.Join(want, "\n") { - t.Fatalf("the build did:\n%s\nwant:\n%s", strings.Join(l.order, "\n"), strings.Join(want, "\n")) - } -} diff --git a/internal/builder/mirror_shared_integration_test.go b/internal/builder/mirror_shared_integration_test.go new file mode 100644 index 00000000..7b5095a6 --- /dev/null +++ b/internal/builder/mirror_shared_integration_test.go @@ -0,0 +1,40 @@ +//go:build integration + +package builder + +import ( + "context" + "strings" + "testing" +) + +func TestABuildSaysWhatItMirroredEvenWhenItFails(t *testing.T) { + // A recipe that reaches for an image nobody declared is refused — after the base was copied. + r, workspace := aRepository(t, anImage, map[string]string{ + "modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}\nCOPY --from=vendor/other:1 /x /x"}) + r.contents["modules/bus/module.json"] = anImage + r.tree = "aaaa" + m := &mirroring{recorded: r, at: "registry-a:5000"} + got, err := Build(context.Background(), r.run, m, "https://forge.invalid/catalogue.git", "modules/bus", "", workspace, + nil, Npmrc{}, GitCredential{}, nil) + if err == nil { + t.Fatal("a recipe fetching an undeclared image was built") + } + want := "registry-a:5000/upstream/docker.io/vendor/server@sha256:" + strings.Repeat("1", 64) + if len(got.Mirrored) != 1 || got.Mirrored[0] != want { + t.Fatalf("a failed build said it mirrored %v, want [%s]", got.Mirrored, want) + } + + // And a build that works says it too. + r2, workspace2 := aRepository(t, anImage, map[string]string{"modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}"}) + r2.contents["modules/bus/module.json"] = anImage + r2.tree = "aaaa" + ok, err := Build(context.Background(), r2.run, &mirroring{recorded: r2, at: "registry-a:5000"}, + "https://forge.invalid/catalogue.git", "modules/bus", "", workspace2, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if len(ok.Mirrored) != 1 || ok.Mirrored[0] != want { + t.Fatalf("a build said it mirrored %v, want [%s]", ok.Mirrored, want) + } +} diff --git a/internal/builder/mirror_shared_test.go b/internal/builder/mirror_shared_test.go index 45d21695..bc986db6 100644 --- a/internal/builder/mirror_shared_test.go +++ b/internal/builder/mirror_shared_test.go @@ -219,36 +219,6 @@ func countPrefix(m map[string][]byte, prefix string) int { // What a build copied is said whether it worked or not: the copy is in the store either way, and a // build that failed after copying is the one record that it is there. -func TestABuildSaysWhatItMirroredEvenWhenItFails(t *testing.T) { - // A recipe that reaches for an image nobody declared is refused — after the base was copied. - r, workspace := aRepository(t, anImage, map[string]string{ - "modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}\nCOPY --from=vendor/other:1 /x /x"}) - r.contents["modules/bus/module.json"] = anImage - r.tree = "aaaa" - m := &mirroring{recorded: r, at: "registry-a:5000"} - got, err := Build(context.Background(), r.run, m, "https://forge.invalid/catalogue.git", "modules/bus", "", workspace, - nil, Npmrc{}, GitCredential{}, nil) - if err == nil { - t.Fatal("a recipe fetching an undeclared image was built") - } - want := "registry-a:5000/upstream/docker.io/vendor/server@sha256:" + strings.Repeat("1", 64) - if len(got.Mirrored) != 1 || got.Mirrored[0] != want { - t.Fatalf("a failed build said it mirrored %v, want [%s]", got.Mirrored, want) - } - - // And a build that works says it too. - r2, workspace2 := aRepository(t, anImage, map[string]string{"modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}"}) - r2.contents["modules/bus/module.json"] = anImage - r2.tree = "aaaa" - ok, err := Build(context.Background(), r2.run, &mirroring{recorded: r2, at: "registry-a:5000"}, - "https://forge.invalid/catalogue.git", "modules/bus", "", workspace2, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if len(ok.Mirrored) != 1 || ok.Mirrored[0] != want { - t.Fatalf("a build said it mirrored %v, want [%s]", ok.Mirrored, want) - } -} // held puts one image in a repository the old way and answers its index digest and the registry. func heldUnderAModule(t *testing.T) (*aRegistryOfRepositories, Registry, string, string, map[string][]byte, func()) { diff --git a/internal/builder/packages_integration_test.go b/internal/builder/packages_integration_test.go new file mode 100644 index 00000000..0933214a --- /dev/null +++ b/internal/builder/packages_integration_test.go @@ -0,0 +1,135 @@ +//go:build integration + +package builder + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestAnImageBuildGetsTheCredentialInTheContextAndHostNetwork(t *testing.T) { + r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch\nCOPY .npmrc ./", "files/x": "y"}) + n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"} + npmrc := filepath.Join(workspace, "source", ".npmrc") + inContext := false + run := func(ctx context.Context, dir, name string, args ...string) (string, error) { + if name == "docker" && len(args) > 0 && args[0] == "build" { + _, err := os.Stat(npmrc) + inContext = err == nil + } + return r.run(ctx, dir, name, args...) + } + if _, err := Build(context.Background(), run, r, + "https://forge.invalid/meshboard.git", "", "", workspace, nil, n, GitCredential{}, nil); err != nil { + t.Fatalf("the build failed: %v", err) + } + + var build string + for _, line := range r.ran { + if strings.HasPrefix(line, "docker build") { + build = line + } + } + if build == "" { + t.Fatal("no docker build ran") + } + if strings.Contains(build, "--secret") { + t.Fatalf("the build used a buildkit secret, which this path avoids: %s", build) + } + if !strings.Contains(build, "--network host") { + t.Fatalf("the build was not given the host network to reach the registry: %s", build) + } + // The .npmrc is written into the build context (the source tree), where a Dockerfile COPYs it, and + // removed with the tree when the build ends: a later check's container mounts the workspace (novox/hq + // issue 462). + if !inContext { + t.Fatal("the credential was not in the build context when the image was built") + } + if _, err := os.Stat(npmrc); !os.IsNotExist(err) { + t.Fatalf("the credential outlives the build in the workspace: %v", err) + } +} + +func TestAnImageBuildWithoutACredentialGetsNoHostNetwork(t *testing.T) { + r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch", "files/x": "y"}) + if _, err := Build(context.Background(), r.run, r, + "https://forge.invalid/meshboard.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil); err != nil { + t.Fatalf("the build failed: %v", err) + } + for _, line := range r.ran { + if strings.HasPrefix(line, "docker build") && (strings.Contains(line, "--secret") || strings.Contains(line, "--network host")) { + t.Fatalf("a build with no credential was still given build-network or a secret: %s", line) + } + } + tree := filepath.Join(workspace, "source") + if _, err := os.Stat(filepath.Join(tree, ".npmrc")); err == nil { + t.Fatal("an .npmrc was written into a build that has no credential") + } +} + +func TestAPackageIsBuiltOnAPublicBaseAndPublishedByVersion(t *testing.T) { + r, workspace := aRepository(t, aPackage, map[string]string{ + "package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`, + }) + n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"} + got, err := Build(context.Background(), r.run, r, + "https://forge.invalid/mesh-sdk.git", "", "", workspace, nil, n, GitCredential{}, nil) + if err != nil { + t.Fatalf("the package did not build: %v", err) + } + if len(got.Built) != 1 || got.Built[0].Reference != "@novox/mesh-sdk@0.1.0" { + t.Fatalf("a package is published by name and version, got %+v", got.Built) + } + if len(r.images) != 0 || len(r.archives) != 0 { + t.Fatal("a package was pushed to the artifact store, which is not where packages live") + } + var ran string + for _, line := range r.ran { + if strings.HasPrefix(line, "docker run") { + ran = line + } + } + if ran == "" { + t.Fatal("nothing ran to build the package") + } + if !strings.Contains(ran, "node:22-bookworm-slim") { + t.Fatalf("a package was not built on a public base: %s", ran) + } + if !strings.Contains(ran, ":/root/.npmrc:ro") { + t.Fatalf("the credential was not mounted read-only for the publish: %s", ran) + } +} + +func TestAPackageWithNoRegistryIsRefused(t *testing.T) { + r, workspace := aRepository(t, aPackage, map[string]string{ + "package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`, + }) + _, err := Build(context.Background(), r.run, r, + "https://forge.invalid/mesh-sdk.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err == nil { + t.Fatal("a package built with no registry to publish to, silently") + } +} + +func TestAnImageThatDoesNotAskForTheCredentialDoesNotGetIt(t *testing.T) { + // A Dockerfile with no .npmrc reference (like the control plane's) must build clean: no .npmrc + // in its context, no host network — so its image stays deterministic and the credential does not + // leak into a build that never resolves a mesh package. + r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch\nCOPY . .", "files/x": "y"}) + n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"} + if _, err := Build(context.Background(), r.run, r, + "https://forge.invalid/meshboard.git", "", "", workspace, nil, n, GitCredential{}, nil); err != nil { + t.Fatalf("the build failed: %v", err) + } + for _, line := range r.ran { + if strings.HasPrefix(line, "docker build") && strings.Contains(line, "--network host") { + t.Fatalf("a build that does not ask for the credential got the host network: %s", line) + } + } + if _, err := os.Stat(filepath.Join(workspace, "source", ".npmrc")); err == nil { + t.Fatal("an .npmrc was written into a build that does not reference it") + } +} diff --git a/internal/builder/packages_test.go b/internal/builder/packages_test.go index 7cccbf16..23bf50fe 100644 --- a/internal/builder/packages_test.go +++ b/internal/builder/packages_test.go @@ -1,9 +1,6 @@ package builder import ( - "context" - "os" - "path/filepath" "strings" "testing" ) @@ -67,65 +64,6 @@ func TestNpmrcDisabledUntilThereIsARegistry(t *testing.T) { // COPY in a stage it does not publish — and the build runs on the host network so a RUN resolving the // registry reaches it where the binding says (novox/hq ADR 0076). Not a buildkit secret, because this // machine's docker may carry no buildx. -func TestAnImageBuildGetsTheCredentialInTheContextAndHostNetwork(t *testing.T) { - r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch\nCOPY .npmrc ./", "files/x": "y"}) - n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"} - npmrc := filepath.Join(workspace, "source", ".npmrc") - inContext := false - run := func(ctx context.Context, dir, name string, args ...string) (string, error) { - if name == "docker" && len(args) > 0 && args[0] == "build" { - _, err := os.Stat(npmrc) - inContext = err == nil - } - return r.run(ctx, dir, name, args...) - } - if _, err := Build(context.Background(), run, r, - "https://forge.invalid/meshboard.git", "", "", workspace, nil, n, GitCredential{}, nil); err != nil { - t.Fatalf("the build failed: %v", err) - } - - var build string - for _, line := range r.ran { - if strings.HasPrefix(line, "docker build") { - build = line - } - } - if build == "" { - t.Fatal("no docker build ran") - } - if strings.Contains(build, "--secret") { - t.Fatalf("the build used a buildkit secret, which this path avoids: %s", build) - } - if !strings.Contains(build, "--network host") { - t.Fatalf("the build was not given the host network to reach the registry: %s", build) - } - // The .npmrc is written into the build context (the source tree), where a Dockerfile COPYs it, and - // removed with the tree when the build ends: a later check's container mounts the workspace (novox/hq - // issue 462). - if !inContext { - t.Fatal("the credential was not in the build context when the image was built") - } - if _, err := os.Stat(npmrc); !os.IsNotExist(err) { - t.Fatalf("the credential outlives the build in the workspace: %v", err) - } -} - -func TestAnImageBuildWithoutACredentialGetsNoHostNetwork(t *testing.T) { - r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch", "files/x": "y"}) - if _, err := Build(context.Background(), r.run, r, - "https://forge.invalid/meshboard.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil); err != nil { - t.Fatalf("the build failed: %v", err) - } - for _, line := range r.ran { - if strings.HasPrefix(line, "docker build") && (strings.Contains(line, "--secret") || strings.Contains(line, "--network host")) { - t.Fatalf("a build with no credential was still given build-network or a secret: %s", line) - } - } - tree := filepath.Join(workspace, "source") - if _, err := os.Stat(filepath.Join(tree, ".npmrc")); err == nil { - t.Fatal("an .npmrc was written into a build that has no credential") - } -} const aPackage = `{"module":"mesh-sdk","version":"1", "build":{"artifacts":[{"name":"lib","kind":"package","language":"typescript"}]}, @@ -133,49 +71,6 @@ const aPackage = `{"module":"mesh-sdk","version":"1", // A package is compiled on a public base and published to the mesh's package registry by version, // with nothing pushed to the artifact store and the credential mounted, not baked (novox/hq ADR 0076). -func TestAPackageIsBuiltOnAPublicBaseAndPublishedByVersion(t *testing.T) { - r, workspace := aRepository(t, aPackage, map[string]string{ - "package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`, - }) - n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"} - got, err := Build(context.Background(), r.run, r, - "https://forge.invalid/mesh-sdk.git", "", "", workspace, nil, n, GitCredential{}, nil) - if err != nil { - t.Fatalf("the package did not build: %v", err) - } - if len(got.Built) != 1 || got.Built[0].Reference != "@novox/mesh-sdk@0.1.0" { - t.Fatalf("a package is published by name and version, got %+v", got.Built) - } - if len(r.images) != 0 || len(r.archives) != 0 { - t.Fatal("a package was pushed to the artifact store, which is not where packages live") - } - var ran string - for _, line := range r.ran { - if strings.HasPrefix(line, "docker run") { - ran = line - } - } - if ran == "" { - t.Fatal("nothing ran to build the package") - } - if !strings.Contains(ran, "node:22-bookworm-slim") { - t.Fatalf("a package was not built on a public base: %s", ran) - } - if !strings.Contains(ran, ":/root/.npmrc:ro") { - t.Fatalf("the credential was not mounted read-only for the publish: %s", ran) - } -} - -func TestAPackageWithNoRegistryIsRefused(t *testing.T) { - r, workspace := aRepository(t, aPackage, map[string]string{ - "package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`, - }) - _, err := Build(context.Background(), r.run, r, - "https://forge.invalid/mesh-sdk.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err == nil { - t.Fatal("a package built with no registry to publish to, silently") - } -} func TestNpmrcRendersBasicAuthWhenGivenAUserAndPassword(t *testing.T) { n := Npmrc{ @@ -210,23 +105,3 @@ func TestNpmrcRefusesWhenGivenNeitherTokenNorPassword(t *testing.T) { t.Fatal("a username with no password rendered an .npmrc") } } - -func TestAnImageThatDoesNotAskForTheCredentialDoesNotGetIt(t *testing.T) { - // A Dockerfile with no .npmrc reference (like the control plane's) must build clean: no .npmrc - // in its context, no host network — so its image stays deterministic and the credential does not - // leak into a build that never resolves a mesh package. - r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch\nCOPY . .", "files/x": "y"}) - n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"} - if _, err := Build(context.Background(), r.run, r, - "https://forge.invalid/meshboard.git", "", "", workspace, nil, n, GitCredential{}, nil); err != nil { - t.Fatalf("the build failed: %v", err) - } - for _, line := range r.ran { - if strings.HasPrefix(line, "docker build") && strings.Contains(line, "--network host") { - t.Fatalf("a build that does not ask for the credential got the host network: %s", line) - } - } - if _, err := os.Stat(filepath.Join(workspace, "source", ".npmrc")); err == nil { - t.Fatal("an .npmrc was written into a build that does not reference it") - } -} diff --git a/internal/builder/predefined_test.go b/internal/builder/predefined_test.go index 71ac4fec..b1315afa 100644 --- a/internal/builder/predefined_test.go +++ b/internal/builder/predefined_test.go @@ -1,3 +1,5 @@ +//go:build integration + package builder import ( diff --git a/internal/builder/registry_unbound_integration_test.go b/internal/builder/registry_unbound_integration_test.go new file mode 100644 index 00000000..eb4689c0 --- /dev/null +++ b/internal/builder/registry_unbound_integration_test.go @@ -0,0 +1,93 @@ +//go:build integration + +package builder + +import ( + "context" + "strings" + "testing" +) + +func TestABuildNeedingTheRegistryIsRefusedOnAnAgentWithoutOne(t *testing.T) { + none := Npmrc{Machine: "workstation"} + cases := map[string]struct { + manifest string + files map[string]string + held map[string]string + }{ + "a package": {aPackage, map[string]string{"package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`}, nil}, + "an image naming .npmrc": {withBoth, map[string]string{ + "Dockerfile": "FROM scratch\nCOPY .npmrc /root/.npmrc\n", "files/x": "y"}, nil}, + "a TypeScript bundle depending on the SDK": {aBundle, map[string]string{ + "index.ts": "console.log(1)", "package.json": `{"dependencies":{"@novox/mesh-sdk":"^0.1.0"}}`}, + map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)}}, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + r, workspace := aRepository(t, c.manifest, c.files) + var said []string + _, err := Build(context.Background(), compiling{r}.run, r, + "https://forge.invalid/m.git", "", "", workspace, c.held, none, GitCredential{}, + func(step, message string) { said = append(said, step+": "+message) }) + if err == nil { + t.Fatal("built with no package registry") + } + for _, want := range []string{"workstation", "npm-package-registry", "unbound", "gitea"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal does not say %q: %v", want, err) + } + } + for _, line := range r.ran { + if strings.HasPrefix(line, "docker build") || strings.HasPrefix(line, "docker run") || + strings.HasPrefix(line, "docker pull") { + t.Errorf("something was built after the refusal: %s", line) + } + } + if len(r.images) != 0 || len(r.archives) != 0 { + t.Errorf("something was published after the refusal: %v %v", r.images, r.archives) + } + if !strings.Contains(strings.Join(said, "\n"), "REFUSED") { + t.Errorf("the build's log does not say it was refused:\n%s", strings.Join(said, "\n")) + } + }) + } + + // A Go artifact builds on the same agent. + files := aSharedRepository("one", "1") + manifest := `{"module":"ctl","version":"1","build":{"artifacts":[ + {"name":"controller","kind":"bundle","language":"go","system":"arch","from":"cmd/ctl","binary":"ctl"}]}, + "resources":[{"id":"controller","type":"process","name":"ctl","artifact":"controller","run":["./ctl"]}]}` + r := &recorded{contents: map[string]string{ManifestName: manifest}} + for k, v := range files { + r.contents[k] = v + } + held := map[string]string{"mesh-tools-go/build": "registry.invalid/mesh-tools-go/build@sha256:" + strings.Repeat("b", 64)} + if _, err := Build(context.Background(), onTrunk{recorded: r}.run, r, + "https://forge.invalid/ctl.git", "", "", t.TempDir(), held, none, GitCredential{}, nil); err != nil { + t.Fatalf("a Go artifact did not build on an agent without the registry: %v", err) + } + // And an image that does not name .npmrc. + r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch\nCOPY . .", "files/x": "y"}) + if _, err := Build(context.Background(), r.run, r, + "https://forge.invalid/meshboard.git", "", "", workspace, nil, none, GitCredential{}, nil); err != nil { + t.Fatalf("an image not naming .npmrc did not build on an agent without the registry: %v", err) + } +} + +func TestABuildSaysWhichRegistryItResolvedFrom(t *testing.T) { + for _, from := range []string{FromBinding, FromGenesis} { + r, workspace := aRepository(t, aPackage, map[string]string{ + "package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`, + }) + n := Npmrc{Scope: "@novox", Registry: "http://127.0.0.1:3000/api/packages/novox/npm/", + Username: "mesh-builder", Password: "p", From: from, Machine: "anchor"} + var said []string + if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/mesh-sdk.git", "", "", + workspace, nil, n, GitCredential{}, func(step, message string) { said = append(said, step+": "+message) }); err != nil { + t.Fatalf("%s: %v", from, err) + } + if !strings.Contains(strings.Join(said, "\n"), "from "+from) { + t.Errorf("the log does not say the registry came from %s:\n%s", from, strings.Join(said, "\n")) + } + } +} diff --git a/internal/builder/registry_unbound_test.go b/internal/builder/registry_unbound_test.go index d766a72d..c5c1f963 100644 --- a/internal/builder/registry_unbound_test.go +++ b/internal/builder/registry_unbound_test.go @@ -1,10 +1,8 @@ package builder import ( - "context" "os" "path/filepath" - "strings" "testing" "github.com/novox/mesh-controller/internal/beside" @@ -17,90 +15,8 @@ import ( // with an `@novox/` dependency each fail before anything is built or published, naming the machine, // the unbound requirement and the module that provides it. A Go artifact, and an image that does not // name .npmrc, build there as before. -func TestABuildNeedingTheRegistryIsRefusedOnAnAgentWithoutOne(t *testing.T) { - none := Npmrc{Machine: "workstation"} - cases := map[string]struct { - manifest string - files map[string]string - held map[string]string - }{ - "a package": {aPackage, map[string]string{"package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`}, nil}, - "an image naming .npmrc": {withBoth, map[string]string{ - "Dockerfile": "FROM scratch\nCOPY .npmrc /root/.npmrc\n", "files/x": "y"}, nil}, - "a TypeScript bundle depending on the SDK": {aBundle, map[string]string{ - "index.ts": "console.log(1)", "package.json": `{"dependencies":{"@novox/mesh-sdk":"^0.1.0"}}`}, - map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)}}, - } - for name, c := range cases { - t.Run(name, func(t *testing.T) { - r, workspace := aRepository(t, c.manifest, c.files) - var said []string - _, err := Build(context.Background(), compiling{r}.run, r, - "https://forge.invalid/m.git", "", "", workspace, c.held, none, GitCredential{}, - func(step, message string) { said = append(said, step+": "+message) }) - if err == nil { - t.Fatal("built with no package registry") - } - for _, want := range []string{"workstation", "npm-package-registry", "unbound", "gitea"} { - if !strings.Contains(err.Error(), want) { - t.Errorf("the refusal does not say %q: %v", want, err) - } - } - for _, line := range r.ran { - if strings.HasPrefix(line, "docker build") || strings.HasPrefix(line, "docker run") || - strings.HasPrefix(line, "docker pull") { - t.Errorf("something was built after the refusal: %s", line) - } - } - if len(r.images) != 0 || len(r.archives) != 0 { - t.Errorf("something was published after the refusal: %v %v", r.images, r.archives) - } - if !strings.Contains(strings.Join(said, "\n"), "REFUSED") { - t.Errorf("the build's log does not say it was refused:\n%s", strings.Join(said, "\n")) - } - }) - } - - // A Go artifact builds on the same agent. - files := aSharedRepository("one", "1") - manifest := `{"module":"ctl","version":"1","build":{"artifacts":[ - {"name":"controller","kind":"bundle","language":"go","system":"arch","from":"cmd/ctl","binary":"ctl"}]}, - "resources":[{"id":"controller","type":"process","name":"ctl","artifact":"controller","run":["./ctl"]}]}` - r := &recorded{contents: map[string]string{ManifestName: manifest}} - for k, v := range files { - r.contents[k] = v - } - held := map[string]string{"mesh-tools-go/build": "registry.invalid/mesh-tools-go/build@sha256:" + strings.Repeat("b", 64)} - if _, err := Build(context.Background(), onTrunk{recorded: r}.run, r, - "https://forge.invalid/ctl.git", "", "", t.TempDir(), held, none, GitCredential{}, nil); err != nil { - t.Fatalf("a Go artifact did not build on an agent without the registry: %v", err) - } - // And an image that does not name .npmrc. - r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch\nCOPY . .", "files/x": "y"}) - if _, err := Build(context.Background(), r.run, r, - "https://forge.invalid/meshboard.git", "", "", workspace, nil, none, GitCredential{}, nil); err != nil { - t.Fatalf("an image not naming .npmrc did not build on an agent without the registry: %v", err) - } -} // **The build's log says which registry it resolved from** (novox/hq ADR 0336 §4). -func TestABuildSaysWhichRegistryItResolvedFrom(t *testing.T) { - for _, from := range []string{FromBinding, FromGenesis} { - r, workspace := aRepository(t, aPackage, map[string]string{ - "package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`, - }) - n := Npmrc{Scope: "@novox", Registry: "http://127.0.0.1:3000/api/packages/novox/npm/", - Username: "mesh-builder", Password: "p", From: from, Machine: "anchor"} - var said []string - if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/mesh-sdk.git", "", "", - workspace, nil, n, GitCredential{}, func(step, message string) { said = append(said, step+": "+message) }); err != nil { - t.Fatalf("%s: %v", from, err) - } - if !strings.Contains(strings.Join(said, "\n"), "from "+from) { - t.Errorf("the log does not say the registry came from %s:\n%s", from, strings.Join(said, "\n")) - } - } -} // The refusal names the module that provides the registry; it is the catalogue's provider. func TestTheRegistryABuildIsRefusedForNamesItsProvider(t *testing.T) { diff --git a/internal/builder/registry_wait_test.go b/internal/builder/registry_wait_test.go index 8bb9be85..c529f716 100644 --- a/internal/builder/registry_wait_test.go +++ b/internal/builder/registry_wait_test.go @@ -76,13 +76,13 @@ func TestABuildWaitsForARegistryHeldStillAndGoesOn(t *testing.T) { waitingFor(t, 5*time.Second) said := saying(t) f := &fakeRegistry{} - r := Registry{Address: heldStill(t, f, 400*time.Millisecond)} + r := Registry{Address: heldStill(t, f, 100*time.Millisecond)} body := []byte("a theme") sum := sha256.Sum256(body) digest := "sha256:" + hex.EncodeToString(sum[:]) if _, err := r.PublishArchive(context.Background(), "shell/config", body, digest); err != nil { - t.Fatalf("a registry refusing for 400ms failed the build: %v", err) + t.Fatalf("a registry refusing for 100ms failed the build: %v", err) } if string(f.blobs[digest]) != "a theme" { t.Fatalf("the registry holds %q", f.blobs[digest]) @@ -97,7 +97,7 @@ func TestABuildWaitsForARegistryHeldStillAndGoesOn(t *testing.T) { } func TestABuildFailsLoudlyOnARegistryRefusingPastTheBound(t *testing.T) { - waitingFor(t, 300*time.Millisecond) + waitingFor(t, 100*time.Millisecond) said := saying(t) f := &fakeRegistry{} r := Registry{Address: heldStill(t, f, -1)} @@ -113,7 +113,7 @@ func TestABuildFailsLoudlyOnARegistryRefusingPastTheBound(t *testing.T) { if !strings.Contains(err.Error(), "refused every connection for") || !strings.Contains(err.Error(), "connection refused") { t.Errorf("the failure does not say it waited and was refused throughout: %v", err) } - if waited := time.Since(started); waited < 300*time.Millisecond { + if waited := time.Since(started); waited < 100*time.Millisecond { t.Errorf("failed after %s, before the bound", waited) } if log := strings.Join(said(), "\n"); !strings.Contains(log, "waits for the registry") { diff --git a/internal/builder/replay310_test.go b/internal/builder/replay310_test.go index 1680588c..4eb05a6e 100644 --- a/internal/builder/replay310_test.go +++ b/internal/builder/replay310_test.go @@ -1,3 +1,5 @@ +//go:build integration + package builder import ( diff --git a/internal/builder/replay341_test.go b/internal/builder/replay341_test.go index 031a903d..1d28498b 100644 --- a/internal/builder/replay341_test.go +++ b/internal/builder/replay341_test.go @@ -1,3 +1,5 @@ +//go:build integration + package builder import ( diff --git a/internal/builder/replay363_test.go b/internal/builder/replay363_test.go index 07271d28..7715e4ea 100644 --- a/internal/builder/replay363_test.go +++ b/internal/builder/replay363_test.go @@ -1,3 +1,5 @@ +//go:build integration + package builder import ( diff --git a/internal/builder/replay713_test.go b/internal/builder/replay713_test.go index 214ae2a2..76566b9e 100644 --- a/internal/builder/replay713_test.go +++ b/internal/builder/replay713_test.go @@ -1,3 +1,5 @@ +//go:build integration + package builder import ( diff --git a/internal/builder/replay716_test.go b/internal/builder/replay716_test.go index 306c3e92..d1991c1a 100644 --- a/internal/builder/replay716_test.go +++ b/internal/builder/replay716_test.go @@ -1,3 +1,5 @@ +//go:build integration + package builder import ( diff --git a/internal/builder/reproducible_integration_test.go b/internal/builder/reproducible_integration_test.go new file mode 100644 index 00000000..7f3c5ac5 --- /dev/null +++ b/internal/builder/reproducible_integration_test.go @@ -0,0 +1,91 @@ +//go:build integration + +package builder + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestAGoBundleBuiltTwiceFromOneBuildSourceIsOneDigest(t *testing.T) { + for _, tool := range []string{"go", "git"} { + if _, err := exec.LookPath(tool); err != nil { + t.Skipf("%s is not on this machine: %v", tool, err) + } + } + chain, err := ToolchainFor("go") + if err != nil { + t.Fatal(err) + } + root := t.TempDir() + repo := filepath.Join(root, "origin") + files := map[string]string{ + "go.mod": "module example.com/ctl\n\ngo 1.22\n", + "cmd/ctl/main.go": "package main\n\nfunc main() { println(\"ctl\") }\n", + "README.md": "one\n", + } + for name, body := range files { + if err := os.MkdirAll(filepath.Dir(filepath.Join(repo, name)), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(repo, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + git := func(dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", append([]string{"-c", "user.name=t", "-c", "user.email=t@example.invalid", + "-c", "commit.gpgsign=false", "-c", "init.defaultBranch=main"}, args...)...) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } + return strings.TrimSpace(string(out)) + } + git(repo, "init", "-q") + git(repo, "add", ".") + git(repo, "commit", "-q", "-m", "one") + first := git(repo, "rev-parse", "HEAD") + if err := os.WriteFile(filepath.Join(repo, "README.md"), []byte("two\n"), 0o644); err != nil { + t.Fatal(err) + } + git(repo, "commit", "-q", "-am", "only the README") + second := git(repo, "rev-parse", "HEAD") + + build := func(commit, at string) string { + t.Helper() + clone := filepath.Join(root, at) + git(root, "clone", "-q", repo, clone) + git(clone, "checkout", "-q", commit) + out := filepath.Join(clone, Out("server"), "ctl") + line := append([]string(nil), chain.Compile...) + line = append(line, "-ldflags", strings.Join(chain.LinkerFlags, " "), chain.OutputFlag, out, "./cmd/ctl") + cmd := exec.Command(line[0], line[1:]...) + cmd.Dir = clone + cmd.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOPROXY=off", "GOWORK=off") + if b, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("%s: %v\n%s", strings.Join(line, " "), err, b) + } + body, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + if bytes.Contains(body, []byte(commit)) { + t.Errorf("the binary built at %s carries the commit it was built at", commit[:12]) + } + sum := sha256.Sum256(body) + return hex.EncodeToString(sum[:]) + } + a, b := build(first, "a"), build(second, "elsewhere/b") + if a != b { + t.Fatalf("one build source, two digests: %s at %s, %s at %s (compiled with %q)", + a[:12], first[:12], b[:12], second[:12], strings.Join(chain.Compile, " ")) + } +} diff --git a/internal/builder/reproducible_test.go b/internal/builder/reproducible_test.go index 345ce055..137d1a03 100644 --- a/internal/builder/reproducible_test.go +++ b/internal/builder/reproducible_test.go @@ -1,11 +1,7 @@ package builder import ( - "bytes" - "crypto/sha256" - "encoding/hex" "os" - "os/exec" "path/filepath" "strings" "testing" @@ -19,82 +15,6 @@ import ( // // The toolchain's own compile line is run here, as the build seat runs it inside the toolchain image: in // two clones at two paths, of two commits that differ outside the program. One digest is wanted. -func TestAGoBundleBuiltTwiceFromOneBuildSourceIsOneDigest(t *testing.T) { - for _, tool := range []string{"go", "git"} { - if _, err := exec.LookPath(tool); err != nil { - t.Skipf("%s is not on this machine: %v", tool, err) - } - } - chain, err := ToolchainFor("go") - if err != nil { - t.Fatal(err) - } - root := t.TempDir() - repo := filepath.Join(root, "origin") - files := map[string]string{ - "go.mod": "module example.com/ctl\n\ngo 1.22\n", - "cmd/ctl/main.go": "package main\n\nfunc main() { println(\"ctl\") }\n", - "README.md": "one\n", - } - for name, body := range files { - if err := os.MkdirAll(filepath.Dir(filepath.Join(repo, name)), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(repo, name), []byte(body), 0o644); err != nil { - t.Fatal(err) - } - } - git := func(dir string, args ...string) string { - t.Helper() - cmd := exec.Command("git", append([]string{"-c", "user.name=t", "-c", "user.email=t@example.invalid", - "-c", "commit.gpgsign=false", "-c", "init.defaultBranch=main"}, args...)...) - cmd.Dir = dir - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) - } - return strings.TrimSpace(string(out)) - } - git(repo, "init", "-q") - git(repo, "add", ".") - git(repo, "commit", "-q", "-m", "one") - first := git(repo, "rev-parse", "HEAD") - if err := os.WriteFile(filepath.Join(repo, "README.md"), []byte("two\n"), 0o644); err != nil { - t.Fatal(err) - } - git(repo, "commit", "-q", "-am", "only the README") - second := git(repo, "rev-parse", "HEAD") - - build := func(commit, at string) string { - t.Helper() - clone := filepath.Join(root, at) - git(root, "clone", "-q", repo, clone) - git(clone, "checkout", "-q", commit) - out := filepath.Join(clone, Out("server"), "ctl") - line := append([]string(nil), chain.Compile...) - line = append(line, "-ldflags", strings.Join(chain.LinkerFlags, " "), chain.OutputFlag, out, "./cmd/ctl") - cmd := exec.Command(line[0], line[1:]...) - cmd.Dir = clone - cmd.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOPROXY=off", "GOWORK=off") - if b, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("%s: %v\n%s", strings.Join(line, " "), err, b) - } - body, err := os.ReadFile(out) - if err != nil { - t.Fatal(err) - } - if bytes.Contains(body, []byte(commit)) { - t.Errorf("the binary built at %s carries the commit it was built at", commit[:12]) - } - sum := sha256.Sum256(body) - return hex.EncodeToString(sum[:]) - } - a, b := build(first, "a"), build(second, "elsewhere/b") - if a != b { - t.Fatalf("one build source, two digests: %s at %s, %s at %s (compiled with %q)", - a[:12], first[:12], b[:12], second[:12], strings.Join(chain.Compile, " ")) - } -} // The toolchain says so itself, so a reader of the compile line sees it without running a build. func TestTheGoToolchainStampsNoVersionControl(t *testing.T) { diff --git a/internal/builder/reuse_integration_test.go b/internal/builder/reuse_integration_test.go new file mode 100644 index 00000000..c7ab6afb --- /dev/null +++ b/internal/builder/reuse_integration_test.go @@ -0,0 +1,336 @@ +//go:build integration + +package builder + +import ( + "reflect" + "strings" + "testing" +) + +func aCheckBuilt(t *testing.T, scratch *aScratch, tree string) CheckBuilt { + t.Helper() + r, run, workspace := aModuleAt(t, tree) + return aCheckBuiltBy(t, scratch, r, run, workspace) +} + +func aCheckBuiltBy(t *testing.T, scratch *aScratch, r *recorded, run Runner, workspace string) CheckBuilt { + t.Helper() + spec := CheckSpec{ID: "build-1", Owner: aPull.Owner, Repo: aPull.Repo, Number: aPull.Number, + Builds: []CheckBuildSpec{{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod", + Ref: "main"}}} + built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: scratch}, workspace, "", "", + CheckVerdict{Head: "abcdefabcdef", Trunk: "123456123456"}, GitCredential{}, nil, logging(nil)) + if len(built) != 1 || built[0].NotBuilt != "" || built[0].Key == "" { + t.Fatalf("the check did not build the module for its merge: %+v", built) + } + if !dockerBuilt(r) { + t.Fatal("the check built no image") + } + for repository := range r.images { + if !strings.HasPrefix(repository, ScratchPrefix) { + t.Fatalf("the check published %s outside the scratch namespace", repository) + } + } + for repository := range r.archives { + if !strings.HasPrefix(repository, ScratchPrefix) { + t.Fatalf("the check published %s outside the scratch namespace", repository) + } + } + return built[0] +} + +func TestAWalkWhoseKeyIsItsChecksBuildsNothingAndRegistersTheChecksDigest(t *testing.T) { + withVersion(t, "builder-1") + scratch := newScratch() + checked := aCheckBuilt(t, scratch, "tree-one") + + r, run, workspace := aModuleAt(t, "tree-one") + got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)}, + "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if dockerBuilt(r) || len(r.images) > 0 || len(r.archives) > 0 { + t.Fatalf("the walk built again: ran %v, published %v %v", r.ran, r.images, r.archives) + } + if got.Reused == nil || got.Why != "" { + t.Fatalf("the walk did not say it reused the check's build: reused %+v, why %q", got.Reused, got.Why) + } + if got.ReuseKey != checked.Key { + t.Fatalf("the walk's key %s is not the check's %s", got.ReuseKey, checked.Key) + } + if len(scratch.took) != 2 { + t.Fatalf("the walk copied %v, not the check's two artifacts", scratch.took) + } + for _, made := range checked.Made { + registered := strings.Replace(made.Reference, ScratchPrefix, "", 1) + found := false + for _, b := range got.Built { + found = found || (b.Reference == registered && b.Digest == made.Digest) + } + if !found { + t.Fatalf("the walk registers %+v, not the check's %s copied", got.Built, made.Reference) + } + } + if image, _ := got.Manifest.Resources[0]["image"].(string); image != "registry.invalid/meshboard/server@sha256:"+strings.Repeat("a", 64) { + t.Fatalf("the manifest pins %q, not the check's digest in the module's namespace", image) + } + words := got.Reused.Words() + if !strings.HasPrefix(words, "reused from the check of novox/meshboard#7 at head abcdefabcdef (merged into trunk 123456123456)") || + !strings.Contains(words, "sha256:") { + t.Fatalf("the walk says %q", words) + } +} + +func TestARecordNoVerdictNamesIsRefusedAndBuilt(t *testing.T) { + withVersion(t, "builder-1") + scratch := newScratch() + checked := aCheckBuilt(t, scratch, "tree-one") + for name, leave := range map[string][]NamedBuild{ + "no verdict": nil, + "another digest": func() []NamedBuild { + n := named(checked) + n[0].Digests["server"] = "sha256:" + strings.Repeat("e", 64) + return n + }(), + "another check": func() []NamedBuild { n := named(checked); n[0].Check = "build-9"; return n }(), + } { + t.Run(name, func(t *testing.T) { + scratch.took = nil + r, run, workspace := aModuleAt(t, "tree-one") + got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: leave}, + "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if got.Reused != nil || len(scratch.took) > 0 || !dockerBuilt(r) { + t.Fatalf("a record no verdict names was taken: %+v, copied %v", got.Reused, scratch.took) + } + if !strings.Contains(got.Why, "no check verdict of the walk's pull requests at the head the forge merged names it") || + !strings.Contains(got.Why, "at head abcdefabcdef") { + t.Fatalf("the walk says %q", got.Why) + } + }) + } +} + +func TestAWalkWhoseKeyDiffersBuildsAndSaysWhy(t *testing.T) { + withVersion(t, "builder-1") + scratch := newScratch() + checked := aCheckBuilt(t, scratch, "tree-one") + + r, run, workspace := aModuleAt(t, "tree-two") + got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)}, + "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if !dockerBuilt(r) || got.Reused != nil { + t.Fatalf("a walk whose source differs did not build: reused %+v", got.Reused) + } + if !strings.Contains(got.Why, "the key differs") || !strings.Contains(got.Why, partTree) { + t.Fatalf("the walk says %q, not which part of the key differs", got.Why) + } + + // And a builder of another version builds too, saying so. + withVersion(t, "builder-2") + r, run, workspace = aModuleAt(t, "tree-one") + got, err = BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)}, + "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if !dockerBuilt(r) || !strings.Contains(got.Why, partBuilder) { + t.Fatalf("a newer builder took the older one's build, or did not say why not: %q", got.Why) + } + + // No check of the walk's pull requests built it: said. + r, run, workspace = aModuleAt(t, "tree-one") + got, _ = BuildReusing(t.Context(), run, r, Reuse{Scratch: newScratch(), Pulls: []PullRef{aPull}}, + "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if !dockerBuilt(r) || !strings.Contains(got.Why, "no check of novox/meshboard#7 built it") { + t.Fatalf("the walk says %q", got.Why) + } +} + +func TestAFailedCopyOfAChecksBuildFallsBackToBuilding(t *testing.T) { + withVersion(t, "builder-1") + scratch := newScratch() + checked := aCheckBuilt(t, scratch, "tree-one") + scratch.failTake = true + + r, run, workspace := aModuleAt(t, "tree-one") + got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)}, + "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatalf("a failed copy failed the walk: %v", err) + } + if !dockerBuilt(r) || got.Reused != nil || len(r.images) == 0 { + t.Fatal("a failed copy did not fall back to a build") + } + if !strings.Contains(got.Why, "copying meshboard's") || !strings.Contains(got.Why, "refused the mount") { + t.Fatalf("the walk says %q", got.Why) + } + for repository := range r.images { + if strings.HasPrefix(repository, ScratchPrefix) { + t.Fatalf("the walk's own build went into the scratch namespace: %s", repository) + } + } +} + +func TestOnlyAWalkOfATrunkCommitTakesAChecksBuild(t *testing.T) { + withVersion(t, "builder-1") + scratch := newScratch() + checked := aCheckBuilt(t, scratch, "tree-one") + + r, run, workspace := aModuleAt(t, "tree-one") + if _, err := Build(t.Context(), run, r, "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, + Npmrc{}, GitCredential{}, nil); err != nil { + t.Fatal(err) + } + if !dockerBuilt(r) || len(scratch.took) > 0 { + t.Fatal("a build not asked to reuse took a check's build") + } + + r, _, workspace = aModuleAt(t, "tree-one") // no trunk said: off it + got, err := BuildReusing(t.Context(), r.run, r, Reuse{Scratch: scratch, Named: named(checked)}, "https://forge.invalid/meshboard.git", + "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if !dockerBuilt(r) || got.Reused != nil || !strings.Contains(got.Why, "not on its trunk") { + t.Fatalf("a commit off the trunk took a check's build: %q", got.Why) + } +} + +func TestACheckThatFailedBuildsNothingForItsMerge(t *testing.T) { + if skipsTheBuilds(CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "fail"}}) == "" { + t.Fatal("a check whose repository layer failed builds for its merge") + } + if skipsTheBuilds(CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "warning"}}) != "" { + t.Fatal("a check that passed with a warning builds nothing for its merge") + } + r, run, workspace := aModuleAt(t, "tree-one") + if built := buildForTheMerge(t.Context(), run, CheckSpec{ID: "build-1"}, CheckBuilding{Publish: r, Scratch: newScratch()}, + workspace, "", "", CheckVerdict{}, GitCredential{}, nil, logging(nil)); built != nil || len(r.ran) > 0 { + t.Fatalf("a check moving no module built %v, ran %v", built, r.ran) + } +} + +func TestACheckBuildsNoModuleWithoutAKeyNorWhatStandsAfterIt(t *testing.T) { + withVersion(t, "builder-1") + r, run, workspace := aModuleAt(t, "tree-one") + r.contents["mod/"+ManifestName] = `{"module":"meshboard","version":"1","build":{"artifacts":[ + {"name":"mirror","kind":"upstream","from":"docker.io/library/alpine:3"}]}}` + spec := CheckSpec{ID: "build-1", Owner: "novox", Repo: "meshboard", Number: 7, Builds: []CheckBuildSpec{ + {Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod", Tier: 0}, + {Module: "after", Repository: "https://forge.invalid/after.git", Tier: 1}}} + built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: newScratch()}, workspace, "", "", + CheckVerdict{}, GitCredential{}, nil, logging(nil)) + if len(built) != 2 || !strings.Contains(built[0].NotBuilt, "named by tag") || + !strings.Contains(built[1].NotBuilt, "stands after meshboard") { + t.Fatalf("built %+v", built) + } + if len(r.images) > 0 { + t.Fatalf("a module without a key was published: %v", r.images) + } +} + +func aCheckAgain(t *testing.T, scratch *aScratch, tree string, leave []NamedBuild) (CheckBuilt, *recorded) { + t.Helper() + return aCheckOf(t, scratch, aPull, "abcdefabcdef", tree, leave) +} + +func aCheckOf(t *testing.T, scratch *aScratch, pull PullRef, head, tree string, leave []NamedBuild) (CheckBuilt, *recorded) { + t.Helper() + r, run, workspace := aModuleAt(t, tree) + spec := CheckSpec{ID: "build-2", Owner: pull.Owner, Repo: pull.Repo, Number: pull.Number, + Builds: []CheckBuildSpec{{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod", + Ref: "main"}}} + built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: scratch, Named: leave}, + workspace, "", "", CheckVerdict{Head: head, Trunk: "654321654321"}, GitCredential{}, nil, logging(nil)) + if len(built) != 1 || built[0].NotBuilt != "" || built[0].Key == "" { + t.Fatalf("the second check did not answer the module for its merge: %+v", built) + } + return built[0], r +} + +func TestASecondCheckWhoseKeyIsUnchangedBuildsNothingAndNamesTheEarlierBuild(t *testing.T) { + withVersion(t, "builder-1") + scratch := newScratch() + first := aCheckBuilt(t, scratch, "tree-one") + + again, r := aCheckAgain(t, scratch, "tree-one", named(first)) + if dockerBuilt(r) || len(r.images) > 0 || len(r.archives) > 0 { + t.Fatalf("the second check built again: ran %v, published %v %v", r.ran, r.images, r.archives) + } + if again.Key != first.Key || again.Kept != "build-1" || !reflect.DeepEqual(again.Made, first.Made) { + t.Fatalf("the second check names %+v, not the first check's build %+v", again, first) + } + // The walk takes it by what the second verdict names. + leave := named(again) + leave[0].Check = again.Kept + w, run, workspace := aModuleAt(t, "tree-one") + got, err := BuildReusing(t.Context(), run, w, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: leave}, + "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil || got.Reused == nil || dockerBuilt(w) { + t.Fatalf("the walk did not take the kept build: %v, why %q", err, got.Why) + } +} + +func TestASecondCheckWhoseKeyChangedBuilds(t *testing.T) { + withVersion(t, "builder-1") + scratch := newScratch() + first := aCheckBuilt(t, scratch, "tree-one") + + again, r := aCheckAgain(t, scratch, "tree-two", named(first)) + if !dockerBuilt(r) || again.Kept != "" || again.Key == first.Key { + t.Fatalf("a check whose key changed did not build: %+v", again) + } +} + +func TestASecondCheckKeepsNoRecordAVerdictDoesNotName(t *testing.T) { + withVersion(t, "builder-1") + scratch := newScratch() + first := aCheckBuilt(t, scratch, "tree-one") + for name, leave := range map[string][]NamedBuild{ + "no verdict": nil, + "another digest": func() []NamedBuild { + n := named(first) + n[0].Digests["server"] = "sha256:" + strings.Repeat("e", 64) + return n + }(), + "another key": func() []NamedBuild { n := named(first); n[0].Key = "r1:" + strings.Repeat("f", 64); return n }(), + } { + t.Run(name, func(t *testing.T) { + again, r := aCheckAgain(t, scratch, "tree-one", leave) + if !dockerBuilt(r) || again.Kept != "" { + t.Fatalf("a record no verdict names was kept: %+v", again) + } + }) + } +} + +func TestACheckKeepsNoBuildOfAnotherHeadOrPullRequest(t *testing.T) { + withVersion(t, "builder-1") + other := PullRef{Owner: "novox", Repo: "meshboard", Number: 8} + for name, c := range map[string]struct { + pull PullRef + head string + }{ + "another head": {aPull, "999999999999"}, + "another pull request": {other, "abcdefabcdef"}, + } { + t.Run(name, func(t *testing.T) { + scratch := newScratch() + first := aCheckBuilt(t, scratch, "tree-one") + leave := named(first) + leave[0].Pull = c.pull // as if its own verdict named it + again, r := aCheckOf(t, scratch, c.pull, c.head, "tree-one", leave) + if !dockerBuilt(r) || again.Kept != "" { + t.Fatalf("a build of %s was kept: %+v", name, again) + } + }) + } +} diff --git a/internal/builder/reuse_test.go b/internal/builder/reuse_test.go index c9cbb229..2f2c5277 100644 --- a/internal/builder/reuse_test.go +++ b/internal/builder/reuse_test.go @@ -6,7 +6,6 @@ import ( "errors" "net/http" "net/http/httptest" - "reflect" "slices" "strconv" "strings" @@ -104,38 +103,8 @@ func dockerBuilt(r *recorded) bool { var aPull = PullRef{Owner: "novox", Repo: "meshboard", Number: 7} // aCheckBuilt is the check of aPull building the module from a tree, into the scratch namespace. -func aCheckBuilt(t *testing.T, scratch *aScratch, tree string) CheckBuilt { - t.Helper() - r, run, workspace := aModuleAt(t, tree) - return aCheckBuiltBy(t, scratch, r, run, workspace) -} // aCheckBuiltBy is aCheckBuilt of the repository given. -func aCheckBuiltBy(t *testing.T, scratch *aScratch, r *recorded, run Runner, workspace string) CheckBuilt { - t.Helper() - spec := CheckSpec{ID: "build-1", Owner: aPull.Owner, Repo: aPull.Repo, Number: aPull.Number, - Builds: []CheckBuildSpec{{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod", - Ref: "main"}}} - built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: scratch}, workspace, "", "", - CheckVerdict{Head: "abcdefabcdef", Trunk: "123456123456"}, GitCredential{}, nil, logging(nil)) - if len(built) != 1 || built[0].NotBuilt != "" || built[0].Key == "" { - t.Fatalf("the check did not build the module for its merge: %+v", built) - } - if !dockerBuilt(r) { - t.Fatal("the check built no image") - } - for repository := range r.images { - if !strings.HasPrefix(repository, ScratchPrefix) { - t.Fatalf("the check published %s outside the scratch namespace", repository) - } - } - for repository := range r.archives { - if !strings.HasPrefix(repository, ScratchPrefix) { - t.Fatalf("the check published %s outside the scratch namespace", repository) - } - } - return built[0] -} // named is the walk's leave naming what a check's verdict says it built, as the controller reads it. func named(c CheckBuilt) []NamedBuild { @@ -159,211 +128,21 @@ func withVersion(t *testing.T, v string) { // A merge whose reuse key equals its check's builds nothing again: the walk copies the check's build by digest // and registers that digest. -func TestAWalkWhoseKeyIsItsChecksBuildsNothingAndRegistersTheChecksDigest(t *testing.T) { - withVersion(t, "builder-1") - scratch := newScratch() - checked := aCheckBuilt(t, scratch, "tree-one") - - r, run, workspace := aModuleAt(t, "tree-one") - got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)}, - "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if dockerBuilt(r) || len(r.images) > 0 || len(r.archives) > 0 { - t.Fatalf("the walk built again: ran %v, published %v %v", r.ran, r.images, r.archives) - } - if got.Reused == nil || got.Why != "" { - t.Fatalf("the walk did not say it reused the check's build: reused %+v, why %q", got.Reused, got.Why) - } - if got.ReuseKey != checked.Key { - t.Fatalf("the walk's key %s is not the check's %s", got.ReuseKey, checked.Key) - } - if len(scratch.took) != 2 { - t.Fatalf("the walk copied %v, not the check's two artifacts", scratch.took) - } - for _, made := range checked.Made { - registered := strings.Replace(made.Reference, ScratchPrefix, "", 1) - found := false - for _, b := range got.Built { - found = found || (b.Reference == registered && b.Digest == made.Digest) - } - if !found { - t.Fatalf("the walk registers %+v, not the check's %s copied", got.Built, made.Reference) - } - } - if image, _ := got.Manifest.Resources[0]["image"].(string); image != "registry.invalid/meshboard/server@sha256:"+strings.Repeat("a", 64) { - t.Fatalf("the manifest pins %q, not the check's digest in the module's namespace", image) - } - words := got.Reused.Words() - if !strings.HasPrefix(words, "reused from the check of novox/meshboard#7 at head abcdefabcdef (merged into trunk 123456123456)") || - !strings.Contains(words, "sha256:") { - t.Fatalf("the walk says %q", words) - } -} // A record no verdict names is refused and built instead, and the walk says so: the store takes writes from anything // on its network, so a record planted under another merge's key is never registered. -func TestARecordNoVerdictNamesIsRefusedAndBuilt(t *testing.T) { - withVersion(t, "builder-1") - scratch := newScratch() - checked := aCheckBuilt(t, scratch, "tree-one") - for name, leave := range map[string][]NamedBuild{ - "no verdict": nil, - "another digest": func() []NamedBuild { - n := named(checked) - n[0].Digests["server"] = "sha256:" + strings.Repeat("e", 64) - return n - }(), - "another check": func() []NamedBuild { n := named(checked); n[0].Check = "build-9"; return n }(), - } { - t.Run(name, func(t *testing.T) { - scratch.took = nil - r, run, workspace := aModuleAt(t, "tree-one") - got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: leave}, - "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if got.Reused != nil || len(scratch.took) > 0 || !dockerBuilt(r) { - t.Fatalf("a record no verdict names was taken: %+v, copied %v", got.Reused, scratch.took) - } - if !strings.Contains(got.Why, "no check verdict of the walk's pull requests at the head the forge merged names it") || - !strings.Contains(got.Why, "at head abcdefabcdef") { - t.Fatalf("the walk says %q", got.Why) - } - }) - } -} // A differing key builds, and says which part differs. -func TestAWalkWhoseKeyDiffersBuildsAndSaysWhy(t *testing.T) { - withVersion(t, "builder-1") - scratch := newScratch() - checked := aCheckBuilt(t, scratch, "tree-one") - - r, run, workspace := aModuleAt(t, "tree-two") - got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)}, - "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if !dockerBuilt(r) || got.Reused != nil { - t.Fatalf("a walk whose source differs did not build: reused %+v", got.Reused) - } - if !strings.Contains(got.Why, "the key differs") || !strings.Contains(got.Why, partTree) { - t.Fatalf("the walk says %q, not which part of the key differs", got.Why) - } - - // And a builder of another version builds too, saying so. - withVersion(t, "builder-2") - r, run, workspace = aModuleAt(t, "tree-one") - got, err = BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)}, - "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if !dockerBuilt(r) || !strings.Contains(got.Why, partBuilder) { - t.Fatalf("a newer builder took the older one's build, or did not say why not: %q", got.Why) - } - - // No check of the walk's pull requests built it: said. - r, run, workspace = aModuleAt(t, "tree-one") - got, _ = BuildReusing(t.Context(), run, r, Reuse{Scratch: newScratch(), Pulls: []PullRef{aPull}}, - "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) - if !dockerBuilt(r) || !strings.Contains(got.Why, "no check of novox/meshboard#7 built it") { - t.Fatalf("the walk says %q", got.Why) - } -} // A copy that fails is said and falls back to a build; it never fails the walk. -func TestAFailedCopyOfAChecksBuildFallsBackToBuilding(t *testing.T) { - withVersion(t, "builder-1") - scratch := newScratch() - checked := aCheckBuilt(t, scratch, "tree-one") - scratch.failTake = true - - r, run, workspace := aModuleAt(t, "tree-one") - got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)}, - "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatalf("a failed copy failed the walk: %v", err) - } - if !dockerBuilt(r) || got.Reused != nil || len(r.images) == 0 { - t.Fatal("a failed copy did not fall back to a build") - } - if !strings.Contains(got.Why, "copying meshboard's") || !strings.Contains(got.Why, "refused the mount") { - t.Fatalf("the walk says %q", got.Why) - } - for repository := range r.images { - if strings.HasPrefix(repository, ScratchPrefix) { - t.Fatalf("the walk's own build went into the scratch namespace: %s", repository) - } - } -} // Only a walk takes a check's build: a plain build — a check's, a dry run's, `build --ref`'s — builds, and a commit // off the trunk never takes one. -func TestOnlyAWalkOfATrunkCommitTakesAChecksBuild(t *testing.T) { - withVersion(t, "builder-1") - scratch := newScratch() - checked := aCheckBuilt(t, scratch, "tree-one") - - r, run, workspace := aModuleAt(t, "tree-one") - if _, err := Build(t.Context(), run, r, "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, - Npmrc{}, GitCredential{}, nil); err != nil { - t.Fatal(err) - } - if !dockerBuilt(r) || len(scratch.took) > 0 { - t.Fatal("a build not asked to reuse took a check's build") - } - - r, _, workspace = aModuleAt(t, "tree-one") // no trunk said: off it - got, err := BuildReusing(t.Context(), r.run, r, Reuse{Scratch: scratch, Named: named(checked)}, "https://forge.invalid/meshboard.git", - "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if !dockerBuilt(r) || got.Reused != nil || !strings.Contains(got.Why, "not on its trunk") { - t.Fatalf("a commit off the trunk took a check's build: %q", got.Why) - } -} // A check that did not pass builds nothing for its merge, and one asked to build nothing builds nothing. -func TestACheckThatFailedBuildsNothingForItsMerge(t *testing.T) { - if skipsTheBuilds(CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "fail"}}) == "" { - t.Fatal("a check whose repository layer failed builds for its merge") - } - if skipsTheBuilds(CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "warning"}}) != "" { - t.Fatal("a check that passed with a warning builds nothing for its merge") - } - r, run, workspace := aModuleAt(t, "tree-one") - if built := buildForTheMerge(t.Context(), run, CheckSpec{ID: "build-1"}, CheckBuilding{Publish: r, Scratch: newScratch()}, - workspace, "", "", CheckVerdict{}, GitCredential{}, nil, logging(nil)); built != nil || len(r.ran) > 0 { - t.Fatalf("a check moving no module built %v, ran %v", built, r.ran) - } -} // A module whose source does not pin its build has no key and is not built by the check; what stands after it // is not built either. -func TestACheckBuildsNoModuleWithoutAKeyNorWhatStandsAfterIt(t *testing.T) { - withVersion(t, "builder-1") - r, run, workspace := aModuleAt(t, "tree-one") - r.contents["mod/"+ManifestName] = `{"module":"meshboard","version":"1","build":{"artifacts":[ - {"name":"mirror","kind":"upstream","from":"docker.io/library/alpine:3"}]}}` - spec := CheckSpec{ID: "build-1", Owner: "novox", Repo: "meshboard", Number: 7, Builds: []CheckBuildSpec{ - {Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod", Tier: 0}, - {Module: "after", Repository: "https://forge.invalid/after.git", Tier: 1}}} - built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: newScratch()}, workspace, "", "", - CheckVerdict{}, GitCredential{}, nil, logging(nil)) - if len(built) != 2 || !strings.Contains(built[0].NotBuilt, "named by tag") || - !strings.Contains(built[1].NotBuilt, "stands after meshboard") { - t.Fatalf("built %+v", built) - } - if len(r.images) > 0 { - t.Fatalf("a module without a key was published: %v", r.images) - } -} // What the key names: every input a fresh build could differ by. func TestTheReuseKeyNamesEveryInputOfTheBuild(t *testing.T) { @@ -642,86 +421,16 @@ func TestABuildTwoChecksShareGoesWithTheLastOfThem(t *testing.T) { // aCheckAgain is a later check (build-2) of aPull building the module from a tree, with the builds the build seat's // verdicts name for it (novox/hq issue 703). -func aCheckAgain(t *testing.T, scratch *aScratch, tree string, leave []NamedBuild) (CheckBuilt, *recorded) { - t.Helper() - return aCheckOf(t, scratch, aPull, "abcdefabcdef", tree, leave) -} // aCheckOf is a later check (build-2) of a pull request at a head, the trunk moved since build-1. -func aCheckOf(t *testing.T, scratch *aScratch, pull PullRef, head, tree string, leave []NamedBuild) (CheckBuilt, *recorded) { - t.Helper() - r, run, workspace := aModuleAt(t, tree) - spec := CheckSpec{ID: "build-2", Owner: pull.Owner, Repo: pull.Repo, Number: pull.Number, - Builds: []CheckBuildSpec{{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod", - Ref: "main"}}} - built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: scratch, Named: leave}, - workspace, "", "", CheckVerdict{Head: head, Trunk: "654321654321"}, GitCredential{}, nil, logging(nil)) - if len(built) != 1 || built[0].NotBuilt != "" || built[0].Key == "" { - t.Fatalf("the second check did not answer the module for its merge: %+v", built) - } - return built[0], r -} // A check asked again when only the trunk moved, whose key is unchanged, builds nothing: it keeps the build an // earlier verdict names and names it by key and digest, so the walk can take it (novox/hq issue 703). -func TestASecondCheckWhoseKeyIsUnchangedBuildsNothingAndNamesTheEarlierBuild(t *testing.T) { - withVersion(t, "builder-1") - scratch := newScratch() - first := aCheckBuilt(t, scratch, "tree-one") - - again, r := aCheckAgain(t, scratch, "tree-one", named(first)) - if dockerBuilt(r) || len(r.images) > 0 || len(r.archives) > 0 { - t.Fatalf("the second check built again: ran %v, published %v %v", r.ran, r.images, r.archives) - } - if again.Key != first.Key || again.Kept != "build-1" || !reflect.DeepEqual(again.Made, first.Made) { - t.Fatalf("the second check names %+v, not the first check's build %+v", again, first) - } - // The walk takes it by what the second verdict names. - leave := named(again) - leave[0].Check = again.Kept - w, run, workspace := aModuleAt(t, "tree-one") - got, err := BuildReusing(t.Context(), run, w, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: leave}, - "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil || got.Reused == nil || dockerBuilt(w) { - t.Fatalf("the walk did not take the kept build: %v, why %q", err, got.Why) - } -} // A check whose key changed builds, as before. -func TestASecondCheckWhoseKeyChangedBuilds(t *testing.T) { - withVersion(t, "builder-1") - scratch := newScratch() - first := aCheckBuilt(t, scratch, "tree-one") - - again, r := aCheckAgain(t, scratch, "tree-two", named(first)) - if !dockerBuilt(r) || again.Kept != "" || again.Key == first.Key { - t.Fatalf("a check whose key changed did not build: %+v", again) - } -} // A scratch record no verdict names is never kept: the store takes writes from anything on its network, so a // check builds rather than name a planted record (novox/hq issue 703, ADR 0350 §2). -func TestASecondCheckKeepsNoRecordAVerdictDoesNotName(t *testing.T) { - withVersion(t, "builder-1") - scratch := newScratch() - first := aCheckBuilt(t, scratch, "tree-one") - for name, leave := range map[string][]NamedBuild{ - "no verdict": nil, - "another digest": func() []NamedBuild { - n := named(first) - n[0].Digests["server"] = "sha256:" + strings.Repeat("e", 64) - return n - }(), - "another key": func() []NamedBuild { n := named(first); n[0].Key = "r1:" + strings.Repeat("f", 64); return n }(), - } { - t.Run(name, func(t *testing.T) { - again, r := aCheckAgain(t, scratch, "tree-one", leave) - if !dockerBuilt(r) || again.Kept != "" { - t.Fatalf("a record no verdict names was kept: %+v", again) - } - }) - } -} // A build a later check kept stays when the check that built it goes: its artifacts and its record, which the later // check's verdict names (novox/hq issue 703, ADR 0350 §5). @@ -767,25 +476,3 @@ func TestABuildALaterCheckKeptStaysWhenTheCheckThatBuiltItGoes(t *testing.T) { // Only an earlier check of the same pull request at the same head is kept from: another head of it, or another pull // request whose verdict names the same key and digests, builds (novox/hq issue 703). -func TestACheckKeepsNoBuildOfAnotherHeadOrPullRequest(t *testing.T) { - withVersion(t, "builder-1") - other := PullRef{Owner: "novox", Repo: "meshboard", Number: 8} - for name, c := range map[string]struct { - pull PullRef - head string - }{ - "another head": {aPull, "999999999999"}, - "another pull request": {other, "abcdefabcdef"}, - } { - t.Run(name, func(t *testing.T) { - scratch := newScratch() - first := aCheckBuilt(t, scratch, "tree-one") - leave := named(first) - leave[0].Pull = c.pull // as if its own verdict named it - again, r := aCheckOf(t, scratch, c.pull, c.head, "tree-one", leave) - if !dockerBuilt(r) || again.Kept != "" { - t.Fatalf("a build of %s was kept: %+v", name, again) - } - }) - } -} diff --git a/internal/builder/siblings_test.go b/internal/builder/siblings_test.go index 98dc8951..5fd1fd2c 100644 --- a/internal/builder/siblings_test.go +++ b/internal/builder/siblings_test.go @@ -1,3 +1,5 @@ +//go:build integration + package builder import ( diff --git a/internal/builder/source_integration_test.go b/internal/builder/source_integration_test.go new file mode 100644 index 00000000..b047ae72 --- /dev/null +++ b/internal/builder/source_integration_test.go @@ -0,0 +1,44 @@ +//go:build integration + +package builder + +import ( + "context" + "strings" + "testing" +) + +func fingerprintOf(t *testing.T, manifest, tree string, mirrored string) string { + t.Helper() + r, workspace := aRepository(t, manifest, map[string]string{"modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}"}) + r.contents["modules/bus/module.json"] = manifest + r.tree = tree + m := &mirroring{recorded: r, at: mirrored} + got, err := Build(context.Background(), r.run, m, "https://forge.invalid/catalogue.git", "modules/bus", "", workspace, + nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + return got.Source +} + +func TestASourceFingerprintNamesWhatABuildWasMadeFrom(t *testing.T) { + one := fingerprintOf(t, anImage, "aaaa", "registry-a:5000") + if !strings.HasPrefix(one, sourcePrefix) { + t.Fatalf("no fingerprint: %q", one) + } + // The same tree and base, the base copied to another registry address: one source. + if again := fingerprintOf(t, anImage, "aaaa", "registry-b:5000"); again != one { + t.Fatalf("one source has two fingerprints: %s %s", one, again) + } + // The module's tree changed: another source. + if changed := fingerprintOf(t, anImage, "bbbb", "registry-a:5000"); changed == one { + t.Fatal("a changed tree kept its fingerprint") + } + // The base moved: another source. + moved := strings.Replace(anImage, "1111111111111111111111111111111111111111111111111111111111111111", + "2222222222222222222222222222222222222222222222222222222222222222", 1) + if changed := fingerprintOf(t, moved, "aaaa", "registry-a:5000"); changed == one { + t.Fatal("a moved base kept its fingerprint") + } +} diff --git a/internal/builder/source_test.go b/internal/builder/source_test.go index 39e97255..ce1e6541 100644 --- a/internal/builder/source_test.go +++ b/internal/builder/source_test.go @@ -14,20 +14,6 @@ const anImage = `{"module":"bus","version":"1", "artifacts":[{"name":"server","kind":"image","from":"Dockerfile"}]}, "resources":[{"id":"svc","type":"container","name":"bus","artifact":"server"}]}` -func fingerprintOf(t *testing.T, manifest, tree string, mirrored string) string { - t.Helper() - r, workspace := aRepository(t, manifest, map[string]string{"modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}"}) - r.contents["modules/bus/module.json"] = manifest - r.tree = tree - m := &mirroring{recorded: r, at: mirrored} - got, err := Build(context.Background(), r.run, m, "https://forge.invalid/catalogue.git", "modules/bus", "", workspace, - nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - return got.Source -} - // mirroring is a store that copies a vendor's image into the mesh's registry, at an address a test says. type mirroring struct { *recorded @@ -39,27 +25,6 @@ func (m *mirroring) MirrorImage(_ context.Context, from, repository string) (str return m.at + "/" + repository + "@" + digest, nil } -func TestASourceFingerprintNamesWhatABuildWasMadeFrom(t *testing.T) { - one := fingerprintOf(t, anImage, "aaaa", "registry-a:5000") - if !strings.HasPrefix(one, sourcePrefix) { - t.Fatalf("no fingerprint: %q", one) - } - // The same tree and base, the base copied to another registry address: one source. - if again := fingerprintOf(t, anImage, "aaaa", "registry-b:5000"); again != one { - t.Fatalf("one source has two fingerprints: %s %s", one, again) - } - // The module's tree changed: another source. - if changed := fingerprintOf(t, anImage, "bbbb", "registry-a:5000"); changed == one { - t.Fatal("a changed tree kept its fingerprint") - } - // The base moved: another source. - moved := strings.Replace(anImage, "1111111111111111111111111111111111111111111111111111111111111111", - "2222222222222222222222222222222222222222222222222222222222222222", 1) - if changed := fingerprintOf(t, moved, "aaaa", "registry-a:5000"); changed == one { - t.Fatal("a moved base kept its fingerprint") - } -} - func TestABuildTheRegistryDecidesHasNoFingerprint(t *testing.T) { s := newSourceInputs("app") s.tree = "aaaa" diff --git a/internal/builder/template_integration_test.go b/internal/builder/template_integration_test.go new file mode 100644 index 00000000..a8880e35 --- /dev/null +++ b/internal/builder/template_integration_test.go @@ -0,0 +1,103 @@ +//go:build integration + +package builder + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestABuildReadsEachTemplateBackFromItsArchive(t *testing.T) { + r, workspace := aRepository(t, templatedModule, templatedFiles) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if text := contentOf(t, got.Manifest, "conf"); text != templatedFiles["files/meshboard.conf"] { + t.Fatalf("the template's text is %q, not the file's", text) + } + if text := contentOf(t, got.Manifest, "deeper"); text != templatedFiles["files/a/deeper.conf"] { + t.Fatalf("the template's text is %q, not the file's", text) + } + if r.archives["meshboard/files"] == "" { + t.Fatalf("the templates archive was not published: %v", r.archives) + } + if problem := catalogue.HeldTemplateProblem(got.Manifest); problem != "" { + t.Fatal(problem) + } +} + +func TestABuildWhoseTemplateIsMissingFailsNamingIt(t *testing.T) { + files := map[string]string{} + for k, v := range templatedFiles { + files[k] = v + } + delete(files, "files/a/deeper.conf") + r, workspace := aRepository(t, templatedModule, files) + _, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err == nil || !strings.Contains(err.Error(), "files/a/deeper.conf") { + t.Fatalf("a build whose template is missing did not fail naming it: %v", err) + } +} + +func TestAWalkTakingAChecksBuildReadsTheTemplatesBackToo(t *testing.T) { + withVersion(t, "builder-1") + contents := map[string]string{"mod/" + ManifestName: templatedModule} + for k, v := range templatedFiles { + contents["mod/"+k] = v + } + scratch := newScratch() + r, run, workspace := aModuleWith(t, "tree-one", contents) + checked := aCheckBuiltBy(t, scratch, r, run, workspace) + + r, run, workspace = aModuleWith(t, "tree-one", contents) + got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)}, + "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if got.Reused == nil || len(r.archives) > 0 { + t.Fatalf("the walk built again: reused %+v, why %q, published %v", got.Reused, got.Why, r.archives) + } + if text := contentOf(t, got.Manifest, "conf"); text != templatedFiles["files/meshboard.conf"] { + t.Fatalf("the template's text is %q, not the file's", text) + } +} + +func TestABuildReadsEachTemplateBackFromTheTemplatesDirectoryItsManifestNames(t *testing.T) { + files := map[string]string{"files/bin/meshboard-hook": "#!/bin/sh\n", "templates/meshboard.conf": "listen = 8080\n"} + r, workspace := aRepository(t, unpackingModule, files) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if text := contentOf(t, got.Manifest, "conf"); text != files["templates/meshboard.conf"] { + t.Fatalf("the template's text is %q, not the file's", text) + } + if got.Manifest.Templates != "" { + t.Fatalf("the built manifest still names its templates directory: %q", got.Manifest.Templates) + } + // files/ packs exactly as it would with no templates beside it. + alone, err := pack(writeTree(t, t.TempDir(), map[string]string{"bin/meshboard-hook": "#!/bin/sh\n"})) + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(alone) + packed := "" + for _, b := range got.Built { + if b.Name == "scripts" { + packed = b.Digest + } + } + if packed != "sha256:"+hex.EncodeToString(sum[:]) { + t.Fatalf("files/ packed to %q, not to what it packs to alone", packed) + } +} diff --git a/internal/builder/template_test.go b/internal/builder/template_test.go index c8c792b2..2bbf8979 100644 --- a/internal/builder/template_test.go +++ b/internal/builder/template_test.go @@ -1,9 +1,6 @@ package builder import ( - "context" - "crypto/sha256" - "encoding/hex" "os" "path/filepath" "strings" @@ -48,66 +45,8 @@ func contentOf(t *testing.T, m catalogue.Manifest, id string) string { return "" } -func TestABuildReadsEachTemplateBackFromItsArchive(t *testing.T) { - r, workspace := aRepository(t, templatedModule, templatedFiles) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, - Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if text := contentOf(t, got.Manifest, "conf"); text != templatedFiles["files/meshboard.conf"] { - t.Fatalf("the template's text is %q, not the file's", text) - } - if text := contentOf(t, got.Manifest, "deeper"); text != templatedFiles["files/a/deeper.conf"] { - t.Fatalf("the template's text is %q, not the file's", text) - } - if r.archives["meshboard/files"] == "" { - t.Fatalf("the templates archive was not published: %v", r.archives) - } - if problem := catalogue.HeldTemplateProblem(got.Manifest); problem != "" { - t.Fatal(problem) - } -} - -func TestABuildWhoseTemplateIsMissingFailsNamingIt(t *testing.T) { - files := map[string]string{} - for k, v := range templatedFiles { - files[k] = v - } - delete(files, "files/a/deeper.conf") - r, workspace := aRepository(t, templatedModule, files) - _, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, - Npmrc{}, GitCredential{}, nil) - if err == nil || !strings.Contains(err.Error(), "files/a/deeper.conf") { - t.Fatalf("a build whose template is missing did not fail naming it: %v", err) - } -} - // A walk that takes its check's build copies the archive by digest and builds nothing: the templates are read back // from the tree packed again, which is the archive only when it packs to the digest taken. -func TestAWalkTakingAChecksBuildReadsTheTemplatesBackToo(t *testing.T) { - withVersion(t, "builder-1") - contents := map[string]string{"mod/" + ManifestName: templatedModule} - for k, v := range templatedFiles { - contents["mod/"+k] = v - } - scratch := newScratch() - r, run, workspace := aModuleWith(t, "tree-one", contents) - checked := aCheckBuiltBy(t, scratch, r, run, workspace) - - r, run, workspace = aModuleWith(t, "tree-one", contents) - got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)}, - "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if got.Reused == nil || len(r.archives) > 0 { - t.Fatalf("the walk built again: reused %+v, why %q, published %v", got.Reused, got.Why, r.archives) - } - if text := contentOf(t, got.Manifest, "conf"); text != templatedFiles["files/meshboard.conf"] { - t.Fatalf("the template's text is %q, not the file's", text) - } -} // The tree packed again must be the archive taken; text read from anything else is not the archive's. func TestTemplateTextsAreRefusedWhenTheTreeDoesNotPackToTheDigestTaken(t *testing.T) { @@ -148,34 +87,3 @@ const unpackingModule = `{"module":"meshboard","version":"1", "resources":[ {"id":"scripts","type":"archive","path":"/usr/local/lib/meshboard","artifact":"scripts"}, {"id":"conf","type":"file","path":"/etc/meshboard/meshboard.conf","template":"templates/meshboard.conf"}]}` - -func TestABuildReadsEachTemplateBackFromTheTemplatesDirectoryItsManifestNames(t *testing.T) { - files := map[string]string{"files/bin/meshboard-hook": "#!/bin/sh\n", "templates/meshboard.conf": "listen = 8080\n"} - r, workspace := aRepository(t, unpackingModule, files) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, - Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if text := contentOf(t, got.Manifest, "conf"); text != files["templates/meshboard.conf"] { - t.Fatalf("the template's text is %q, not the file's", text) - } - if got.Manifest.Templates != "" { - t.Fatalf("the built manifest still names its templates directory: %q", got.Manifest.Templates) - } - // files/ packs exactly as it would with no templates beside it. - alone, err := pack(writeTree(t, t.TempDir(), map[string]string{"bin/meshboard-hook": "#!/bin/sh\n"})) - if err != nil { - t.Fatal(err) - } - sum := sha256.Sum256(alone) - packed := "" - for _, b := range got.Built { - if b.Name == "scripts" { - packed = b.Digest - } - } - if packed != "sha256:"+hex.EncodeToString(sum[:]) { - t.Fatalf("files/ packed to %q, not to what it packs to alone", packed) - } -} diff --git a/internal/builder/upstream_file.go b/internal/builder/upstream_file.go index c4d3a730..5b3a0ebf 100644 --- a/internal/builder/upstream_file.go +++ b/internal/builder/upstream_file.go @@ -41,7 +41,8 @@ const maxUpstream = 512 << 20 // maxEntries is the most entries an upstream archive may hold: millions of empty ones would fill the build // seat's inodes and the node's. Betterbird's release holds 64. -const maxEntries = 100_000 +// It is a variable so that a test can lower it instead of writing a hundred thousand headers. +var maxEntries = 100_000 // noDowngrade refuses a redirect off https: the digest still guards the bytes, and a fetch in the clear says // to whoever is on the path what the build seat fetches. diff --git a/internal/builder/upstream_file_integration_test.go b/internal/builder/upstream_file_integration_test.go new file mode 100644 index 00000000..31a1dfad --- /dev/null +++ b/internal/builder/upstream_file_integration_test.go @@ -0,0 +1,180 @@ +//go:build integration + +package builder + +import ( + "archive/tar" + "bytes" + "context" + "os/exec" + "path/filepath" + "sort" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestAnArchivePinnedByURLIsFetchedCheckedAndPackedWithoutItsOwnDirectory(t *testing.T) { + body := release(t) + v := &vendor{files: map[string][]byte{"/release.tar.gz": body}} + base := serveVendor(t, v) + r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + digest, _ := got.Manifest.Resources[0]["digest"].(string) + if !strings.HasPrefix(digest, "sha256:") || r.archives["betterbird/archive"] != digest { + t.Fatalf("the release was not published and pinned as the module's archive: %v, published %v", + got.Manifest.Resources[0], r.archives) + } + + packed, err := upstreamArchive(context.Background(), workspace, catalogue.Artifact{Name: "archive", Kind: "archive", + URL: base + "/release.tar.gz", SHA256: sumHex(body), Version: "153.4.0esr-bb10", Within: "betterbird"}, + func(string, string, ...any) {}) + if err != nil { + t.Fatal(err) + } + if "sha256:"+sumHex(packed) != digest { + t.Fatalf("a second packing of one pin gave another digest: %s and %s", digest, sumHex(packed)) + } + entries := entriesOf(t, packed) + want := map[string]string{ + "application.ini": "[App]\nVersion=153.4.0\n -rw-r--r--", + "betterbird": "#!/bin/sh\n -rwxr-xr-x", + "defaults/pref/channel-prefs.js": "pref(\"app.update.channel\", \"default\");\n -rw-r--r--", + } + if len(entries) != len(want) { + t.Fatalf("the archive holds %v, want %v", entries, want) + } + for name, w := range want { + if entries[name] != w { + t.Errorf("%s is %q in the archive, want %q", name, entries[name], w) + } + } +} + +func TestAnUpstreamFileIsFetchedOnceAndKeptByItsDigest(t *testing.T) { + // The vendor of an add-on serves only its newest version at one address: a pin that was fetched once + // must still build after the address moved on. + body := release(t) + v := &vendor{files: map[string][]byte{"/release.tar.gz": body}} + base := serveVendor(t, v) + r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil) + first, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + v.files["/release.tar.gz"] = []byte("the vendor's next release") + second, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatalf("a pin fetched before failed to build once the address served something else: %v", err) + } + if v.asked != 1 { + t.Fatalf("the vendor was asked %d times for one pin, want once", v.asked) + } + if first.Manifest.Resources[0]["digest"] != second.Manifest.Resources[0]["digest"] { + t.Fatal("two builds of one pin gave two digests") + } +} + +func TestAnUpstreamFileWhoseDownloadDoesNotMatchItsPinFailsAndNamesBothDigests(t *testing.T) { + pinned := release(t) + served := gzipped(t, tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "not what was reviewed", mode: 0o755}})) + v := &vendor{files: map[string][]byte{"/release.tar.gz": served}} + base := serveVendor(t, v) + r, workspace := aRepository(t, pinnedManifest(base, sumHex(pinned)), nil) + _, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err == nil { + t.Fatal("a download that is not the pinned file was built") + } + for _, digest := range []string{sumHex(pinned), sumHex(served)} { + if !strings.Contains(err.Error(), "sha256:"+digest) { + t.Errorf("the refusal does not name sha256:%s: %v", digest, err) + } + } + if len(r.archives) != 0 { + t.Fatalf("something was published: %v", r.archives) + } + kept, _ := filepath.Glob(filepath.Join(upstreamDir(workspace), "*")) + if len(kept) != 0 { + t.Fatalf("the refused download was kept: %v", kept) + } +} + +func TestAnUpstreamArchiveCompressedWithXzIsRead(t *testing.T) { + // Betterbird publishes its Linux release as .tar.xz; the build seat's image carries xz for it. + if _, err := exec.LookPath("xz"); err != nil { + t.Skip("NOT TESTED here: no xz on this machine (the build seat's image carries it)") + } + plain := tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "#!/bin/sh\n", mode: 0o755}}) + cmd := exec.Command("xz", "-zc") + cmd.Stdin = bytes.NewReader(plain) + compressed, err := cmd.Output() + if err != nil { + t.Fatal(err) + } + v := &vendor{files: map[string][]byte{"/r.tar.xz": compressed}} + base := serveVendor(t, v) + packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive", + URL: base + "/r.tar.xz", SHA256: sumHex(compressed), Version: "1", Within: "betterbird"}, func(string, string, ...any) {}) + if err != nil { + t.Fatal(err) + } + var names []string + for n := range entriesOf(t, packed) { + names = append(names, n) + } + sort.Strings(names) + if strings.Join(names, ",") != "betterbird" { + t.Fatalf("the xz release unpacked to %v", names) + } +} + +func TestAnUpstreamTarWithAGlobalHeaderNestedWithinAndTrailingDataIsRead(t *testing.T) { + // git archive writes a pax global header; a release may unpack under a/b with a/ listed first; xz may be + // followed by bytes after the tar's end, which must be read before xz is waited for. + var raw bytes.Buffer + w := tar.NewWriter(&raw) + w.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, Name: "pax_global_header", PAXRecords: map[string]string{"comment": "abc"}}) + w.WriteHeader(&tar.Header{Name: "a/", Typeflag: tar.TypeDir, Mode: 0o755}) + w.WriteHeader(&tar.Header{Name: "a/b/", Typeflag: tar.TypeDir, Mode: 0o755}) + w.WriteHeader(&tar.Header{Name: "a/b/f", Typeflag: tar.TypeReg, Mode: 0o644, Size: 1}) + w.Write([]byte("x")) + w.Close() + body := gzipped(t, raw.Bytes()) + v := &vendor{files: map[string][]byte{"/r.tar.gz": body}} + base := serveVendor(t, v) + packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive", + URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1", Within: "a/b"}, func(string, string, ...any) {}) + if err != nil { + t.Fatal(err) + } + if e := entriesOf(t, packed); len(e) != 1 || e["f"] == "" { + t.Fatalf("%v", e) + } + if _, err := exec.LookPath("xz"); err != nil { + t.Skip("NOT TESTED here: no xz") + } + // A megabyte after the tar's end: more than a pipe holds. + trailing := append(append([]byte{}, raw.Bytes()...), bytes.Repeat([]byte{0}, 1<<20)...) + cmd := exec.Command("xz", "-zc") + cmd.Stdin = bytes.NewReader(trailing) + compressed, err := cmd.Output() + if err != nil { + t.Fatal(err) + } + v.files["/r.tar.xz"] = compressed + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if _, err := upstreamArchive(ctx, t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive", URL: base + "/r.tar.xz", + SHA256: sumHex(compressed), Version: "1", Within: "a/b"}, func(string, string, ...any) {}); err != nil { + t.Fatal(err) + } +} diff --git a/internal/builder/upstream_file_test.go b/internal/builder/upstream_file_test.go index 34e82f1e..4cb07a49 100644 --- a/internal/builder/upstream_file_test.go +++ b/internal/builder/upstream_file_test.go @@ -15,13 +15,10 @@ import ( "net/http" "net/http/httptest" "os" - "os/exec" "path/filepath" "runtime" - "sort" "strings" "testing" - "time" "github.com/novox/mesh-controller/internal/catalogue" ) @@ -153,98 +150,6 @@ func pinnedManifest(base, sum string) string { "resources":[{"id":"installation","type":"archive","path":"/opt/betterbird","artifact":"archive"}]}` } -func TestAnArchivePinnedByURLIsFetchedCheckedAndPackedWithoutItsOwnDirectory(t *testing.T) { - body := release(t) - v := &vendor{files: map[string][]byte{"/release.tar.gz": body}} - base := serveVendor(t, v) - r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil, - Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - digest, _ := got.Manifest.Resources[0]["digest"].(string) - if !strings.HasPrefix(digest, "sha256:") || r.archives["betterbird/archive"] != digest { - t.Fatalf("the release was not published and pinned as the module's archive: %v, published %v", - got.Manifest.Resources[0], r.archives) - } - - packed, err := upstreamArchive(context.Background(), workspace, catalogue.Artifact{Name: "archive", Kind: "archive", - URL: base + "/release.tar.gz", SHA256: sumHex(body), Version: "153.4.0esr-bb10", Within: "betterbird"}, - func(string, string, ...any) {}) - if err != nil { - t.Fatal(err) - } - if "sha256:"+sumHex(packed) != digest { - t.Fatalf("a second packing of one pin gave another digest: %s and %s", digest, sumHex(packed)) - } - entries := entriesOf(t, packed) - want := map[string]string{ - "application.ini": "[App]\nVersion=153.4.0\n -rw-r--r--", - "betterbird": "#!/bin/sh\n -rwxr-xr-x", - "defaults/pref/channel-prefs.js": "pref(\"app.update.channel\", \"default\");\n -rw-r--r--", - } - if len(entries) != len(want) { - t.Fatalf("the archive holds %v, want %v", entries, want) - } - for name, w := range want { - if entries[name] != w { - t.Errorf("%s is %q in the archive, want %q", name, entries[name], w) - } - } -} - -func TestAnUpstreamFileIsFetchedOnceAndKeptByItsDigest(t *testing.T) { - // The vendor of an add-on serves only its newest version at one address: a pin that was fetched once - // must still build after the address moved on. - body := release(t) - v := &vendor{files: map[string][]byte{"/release.tar.gz": body}} - base := serveVendor(t, v) - r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil) - first, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil, - Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - v.files["/release.tar.gz"] = []byte("the vendor's next release") - second, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil, - Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatalf("a pin fetched before failed to build once the address served something else: %v", err) - } - if v.asked != 1 { - t.Fatalf("the vendor was asked %d times for one pin, want once", v.asked) - } - if first.Manifest.Resources[0]["digest"] != second.Manifest.Resources[0]["digest"] { - t.Fatal("two builds of one pin gave two digests") - } -} - -func TestAnUpstreamFileWhoseDownloadDoesNotMatchItsPinFailsAndNamesBothDigests(t *testing.T) { - pinned := release(t) - served := gzipped(t, tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "not what was reviewed", mode: 0o755}})) - v := &vendor{files: map[string][]byte{"/release.tar.gz": served}} - base := serveVendor(t, v) - r, workspace := aRepository(t, pinnedManifest(base, sumHex(pinned)), nil) - _, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil, - Npmrc{}, GitCredential{}, nil) - if err == nil { - t.Fatal("a download that is not the pinned file was built") - } - for _, digest := range []string{sumHex(pinned), sumHex(served)} { - if !strings.Contains(err.Error(), "sha256:"+digest) { - t.Errorf("the refusal does not name sha256:%s: %v", digest, err) - } - } - if len(r.archives) != 0 { - t.Fatalf("something was published: %v", r.archives) - } - kept, _ := filepath.Glob(filepath.Join(upstreamDir(workspace), "*")) - if len(kept) != 0 { - t.Fatalf("the refused download was kept: %v", kept) - } -} - func TestAnUpstreamFileThatIsNotAnArchiveIsKeptWholeUnderItsName(t *testing.T) { xpi := []byte("PK\x03\x04 an add-on") v := &vendor{files: map[string][]byte{"/owl.xpi": xpi}} @@ -282,78 +187,10 @@ func TestAnUpstreamArchiveHoldingALinkOrAnEntryOutsideItsDirectoryIsRefused(t *t } } -func TestAnUpstreamArchiveCompressedWithXzIsRead(t *testing.T) { - // Betterbird publishes its Linux release as .tar.xz; the build seat's image carries xz for it. - if _, err := exec.LookPath("xz"); err != nil { - t.Skip("NOT TESTED here: no xz on this machine (the build seat's image carries it)") - } - plain := tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "#!/bin/sh\n", mode: 0o755}}) - cmd := exec.Command("xz", "-zc") - cmd.Stdin = bytes.NewReader(plain) - compressed, err := cmd.Output() - if err != nil { - t.Fatal(err) - } - v := &vendor{files: map[string][]byte{"/r.tar.xz": compressed}} - base := serveVendor(t, v) - packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive", - URL: base + "/r.tar.xz", SHA256: sumHex(compressed), Version: "1", Within: "betterbird"}, func(string, string, ...any) {}) - if err != nil { - t.Fatal(err) - } - var names []string - for n := range entriesOf(t, packed) { - names = append(names, n) - } - sort.Strings(names) - if strings.Join(names, ",") != "betterbird" { - t.Fatalf("the xz release unpacked to %v", names) - } -} - -func TestAnUpstreamTarWithAGlobalHeaderNestedWithinAndTrailingDataIsRead(t *testing.T) { - // git archive writes a pax global header; a release may unpack under a/b with a/ listed first; xz may be - // followed by bytes after the tar's end, which must be read before xz is waited for. - var raw bytes.Buffer - w := tar.NewWriter(&raw) - w.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, Name: "pax_global_header", PAXRecords: map[string]string{"comment": "abc"}}) - w.WriteHeader(&tar.Header{Name: "a/", Typeflag: tar.TypeDir, Mode: 0o755}) - w.WriteHeader(&tar.Header{Name: "a/b/", Typeflag: tar.TypeDir, Mode: 0o755}) - w.WriteHeader(&tar.Header{Name: "a/b/f", Typeflag: tar.TypeReg, Mode: 0o644, Size: 1}) - w.Write([]byte("x")) - w.Close() - body := gzipped(t, raw.Bytes()) - v := &vendor{files: map[string][]byte{"/r.tar.gz": body}} - base := serveVendor(t, v) - packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive", - URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1", Within: "a/b"}, func(string, string, ...any) {}) - if err != nil { - t.Fatal(err) - } - if e := entriesOf(t, packed); len(e) != 1 || e["f"] == "" { - t.Fatalf("%v", e) - } - if _, err := exec.LookPath("xz"); err != nil { - t.Skip("NOT TESTED here: no xz") - } - // A megabyte after the tar's end: more than a pipe holds. - trailing := append(append([]byte{}, raw.Bytes()...), bytes.Repeat([]byte{0}, 1<<20)...) - cmd := exec.Command("xz", "-zc") - cmd.Stdin = bytes.NewReader(trailing) - compressed, err := cmd.Output() - if err != nil { - t.Fatal(err) - } - v.files["/r.tar.xz"] = compressed - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - if _, err := upstreamArchive(ctx, t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive", URL: base + "/r.tar.xz", - SHA256: sumHex(compressed), Version: "1", Within: "a/b"}, func(string, string, ...any) {}); err != nil { - t.Fatal(err) - } -} - func TestAnUpstreamArchiveWithTooManyEntriesIsRefused(t *testing.T) { + before := maxEntries + maxEntries = 100 + t.Cleanup(func() { maxEntries = before }) var raw bytes.Buffer w := tar.NewWriter(&raw) for i := 0; i <= maxEntries; i++ { diff --git a/internal/builder/xz_framing_integration_test.go b/internal/builder/xz_framing_integration_test.go new file mode 100644 index 00000000..8192e3e9 --- /dev/null +++ b/internal/builder/xz_framing_integration_test.go @@ -0,0 +1,40 @@ +//go:build integration + +package builder + +import ( + "bytes" + "context" + "os" + "os/exec" + "path/filepath" + "testing" +) + +func TestAnXzFileOfAFewDeltaBlocksAndTwoStreamsIsRead(t *testing.T) { + // The streams this test builds are ones the reader decodes, so what the walk refuses below is the + // same framing the reader follows. + body := bytes.Repeat([]byte("betterbird "), 300) + whole := tarHolding(t, body) + half := len(whole) / 2 + for name, file := range map[string][]byte{ + "one stream of delta blocks": xzStream(whole, 512, true, -1), + "two streams with padding": append(append(xzStream(whole[:half], 700, true, -1), 0, 0, 0, 0), + xzStream(whole[half:], 100, false, -1)...), + } { + path := writeTemp(t, file) + if xzPath, err := exec.LookPath("xz"); err == nil { + if out, err := exec.Command(xzPath, "-t", path).CombinedOutput(); err != nil { + t.Fatalf("%s: xz itself refuses the built stream: %v %s", name, err, out) + } + } + into := t.TempDir() + if err := unpackUpstream(context.Background(), path, into, "betterbird"); err != nil { + t.Fatalf("%s: %v", name, err) + } + got, err := os.ReadFile(filepath.Join(into, "data")) + if err != nil || !bytes.Equal(got, body) { + t.Fatalf("%s: unpacked %d bytes, want %d: %v", name, len(got), len(body), err) + } + } +} diff --git a/internal/builder/xz_framing_test.go b/internal/builder/xz_framing_test.go index a672cb80..558ded0f 100644 --- a/internal/builder/xz_framing_test.go +++ b/internal/builder/xz_framing_test.go @@ -9,7 +9,6 @@ import ( "errors" "hash/crc32" "os" - "os/exec" "path/filepath" "runtime" "strings" @@ -125,34 +124,6 @@ func writeTemp(t *testing.T, body []byte) string { return file } -func TestAnXzFileOfAFewDeltaBlocksAndTwoStreamsIsRead(t *testing.T) { - // The streams this test builds are ones the reader decodes, so what the walk refuses below is the - // same framing the reader follows. - body := bytes.Repeat([]byte("betterbird "), 300) - whole := tarHolding(t, body) - half := len(whole) / 2 - for name, file := range map[string][]byte{ - "one stream of delta blocks": xzStream(whole, 512, true, -1), - "two streams with padding": append(append(xzStream(whole[:half], 700, true, -1), 0, 0, 0, 0), - xzStream(whole[half:], 100, false, -1)...), - } { - path := writeTemp(t, file) - if xzPath, err := exec.LookPath("xz"); err == nil { - if out, err := exec.Command(xzPath, "-t", path).CombinedOutput(); err != nil { - t.Fatalf("%s: xz itself refuses the built stream: %v %s", name, err, out) - } - } - into := t.TempDir() - if err := unpackUpstream(context.Background(), path, into, "betterbird"); err != nil { - t.Fatalf("%s: %v", name, err) - } - got, err := os.ReadFile(filepath.Join(into, "data")) - if err != nil || !bytes.Equal(got, body) { - t.Fatalf("%s: unpacked %d bytes, want %d: %v", name, len(got), len(body), err) - } - } -} - func TestAnXzFileOfManyDeltaBlocksIsRefusedBeforeItIsDecoded(t *testing.T) { // Review 68 on pull request 264: `xz --block-size=1 --delta` of a 200 kB tar kept about 67 MiB live in // the reader, one delta decoder per block. Its index here lists one block, as a file crafted to pass a diff --git a/internal/catalogue/environment_into_integration_test.go b/internal/catalogue/environment_into_integration_test.go new file mode 100644 index 00000000..b41864ec --- /dev/null +++ b/internal/catalogue/environment_into_integration_test.go @@ -0,0 +1,91 @@ +//go:build integration + +package catalogue + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestThePOSIXEnvironmentSourcedTwiceLeavesPATHAsOnce(t *testing.T) { + sh, err := exec.LookPath("sh") + if err != nil { + t.Skip("no sh on this machine") + } + script := "PATH=/usr/bin:/bin\n" + composedPOSIX + "once=$PATH\n" + composedPOSIX + + `[ "$PATH" = "$once" ] || { echo "changed: $once -> $PATH"; exit 1; }` + "\n" + + `echo "$PATH"; echo "$GOPATH"` + out, err := exec.Command(sh, "-c", script).CombinedOutput() + if err != nil { + t.Fatalf("sourcing twice: %v\n%s", err, out) + } + lines := strings.Split(strings.TrimSpace(string(out)), "\n") + want := "/home/op/go/bin:/usr/local/go/bin:/home/op/.local/bin:/home/op/bin:/usr/bin:/bin:/opt/agent/bin:/opt/scripts" + if lines[0] != want { + t.Fatalf("PATH is %s, not %s", lines[0], want) + } + if lines[1] != "/home/op/go" { + t.Fatalf("GOPATH was not exported: %q", lines[1]) + } + // And an entry the account already has stays where it is, and once. + out, err = exec.Command(sh, "-c", "PATH=/opt/scripts:/usr/bin\n"+composedPOSIX+`echo "$PATH"`).CombinedOutput() + if err != nil { + t.Fatalf("%v\n%s", err, out) + } + if got := strings.TrimSpace(string(out)); got != + "/home/op/go/bin:/usr/local/go/bin:/home/op/.local/bin:/home/op/bin:/opt/scripts:/usr/bin:/opt/agent/bin" { + t.Fatalf("an entry already on PATH was added again or moved: %s", got) + } +} + +func TestTheServiceManagerReadsTheSystemdRenderingAsMeant(t *testing.T) { + generator := "/usr/lib/systemd/user-environment-generators/30-systemd-environment-d-generator" + if _, err := os.Stat(generator); err != nil { + t.Skip("NOT JUDGED: no environment.d generator on this machine, so the service manager's reading " + + "of the rendering is judged only where systemd runs; the rendering itself is held byte for byte " + + "by TestTheEnvironmentRendersForTheServiceManagerByteForByte") + } + const base = "/usr/bin:/bin" + read := func(conf string) map[string]string { + t.Helper() + config := t.TempDir() + if err := os.MkdirAll(filepath.Join(config, "environment.d"), 0o755); err != nil { + t.Fatal(err) + } + if conf != "" { + if err := os.WriteFile(filepath.Join(config, "environment.d", "50-mesh.conf"), []byte(conf), 0o644); err != nil { + t.Fatal(err) + } + } + cmd := exec.Command(generator) + cmd.Env = []string{"PATH=" + base, "HOME=" + config, "XDG_CONFIG_HOME=" + config} + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("%v\n%s", err, out) + } + vars := map[string]string{} + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + if k, v, ok := strings.Cut(line, "="); ok { + vars[k] = v + } + } + return vars + } + machine, read50 := read(""), read(composedSystemd) + + // What the machine's own files do to PATH: nothing, or append to it. One that replaces or prepends + // leaves nothing this test can say about the mesh's file, and says so rather than guess. + added, ok := strings.CutPrefix(machine["PATH"], base) + if machine["PATH"] != "" && !ok { + t.Skipf("NOT JUDGED: this machine's own environment.d sets PATH to %s, not %s with something after it, so "+ + "what the mesh's file adds cannot be told apart from it", machine["PATH"], base) + } + want := "/home/op/go/bin:/usr/local/go/bin:/home/op/.local/bin:/home/op/bin:/usr/bin:/bin:/opt/agent/bin:/opt/scripts" + added + if read50["PATH"] != want || read50["GOPATH"] != "/home/op/go" { + t.Fatalf("the service manager read PATH=%s GOPATH=%s, not PATH=%s GOPATH=/home/op/go (the machine's own "+ + "files add %q to PATH)", read50["PATH"], read50["GOPATH"], want, added) + } +} diff --git a/internal/catalogue/environment_into_test.go b/internal/catalogue/environment_into_test.go index 1d428f5c..ca112e9b 100644 --- a/internal/catalogue/environment_into_test.go +++ b/internal/catalogue/environment_into_test.go @@ -1,9 +1,6 @@ package catalogue import ( - "os" - "os/exec" - "path/filepath" "strings" "testing" ) @@ -101,36 +98,6 @@ func TestTheEnvironmentRendersForTheServiceManagerByteForByte(t *testing.T) { // Sourcing twice changes nothing (ADR 0203 §3): a login shell that starts another reads the file // again, and a PATH that grew each time would be the symptom. Run by a real `sh`, because the claim // is about what a shell does with the file, not about what the file looks like. -func TestThePOSIXEnvironmentSourcedTwiceLeavesPATHAsOnce(t *testing.T) { - sh, err := exec.LookPath("sh") - if err != nil { - t.Skip("no sh on this machine") - } - script := "PATH=/usr/bin:/bin\n" + composedPOSIX + "once=$PATH\n" + composedPOSIX + - `[ "$PATH" = "$once" ] || { echo "changed: $once -> $PATH"; exit 1; }` + "\n" + - `echo "$PATH"; echo "$GOPATH"` - out, err := exec.Command(sh, "-c", script).CombinedOutput() - if err != nil { - t.Fatalf("sourcing twice: %v\n%s", err, out) - } - lines := strings.Split(strings.TrimSpace(string(out)), "\n") - want := "/home/op/go/bin:/usr/local/go/bin:/home/op/.local/bin:/home/op/bin:/usr/bin:/bin:/opt/agent/bin:/opt/scripts" - if lines[0] != want { - t.Fatalf("PATH is %s, not %s", lines[0], want) - } - if lines[1] != "/home/op/go" { - t.Fatalf("GOPATH was not exported: %q", lines[1]) - } - // And an entry the account already has stays where it is, and once. - out, err = exec.Command(sh, "-c", "PATH=/opt/scripts:/usr/bin\n"+composedPOSIX+`echo "$PATH"`).CombinedOutput() - if err != nil { - t.Fatalf("%v\n%s", err, out) - } - if got := strings.TrimSpace(string(out)); got != - "/home/op/go/bin:/usr/local/go/bin:/home/op/.local/bin:/home/op/bin:/opt/scripts:/usr/bin:/opt/agent/bin" { - t.Fatalf("an entry already on PATH was added again or moved: %s", got) - } -} // The environment.d rendering, read by the service manager's own generator where this machine has // one — the same reader an account's user manager runs, so the PATH it composes is the one asserted. @@ -143,54 +110,6 @@ func TestThePOSIXEnvironmentSourcedTwiceLeavesPATHAsOnce(t *testing.T) { // A machine without the generator — the build seat's toolchain image holds no systemd — cannot judge // this and says so: there the rendering is held byte for byte by // TestTheEnvironmentRendersForTheServiceManagerByteForByte, and this reading only where systemd runs. -func TestTheServiceManagerReadsTheSystemdRenderingAsMeant(t *testing.T) { - generator := "/usr/lib/systemd/user-environment-generators/30-systemd-environment-d-generator" - if _, err := os.Stat(generator); err != nil { - t.Skip("NOT JUDGED: no environment.d generator on this machine, so the service manager's reading " + - "of the rendering is judged only where systemd runs; the rendering itself is held byte for byte " + - "by TestTheEnvironmentRendersForTheServiceManagerByteForByte") - } - const base = "/usr/bin:/bin" - read := func(conf string) map[string]string { - t.Helper() - config := t.TempDir() - if err := os.MkdirAll(filepath.Join(config, "environment.d"), 0o755); err != nil { - t.Fatal(err) - } - if conf != "" { - if err := os.WriteFile(filepath.Join(config, "environment.d", "50-mesh.conf"), []byte(conf), 0o644); err != nil { - t.Fatal(err) - } - } - cmd := exec.Command(generator) - cmd.Env = []string{"PATH=" + base, "HOME=" + config, "XDG_CONFIG_HOME=" + config} - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("%v\n%s", err, out) - } - vars := map[string]string{} - for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { - if k, v, ok := strings.Cut(line, "="); ok { - vars[k] = v - } - } - return vars - } - machine, read50 := read(""), read(composedSystemd) - - // What the machine's own files do to PATH: nothing, or append to it. One that replaces or prepends - // leaves nothing this test can say about the mesh's file, and says so rather than guess. - added, ok := strings.CutPrefix(machine["PATH"], base) - if machine["PATH"] != "" && !ok { - t.Skipf("NOT JUDGED: this machine's own environment.d sets PATH to %s, not %s with something after it, so "+ - "what the mesh's file adds cannot be told apart from it", machine["PATH"], base) - } - want := "/home/op/go/bin:/usr/local/go/bin:/home/op/.local/bin:/home/op/bin:/usr/bin:/bin:/opt/agent/bin:/opt/scripts" + added - if read50["PATH"] != want || read50["GOPATH"] != "/home/op/go" { - t.Fatalf("the service manager read PATH=%s GOPATH=%s, not PATH=%s GOPATH=/home/op/go (the machine's own "+ - "files add %q to PATH)", read50["PATH"], read50["GOPATH"], want, added) - } -} // Nothing contributed renders nothing, in both formats — not an empty `export PATH`. func TestNoContributionsRenderNothing(t *testing.T) { diff --git a/internal/catalogue/mounts_test.go b/internal/catalogue/mounts_test.go index d0e94a84..f929a5d3 100644 --- a/internal/catalogue/mounts_test.go +++ b/internal/catalogue/mounts_test.go @@ -75,8 +75,9 @@ func TestEveryCatalogueManifestDeclaresWhatItMounts(t *testing.T) { if len(files) == 0 { t.Fatal("no manifests in the catalogue, so this proved nothing") } + root := beside.Dir(t, "mesh-catalog") // once: it copies the captured tree for _, file := range files { - if _, err := ReadTreeManifest(beside.Dir(t, "mesh-catalog"), filepath.Dir(file)); err != nil { + if _, err := ReadTreeManifest(root, filepath.Dir(file)); err != nil { t.Errorf("%s: %v", filepath.Base(filepath.Dir(file)), err) } } diff --git a/internal/inventory/adoption_test.go b/internal/inventory/adoption_test.go index c779d322..b2f83741 100644 --- a/internal/inventory/adoption_test.go +++ b/internal/inventory/adoption_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/agent_account_test.go b/internal/inventory/agent_account_test.go index 44139bb8..f7ccab1a 100644 --- a/internal/inventory/agent_account_test.go +++ b/internal/inventory/agent_account_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/artifacts_test.go b/internal/inventory/artifacts_test.go index 5deef9cc..3160866e 100644 --- a/internal/inventory/artifacts_test.go +++ b/internal/inventory/artifacts_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/bindings_test.go b/internal/inventory/bindings_test.go index 4767cbf1..9e1e778f 100644 --- a/internal/inventory/bindings_test.go +++ b/internal/inventory/bindings_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/builds_test.go b/internal/inventory/builds_test.go index 8608ff0b..fbdfd08a 100644 --- a/internal/inventory/builds_test.go +++ b/internal/inventory/builds_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/busrecords_integration_test.go b/internal/inventory/busrecords_integration_test.go new file mode 100644 index 00000000..02fad1cf --- /dev/null +++ b/internal/inventory/busrecords_integration_test.go @@ -0,0 +1,316 @@ +//go:build integration + +package inventory + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "golang.org/x/crypto/bcrypt" +) + +func aMeshWith(t *testing.T, manifests ...catalogue.Manifest) (*Inventory, context.Context) { + t.Helper() + inv := ForTest(t) + ctx := context.Background() + for _, m := range manifests { + if err := inv.RegisterModule(ctx, m, Source{Repository: "/r"}); err != nil { + t.Fatal(err) + } + } + return inv, ctx +} + +func TestAnAssignedModuleBecomesAUserWithWhatItDeclared(t *testing.T) { + // It declares where its account is delivered: a module with no own secret named broker can never + // be issued one, and is no user at all (novox/hq issue 195) — the case asserted below. + shop := catalogue.Manifest{ + Module: "shop", Version: "1", + Emits: []string{"order.placed"}, Tools: []string{"price"}, + Uses: []string{"telegram-sender"}, + OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/run/broker"}}, + } + quiet := catalogue.Manifest{Module: "quiet", Version: "1", Emits: []string{"thing.happened"}} + inv, ctx := aMeshWith(t, theSeatDeclarer(), shop, quiet) + if _, err := inv.AddNode(ctx, "one"); err != nil { + t.Fatal(err) + } + for _, module := range []string{"shop", "quiet"} { + if _, err := inv.Assign(ctx, "one", module); err != nil { + t.Fatal(err) + } + } + + records, err := inv.BusRecords(ctx) + if err != nil { + t.Fatal(err) + } + var on []broker.Declared + for _, d := range records.Assigned["one"] { + if d.Module == "shop" { + on = append(on, d) + } + } + if len(on) != 1 || on[0].NoAccount { + t.Fatalf("the machine's modules read as %+v", records.Assigned["one"]) + } + if len(on[0].Uses) != 1 || on[0].Uses[0].Accepts[0] != "send" { + t.Fatalf("the seat it uses carries no protocol: %+v — so it would be granted nothing on a "+ + "seat it was assigned to send to", on[0].Uses) + } + if len(on[0].Serves) != 1 || on[0].Serves[0] != "price" { + t.Fatalf("its tools read as %v, and a module that cannot subscribe its own tool subject "+ + "serves nothing", on[0].Serves) + } + + // And it derives into a user the server would accept. + users, err := broker.Users(records) + if err != nil { + t.Fatal(err) + } + var found bool + for _, u := range users { + if u.Username() != "one.shop" { + continue + } + found = true + perms, err := broker.PermissionsFor(u) + if err != nil { + t.Fatal(err) + } + // Its tools are every one under its own name — the list in the manifest is a person's + // vocabulary for asking, not the module's permission to answer. + if !granted(perms.Publish, "mesh.mod.shop.event.order.placed") || + !granted(perms.Publish, "mesh.seat.telegram-sender.accept.send") || + !granted(perms.Subscribe, "mesh.mod.shop.tool.>") { + t.Fatalf("one.shop's authority is not what it declared: %+v", perms) + } + } + if !found { + t.Fatal("no user was derived for the assigned module") + } + for _, u := range users { + if u.Username() == "one.quiet" { + t.Fatal("a module with nowhere to read an account was made a user (novox/hq issue 195)") + } + } +} + +func TestOnlyAMachineWithALiveTokenHasAnEnrolmentUser(t *testing.T) { + inv, ctx := aMeshWith(t) + for _, name := range []string{"live", "expired", "none"} { + if _, err := inv.AddNode(ctx, name); err != nil { + t.Fatal(err) + } + } + if _, err := inv.IssueToken(ctx, "live", time.Hour); err != nil { + t.Fatal(err) + } + // Briefly, then waited out: a token with no lifetime is refused at issue, which is the right + // refusal and leaves this as the way to have an expired one. + if _, err := inv.IssueToken(ctx, "expired", 10*time.Millisecond); err != nil { + t.Fatal(err) + } + time.Sleep(50 * time.Millisecond) + + records, err := inv.BusRecords(ctx) + if err != nil { + t.Fatal(err) + } + if strings.Join(records.Enrolling, ",") != "live" { + t.Fatalf("machines with a live token read as %v", records.Enrolling) + } +} + +func TestIssuingATokenRecordsTheAccountItIsThePasswordOf(t *testing.T) { + inv, ctx := aMeshWith(t) + if _, err := inv.AddNode(ctx, "joiner"); err != nil { + t.Fatal(err) + } + issued, err := inv.IssueToken(ctx, "joiner", time.Hour) + if err != nil { + t.Fatal(err) + } + + name := broker.Principal{Kind: broker.KindEnrolment, Node: "joiner"}.Username() + users, err := inv.BusUsers(ctx) + if err != nil { + t.Fatal(err) + } + user, has := users[name] + if !has { + t.Fatalf("no bus account for %q; the composer would leave the enrolment out and the "+ + "machine would be refused before the mesh heard of it: %v", name, users) + } + if user.Kind != BusEnrolment || user.Node != "joiner" { + t.Errorf("the account is %+v, not this node's enrolment", user) + } + if err := bcrypt.CompareHashAndPassword([]byte(user.PasswordHash), []byte(issued.Secret)); err != nil { + t.Error("the account does not accept the secret the token carries, so presenting the " + + "token would be refused by the server") + } + + // And the composition contains it, which is the thing the server reads. + records, err := inv.BusRecords(ctx) + if err != nil { + t.Fatal(err) + } + derived, err := broker.Users(records) + if err != nil { + t.Fatal(err) + } + hashes := map[string]string{} + for n, u := range users { + hashes[n] = u.PasswordHash + } + _, missing := broker.WithPasswords(derived, hashes) + for _, m := range missing { + if m == name { + t.Fatal("the enrolment user is composed without a password, which is a user nobody can be") + } + } +} + +func TestTheBussOwnModuleBecomesTheSnapshotUser(t *testing.T) { + bus := catalogue.Manifest{ + Module: "nats", Version: "1", + Provides: []catalogue.Offer{{Name: "mesh-bus", Scope: catalogue.ScopeMesh}}, + Claims: []catalogue.Claim{{Name: catalogue.BrokerSeat, Scope: catalogue.ScopeMesh}}, + Tools: []string{"nats_streams"}, + OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/run/broker"}}, + } + shop := catalogue.Manifest{Module: "shop", Version: "1", Emits: []string{"order.placed"}, + OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/run/broker"}}} + inv, ctx := aMeshWith(t, bus, shop) + if _, err := inv.AddNode(ctx, "one"); err != nil { + t.Fatal(err) + } + for _, module := range []string{"nats", "shop"} { + if _, err := inv.Assign(ctx, "one", module); err != nil { + t.Fatal(err) + } + } + records, err := inv.BusRecords(ctx) + if err != nil { + t.Fatal(err) + } + users, err := broker.Users(records) + if err != nil { + t.Fatal(err) + } + seen := map[string]bool{} + for _, u := range users { + perms, err := broker.PermissionsFor(u) + if err != nil { + t.Fatal(err) + } + snapshots := granted(perms.Publish, "$JS.API.STREAM.SNAPSHOT.*") + switch u.Username() { + case "one.nats": + seen[u.Username()] = true + if !snapshots || len(perms.Publish) != len(broker.BusSnapshotGrants.Publish) { + t.Fatalf("the bus's own user is granted %v, not the snapshot API alone", perms.Publish) + } + case "one.shop": + seen[u.Username()] = true + if snapshots { + t.Fatal("a module that is not the bus may snapshot it") + } + } + } + if !seen["one.nats"] || !seen["one.shop"] { + t.Fatalf("users derived: %v", seen) + } +} + +func TestABarIsGrantedTheStateItsMachinesModulesOfferIt(t *testing.T) { + bar := catalogue.Manifest{Module: "a-bar", Version: "1", + Claims: []catalogue.Claim{{Name: catalogue.BarSeat, Scope: catalogue.ScopeNode}}} + power := catalogue.Manifest{Module: "power", Version: "1", + State: []catalogue.StateDeclaration{{Name: "draw"}}, + Contributions: []catalogue.SeatContribution{{Seat: catalogue.BarSeat, Kind: catalogue.BarKindBlock, + Data: map[string]any{"bar": "bottom", "place": "status", "shows": "state", + "options": map[string]any{"state": "power.draw"}}}}} + inv, ctx := aMeshWith(t, bar, power) + for node, modules := range map[string][]string{"laptop": {"a-bar", "power"}, "desk": {"a-bar"}} { + if _, err := inv.AddNode(ctx, node); err != nil { + t.Fatal(err) + } + for _, module := range modules { + if _, err := inv.Assign(ctx, node, module); err != nil { + t.Fatal(err) + } + } + } + records, err := inv.BusRecords(ctx) + if err != nil { + t.Fatal(err) + } + reads := func(node, module string) []broker.KeyedRead { + for _, d := range records.Assigned[node] { + if d.Module == module { + if len(d.Reads) != 0 { + t.Fatalf("%s on %s reads whole buckets: %v", module, node, d.Reads) + } + return d.KeyedReads + } + } + t.Fatalf("%s is not on %s", module, node) + return nil + } + // The laptop's own key, and no other (novox/hq ADR 0260). + if got := reads("laptop", "a-bar"); len(got) != 1 || got[0] != (broker.KeyedRead{Read: "power.draw", Key: "laptop"}) { + t.Fatalf("the laptop's bar reads %v", got) + } + if got := reads("desk", "a-bar"); len(got) != 0 { + t.Fatalf("a bar with no power beside it reads %v", got) + } + if got := reads("laptop", "power"); len(got) != 0 { + t.Fatalf("power reads %v", got) + } +} + +func TestTheProvisionerClientsMachineHasAnAccountOfItsOwn(t *testing.T) { + provisioner := catalogue.Manifest{ + Module: "mesh-provisioner", Version: "1", + DefinesSeats: []catalogue.SeatDeclaration{{Name: broker.NodeProvisionerSeat, Scope: catalogue.ScopeNode, + Serves: []catalogue.Verb{{Name: "get"}}}}, + Claims: []catalogue.Claim{{Name: broker.NodeProvisionerSeat, Scope: catalogue.ScopeNode}}, + OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/run/broker"}, + broker.ProvisionerSecret: {Path: "/run/provisioner-broker"}}, + } + unread := provisioner + unread.Module = "provisioner-unread" + unread.DefinesSeats = nil + unread.OwnSecrets = catalogue.OwnSecrets{"broker": {Path: "/run/broker"}} + inv, ctx := aMeshWith(t, provisioner, unread) + for node, module := range map[string]string{"one": "mesh-provisioner", "two": "provisioner-unread"} { + if _, err := inv.AddNode(ctx, node); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, node, module); err != nil { + t.Fatal(err) + } + } + records, err := inv.BusRecords(ctx) + if err != nil { + t.Fatal(err) + } + users, err := broker.Users(records) + if err != nil { + t.Fatal(err) + } + var clients []string + for _, u := range users { + if u.Kind == broker.KindProvisioner { + clients = append(clients, u.Username()+" for "+u.Module) + } + } + if strings.Join(clients, ",") != "provisioner.one for mesh-provisioner" { + t.Fatalf("the provisioner clients' accounts are %v, want one for the machine whose module reads it", clients) + } +} diff --git a/internal/inventory/busrecords_test.go b/internal/inventory/busrecords_test.go index 4f1b3f93..3dd19b98 100644 --- a/internal/inventory/busrecords_test.go +++ b/internal/inventory/busrecords_test.go @@ -1,14 +1,11 @@ package inventory import ( - "context" "strings" "testing" - "time" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/catalogue" - "golang.org/x/crypto/bcrypt" ) // Reading the bus's user list out of the mesh's records, against a real store. @@ -17,18 +14,6 @@ import ( // the server reads whatever file it is given, and a module whose user is absent fails on its first // publish with an authorisation error that says nothing about a missing assignment. -func aMeshWith(t *testing.T, manifests ...catalogue.Manifest) (*Inventory, context.Context) { - t.Helper() - inv := ForTest(t) - ctx := context.Background() - for _, m := range manifests { - if err := inv.RegisterModule(ctx, m, Source{Repository: "/r"}); err != nil { - t.Fatal(err) - } - } - return inv, ctx -} - func theSeatDeclarer() catalogue.Manifest { return catalogue.Manifest{ Module: "telegram", Version: "1", @@ -41,108 +26,9 @@ func theSeatDeclarer() catalogue.Manifest { // A module assigned to a machine becomes a user with the authority its manifest declared — and the // protocol of a seat declared by a *different* module, which is the whole reason a seat exists. -func TestAnAssignedModuleBecomesAUserWithWhatItDeclared(t *testing.T) { - // It declares where its account is delivered: a module with no own secret named broker can never - // be issued one, and is no user at all (novox/hq issue 195) — the case asserted below. - shop := catalogue.Manifest{ - Module: "shop", Version: "1", - Emits: []string{"order.placed"}, Tools: []string{"price"}, - Uses: []string{"telegram-sender"}, - OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/run/broker"}}, - } - quiet := catalogue.Manifest{Module: "quiet", Version: "1", Emits: []string{"thing.happened"}} - inv, ctx := aMeshWith(t, theSeatDeclarer(), shop, quiet) - if _, err := inv.AddNode(ctx, "one"); err != nil { - t.Fatal(err) - } - for _, module := range []string{"shop", "quiet"} { - if _, err := inv.Assign(ctx, "one", module); err != nil { - t.Fatal(err) - } - } - - records, err := inv.BusRecords(ctx) - if err != nil { - t.Fatal(err) - } - var on []broker.Declared - for _, d := range records.Assigned["one"] { - if d.Module == "shop" { - on = append(on, d) - } - } - if len(on) != 1 || on[0].NoAccount { - t.Fatalf("the machine's modules read as %+v", records.Assigned["one"]) - } - if len(on[0].Uses) != 1 || on[0].Uses[0].Accepts[0] != "send" { - t.Fatalf("the seat it uses carries no protocol: %+v — so it would be granted nothing on a "+ - "seat it was assigned to send to", on[0].Uses) - } - if len(on[0].Serves) != 1 || on[0].Serves[0] != "price" { - t.Fatalf("its tools read as %v, and a module that cannot subscribe its own tool subject "+ - "serves nothing", on[0].Serves) - } - - // And it derives into a user the server would accept. - users, err := broker.Users(records) - if err != nil { - t.Fatal(err) - } - var found bool - for _, u := range users { - if u.Username() != "one.shop" { - continue - } - found = true - perms, err := broker.PermissionsFor(u) - if err != nil { - t.Fatal(err) - } - // Its tools are every one under its own name — the list in the manifest is a person's - // vocabulary for asking, not the module's permission to answer. - if !granted(perms.Publish, "mesh.mod.shop.event.order.placed") || - !granted(perms.Publish, "mesh.seat.telegram-sender.accept.send") || - !granted(perms.Subscribe, "mesh.mod.shop.tool.>") { - t.Fatalf("one.shop's authority is not what it declared: %+v", perms) - } - } - if !found { - t.Fatal("no user was derived for the assigned module") - } - for _, u := range users { - if u.Username() == "one.quiet" { - t.Fatal("a module with nowhere to read an account was made a user (novox/hq issue 195)") - } - } -} // A machine holding a live token gets an enrolment user; one whose token is spent or expired does // not. **An enrolment user outliving its token is a right to join that nobody issued.** -func TestOnlyAMachineWithALiveTokenHasAnEnrolmentUser(t *testing.T) { - inv, ctx := aMeshWith(t) - for _, name := range []string{"live", "expired", "none"} { - if _, err := inv.AddNode(ctx, name); err != nil { - t.Fatal(err) - } - } - if _, err := inv.IssueToken(ctx, "live", time.Hour); err != nil { - t.Fatal(err) - } - // Briefly, then waited out: a token with no lifetime is refused at issue, which is the right - // refusal and leaves this as the way to have an expired one. - if _, err := inv.IssueToken(ctx, "expired", 10*time.Millisecond); err != nil { - t.Fatal(err) - } - time.Sleep(50 * time.Millisecond) - - records, err := inv.BusRecords(ctx) - if err != nil { - t.Fatal(err) - } - if strings.Join(records.Enrolling, ",") != "live" { - t.Fatalf("machines with a live token read as %v", records.Enrolling) - } -} // A module assigned and absent from the catalogue is refused rather than composed with no authority. // @@ -193,199 +79,14 @@ func granted(all []string, one string) bool { // exactly that string. So this checks the two halves that make the account usable — that a row // exists under the name the composer asks for, and that the secret handed out is what that row // accepts. -func TestIssuingATokenRecordsTheAccountItIsThePasswordOf(t *testing.T) { - inv, ctx := aMeshWith(t) - if _, err := inv.AddNode(ctx, "joiner"); err != nil { - t.Fatal(err) - } - issued, err := inv.IssueToken(ctx, "joiner", time.Hour) - if err != nil { - t.Fatal(err) - } - - name := broker.Principal{Kind: broker.KindEnrolment, Node: "joiner"}.Username() - users, err := inv.BusUsers(ctx) - if err != nil { - t.Fatal(err) - } - user, has := users[name] - if !has { - t.Fatalf("no bus account for %q; the composer would leave the enrolment out and the "+ - "machine would be refused before the mesh heard of it: %v", name, users) - } - if user.Kind != BusEnrolment || user.Node != "joiner" { - t.Errorf("the account is %+v, not this node's enrolment", user) - } - if err := bcrypt.CompareHashAndPassword([]byte(user.PasswordHash), []byte(issued.Secret)); err != nil { - t.Error("the account does not accept the secret the token carries, so presenting the " + - "token would be refused by the server") - } - - // And the composition contains it, which is the thing the server reads. - records, err := inv.BusRecords(ctx) - if err != nil { - t.Fatal(err) - } - derived, err := broker.Users(records) - if err != nil { - t.Fatal(err) - } - hashes := map[string]string{} - for n, u := range users { - hashes[n] = u.PasswordHash - } - _, missing := broker.WithPasswords(derived, hashes) - for _, m := range missing { - if m == name { - t.Fatal("the enrolment user is composed without a password, which is a user nobody can be") - } - } -} // The module holding mesh-broker is the bus, and its user is the bus's own: granted the snapshot API // for the night's backup and nothing else, whatever tools it declares (novox/hq ADR 0235). Another // module on the same machine is granted none of it. -func TestTheBussOwnModuleBecomesTheSnapshotUser(t *testing.T) { - bus := catalogue.Manifest{ - Module: "nats", Version: "1", - Provides: []catalogue.Offer{{Name: "mesh-bus", Scope: catalogue.ScopeMesh}}, - Claims: []catalogue.Claim{{Name: catalogue.BrokerSeat, Scope: catalogue.ScopeMesh}}, - Tools: []string{"nats_streams"}, - OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/run/broker"}}, - } - shop := catalogue.Manifest{Module: "shop", Version: "1", Emits: []string{"order.placed"}, - OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/run/broker"}}} - inv, ctx := aMeshWith(t, bus, shop) - if _, err := inv.AddNode(ctx, "one"); err != nil { - t.Fatal(err) - } - for _, module := range []string{"nats", "shop"} { - if _, err := inv.Assign(ctx, "one", module); err != nil { - t.Fatal(err) - } - } - records, err := inv.BusRecords(ctx) - if err != nil { - t.Fatal(err) - } - users, err := broker.Users(records) - if err != nil { - t.Fatal(err) - } - seen := map[string]bool{} - for _, u := range users { - perms, err := broker.PermissionsFor(u) - if err != nil { - t.Fatal(err) - } - snapshots := granted(perms.Publish, "$JS.API.STREAM.SNAPSHOT.*") - switch u.Username() { - case "one.nats": - seen[u.Username()] = true - if !snapshots || len(perms.Publish) != len(broker.BusSnapshotGrants.Publish) { - t.Fatalf("the bus's own user is granted %v, not the snapshot API alone", perms.Publish) - } - case "one.shop": - seen[u.Username()] = true - if snapshots { - t.Fatal("a module that is not the bus may snapshot it") - } - } - } - if !seen["one.nats"] || !seen["one.shop"] { - t.Fatalf("users derived: %v", seen) - } -} // The bar's holder reads the state the modules beside it offer in its blocks (novox/hq ADR 0255), // without naming them: granted on the machine where both are, and nowhere else. -func TestABarIsGrantedTheStateItsMachinesModulesOfferIt(t *testing.T) { - bar := catalogue.Manifest{Module: "a-bar", Version: "1", - Claims: []catalogue.Claim{{Name: catalogue.BarSeat, Scope: catalogue.ScopeNode}}} - power := catalogue.Manifest{Module: "power", Version: "1", - State: []catalogue.StateDeclaration{{Name: "draw"}}, - Contributions: []catalogue.SeatContribution{{Seat: catalogue.BarSeat, Kind: catalogue.BarKindBlock, - Data: map[string]any{"bar": "bottom", "place": "status", "shows": "state", - "options": map[string]any{"state": "power.draw"}}}}} - inv, ctx := aMeshWith(t, bar, power) - for node, modules := range map[string][]string{"laptop": {"a-bar", "power"}, "desk": {"a-bar"}} { - if _, err := inv.AddNode(ctx, node); err != nil { - t.Fatal(err) - } - for _, module := range modules { - if _, err := inv.Assign(ctx, node, module); err != nil { - t.Fatal(err) - } - } - } - records, err := inv.BusRecords(ctx) - if err != nil { - t.Fatal(err) - } - reads := func(node, module string) []broker.KeyedRead { - for _, d := range records.Assigned[node] { - if d.Module == module { - if len(d.Reads) != 0 { - t.Fatalf("%s on %s reads whole buckets: %v", module, node, d.Reads) - } - return d.KeyedReads - } - } - t.Fatalf("%s is not on %s", module, node) - return nil - } - // The laptop's own key, and no other (novox/hq ADR 0260). - if got := reads("laptop", "a-bar"); len(got) != 1 || got[0] != (broker.KeyedRead{Read: "power.draw", Key: "laptop"}) { - t.Fatalf("the laptop's bar reads %v", got) - } - if got := reads("desk", "a-bar"); len(got) != 0 { - t.Fatalf("a bar with no power beside it reads %v", got) - } - if got := reads("laptop", "power"); len(got) != 0 { - t.Fatalf("power reads %v", got) - } -} // A module running the machine's provisioner client gives that machine a client account of its own, apart from the // module's and the tool runner's (novox/hq ADR 0349 clause 11, design 57 P3); one with nowhere to read the account // gives none. -func TestTheProvisionerClientsMachineHasAnAccountOfItsOwn(t *testing.T) { - provisioner := catalogue.Manifest{ - Module: "mesh-provisioner", Version: "1", - DefinesSeats: []catalogue.SeatDeclaration{{Name: broker.NodeProvisionerSeat, Scope: catalogue.ScopeNode, - Serves: []catalogue.Verb{{Name: "get"}}}}, - Claims: []catalogue.Claim{{Name: broker.NodeProvisionerSeat, Scope: catalogue.ScopeNode}}, - OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/run/broker"}, - broker.ProvisionerSecret: {Path: "/run/provisioner-broker"}}, - } - unread := provisioner - unread.Module = "provisioner-unread" - unread.DefinesSeats = nil - unread.OwnSecrets = catalogue.OwnSecrets{"broker": {Path: "/run/broker"}} - inv, ctx := aMeshWith(t, provisioner, unread) - for node, module := range map[string]string{"one": "mesh-provisioner", "two": "provisioner-unread"} { - if _, err := inv.AddNode(ctx, node); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, node, module); err != nil { - t.Fatal(err) - } - } - records, err := inv.BusRecords(ctx) - if err != nil { - t.Fatal(err) - } - users, err := broker.Users(records) - if err != nil { - t.Fatal(err) - } - var clients []string - for _, u := range users { - if u.Kind == broker.KindProvisioner { - clients = append(clients, u.Username()+" for "+u.Module) - } - } - if strings.Join(clients, ",") != "provisioner.one for mesh-provisioner" { - t.Fatalf("the provisioner clients' accounts are %v, want one for the machine whose module reads it", clients) - } -} diff --git a/internal/inventory/bususers_integration_test.go b/internal/inventory/bususers_integration_test.go new file mode 100644 index 00000000..390a8ac4 --- /dev/null +++ b/internal/inventory/bususers_integration_test.go @@ -0,0 +1,143 @@ +//go:build integration + +package inventory + +import ( + "context" + "testing" + + "golang.org/x/crypto/bcrypt" +) + +func TestABusPasswordIsReturnedOnceAndOnlyItsHashIsKept(t *testing.T) { + inv := ForTest(t) + ctx := context.Background() + + password, err := inv.MintBusPassword(ctx, aBusUser("shop")) + if err != nil { + t.Fatal(err) + } + if password == "" { + t.Fatal("no password came back, so nothing can be sealed to the module") + } + + hash, known, err := inv.BusUserHash(ctx, "one.shop") + if err != nil || !known { + t.Fatalf("the user was not recorded: %v %v", known, err) + } + if hash == password { + t.Fatal("the store holds the password itself") + } + if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(password)); err != nil { + t.Fatalf("the recorded hash does not verify the password it was made from: %v", err) + } +} + +func TestMintingAgainRotatesRatherThanAddsAUser(t *testing.T) { + inv := ForTest(t) + ctx := context.Background() + + first, err := inv.MintBusPassword(ctx, aBusUser("shop")) + if err != nil { + t.Fatal(err) + } + second, err := inv.MintBusPassword(ctx, aBusUser("shop")) + if err != nil { + t.Fatal(err) + } + if first == second { + t.Fatal("minting twice produced the same password") + } + users, err := inv.BusUsers(ctx) + if err != nil { + t.Fatal(err) + } + if len(users) != 1 { + t.Fatalf("%d users after two mints for one name", len(users)) + } + hash := users["one.shop"].PasswordHash + if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(second)); err != nil { + t.Fatal("the kept hash is not the newest password's") + } + if bcrypt.CompareHashAndPassword([]byte(hash), []byte(first)) == nil { + t.Fatal("the previous password still verifies, so a rotation revoked nothing") + } +} + +func TestForgettingANodeTakesItsBusUsersWithIt(t *testing.T) { + inv := ForTest(t) + ctx := context.Background() + + for _, u := range []BusUser{ + {Username: "node.one", Kind: BusNode, Node: "one"}, + aBusUser("shop"), + {Username: "node.two", Kind: BusNode, Node: "two"}, + {Username: "controller", Kind: BusController}, + } { + if _, err := inv.MintBusPassword(ctx, u); err != nil { + t.Fatal(err) + } + } + if err := inv.ForgetBusUsersOf(ctx, "one"); err != nil { + t.Fatal(err) + } + users, err := inv.BusUsers(ctx) + if err != nil { + t.Fatal(err) + } + if _, still := users["node.one"]; still { + t.Fatal("a forgotten node's host credential still works") + } + if _, still := users["one.shop"]; still { + t.Fatal("a module on a forgotten node still has a credential") + } + // And nothing else went with it: the controller has no node, and another machine's user is + // another machine's. + for _, kept := range []string{"node.two", "controller"} { + if _, ok := users[kept]; !ok { + t.Fatalf("%s was removed with another node's users", kept) + } + } +} + +func TestForgettingTheUsersOfNoNodeIsRefused(t *testing.T) { + inv := ForTest(t) + if err := inv.ForgetBusUsersOf(context.Background(), ""); err == nil { + t.Fatal("forgetting the bus users of no node was allowed") + } +} + +func TestACredentialTheMeshDidNotMintIsRecordedOnceAndNotOverwritten(t *testing.T) { + inv := ForTest(t) + ctx := context.Background() + + if err := inv.SeedBusUser(ctx, BusUser{Username: "controller", Kind: BusController}, + "bootstrap"); err != nil { + t.Fatal(err) + } + hash, known, err := inv.BusUserHash(ctx, "controller") + if err != nil || !known { + t.Fatalf("the credential was not recorded: %v %v", known, err) + } + if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte("bootstrap")); err != nil { + t.Fatalf("what was recorded does not verify the credential given: %v", err) + } + + // Minted since, then seeded again — which is what a restart does. The rotation must stand, or + // every restart would put the bootstrap password back over it. + minted, err := inv.MintBusPassword(ctx, BusUser{Username: "controller", Kind: BusController}) + if err != nil { + t.Fatal(err) + } + if err := inv.SeedBusUser(ctx, BusUser{Username: "controller", Kind: BusController}, + "bootstrap"); err != nil { + t.Fatal(err) + } + hash, _, err = inv.BusUserHash(ctx, "controller") + if err != nil { + t.Fatal(err) + } + if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(minted)); err != nil { + t.Fatal("a restart put the bootstrap credential back over a rotated one") + } +} diff --git a/internal/inventory/bususers_test.go b/internal/inventory/bususers_test.go index 401c6220..1bbb5aee 100644 --- a/internal/inventory/bususers_test.go +++ b/internal/inventory/bususers_test.go @@ -1,7 +1,6 @@ package inventory import ( - "context" "testing" "golang.org/x/crypto/bcrypt" @@ -17,147 +16,19 @@ func aBusUser(module string) BusUser { // The plaintext comes back once and the store keeps only a hash that verifies against it. **A // credential recoverable from the mesh's store is one whose blast radius is the store's**, so what // is asserted is that the password is not in there. -func TestABusPasswordIsReturnedOnceAndOnlyItsHashIsKept(t *testing.T) { - inv := ForTest(t) - ctx := context.Background() - - password, err := inv.MintBusPassword(ctx, aBusUser("shop")) - if err != nil { - t.Fatal(err) - } - if password == "" { - t.Fatal("no password came back, so nothing can be sealed to the module") - } - - hash, known, err := inv.BusUserHash(ctx, "one.shop") - if err != nil || !known { - t.Fatalf("the user was not recorded: %v %v", known, err) - } - if hash == password { - t.Fatal("the store holds the password itself") - } - if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(password)); err != nil { - t.Fatalf("the recorded hash does not verify the password it was made from: %v", err) - } -} // Minting again replaces what was there rather than failing or adding a second row: that is a // rotation, and the old credential stops working at the next composition. -func TestMintingAgainRotatesRatherThanAddsAUser(t *testing.T) { - inv := ForTest(t) - ctx := context.Background() - - first, err := inv.MintBusPassword(ctx, aBusUser("shop")) - if err != nil { - t.Fatal(err) - } - second, err := inv.MintBusPassword(ctx, aBusUser("shop")) - if err != nil { - t.Fatal(err) - } - if first == second { - t.Fatal("minting twice produced the same password") - } - users, err := inv.BusUsers(ctx) - if err != nil { - t.Fatal(err) - } - if len(users) != 1 { - t.Fatalf("%d users after two mints for one name", len(users)) - } - hash := users["one.shop"].PasswordHash - if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(second)); err != nil { - t.Fatal("the kept hash is not the newest password's") - } - if bcrypt.CompareHashAndPassword([]byte(hash), []byte(first)) == nil { - t.Fatal("the previous password still verifies, so a rotation revoked nothing") - } -} // Forgetting a node takes every credential that belonged to it — its host's and every module // assigned to it. What a forgotten node leaves behind otherwise is a working set of credentials for // a machine the mesh no longer knows. -func TestForgettingANodeTakesItsBusUsersWithIt(t *testing.T) { - inv := ForTest(t) - ctx := context.Background() - - for _, u := range []BusUser{ - {Username: "node.one", Kind: BusNode, Node: "one"}, - aBusUser("shop"), - {Username: "node.two", Kind: BusNode, Node: "two"}, - {Username: "controller", Kind: BusController}, - } { - if _, err := inv.MintBusPassword(ctx, u); err != nil { - t.Fatal(err) - } - } - if err := inv.ForgetBusUsersOf(ctx, "one"); err != nil { - t.Fatal(err) - } - users, err := inv.BusUsers(ctx) - if err != nil { - t.Fatal(err) - } - if _, still := users["node.one"]; still { - t.Fatal("a forgotten node's host credential still works") - } - if _, still := users["one.shop"]; still { - t.Fatal("a module on a forgotten node still has a credential") - } - // And nothing else went with it: the controller has no node, and another machine's user is - // another machine's. - for _, kept := range []string{"node.two", "controller"} { - if _, ok := users[kept]; !ok { - t.Fatalf("%s was removed with another node's users", kept) - } - } -} // Forgetting the users of no node would forget every user that has none — the controller and every // person — so it is refused rather than run. -func TestForgettingTheUsersOfNoNodeIsRefused(t *testing.T) { - inv := ForTest(t) - if err := inv.ForgetBusUsersOf(context.Background(), ""); err == nil { - t.Fatal("forgetting the bus users of no node was allowed") - } -} // The controller's own user is created by the installer, so the mesh has to be able to record a // credential it did not mint — or the first composition leaves the writer out of the file it writes. -func TestACredentialTheMeshDidNotMintIsRecordedOnceAndNotOverwritten(t *testing.T) { - inv := ForTest(t) - ctx := context.Background() - - if err := inv.SeedBusUser(ctx, BusUser{Username: "controller", Kind: BusController}, - "bootstrap"); err != nil { - t.Fatal(err) - } - hash, known, err := inv.BusUserHash(ctx, "controller") - if err != nil || !known { - t.Fatalf("the credential was not recorded: %v %v", known, err) - } - if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte("bootstrap")); err != nil { - t.Fatalf("what was recorded does not verify the credential given: %v", err) - } - - // Minted since, then seeded again — which is what a restart does. The rotation must stand, or - // every restart would put the bootstrap password back over it. - minted, err := inv.MintBusPassword(ctx, BusUser{Username: "controller", Kind: BusController}) - if err != nil { - t.Fatal(err) - } - if err := inv.SeedBusUser(ctx, BusUser{Username: "controller", Kind: BusController}, - "bootstrap"); err != nil { - t.Fatal(err) - } - hash, _, err = inv.BusUserHash(ctx, "controller") - if err != nil { - t.Fatal(err) - } - if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(minted)); err != nil { - t.Fatal("a restart put the bootstrap credential back over a rotated one") - } -} // Outside a test a bus password is hashed at bcrypt's default cost: only ForTest lowers it (novox/hq issue 591). func TestABusPasswordIsHashedAtTheDefaultCostOutsideATest(t *testing.T) { diff --git a/internal/inventory/busy_integration_test.go b/internal/inventory/busy_integration_test.go new file mode 100644 index 00000000..61283905 --- /dev/null +++ b/internal/inventory/busy_integration_test.go @@ -0,0 +1,66 @@ +//go:build integration + +package inventory + +import ( + "testing" + "time" +) + +func TestAReportOfABusyContainerIsKeptAndReadBack(t *testing.T) { + inv, node := aNodeWithModules(t) + ctx := t.Context() + record, err := inv.NodeByName(ctx, node) + if err != nil { + t.Fatal(err) + } + since := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC) + waiting := []BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: BusyWaiting, + Since: since, Until: since.Add(15 * time.Minute), Said: "busy: a check"}} + keep := func(busy []BusyWait) bool { + t.Helper() + news, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: "d1", Applied: 1, + Busy: busy}, ReportOrder{Epoch: 1, Sequence: 1}, func(ReportOrder) bool { return false }) + if err != nil { + t.Fatal(err) + } + return news + } + read := func() []BusyWait { + t.Helper() + reports, err := inv.LastReports(ctx) + if err != nil { + t.Fatal(err) + } + for _, r := range reports { + if r.Node == node { + return r.Busy + } + } + t.Fatal("the machine's report is not read back") + return nil + } + keep(nil) + if got := read(); len(got) != 0 { + t.Fatalf("a report with nothing busy reads back %+v", got) + } + if !keep(waiting) { + t.Error("a container starting to wait is not news") + } + got := read() + if b, ok := WaitingFor(got, "build-agent"); !ok || !b.Since.Equal(since) || b.Said != "busy: a check" { + t.Fatalf("the wait is not read back: %+v", got) + } + again := []BusyWait{waiting[0]} + again[0].Said = "busy: a check, a minute on" + if keep(again) { + t.Error("a wait whose output changed is news") + } + if !keep([]BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: "replaced-when-idle", + Since: since, Until: since.Add(15 * time.Minute)}}) { + t.Error("a container replaced after it waited is not news") + } + if _, ok := WaitingFor(read(), "build-agent"); ok { + t.Error("a container replaced still reads as waiting") + } +} diff --git a/internal/inventory/busy_test.go b/internal/inventory/busy_test.go index bd184a3e..7170119d 100644 --- a/internal/inventory/busy_test.go +++ b/internal/inventory/busy_test.go @@ -83,63 +83,6 @@ func TestABusyContainerIsItsModules(t *testing.T) { // A machine's report of a busy container is kept with its last account and read back beside it, and its // starting and its end are news; its output alone is not (novox/hq ADR 0335). Needs the store. -func TestAReportOfABusyContainerIsKeptAndReadBack(t *testing.T) { - inv, node := aNodeWithModules(t) - ctx := t.Context() - record, err := inv.NodeByName(ctx, node) - if err != nil { - t.Fatal(err) - } - since := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC) - waiting := []BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: BusyWaiting, - Since: since, Until: since.Add(15 * time.Minute), Said: "busy: a check"}} - keep := func(busy []BusyWait) bool { - t.Helper() - news, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: "d1", Applied: 1, - Busy: busy}, ReportOrder{Epoch: 1, Sequence: 1}, func(ReportOrder) bool { return false }) - if err != nil { - t.Fatal(err) - } - return news - } - read := func() []BusyWait { - t.Helper() - reports, err := inv.LastReports(ctx) - if err != nil { - t.Fatal(err) - } - for _, r := range reports { - if r.Node == node { - return r.Busy - } - } - t.Fatal("the machine's report is not read back") - return nil - } - keep(nil) - if got := read(); len(got) != 0 { - t.Fatalf("a report with nothing busy reads back %+v", got) - } - if !keep(waiting) { - t.Error("a container starting to wait is not news") - } - got := read() - if b, ok := WaitingFor(got, "build-agent"); !ok || !b.Since.Equal(since) || b.Said != "busy: a check" { - t.Fatalf("the wait is not read back: %+v", got) - } - again := []BusyWait{waiting[0]} - again[0].Said = "busy: a check, a minute on" - if keep(again) { - t.Error("a wait whose output changed is news") - } - if !keep([]BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: "replaced-when-idle", - Since: since, Until: since.Add(15 * time.Minute)}}) { - t.Error("a container replaced after it waited is not news") - } - if _, ok := WaitingFor(read(), "build-agent"); ok { - t.Error("a container replaced still reads as waiting") - } -} // A machine of the rest that answered and waits for a busy container is never read as silent, its wait is kept // out of its apply time, and once replaced its report is the end of the wait (novox/hq ADR 0335). diff --git a/internal/inventory/catalogue_test.go b/internal/inventory/catalogue_test.go index 39f66cc6..f956ba31 100644 --- a/internal/inventory/catalogue_test.go +++ b/internal/inventory/catalogue_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/collection_test.go b/internal/inventory/collection_test.go index e05a2a3f..b608facf 100644 --- a/internal/inventory/collection_test.go +++ b/internal/inventory/collection_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/config_move_test.go b/internal/inventory/config_move_test.go index 1e321bee..a4912522 100644 --- a/internal/inventory/config_move_test.go +++ b/internal/inventory/config_move_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/core_box_test.go b/internal/inventory/core_box_test.go index 5a07a755..8d08c990 100644 --- a/internal/inventory/core_box_test.go +++ b/internal/inventory/core_box_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/data_test.go b/internal/inventory/data_test.go index 9d23e49e..22cc398f 100644 --- a/internal/inventory/data_test.go +++ b/internal/inventory/data_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/doing_test.go b/internal/inventory/doing_test.go index 76d6a203..b56c8239 100644 --- a/internal/inventory/doing_test.go +++ b/internal/inventory/doing_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/durations_test.go b/internal/inventory/durations_test.go index f3c84776..356618f5 100644 --- a/internal/inventory/durations_test.go +++ b/internal/inventory/durations_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/forget_test.go b/internal/inventory/forget_test.go index b28e317c..ff79f139 100644 --- a/internal/inventory/forget_test.go +++ b/internal/inventory/forget_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/given_test.go b/internal/inventory/given_test.go index c6544297..a79a7877 100644 --- a/internal/inventory/given_test.go +++ b/internal/inventory/given_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/heals_test.go b/internal/inventory/heals_test.go index 8069c580..4a3a479c 100644 --- a/internal/inventory/heals_test.go +++ b/internal/inventory/heals_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/hold_plans_test.go b/internal/inventory/hold_plans_test.go index 854451a8..daa4744a 100644 --- a/internal/inventory/hold_plans_test.go +++ b/internal/inventory/hold_plans_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/hold_test.go b/internal/inventory/hold_test.go index 50c187c9..19cc4132 100644 --- a/internal/inventory/hold_test.go +++ b/internal/inventory/hold_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/holdings_test.go b/internal/inventory/holdings_test.go index 2a32a2e9..b298a59d 100644 --- a/internal/inventory/holdings_test.go +++ b/internal/inventory/holdings_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go index daabd806..178851e1 100644 --- a/internal/inventory/inventory_test.go +++ b/internal/inventory/inventory_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/issued_test.go b/internal/inventory/issued_test.go index f8725f20..65c40254 100644 --- a/internal/inventory/issued_test.go +++ b/internal/inventory/issued_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/looks_test.go b/internal/inventory/looks_test.go index 9b41fdda..9d4e91a4 100644 --- a/internal/inventory/looks_test.go +++ b/internal/inventory/looks_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/mirrored_test.go b/internal/inventory/mirrored_test.go index 3fb13464..e58bf53d 100644 --- a/internal/inventory/mirrored_test.go +++ b/internal/inventory/mirrored_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/named_peer_test.go b/internal/inventory/named_peer_test.go index 52c669ed..81ccc2e7 100644 --- a/internal/inventory/named_peer_test.go +++ b/internal/inventory/named_peer_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory // A carried peer is nameable (novox/hq issue 112): the operator states which machine a carried diff --git a/internal/inventory/older_build_test.go b/internal/inventory/older_build_test.go index a0d3c7a1..5c3f861f 100644 --- a/internal/inventory/older_build_test.go +++ b/internal/inventory/older_build_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/operator_test.go b/internal/inventory/operator_test.go index bfa71e77..9b1a9b13 100644 --- a/internal/inventory/operator_test.go +++ b/internal/inventory/operator_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/order_test.go b/internal/inventory/order_test.go index bd09d191..910d783a 100644 --- a/internal/inventory/order_test.go +++ b/internal/inventory/order_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/outside_test.go b/internal/inventory/outside_test.go index b45ad0fd..70019a76 100644 --- a/internal/inventory/outside_test.go +++ b/internal/inventory/outside_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/pending_test.go b/internal/inventory/pending_test.go index 1ae3b8dd..a3a25572 100644 --- a/internal/inventory/pending_test.go +++ b/internal/inventory/pending_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/people_test.go b/internal/inventory/people_test.go index 5919607e..86351b5e 100644 --- a/internal/inventory/people_test.go +++ b/internal/inventory/people_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/pin_completed_test.go b/internal/inventory/pin_completed_test.go index e090993b..9fac8387 100644 --- a/internal/inventory/pin_completed_test.go +++ b/internal/inventory/pin_completed_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/placement_test.go b/internal/inventory/placement_test.go index 0483fe7b..0acda5c2 100644 --- a/internal/inventory/placement_test.go +++ b/internal/inventory/placement_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/placement_words_test.go b/internal/inventory/placement_words_test.go index d4645ffb..e3a8ee7f 100644 --- a/internal/inventory/placement_words_test.go +++ b/internal/inventory/placement_words_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/plan_id_test.go b/internal/inventory/plan_id_test.go index 76323226..88cd0021 100644 --- a/internal/inventory/plan_id_test.go +++ b/internal/inventory/plan_id_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/plans_test.go b/internal/inventory/plans_test.go index 027600eb..831a63b5 100644 --- a/internal/inventory/plans_test.go +++ b/internal/inventory/plans_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/ports_test.go b/internal/inventory/ports_test.go index 60f0665b..eab01635 100644 --- a/internal/inventory/ports_test.go +++ b/internal/inventory/ports_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/predefined_test.go b/internal/inventory/predefined_test.go index d2ab52d2..14d4fa5e 100644 --- a/internal/inventory/predefined_test.go +++ b/internal/inventory/predefined_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/public_domain_test.go b/internal/inventory/public_domain_test.go index 19190b46..eb81eec7 100644 --- a/internal/inventory/public_domain_test.go +++ b/internal/inventory/public_domain_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/reachable_integration_test.go b/internal/inventory/reachable_integration_test.go new file mode 100644 index 00000000..8b222092 --- /dev/null +++ b/internal/inventory/reachable_integration_test.go @@ -0,0 +1,33 @@ +//go:build integration + +package inventory + +import ( + "context" + "os" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgconn" +) + +func TestAWrongPasswordIsAnAnswerNotAnOutage(t *testing.T) { + dsn := os.Getenv("MESH_TEST_POSTGRES") + if dsn == "" { + t.Skip("MESH_TEST_POSTGRES is not set") + } + wrong := strings.Replace(dsn, "postgres:check@", "postgres:not-the-password@", 1) + if wrong == dsn { + t.Skip("the test store's address does not carry the expected credential") + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err := pgconn.Connect(ctx, wrong) + if err == nil { + t.Fatal("a wrong password connected") + } + if Unreachable(err) { + t.Fatalf("a wrong password was taken for an outage and would be retried for ever: %v", err) + } +} diff --git a/internal/inventory/reachable_test.go b/internal/inventory/reachable_test.go index 04543587..701d9a9f 100644 --- a/internal/inventory/reachable_test.go +++ b/internal/inventory/reachable_test.go @@ -5,8 +5,6 @@ import ( "errors" "fmt" "io" - "os" - "strings" "testing" "time" @@ -55,22 +53,3 @@ func TestAStoreThatRefusesTheConnectionIsUnreachable(t *testing.T) { // A connection the store refused on a wrong password is an answer, even though it arrives as a // failed connection — asked again for ever, it would hold every message behind it. -func TestAWrongPasswordIsAnAnswerNotAnOutage(t *testing.T) { - dsn := os.Getenv("MESH_TEST_POSTGRES") - if dsn == "" { - t.Skip("MESH_TEST_POSTGRES is not set") - } - wrong := strings.Replace(dsn, "postgres:check@", "postgres:not-the-password@", 1) - if wrong == dsn { - t.Skip("the test store's address does not carry the expected credential") - } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - _, err := pgconn.Connect(ctx, wrong) - if err == nil { - t.Fatal("a wrong password connected") - } - if Unreachable(err) { - t.Fatalf("a wrong password was taken for an outage and would be retried for ever: %v", err) - } -} diff --git a/internal/inventory/record_mirrors_integration_test.go b/internal/inventory/record_mirrors_integration_test.go new file mode 100644 index 00000000..b2dc6685 --- /dev/null +++ b/internal/inventory/record_mirrors_integration_test.go @@ -0,0 +1,37 @@ +//go:build integration + +package inventory + +import ( + "encoding/json" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestAModuleInTheConfigFormIsMirroredWithItsWholeDeclaration(t *testing.T) { + inv := fresh(t) + var m catalogue.Manifest + if err := json.Unmarshal([]byte(`{"module": "proxy", "version": "1", "config": {"port": {"type": "number", + "description": "the port it listens on, a TCP port number", "default": 8080, "minimum": 1, + "maximum": 65535, "enum": [8080, 8443], "predefined": "http.port"}}}`), &m); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(t.Context(), m, Source{}); err != nil { + t.Fatal(err) + } + values, err := inv.DeclaredConfigMirror(t.Context()) + if err != nil { + t.Fatal(err) + } + var got DeclaredConfigMirrored + if err := json.Unmarshal(values["proxy"], &got); err != nil { + t.Fatal(err) + } + port, has := got.Settings["port"] + if !has || port.Type != catalogue.ConfigNumber || port.Description == "" || string(port.Default) != "8080" || + port.Minimum == nil || *port.Minimum != 1 || port.Maximum == nil || *port.Maximum != 65535 || + string(port.Enum) != "[8080,8443]" || port.Predefined != "http.port" { + t.Errorf("a module in the config form is mirrored as %s", values["proxy"]) + } +} diff --git a/internal/inventory/record_mirrors_test.go b/internal/inventory/record_mirrors_test.go index cb329ae1..013d94e5 100644 --- a/internal/inventory/record_mirrors_test.go +++ b/internal/inventory/record_mirrors_test.go @@ -85,29 +85,3 @@ func TestADeclarationUsingEveryFieldRoundTripsThroughTheMirror(t *testing.T) { // **A module moved to the config form is mirrored from it** (ADR 0348 §1): the store keeps its manifest in the // form it was written in, under `config`, and the declared-config mirror reads its declarations from there with // the config form's fields. -func TestAModuleInTheConfigFormIsMirroredWithItsWholeDeclaration(t *testing.T) { - inv := fresh(t) - var m catalogue.Manifest - if err := json.Unmarshal([]byte(`{"module": "proxy", "version": "1", "config": {"port": {"type": "number", - "description": "the port it listens on, a TCP port number", "default": 8080, "minimum": 1, - "maximum": 65535, "enum": [8080, 8443], "predefined": "http.port"}}}`), &m); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(t.Context(), m, Source{}); err != nil { - t.Fatal(err) - } - values, err := inv.DeclaredConfigMirror(t.Context()) - if err != nil { - t.Fatal(err) - } - var got DeclaredConfigMirrored - if err := json.Unmarshal(values["proxy"], &got); err != nil { - t.Fatal(err) - } - port, has := got.Settings["port"] - if !has || port.Type != catalogue.ConfigNumber || port.Description == "" || string(port.Default) != "8080" || - port.Minimum == nil || *port.Minimum != 1 || port.Maximum == nil || *port.Maximum != 65535 || - string(port.Enum) != "[8080,8443]" || port.Predefined != "http.port" { - t.Errorf("a module in the config form is mirrored as %s", values["proxy"]) - } -} diff --git a/internal/inventory/registered_held_integration_test.go b/internal/inventory/registered_held_integration_test.go new file mode 100644 index 00000000..cf299498 --- /dev/null +++ b/internal/inventory/registered_held_integration_test.go @@ -0,0 +1,59 @@ +//go:build integration + +package inventory + +import ( + "encoding/json" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestWhatRunsIsTheRegisteredBuildNotANewerOnePutBack(t *testing.T) { + inv := fresh(t) + ctx := t.Context() + manifestOf := func(image string) catalogue.Manifest { + return catalogue.Manifest{Module: "one", Version: "1", Resources: []map[string]any{ + {"id": "app", "type": "container", "name": "one", "image": image}}} + } + record := func(id, image string, asked time.Time) Build { + t.Helper() + raw, _ := json.Marshal(manifestOf(image)) + b := Build{ID: id, Module: "one", Commit: id + "aaaaaaaa", Repository: "novox/one", Manifest: raw, + Asked: asked, At: asked, Made: []Artifact{{Name: "one", Kind: "image", Reference: image}}} + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, manifestOf(image), Source{Repository: "novox/one", BuiltFrom: b.Commit, + Asked: asked}); err != nil { + t.Fatal(err) + } + return b + } + x := "store/one@sha256:" + "1111111111111111111111111111111111111111111111111111111111111111" + y := "store/one@sha256:" + "2222222222222222222222222222222222222222222222222222222222222222" + before := record("x", x, time.Now().Add(-2*time.Hour)) + failed := record("y", y, time.Now().Add(-time.Minute)) + if err := inv.RecordGate(ctx, GateVerdict{Build: failed.ID, Module: "one", Commit: failed.Commit, + Machines: []string{"anchor"}, Verdict: GateFailed}); err != nil { + t.Fatal(err) + } + if err := inv.RestoreModule(ctx, before); err != nil { + t.Fatal(err) + } + held, err := inv.Held(ctx) + if err != nil { + t.Fatal(err) + } + running, err := inv.RegisteredHeld(ctx) + if err != nil { + t.Fatal(err) + } + if running["one/one"] == y { + t.Fatalf("what runs is read as the build whose gate failed (%s); Held says %s", y, held["one/one"]) + } + if got := running["one/one"]; got != "" && got != x { + t.Fatalf("what runs is read as %s, want the build put back %s or nothing", got, x) + } +} diff --git a/internal/inventory/registered_held_test.go b/internal/inventory/registered_held_test.go index ecb86043..a471b959 100644 --- a/internal/inventory/registered_held_test.go +++ b/internal/inventory/registered_held_test.go @@ -1,63 +1,12 @@ package inventory import ( - "encoding/json" "testing" - "time" - - "github.com/novox/mesh-controller/internal/catalogue" ) // TestWhatRunsIsTheRegisteredBuildNotANewerOnePutBack is novox/hq issue 721's review: the newest build of a module // failed its gate and the mesh put the build before it back. Held names the newest; what runs, and what a merge's build // is compared with, is the build put back — so a merge whose bytes equal the failed build is never said unchanged. -func TestWhatRunsIsTheRegisteredBuildNotANewerOnePutBack(t *testing.T) { - inv := fresh(t) - ctx := t.Context() - manifestOf := func(image string) catalogue.Manifest { - return catalogue.Manifest{Module: "one", Version: "1", Resources: []map[string]any{ - {"id": "app", "type": "container", "name": "one", "image": image}}} - } - record := func(id, image string, asked time.Time) Build { - t.Helper() - raw, _ := json.Marshal(manifestOf(image)) - b := Build{ID: id, Module: "one", Commit: id + "aaaaaaaa", Repository: "novox/one", Manifest: raw, - Asked: asked, At: asked, Made: []Artifact{{Name: "one", Kind: "image", Reference: image}}} - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, manifestOf(image), Source{Repository: "novox/one", BuiltFrom: b.Commit, - Asked: asked}); err != nil { - t.Fatal(err) - } - return b - } - x := "store/one@sha256:" + "1111111111111111111111111111111111111111111111111111111111111111" - y := "store/one@sha256:" + "2222222222222222222222222222222222222222222222222222222222222222" - before := record("x", x, time.Now().Add(-2*time.Hour)) - failed := record("y", y, time.Now().Add(-time.Minute)) - if err := inv.RecordGate(ctx, GateVerdict{Build: failed.ID, Module: "one", Commit: failed.Commit, - Machines: []string{"anchor"}, Verdict: GateFailed}); err != nil { - t.Fatal(err) - } - if err := inv.RestoreModule(ctx, before); err != nil { - t.Fatal(err) - } - held, err := inv.Held(ctx) - if err != nil { - t.Fatal(err) - } - running, err := inv.RegisteredHeld(ctx) - if err != nil { - t.Fatal(err) - } - if running["one/one"] == y { - t.Fatalf("what runs is read as the build whose gate failed (%s); Held says %s", y, held["one/one"]) - } - if got := running["one/one"]; got != "" && got != x { - t.Fatalf("what runs is read as %s, want the build put back %s or nothing", got, x) - } -} // TestOnlyArtifactsTheRegisteredManifestNamesAreRunning: registeredOnly keeps a held artifact only when its digest is // in its module's registered manifest. diff --git a/internal/inventory/retire_test.go b/internal/inventory/retire_test.go index 1956982c..e90b9b77 100644 --- a/internal/inventory/retire_test.go +++ b/internal/inventory/retire_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/seats_test.go b/internal/inventory/seats_test.go index 1dfac4b2..ff22f6a2 100644 --- a/internal/inventory/seats_test.go +++ b/internal/inventory/seats_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/secret_ask_test.go b/internal/inventory/secret_ask_test.go index 51a21af5..6227c8c9 100644 --- a/internal/inventory/secret_ask_test.go +++ b/internal/inventory/secret_ask_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/secret_family_integration_test.go b/internal/inventory/secret_family_integration_test.go new file mode 100644 index 00000000..36f93ed3 --- /dev/null +++ b/internal/inventory/secret_family_integration_test.go @@ -0,0 +1,58 @@ +//go:build integration + +package inventory + +import ( + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestAMemberGivenIsReadAsGivenAndNothingElseIs(t *testing.T) { + inv := fresh(t) + ctx := t.Context() + node, err := inv.AddNode(ctx, "workstation") + if err != nil { + t.Fatal(err) + } + key, _ := aSealingKey(t) + if err := inv.RecordSealingKey(ctx, node.ID, key); err != nil { + t.Fatal(err) + } + m := catalogue.Manifest{Module: "mounts", Version: "1", OwnSecrets: catalogue.OwnSecrets{ + "smb-password-*": {Path: "/s/smb-password-*.secret", IssuedBy: catalogue.IssuedOutside}, + "token": {Path: "/s/token", IssuedBy: catalogue.IssuedOutside}, + }} + if err := inv.RegisterModule(ctx, m, Source{}); err != nil { + t.Fatal(err) + } + if err := inv.AcceptSecretForModule(ctx, "workstation", "mounts", "smb-password-games", "hunter2"); err != nil { + t.Fatalf("a member was refused: %v", err) + } + if err := inv.AcceptSecretForModule(ctx, "workstation", "mounts", "smb-pass", "x"); err == nil { + t.Fatal("a name of no family was accepted") + } + members, err := inv.GivenMembers(ctx, "workstation", "mounts", "smb-password-*") + if err != nil { + t.Fatal(err) + } + if len(members) != 1 || members[0].Name != "smb-password-games" || !members[0].Current || members[0].Sealed == "" || + strings.Contains(members[0].Sealed, "hunter2") { + t.Fatalf("members: %+v", members) + } + given, err := inv.GivenOwnSecrets(ctx, "workstation", "mounts") + if err != nil { + t.Fatal(err) + } + if _, was := given["smb-password-games"]; !was || len(given) != 1 { + t.Fatalf("given: %v", given) + } + // A value the mesh made is not one a person gave. + if _, err := inv.SecretForModule(ctx, "workstation", "mounts", "token"); err != nil { + t.Fatal(err) + } + if given, _ := inv.GivenOwnSecrets(ctx, "workstation", "mounts"); len(given) != 1 { + t.Fatalf("a value the mesh made counted as given: %v", given) + } +} diff --git a/internal/inventory/secret_family_test.go b/internal/inventory/secret_family_test.go index 1021dc63..85756a9b 100644 --- a/internal/inventory/secret_family_test.go +++ b/internal/inventory/secret_family_test.go @@ -27,50 +27,3 @@ func TestAMemberIsGivableAtTheDeskAndANameOutsideTheFamilyIsNot(t *testing.T) { } // A member given is read back as given, sealed to the machine's key; the mesh makes none, so nothing else is. -func TestAMemberGivenIsReadAsGivenAndNothingElseIs(t *testing.T) { - inv := fresh(t) - ctx := t.Context() - node, err := inv.AddNode(ctx, "workstation") - if err != nil { - t.Fatal(err) - } - key, _ := aSealingKey(t) - if err := inv.RecordSealingKey(ctx, node.ID, key); err != nil { - t.Fatal(err) - } - m := catalogue.Manifest{Module: "mounts", Version: "1", OwnSecrets: catalogue.OwnSecrets{ - "smb-password-*": {Path: "/s/smb-password-*.secret", IssuedBy: catalogue.IssuedOutside}, - "token": {Path: "/s/token", IssuedBy: catalogue.IssuedOutside}, - }} - if err := inv.RegisterModule(ctx, m, Source{}); err != nil { - t.Fatal(err) - } - if err := inv.AcceptSecretForModule(ctx, "workstation", "mounts", "smb-password-games", "hunter2"); err != nil { - t.Fatalf("a member was refused: %v", err) - } - if err := inv.AcceptSecretForModule(ctx, "workstation", "mounts", "smb-pass", "x"); err == nil { - t.Fatal("a name of no family was accepted") - } - members, err := inv.GivenMembers(ctx, "workstation", "mounts", "smb-password-*") - if err != nil { - t.Fatal(err) - } - if len(members) != 1 || members[0].Name != "smb-password-games" || !members[0].Current || members[0].Sealed == "" || - strings.Contains(members[0].Sealed, "hunter2") { - t.Fatalf("members: %+v", members) - } - given, err := inv.GivenOwnSecrets(ctx, "workstation", "mounts") - if err != nil { - t.Fatal(err) - } - if _, was := given["smb-password-games"]; !was || len(given) != 1 { - t.Fatalf("given: %v", given) - } - // A value the mesh made is not one a person gave. - if _, err := inv.SecretForModule(ctx, "workstation", "mounts", "token"); err != nil { - t.Fatal(err) - } - if given, _ := inv.GivenOwnSecrets(ctx, "workstation", "mounts"); len(given) != 1 { - t.Fatalf("a value the mesh made counted as given: %v", given) - } -} diff --git a/internal/inventory/secret_move_test.go b/internal/inventory/secret_move_test.go index 3f964db3..9050ad15 100644 --- a/internal/inventory/secret_move_test.go +++ b/internal/inventory/secret_move_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/secrets_test.go b/internal/inventory/secrets_test.go index 760db472..69abd1fa 100644 --- a/internal/inventory/secrets_test.go +++ b/internal/inventory/secrets_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( @@ -6,12 +8,13 @@ import ( "crypto/rand" "encoding/base64" "errors" - "github.com/novox/mesh-controller/internal/catalogue" - "github.com/novox/mesh-controller/internal/secrets" "reflect" "strings" "testing" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/secrets" + "golang.org/x/crypto/nacl/box" ) diff --git a/internal/inventory/sends_test.go b/internal/inventory/sends_test.go index 37aaafd6..b08ab198 100644 --- a/internal/inventory/sends_test.go +++ b/internal/inventory/sends_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/sent_builds_test.go b/internal/inventory/sent_builds_test.go index 44015d8e..f28ab693 100644 --- a/internal/inventory/sent_builds_test.go +++ b/internal/inventory/sent_builds_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/sent_bus_users_test.go b/internal/inventory/sent_bus_users_test.go index 57175714..33439454 100644 --- a/internal/inventory/sent_bus_users_test.go +++ b/internal/inventory/sent_bus_users_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import "testing" diff --git a/internal/inventory/sent_plans_test.go b/internal/inventory/sent_plans_test.go index b87f81be..63a82deb 100644 --- a/internal/inventory/sent_plans_test.go +++ b/internal/inventory/sent_plans_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/settings_judged_test.go b/internal/inventory/settings_judged_test.go index 1563fa17..ca1b8488 100644 --- a/internal/inventory/settings_judged_test.go +++ b/internal/inventory/settings_judged_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/store_as_run_test.go b/internal/inventory/store_as_run_test.go index 9cb37f6c..59172256 100644 --- a/internal/inventory/store_as_run_test.go +++ b/internal/inventory/store_as_run_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/stored_silent_file_test.go b/internal/inventory/stored_silent_file_test.go index d5029b9e..a110b235 100644 --- a/internal/inventory/stored_silent_file_test.go +++ b/internal/inventory/stored_silent_file_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/template_test.go b/internal/inventory/template_test.go index b9f58cc8..75c8c12e 100644 --- a/internal/inventory/template_test.go +++ b/internal/inventory/template_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/tunnel_test.go b/internal/inventory/tunnel_test.go index 9498ad4b..d6e7980a 100644 --- a/internal/inventory/tunnel_test.go +++ b/internal/inventory/tunnel_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/unknown_field_test.go b/internal/inventory/unknown_field_test.go index fd163f86..0995f182 100644 --- a/internal/inventory/unknown_field_test.go +++ b/internal/inventory/unknown_field_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/unseen_test.go b/internal/inventory/unseen_test.go index 73a98031..e19b0752 100644 --- a/internal/inventory/unseen_test.go +++ b/internal/inventory/unseen_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/walkphases_582_integration_test.go b/internal/inventory/walkphases_582_integration_test.go new file mode 100644 index 00000000..643d2510 --- /dev/null +++ b/internal/inventory/walkphases_582_integration_test.go @@ -0,0 +1,49 @@ +//go:build integration + +package inventory + +import ( + "testing" + "time" +) + +func TestAWalksRestIsReadFromEachMachinesOwnSend(t *testing.T) { + inv := ForTest(t) + ctx := t.Context() + record, err := inv.AddNode(ctx, "g14") + if err != nil { + t.Fatal(err) + } + if _, err := inv.NextSequence(ctx, record.ID); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, record.ID, "d-582", map[string]string{"build-agent": "c1"}); err != nil { + t.Fatal(err) + } + sent, found, err := inv.SentTo(ctx, "g14") + if err != nil || !found || sent.At.IsZero() || sent.Digest != "d-582" { + t.Fatalf("the machine's own send time: %+v %v %v", sent, found, err) + } + if _, ok, err := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second)); err != nil || ok { + t.Fatalf("no report yet: %v %v", ok, err) + } + if err := inv.RecordApplyDuration(ctx, "g14", "d-582", OutcomeApplied); err != nil { + t.Fatal(err) + } + // The walk's record of its send, 20 s after the machine's own: still found, by the machine's own send. + r, ok, err := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second)) + if err != nil || !ok || r.Outcome != OutcomeApplied || r.At.Before(sent.At) { + t.Fatalf("the machine's own report of its send: %+v %v %v", r, ok, err) + } + // A later send, reported: never this walk's. + if err := inv.RecordSent(ctx, record.ID, "d-next", map[string]string{"build-agent": "c2"}); err != nil { + t.Fatal(err) + } + if err := inv.RecordApplyDuration(ctx, "g14", "d-next", OutcomeApplied); err != nil { + t.Fatal(err) + } + again, _, _ := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second)) + if !again.At.Equal(r.At) { + t.Fatalf("a later send's report read as this walk's: %+v, was %+v", again, r) + } +} diff --git a/internal/inventory/walkphases_582_test.go b/internal/inventory/walkphases_582_test.go index 98d8cab2..0b363f87 100644 --- a/internal/inventory/walkphases_582_test.go +++ b/internal/inventory/walkphases_582_test.go @@ -146,46 +146,6 @@ func TestAMachineSentTwiceByOneWalkIsHeardByItsLaterSend(t *testing.T) { // The store: a machine's own send time is read with what it was sent, and its `apply` duration is found by it — // the send and the duration start at one moment, the node's sent_at. -func TestAWalksRestIsReadFromEachMachinesOwnSend(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - record, err := inv.AddNode(ctx, "g14") - if err != nil { - t.Fatal(err) - } - if _, err := inv.NextSequence(ctx, record.ID); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, record.ID, "d-582", map[string]string{"build-agent": "c1"}); err != nil { - t.Fatal(err) - } - sent, found, err := inv.SentTo(ctx, "g14") - if err != nil || !found || sent.At.IsZero() || sent.Digest != "d-582" { - t.Fatalf("the machine's own send time: %+v %v %v", sent, found, err) - } - if _, ok, err := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second)); err != nil || ok { - t.Fatalf("no report yet: %v %v", ok, err) - } - if err := inv.RecordApplyDuration(ctx, "g14", "d-582", OutcomeApplied); err != nil { - t.Fatal(err) - } - // The walk's record of its send, 20 s after the machine's own: still found, by the machine's own send. - r, ok, err := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second)) - if err != nil || !ok || r.Outcome != OutcomeApplied || r.At.Before(sent.At) { - t.Fatalf("the machine's own report of its send: %+v %v %v", r, ok, err) - } - // A later send, reported: never this walk's. - if err := inv.RecordSent(ctx, record.ID, "d-next", map[string]string{"build-agent": "c2"}); err != nil { - t.Fatal(err) - } - if err := inv.RecordApplyDuration(ctx, "g14", "d-next", OutcomeApplied); err != nil { - t.Fatal(err) - } - again, _, _ := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second)) - if !again.At.Equal(r.At) { - t.Fatalf("a later send's report read as this walk's: %+v, was %+v", again, r) - } -} // Issue 840 (ADR 0362): the walk's report of a machine is asked for with only the provisioned modules the walk sent // there, never every module the machine's declaration hands to the provisioner. diff --git a/internal/inventory/walkphases_integration_test.go b/internal/inventory/walkphases_integration_test.go new file mode 100644 index 00000000..f7cbb108 --- /dev/null +++ b/internal/inventory/walkphases_integration_test.go @@ -0,0 +1,77 @@ +//go:build integration + +package inventory + +import ( + "testing" + "time" +) + +func TestAWalksMomentsAreKeptAndItsRestsReportRead(t *testing.T) { + inv := ForTest(t) + ctx := t.Context() + p := walkOf(t, recordedWalk) + p.Revision, p.Epoch = 0, 0 + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + back, err := inv.PlanByID(ctx, p.ID) + if err != nil || back.Times == nil || back.Times.Class != ClassCore || back.Times.WindowClosed == nil || + !back.Times.WindowClosed.Equal(*p.Times.WindowClosed) { + t.Fatalf("the walk's moments were not kept: %v %+v", err, back.Times) + } + if back.Delivery.Merges[0].Merged.IsZero() { + t.Fatal("a merge's time was not kept") + } + sent := time.Now().UTC().Add(-time.Minute).Truncate(time.Millisecond) + if _, ok, err := inv.FirstAppliedAfter(ctx, "ace", SentDeclaration{}, sent); err != nil || ok { + t.Fatalf("no report yet: %v %v", ok, err) + } + for _, d := range []Duration{ + {Kind: DurationApply, Subject: "ace", Node: "ace", Ref: "old@1", Started: sent.Add(-time.Hour), Took: time.Second, Detail: OutcomeApplied}, + {Kind: DurationApply, Subject: "ace", Node: "ace", Ref: "d1@2", Started: sent.Add(-time.Second), Took: 12 * time.Second, Detail: OutcomeApplied}, + } { + if err := inv.RecordDuration(ctx, d); err != nil { + t.Fatal(err) + } + } + r, ok, err := inv.FirstAppliedAfter(ctx, "ace", SentDeclaration{}, sent) + if err != nil || !ok || r.Outcome != OutcomeApplied || !r.At.Equal(sent.Add(11*time.Second)) { + t.Fatalf("the report after the send: %+v %v %v", r, ok, err) + } + if anyKept, total, err := inv.WalkPhasesKept(ctx, p.ID); err != nil || anyKept || total { + t.Fatalf("nothing kept yet: %v %v %v", anyKept, total, err) + } + if err := inv.RecordDuration(ctx, Duration{Kind: DurationWalkPhase, Subject: "core/total", Ref: p.ID + "/total", + Started: sent, Took: time.Minute}); err != nil { + t.Fatal(err) + } + if anyKept, total, err := inv.WalkPhasesKept(ctx, p.ID); err != nil || !anyKept || !total { + t.Fatalf("the total kept: %v %v %v", anyKept, total, err) + } +} + +func TestALateReportIsSilentNotTheEnd(t *testing.T) { + sent := time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC) + p := walkOf(t, recordedWalk) + p.Modules["mesh-delivery"].SentAt = &sent + p.Modules["mesh-delivery"].Rest = map[string]SentDeclaration{"laptop": {Digest: "d"}, "server": {Digest: "e"}} + w := p.WalkPhases(sent.Add(3*time.Hour), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) { + if node == "server" { + return AppliedReport{At: sent.Add(20 * time.Second), Outcome: OutcomeApplied}, true + } + return AppliedReport{At: sent.Add(2 * time.Hour), Outcome: OutcomeApplied}, true + }) + if len(w.Silent) != 1 || w.Silent[0] != "laptop" || w.End == nil || !w.End.Equal(sent.Add(20*time.Second)) { + t.Fatalf("a late report: silent %v, end %v", w.Silent, w.End) + } + inv := ForTest(t) + ctx := t.Context() + if err := inv.RecordDuration(ctx, Duration{Kind: DurationApply, Subject: "laptop", Node: "laptop", Ref: "late@1", + Started: sent.Add(time.Hour), Took: time.Second, Detail: OutcomeApplied}); err != nil { + t.Fatal(err) + } + if _, ok, err := inv.FirstAppliedAfter(ctx, "laptop", SentDeclaration{}, sent); err != nil || ok { + t.Fatalf("a send past the bound read as the walk's: %v %v", ok, err) + } +} diff --git a/internal/inventory/walkphases_test.go b/internal/inventory/walkphases_test.go index 533e66a0..216d6d5a 100644 --- a/internal/inventory/walkphases_test.go +++ b/internal/inventory/walkphases_test.go @@ -230,76 +230,9 @@ func TestAGateKeepsItsReadings(t *testing.T) { // A walk's moments are kept and read back with it; a machine's first report after a send is read from the // controller's apply durations. -func TestAWalksMomentsAreKeptAndItsRestsReportRead(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - p := walkOf(t, recordedWalk) - p.Revision, p.Epoch = 0, 0 - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - back, err := inv.PlanByID(ctx, p.ID) - if err != nil || back.Times == nil || back.Times.Class != ClassCore || back.Times.WindowClosed == nil || - !back.Times.WindowClosed.Equal(*p.Times.WindowClosed) { - t.Fatalf("the walk's moments were not kept: %v %+v", err, back.Times) - } - if back.Delivery.Merges[0].Merged.IsZero() { - t.Fatal("a merge's time was not kept") - } - sent := time.Now().UTC().Add(-time.Minute).Truncate(time.Millisecond) - if _, ok, err := inv.FirstAppliedAfter(ctx, "ace", SentDeclaration{}, sent); err != nil || ok { - t.Fatalf("no report yet: %v %v", ok, err) - } - for _, d := range []Duration{ - {Kind: DurationApply, Subject: "ace", Node: "ace", Ref: "old@1", Started: sent.Add(-time.Hour), Took: time.Second, Detail: OutcomeApplied}, - {Kind: DurationApply, Subject: "ace", Node: "ace", Ref: "d1@2", Started: sent.Add(-time.Second), Took: 12 * time.Second, Detail: OutcomeApplied}, - } { - if err := inv.RecordDuration(ctx, d); err != nil { - t.Fatal(err) - } - } - r, ok, err := inv.FirstAppliedAfter(ctx, "ace", SentDeclaration{}, sent) - if err != nil || !ok || r.Outcome != OutcomeApplied || !r.At.Equal(sent.Add(11*time.Second)) { - t.Fatalf("the report after the send: %+v %v %v", r, ok, err) - } - if anyKept, total, err := inv.WalkPhasesKept(ctx, p.ID); err != nil || anyKept || total { - t.Fatalf("nothing kept yet: %v %v %v", anyKept, total, err) - } - if err := inv.RecordDuration(ctx, Duration{Kind: DurationWalkPhase, Subject: "core/total", Ref: p.ID + "/total", - Started: sent, Took: time.Minute}); err != nil { - t.Fatal(err) - } - if anyKept, total, err := inv.WalkPhasesKept(ctx, p.ID); err != nil || !anyKept || !total { - t.Fatalf("the total kept: %v %v %v", anyKept, total, err) - } -} // A machine that slept past the bound and reported later is listed silent: its late report never moves the // walk's end; and a report sent past the bound is not read as the walk's. -func TestALateReportIsSilentNotTheEnd(t *testing.T) { - sent := time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC) - p := walkOf(t, recordedWalk) - p.Modules["mesh-delivery"].SentAt = &sent - p.Modules["mesh-delivery"].Rest = map[string]SentDeclaration{"laptop": {Digest: "d"}, "server": {Digest: "e"}} - w := p.WalkPhases(sent.Add(3*time.Hour), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) { - if node == "server" { - return AppliedReport{At: sent.Add(20 * time.Second), Outcome: OutcomeApplied}, true - } - return AppliedReport{At: sent.Add(2 * time.Hour), Outcome: OutcomeApplied}, true - }) - if len(w.Silent) != 1 || w.Silent[0] != "laptop" || w.End == nil || !w.End.Equal(sent.Add(20*time.Second)) { - t.Fatalf("a late report: silent %v, end %v", w.Silent, w.End) - } - inv := ForTest(t) - ctx := t.Context() - if err := inv.RecordDuration(ctx, Duration{Kind: DurationApply, Subject: "laptop", Node: "laptop", Ref: "late@1", - Started: sent.Add(time.Hour), Took: time.Second, Detail: OutcomeApplied}); err != nil { - t.Fatal(err) - } - if _, ok, err := inv.FirstAppliedAfter(ctx, "laptop", SentDeclaration{}, sent); err != nil || ok { - t.Fatalf("a send past the bound read as the walk's: %v %v", ok, err) - } -} // Moments out of order — #206's own walk on 2026-10-10 kept its first send 783 ms before its build — are said // unknown, never a negative phase: the phase before is unknown and its time joins the unknown time, the phase diff --git a/internal/inventory/walksaid_integration_test.go b/internal/inventory/walksaid_integration_test.go new file mode 100644 index 00000000..6bd21f1f --- /dev/null +++ b/internal/inventory/walksaid_integration_test.go @@ -0,0 +1,50 @@ +//go:build integration + +package inventory + +import ( + "testing" + "time" +) + +func TestAWalksStartIsKeptAndARefusedSaveLeavesItUnsaid(t *testing.T) { + inv := ForTest(t) + ctx := t.Context() + cut := time.Now().UTC().Add(-time.Minute) + p := Plan{ID: "plan-308", Repository: "novox/mesh-catalog", Commit: "abc", Created: cut, State: PlanBuilding, + Tiers: [][]string{{"messenger"}}, Modules: map[string]*PlanModule{"messenger": {}}, + Times: &PlanTimes{Cut: &cut, Class: ClassLeaf}} + var said []string + was := PlanSaved + PlanSaved = func(_ *Inventory, p Plan) { said = append(said, p.WalkSaid) } + t.Cleanup(func() { PlanSaved = was }) + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + back, err := inv.PlanByID(ctx, p.ID) + if err != nil || back.Times == nil || back.Times.Started == nil || back.Times.Class != ClassLeaf { + t.Fatalf("the start is kept with the walk: %v %+v", err, back.Times) + } + // Read back by another controller and saved: not said again. + back.State = PlanRolling + if err := inv.SavePlan(ctx, &back); err != nil { + t.Fatal(err) + } + // A stale copy ending it is refused, and unmarked. + stale := p + stale.Revision = 1 + stale.State = PlanDone + if err := inv.SavePlan(ctx, &stale); err == nil { + t.Fatal("a stale save is refused") + } + if stale.WalkSaid != "" || stale.Times.Ended != nil { + t.Fatalf("a refused save is unmarked: %q %+v", stale.WalkSaid, stale.Times) + } + back.State = PlanDone + if err := inv.SavePlan(ctx, &back); err != nil { + t.Fatal(err) + } + if len(said) != 3 || said[0] != WalkStarted || said[1] != "" || said[2] != WalkEnded { + t.Fatalf("said: %q, want started, nothing, ended", said) + } +} diff --git a/internal/inventory/walksaid_test.go b/internal/inventory/walksaid_test.go index 69cf2e5e..6c0a5275 100644 --- a/internal/inventory/walksaid_test.go +++ b/internal/inventory/walksaid_test.go @@ -87,44 +87,3 @@ func TestAWalksOutcome(t *testing.T) { // The marks are kept with the walk, so a controller replaced mid-walk neither says its start again nor misses // its end; and a save refused leaves the walk unmarked, so the save that is made says it. -func TestAWalksStartIsKeptAndARefusedSaveLeavesItUnsaid(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - cut := time.Now().UTC().Add(-time.Minute) - p := Plan{ID: "plan-308", Repository: "novox/mesh-catalog", Commit: "abc", Created: cut, State: PlanBuilding, - Tiers: [][]string{{"messenger"}}, Modules: map[string]*PlanModule{"messenger": {}}, - Times: &PlanTimes{Cut: &cut, Class: ClassLeaf}} - var said []string - was := PlanSaved - PlanSaved = func(_ *Inventory, p Plan) { said = append(said, p.WalkSaid) } - t.Cleanup(func() { PlanSaved = was }) - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - back, err := inv.PlanByID(ctx, p.ID) - if err != nil || back.Times == nil || back.Times.Started == nil || back.Times.Class != ClassLeaf { - t.Fatalf("the start is kept with the walk: %v %+v", err, back.Times) - } - // Read back by another controller and saved: not said again. - back.State = PlanRolling - if err := inv.SavePlan(ctx, &back); err != nil { - t.Fatal(err) - } - // A stale copy ending it is refused, and unmarked. - stale := p - stale.Revision = 1 - stale.State = PlanDone - if err := inv.SavePlan(ctx, &stale); err == nil { - t.Fatal("a stale save is refused") - } - if stale.WalkSaid != "" || stale.Times.Ended != nil { - t.Fatalf("a refused save is unmarked: %q %+v", stale.WalkSaid, stale.Times) - } - back.State = PlanDone - if err := inv.SavePlan(ctx, &back); err != nil { - t.Fatal(err) - } - if len(said) != 3 || said[0] != WalkStarted || said[1] != "" || said[2] != WalkEnded { - t.Fatalf("said: %q, want started, nothing, ended", said) - } -} diff --git a/internal/link/advisories_integration_test.go b/internal/link/advisories_integration_test.go new file mode 100644 index 00000000..7ce43efe --- /dev/null +++ b/internal/link/advisories_integration_test.go @@ -0,0 +1,59 @@ +//go:build integration + +package link + +import ( + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestNatsARefusedQuestionIsSaidAtOnce(t *testing.T) { + url := os.Getenv("MESH_TEST_NATS_REFUSING") + if url == "" { + t.Skip("MESH_TEST_NATS_REFUSING unset: a server whose user may not publish mesh.seat.>") + } + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + started := time.Now() + _, err = AskSeatTool(t.Context(), conn, "node-intrusion-prevention", "banned", "anchor", map[string]any{}, 10*time.Second) + if err == nil || !strings.Contains(err.Error(), "the bus refused") { + t.Fatalf("answered %v", err) + } + if took := time.Since(started); took > 3*time.Second { + t.Fatalf("a refusal took %s to be known", took) + } +} + +func TestNatsARefusalReachesItsWaiter(t *testing.T) { + url := testbus.URL(t) + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + mine, stop := refusalsOf(conn, "mesh.seat.s.tool.v.anchor") + defer stop() + other, stopOther := refusalsOf(conn, "mesh.seat.s.tool.v.laptop") + defer stopOther() + conn.ErrorHandler()(conn, nil, fmt.Errorf("%w: Permissions Violation for Publish to %q", + nats.ErrPermissionViolation, "mesh.seat.s.tool.v.anchor")) + select { + case <-mine: + case <-time.After(time.Second): + t.Fatal("the waiter was not told") + } + select { + case <-other: + t.Fatal("another subject's waiter was told") + default: + } +} diff --git a/internal/link/advisories_test.go b/internal/link/advisories_test.go index 56d174f2..06a0a759 100644 --- a/internal/link/advisories_test.go +++ b/internal/link/advisories_test.go @@ -1,15 +1,7 @@ package link import ( - "fmt" - "os" - "strings" "testing" - "time" - - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/testbus" ) // **A reader's own consumer gone is not a consumer lost**: every bucket watch and stream read-back @@ -45,48 +37,5 @@ func TestOnlyTheMeshsOwnConsumersAreSaidLost(t *testing.T) { // **A question the bus refuses is answered as refused at once**, not after its timeout: against a // server whose only user may not publish where it asks. -func TestNatsARefusedQuestionIsSaidAtOnce(t *testing.T) { - url := os.Getenv("MESH_TEST_NATS_REFUSING") - if url == "" { - t.Skip("MESH_TEST_NATS_REFUSING unset: a server whose user may not publish mesh.seat.>") - } - conn, err := nats.Connect(url) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - started := time.Now() - _, err = AskSeatTool(t.Context(), conn, "node-intrusion-prevention", "banned", "anchor", map[string]any{}, 10*time.Second) - if err == nil || !strings.Contains(err.Error(), "the bus refused") { - t.Fatalf("answered %v", err) - } - if took := time.Since(started); took > 3*time.Second { - t.Fatalf("a refusal took %s to be known", took) - } -} // The refusal reaches whoever waits on that subject, and only them. -func TestNatsARefusalReachesItsWaiter(t *testing.T) { - url := testbus.URL(t) - conn, err := nats.Connect(url) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - mine, stop := refusalsOf(conn, "mesh.seat.s.tool.v.anchor") - defer stop() - other, stopOther := refusalsOf(conn, "mesh.seat.s.tool.v.laptop") - defer stopOther() - conn.ErrorHandler()(conn, nil, fmt.Errorf("%w: Permissions Violation for Publish to %q", - nats.ErrPermissionViolation, "mesh.seat.s.tool.v.anchor")) - select { - case <-mine: - case <-time.After(time.Second): - t.Fatal("the waiter was not told") - } - select { - case <-other: - t.Fatal("another subject's waiter was told") - default: - } -} diff --git a/internal/link/builds_nats_test.go b/internal/link/builds_nats_test.go index cc5931fb..e06038fa 100644 --- a/internal/link/builds_nats_test.go +++ b/internal/link/builds_nats_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/bus_integration_test.go b/internal/link/bus_integration_test.go new file mode 100644 index 00000000..38627827 --- /dev/null +++ b/internal/link/bus_integration_test.go @@ -0,0 +1,124 @@ +//go:build integration + +package link + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestTheNatsBusEmitsTheEnvelopeTheFixturePins(t *testing.T) { + url := testbus.URL(t) + f := loadFixture(t, "events/module-event.json") + + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + js, err := conn.JetStream() + if err != nil { + t.Fatal(err) + } + if _, err := js.AddStream(&nats.StreamConfig{ + Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}, + }); err != nil && err != nats.ErrStreamNameAlreadyInUse { + t.Fatal(err) + } + + bus := OverNATS{JS: js} + body, _ := json.Marshal(f.Given.Body) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := EmitEvent(ctx, bus, f.Given.Key, f.Given.Module, f.Given.Node, f.Given.Body); err != nil { + t.Fatal(err) + } + + // Read back from the stream, not from the thing that wrote it. + raw, err := js.GetLastMsg("EVENTS", f.Wire.Subject) + if err != nil { + t.Fatalf("nothing landed on %s, which the fixture names: %v", f.Wire.Subject, err) + } + for _, h := range f.Wire.RequiredHeaders { + if raw.Header.Get(h) == "" { + t.Errorf("%s is not set on the wire, and the fixture requires it", h) + } + } + if got := raw.Header.Get("x-source"); got != f.Given.Module { + t.Errorf("x-source is %q; the subject says %q", got, f.Given.Module) + } + if string(raw.Data) != string(body) { + t.Errorf("the payload is %s, expected the body alone: %s", raw.Data, body) + } + // The envelope must not also be nested inside the payload. + var nested map[string]any + if json.Unmarshal(raw.Data, &nested) == nil { + if _, has := nested["key"]; has { + t.Error("the payload carries the envelope's own fields, which the fixture refuses") + } + } +} + +func TestAskingAToolNobodyServesFailsAtOnce(t *testing.T) { + url := testbus.URL(t) + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + bus := OverNATS{Conn: conn} + began := time.Now() + _, err = bus.AskTool(context.Background(), "nobody", "home", nil, 30*time.Second) + if err == nil { + t.Fatal("asking a tool nothing serves succeeded") + } + if took := time.Since(began); took > 2*time.Second { + t.Errorf("took %v to say nothing serves it; the caller waited out the timeout", took) + } + if !strings.Contains(err.Error(), "nothing serves") { + t.Errorf("the refusal does not say nobody is there: %v", err) + } + // And it is ErrNothingServes, through Ask too, so an asker can wait out a tool runner re-serving and + // never a refusal (novox/hq issue 486). + if !errors.Is(err, ErrNothingServes) { + t.Errorf("nothing serving the tool is not ErrNothingServes: %v", err) + } + if _, err := Ask(context.Background(), bus, "nobody", "home", nil, 30*time.Second); !errors.Is(err, ErrNothingServes) || + !strings.Contains(err.Error(), "nothing serves nobody.home") { + t.Errorf("Ask does not say ErrNothingServes in its words: %v", err) + } +} + +func TestAskingAServedToolAnswers(t *testing.T) { + url := testbus.URL(t) + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + sub, err := conn.Subscribe(ToolSubject("shop", "price"), func(m *nats.Msg) { + _ = m.Respond([]byte(`{"total":12}`)) + }) + if err != nil { + t.Fatal(err) + } + defer sub.Unsubscribe() + + got, err := OverNATS{Conn: conn}.AskTool(context.Background(), "shop", "price", + []byte(`{"qty":4}`), 5*time.Second) + if err != nil { + t.Fatal(err) + } + if string(got) != `{"total":12}` { + t.Fatalf("the answer came back as %s", got) + } +} diff --git a/internal/link/bus_test.go b/internal/link/bus_test.go index a8fa0745..24387a45 100644 --- a/internal/link/bus_test.go +++ b/internal/link/bus_test.go @@ -1,16 +1,7 @@ package link import ( - "context" - "encoding/json" - "errors" - "strings" "testing" - "time" - - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/testbus" ) // **Both implementations, one fixture.** The point of the seam is not that the transport can be @@ -20,57 +11,6 @@ import ( // Against a real server, because what the fixture pins is what reaches the wire: // // go test ./internal/link/ -run TestTheNatsBus (each test on a bus of its own: internal/testbus) -func TestTheNatsBusEmitsTheEnvelopeTheFixturePins(t *testing.T) { - url := testbus.URL(t) - f := loadFixture(t, "events/module-event.json") - - conn, err := nats.Connect(url) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - js, err := conn.JetStream() - if err != nil { - t.Fatal(err) - } - if _, err := js.AddStream(&nats.StreamConfig{ - Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}, - }); err != nil && err != nats.ErrStreamNameAlreadyInUse { - t.Fatal(err) - } - - bus := OverNATS{JS: js} - body, _ := json.Marshal(f.Given.Body) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if err := EmitEvent(ctx, bus, f.Given.Key, f.Given.Module, f.Given.Node, f.Given.Body); err != nil { - t.Fatal(err) - } - - // Read back from the stream, not from the thing that wrote it. - raw, err := js.GetLastMsg("EVENTS", f.Wire.Subject) - if err != nil { - t.Fatalf("nothing landed on %s, which the fixture names: %v", f.Wire.Subject, err) - } - for _, h := range f.Wire.RequiredHeaders { - if raw.Header.Get(h) == "" { - t.Errorf("%s is not set on the wire, and the fixture requires it", h) - } - } - if got := raw.Header.Get("x-source"); got != f.Given.Module { - t.Errorf("x-source is %q; the subject says %q", got, f.Given.Module) - } - if string(raw.Data) != string(body) { - t.Errorf("the payload is %s, expected the body alone: %s", raw.Data, body) - } - // The envelope must not also be nested inside the payload. - var nested map[string]any - if json.Unmarshal(raw.Data, &nested) == nil { - if _, has := nested["key"]; has { - t.Error("the payload carries the envelope's own fields, which the fixture refuses") - } - } -} // The subject a declaration lands on is one node's, and nothing else's — the state shape. func TestADeclarationIsAddressedToOneNode(t *testing.T) { @@ -96,60 +36,5 @@ func TestAnEventsSubjectIsDerivedFromItsSource(t *testing.T) { // A tool nobody serves says so at once. The difference between "that module is down" and "that // tool is slow" is the first thing a person asking wants, and waiting out the timeout to say it // is how a fast answer becomes a slow non-answer. -func TestAskingAToolNobodyServesFailsAtOnce(t *testing.T) { - url := testbus.URL(t) - conn, err := nats.Connect(url) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - - bus := OverNATS{Conn: conn} - began := time.Now() - _, err = bus.AskTool(context.Background(), "nobody", "home", nil, 30*time.Second) - if err == nil { - t.Fatal("asking a tool nothing serves succeeded") - } - if took := time.Since(began); took > 2*time.Second { - t.Errorf("took %v to say nothing serves it; the caller waited out the timeout", took) - } - if !strings.Contains(err.Error(), "nothing serves") { - t.Errorf("the refusal does not say nobody is there: %v", err) - } - // And it is ErrNothingServes, through Ask too, so an asker can wait out a tool runner re-serving and - // never a refusal (novox/hq issue 486). - if !errors.Is(err, ErrNothingServes) { - t.Errorf("nothing serving the tool is not ErrNothingServes: %v", err) - } - if _, err := Ask(context.Background(), bus, "nobody", "home", nil, 30*time.Second); !errors.Is(err, ErrNothingServes) || - !strings.Contains(err.Error(), "nothing serves nobody.home") { - t.Errorf("Ask does not say ErrNothingServes in its words: %v", err) - } -} // And a served tool answers, with no reply queue to declare and no correlation to check. -func TestAskingAServedToolAnswers(t *testing.T) { - url := testbus.URL(t) - conn, err := nats.Connect(url) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - - sub, err := conn.Subscribe(ToolSubject("shop", "price"), func(m *nats.Msg) { - _ = m.Respond([]byte(`{"total":12}`)) - }) - if err != nil { - t.Fatal(err) - } - defer sub.Unsubscribe() - - got, err := OverNATS{Conn: conn}.AskTool(context.Background(), "shop", "price", - []byte(`{"qty":4}`), 5*time.Second) - if err != nil { - t.Fatal(err) - } - if string(got) != `{"total":12}` { - t.Fatalf("the answer came back as %s", got) - } -} diff --git a/internal/link/busfaults_integration_test.go b/internal/link/busfaults_integration_test.go new file mode 100644 index 00000000..bed450e7 --- /dev/null +++ b/internal/link/busfaults_integration_test.go @@ -0,0 +1,96 @@ +//go:build integration + +package link + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/testbus" +) + +func aBusWithEvents(t *testing.T) *Server { + t.Helper() + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + return ConnectNats(js, nil, nil) +} + +func TestNatsBusFaultsAreOnePerUserAndKindEachClosureCountedOnce(t *testing.T) { + s := aBusWithEvents(t) + now := time.Now().UTC() + slow := BusClosure{User: "laptop.claude-code", Name: "laptop.claude-code", CID: 10, Reason: "Slow Consumer (Write Deadline)", + Opened: now.Add(-20 * time.Minute), Closed: now.Add(-2 * time.Minute), Pending: 334000} + sayClosure(t, s, BusSlowConsumerEvent, slow) + sayClosure(t, s, BusSlowConsumerEvent, slow) // said again: the module started again + again := slow + again.CID, again.Opened, again.Closed = 31, now.Add(-90*time.Second), now.Add(-time.Minute) + sayClosure(t, s, BusSlowConsumerEvent, again) + sayClosure(t, s, BusConnectionClosedEvent, BusClosure{User: "anchor.desk", CID: 7, + Reason: "Maximum Message Payload Exceeded", Opened: now.Add(-time.Hour / 2), Closed: now.Add(-30 * time.Second)}) + // Older than the hour: S9 no longer says it. + sayClosure(t, s, BusSlowConsumerEvent, BusClosure{User: "anchor.media", CID: 3, Reason: "Slow Consumer (Pending Bytes)", + Opened: now.Add(-3 * time.Hour), Closed: now.Add(-2 * time.Hour)}) + + got, err := s.BusFaults(t.Context(), now.Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 { + t.Fatalf("read %d advisories, want one for each user within the hour: %+v", len(got), got) + } + for _, a := range got { + switch a.User { + case "laptop.claude-code": + if a.Kind != AdvisorySlowConsumer || a.ID != "laptop.claude-code" || a.Count != 2 || + !a.Last.Equal(again.Closed) || !a.First.Equal(slow.Closed) || !strings.Contains(a.Said, "connection 31") { + t.Errorf("the slow consumer read as %+v: two closures, the one said twice counted once", a) + } + case "anchor.desk": + if a.Kind != AdvisoryConnectionClosed || a.Count != 1 || a.Reason != "Maximum Message Payload Exceeded" { + t.Errorf("the closed connection read as %+v", a) + } + default: + t.Errorf("an advisory for %s", a.User) + } + } +} + +func TestNatsBusFaultsReadNothingWhenTheBusModuleSaidNothing(t *testing.T) { + s := aBusWithEvents(t) + started := time.Now() + got, err := s.BusFaults(t.Context(), time.Now().Add(-time.Hour)) + if err != nil || len(got) != 0 { + t.Fatalf("read %v, %v from a stream the bus module never wrote", got, err) + } + if took := time.Since(started); took > time.Second { + t.Fatalf("reading nothing took %s: a tick must not wait on an empty stream", took) + } +} + +func TestNatsBusFaultsReadAtMostTheBoundAndSaySo(t *testing.T) { + s := aBusWithEvents(t) + was := MaxBusEventsRead + MaxBusEventsRead = 2 + t.Cleanup(func() { MaxBusEventsRead = was }) + now := time.Now().UTC() + for i := range 3 { + sayClosure(t, s, BusSlowConsumerEvent, BusClosure{User: "laptop.claude-code", CID: uint64(10 + i), + Reason: "Slow Consumer (Pending Bytes)", Opened: now.Add(-time.Hour / 2), Closed: now.Add(-time.Duration(3-i) * time.Minute)}) + } + got, err := s.BusFaults(t.Context(), now.Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].Count != 2 || !strings.Contains(got[0].Said, "at least: the read stopped at 2") { + t.Fatalf("a read past its bound read as %+v", got) + } +} diff --git a/internal/link/busfaults_test.go b/internal/link/busfaults_test.go index d1eda4ef..ccc1f0bd 100644 --- a/internal/link/busfaults_test.go +++ b/internal/link/busfaults_test.go @@ -2,12 +2,8 @@ package link import ( "encoding/json" - "strings" "testing" "time" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/testbus" ) // What the bus module says of its users' connections, read back as advisories (novox/hq ADR 0357, issue @@ -15,19 +11,6 @@ import ( // // go test ./internal/link/ -run TestNatsBusFaults (each test on a bus of its own: internal/testbus) -func aBusWithEvents(t *testing.T) *Server { - t.Helper() - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - return ConnectNats(js, nil, nil) -} - func sayClosure(t *testing.T, s *Server, event string, c BusClosure) { t.Helper() body, _ := json.Marshal(c) @@ -36,58 +19,6 @@ func sayClosure(t *testing.T, s *Server, event string, c BusClosure) { } } -func TestNatsBusFaultsAreOnePerUserAndKindEachClosureCountedOnce(t *testing.T) { - s := aBusWithEvents(t) - now := time.Now().UTC() - slow := BusClosure{User: "laptop.claude-code", Name: "laptop.claude-code", CID: 10, Reason: "Slow Consumer (Write Deadline)", - Opened: now.Add(-20 * time.Minute), Closed: now.Add(-2 * time.Minute), Pending: 334000} - sayClosure(t, s, BusSlowConsumerEvent, slow) - sayClosure(t, s, BusSlowConsumerEvent, slow) // said again: the module started again - again := slow - again.CID, again.Opened, again.Closed = 31, now.Add(-90*time.Second), now.Add(-time.Minute) - sayClosure(t, s, BusSlowConsumerEvent, again) - sayClosure(t, s, BusConnectionClosedEvent, BusClosure{User: "anchor.desk", CID: 7, - Reason: "Maximum Message Payload Exceeded", Opened: now.Add(-time.Hour / 2), Closed: now.Add(-30 * time.Second)}) - // Older than the hour: S9 no longer says it. - sayClosure(t, s, BusSlowConsumerEvent, BusClosure{User: "anchor.media", CID: 3, Reason: "Slow Consumer (Pending Bytes)", - Opened: now.Add(-3 * time.Hour), Closed: now.Add(-2 * time.Hour)}) - - got, err := s.BusFaults(t.Context(), now.Add(-time.Hour)) - if err != nil { - t.Fatal(err) - } - if len(got) != 2 { - t.Fatalf("read %d advisories, want one for each user within the hour: %+v", len(got), got) - } - for _, a := range got { - switch a.User { - case "laptop.claude-code": - if a.Kind != AdvisorySlowConsumer || a.ID != "laptop.claude-code" || a.Count != 2 || - !a.Last.Equal(again.Closed) || !a.First.Equal(slow.Closed) || !strings.Contains(a.Said, "connection 31") { - t.Errorf("the slow consumer read as %+v: two closures, the one said twice counted once", a) - } - case "anchor.desk": - if a.Kind != AdvisoryConnectionClosed || a.Count != 1 || a.Reason != "Maximum Message Payload Exceeded" { - t.Errorf("the closed connection read as %+v", a) - } - default: - t.Errorf("an advisory for %s", a.User) - } - } -} - -func TestNatsBusFaultsReadNothingWhenTheBusModuleSaidNothing(t *testing.T) { - s := aBusWithEvents(t) - started := time.Now() - got, err := s.BusFaults(t.Context(), time.Now().Add(-time.Hour)) - if err != nil || len(got) != 0 { - t.Fatalf("read %v, %v from a stream the bus module never wrote", got, err) - } - if took := time.Since(started); took > time.Second { - t.Fatalf("reading nothing took %s: a tick must not wait on an empty stream", took) - } -} - func TestTheControllersOwnSlowConsumerAndItsClosedConnectionAreOneAdvisory(t *testing.T) { at := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC) own := []Advisory{{Kind: AdvisorySlowConsumer, ID: "controller", Said: "the controller fell behind", First: at, Last: at, Count: 1}} @@ -101,21 +32,3 @@ func TestTheControllersOwnSlowConsumerAndItsClosedConnectionAreOneAdvisory(t *te // **A busy hour is read up to a bound, and says it was cut** (review of #326): a storm of closures must not // spend a watchdog tick, and a count cut short is a floor, said as one. -func TestNatsBusFaultsReadAtMostTheBoundAndSaySo(t *testing.T) { - s := aBusWithEvents(t) - was := MaxBusEventsRead - MaxBusEventsRead = 2 - t.Cleanup(func() { MaxBusEventsRead = was }) - now := time.Now().UTC() - for i := range 3 { - sayClosure(t, s, BusSlowConsumerEvent, BusClosure{User: "laptop.claude-code", CID: uint64(10 + i), - Reason: "Slow Consumer (Pending Bytes)", Opened: now.Add(-time.Hour / 2), Closed: now.Add(-time.Duration(3-i) * time.Minute)}) - } - got, err := s.BusFaults(t.Context(), now.Add(-time.Hour)) - if err != nil { - t.Fatal(err) - } - if len(got) != 1 || got[0].Count != 2 || !strings.Contains(got[0].Said, "at least: the read stopped at 2") { - t.Fatalf("a read past its bound read as %+v", got) - } -} diff --git a/internal/link/calls_integration_test.go b/internal/link/calls_integration_test.go new file mode 100644 index 00000000..500d1c31 --- /dev/null +++ b/internal/link/calls_integration_test.go @@ -0,0 +1,38 @@ +//go:build integration + +package link + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestAHandlerThatPanicsAnswersAnError(t *testing.T) { + conn, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + stop, err := OverNATS{Conn: conn}.ServeSeatTools("panicky", map[string]ToolHandler{ + "boom": func(context.Context, json.RawMessage) (any, error) { panic("nil map") }, + "fine": func(context.Context, json.RawMessage) (any, error) { return "ok", nil }, + }, nil) + if err != nil { + t.Fatal(err) + } + defer stop() + answer, err := AskMeshSeatTool(context.Background(), conn, "panicky", "boom", map[string]any{}, 5*time.Second) + if err != nil || !strings.Contains(answer.Error, "failed inside the controller") { + t.Fatalf("a panic answered %+v (%v)", answer, err) + } + answer, err = AskMeshSeatTool(context.Background(), conn, "panicky", "fine", map[string]any{}, 5*time.Second) + if err != nil || answer.Error != "" { + t.Fatalf("the call after a panic answered %+v (%v)", answer, err) + } +} diff --git a/internal/link/calls_kept_integration_test.go b/internal/link/calls_kept_integration_test.go new file mode 100644 index 00000000..db13270c --- /dev/null +++ b/internal/link/calls_kept_integration_test.go @@ -0,0 +1,219 @@ +//go:build integration + +package link + +import ( + "bytes" + "context" + "encoding/json" + "log" + "strings" + "testing" + + "github.com/nats-io/nats.go/jetstream" + "github.com/novox/mesh-controller/internal/broker" +) + +func callsBucket(t *testing.T) *BusCalls { + t.Helper() + js := aBus(t) + api, err := jetstream.New(js.Conn()) + if err != nil { + t.Fatal(err) + } + _ = api.DeleteKeyValue(t.Context(), broker.CallsBucket) + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + keeper, err := CallsOnTheBus(t.Context(), js.Conn()) + if err != nil { + t.Fatal(err) + } + return keeper +} + +func TestNatsACallsOutcomeOutlivesItsController(t *testing.T) { + keeper := callsBucket(t) + first := NewCallLog() + if err := first.Durably(t.Context(), keeper, "controller@one", nil); err != nil { + t.Fatal(err) + } + a := newAnswers(t) + first.serveCall("mesh-controller", "push", json.RawMessage(`{"node":"anchor","values":"secret"}`), + "_INBOX.node-tools.g14.Pe3sGzAtv8jUBYQ6sKSvKz.1", + func(context.Context, json.RawMessage) (any, error) { return "anchor told", nil }, a.respond, nil) + recent, _ := first.Recent() + finished := waitKept(t, keeper, recent[0].ID, CallAnswered) + // A second call, still running when its controller stops. + left := first.begin("mesh-controller", "plans", json.RawMessage(`{}`), "") + waitKept(t, keeper, left.ID, CallRunning) + + var said bytes.Buffer + second := NewCallLog() + if err := second.Durably(t.Context(), keeper, "controller@two", log.New(&said, "", 0)); err != nil { + t.Fatal(err) + } + c, found, err := second.Get(finished.ID) + if err != nil || !found { + t.Fatalf("the next controller cannot read %s: %v %v", finished.ID, found, err) + } + if !strings.Contains(string(c.Answer), "anchor told") || c.Caller != "node-tools.g14" || c.Holder != "controller@one" { + t.Fatalf("kept %+v", c) + } + if strings.Contains(string(c.Args), "secret") { + t.Fatalf("a setting was kept: %s", c.Args) + } + abandoned, _, _ := second.Get(left.ID) + if abandoned.State != CallAbandoned || !strings.Contains(said.String(), left.ID) { + t.Fatalf("a call left running reads %q, and was said: %q", abandoned.State, said.String()) + } + listed, err := second.Recent() + if err != nil || len(listed) != 2 { + t.Fatalf("the next controller lists %d calls: %v", len(listed), err) + } + // Its own running calls are not its predecessor's: a controller starting again under the same + // name leaves them alone. + mine := second.begin("mesh-controller", "status", nil, "") + waitKept(t, keeper, mine.ID, CallRunning) + if err := second.Durably(t.Context(), keeper, "controller@two", nil); err != nil { + t.Fatal(err) + } + if c, _, _ := keeper.Kept(t.Context(), mine.ID); c.State != CallRunning { + t.Fatalf("a controller marked its own running call %q", c.State) + } +} + +func TestNatsAFinishRacingTheSweepsSnapshotReadsFinished(t *testing.T) { + keeper := callsBucket(t) + running := Call{ID: "call-711-1", Seat: "mesh-controller", Verb: "push", State: CallRunning, Holder: "controller@one"} + if err := keeper.Keep(t.Context(), running); err != nil { + t.Fatal(err) + } + waitKept(t, keeper, running.ID, CallRunning) + + finished := running + finished.State = CallAnswered + finished.Answer = json.RawMessage(`"anchor told"`) + wrapped := racingRecent{CallKeeper: keeper, thenLand: func() { + if err := keeper.Keep(context.Background(), finished); err != nil { + t.Error(err) + } + waitKept(t, keeper, running.ID, CallAnswered) + }} + + var said bytes.Buffer + second := NewCallLog() + if err := second.Durably(t.Context(), wrapped, "controller@two", log.New(&said, "", 0)); err != nil { + t.Fatal(err) + } + got, found, err := second.Get(running.ID) + if err != nil || !found || got.State != CallAnswered || !strings.Contains(string(got.Answer), "anchor told") { + t.Fatalf("its finish did not stand: %+v %v %v", got, found, err) + } + if strings.Contains(said.String(), running.ID) { + t.Errorf("a finished call was said abandoned: %q", said.String()) + } +} + +func TestNatsARunningCallWithNoFinishIsStillAbandoned(t *testing.T) { + keeper := callsBucket(t) + running := Call{ID: "call-711-2", Seat: "mesh-controller", Verb: "plans", State: CallRunning, Holder: "controller@one"} + if err := keeper.Keep(t.Context(), running); err != nil { + t.Fatal(err) + } + waitKept(t, keeper, running.ID, CallRunning) + + var said bytes.Buffer + second := NewCallLog() + if err := second.Durably(t.Context(), keeper, "controller@two", log.New(&said, "", 0)); err != nil { + t.Fatal(err) + } + got, found, err := second.Get(running.ID) + if err != nil || !found || got.State != CallAbandoned || !strings.Contains(said.String(), running.ID) { + t.Fatalf("got %+v %v %v; said %q", got, found, err, said.String()) + } +} + +func TestNatsAFinishRacingTheSweepsWriteWins(t *testing.T) { + keeper := callsBucket(t) + running := Call{ID: "call-711-3", Seat: "mesh-controller", Verb: "push", State: CallRunning, Holder: "controller@one"} + if err := keeper.Keep(t.Context(), running); err != nil { + t.Fatal(err) + } + waitKept(t, keeper, running.ID, CallRunning) + + finished := running + finished.State = CallFinishedAfter + finished.Answer = json.RawMessage(`"anchor told"`) + landed := false + racey := &BusCalls{kv: delayedKV{KeyValue: keeper.kv, before: func() { + if landed { + return + } + landed = true + if err := keeper.Keep(context.Background(), finished); err != nil { + t.Error(err) + } + waitKept(t, keeper, running.ID, CallFinishedAfter) + }}} + + var said bytes.Buffer + second := NewCallLog() + if err := second.Durably(t.Context(), racey, "controller@two", log.New(&said, "", 0)); err != nil { + t.Fatal(err) + } + got, found, err := second.Get(running.ID) + if err != nil || !found || got.State != CallFinishedAfter || !strings.Contains(string(got.Answer), "anchor told") { + t.Fatalf("the finish did not win the race: %+v %v %v", got, found, err) + } + if strings.Contains(said.String(), running.ID) { + t.Errorf("a finished call was said abandoned: %q", said.String()) + } +} + +func TestNatsAMarkAbandonedGivesUpAfterTenTries(t *testing.T) { + keeper := callsBucket(t) + running := Call{ID: "call-711-4", Seat: "mesh-controller", Verb: "push", State: CallRunning, Holder: "controller@one"} + if err := keeper.Keep(t.Context(), running); err != nil { + t.Fatal(err) + } + waitKept(t, keeper, running.ID, CallRunning) + + tries := 0 + stuck := &BusCalls{kv: alwaysMismatchKV{KeyValue: keeper.kv, tries: &tries}} + _, err := stuck.MarkAbandoned(t.Context(), running.ID) + if err == nil || !strings.Contains(err.Error(), running.ID) { + t.Fatalf("expected an error naming %s, got %v", running.ID, err) + } + if tries != markAbandonedTries { + t.Fatalf("tried %d times, want %d", tries, markAbandonedTries) + } +} + +func TestNatsTheCallsBucketIsBounded(t *testing.T) { + callsBucket(t) + js := aBus(t) + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + info, err := js.Context().StreamInfo("KV_" + broker.CallsBucket) + if err != nil { + t.Fatal(err) + } + if info.Config.MaxMsgs != 2*broker.KeptCallsDurably || info.Config.MaxAge != broker.CallsKeptFor { + t.Fatalf("kept %d messages for %s", info.Config.MaxMsgs, info.Config.MaxAge) + } +} + +func TestNatsAnAnswerLargerThanTheBoundIsCut(t *testing.T) { + keeper := callsBucket(t) + big, _ := json.Marshal(map[string]string{"result": strings.Repeat("x", broker.CallAnswerBytes+10)}) + c := Call{ID: "call-1-1", Seat: "mesh-controller", Verb: "plan", State: CallAnswered, Answer: big} + if err := keeper.Keep(t.Context(), c); err != nil { + t.Fatal(err) + } + got, found, err := keeper.Kept(t.Context(), "call-1-1") + if err != nil || !found || !strings.Contains(string(got.Answer), "the first") { + t.Fatalf("%v %v %.200s", found, err, got.Answer) + } +} diff --git a/internal/link/calls_kept_test.go b/internal/link/calls_kept_test.go index 175d14a2..1b285137 100644 --- a/internal/link/calls_kept_test.go +++ b/internal/link/calls_kept_test.go @@ -1,40 +1,16 @@ package link import ( - "bytes" "context" - "encoding/json" - "log" - "strings" "testing" "time" "github.com/nats-io/nats.go/jetstream" - - "github.com/novox/mesh-controller/internal/broker" ) // The calls bucket against a real server (novox/hq to-be 45 §6): whether a call's outcome is // readable by id from a controller that did not serve it is a claim about what the bus keeps. -func callsBucket(t *testing.T) *BusCalls { - t.Helper() - js := aBus(t) - api, err := jetstream.New(js.Conn()) - if err != nil { - t.Fatal(err) - } - _ = api.DeleteKeyValue(t.Context(), broker.CallsBucket) - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - keeper, err := CallsOnTheBus(t.Context(), js.Conn()) - if err != nil { - t.Fatal(err) - } - return keeper -} - // waitKept waits until the keeper holds a call in a state, the writes being queued. func waitKept(t *testing.T, keeper CallKeeper, id, state string) Call { t.Helper() @@ -54,56 +30,6 @@ func waitKept(t *testing.T, keeper CallKeeper, id, state string) Call { // **A controller restart keeps every call's outcome** (to-be 45 Phase 0): a call one controller // answered is read whole by id from the next, and a call it left running is said abandoned rather // than running for ever. -func TestNatsACallsOutcomeOutlivesItsController(t *testing.T) { - keeper := callsBucket(t) - first := NewCallLog() - if err := first.Durably(t.Context(), keeper, "controller@one", nil); err != nil { - t.Fatal(err) - } - a := newAnswers(t) - first.serveCall("mesh-controller", "push", json.RawMessage(`{"node":"anchor","values":"secret"}`), - "_INBOX.node-tools.g14.Pe3sGzAtv8jUBYQ6sKSvKz.1", - func(context.Context, json.RawMessage) (any, error) { return "anchor told", nil }, a.respond, nil) - recent, _ := first.Recent() - finished := waitKept(t, keeper, recent[0].ID, CallAnswered) - // A second call, still running when its controller stops. - left := first.begin("mesh-controller", "plans", json.RawMessage(`{}`), "") - waitKept(t, keeper, left.ID, CallRunning) - - var said bytes.Buffer - second := NewCallLog() - if err := second.Durably(t.Context(), keeper, "controller@two", log.New(&said, "", 0)); err != nil { - t.Fatal(err) - } - c, found, err := second.Get(finished.ID) - if err != nil || !found { - t.Fatalf("the next controller cannot read %s: %v %v", finished.ID, found, err) - } - if !strings.Contains(string(c.Answer), "anchor told") || c.Caller != "node-tools.g14" || c.Holder != "controller@one" { - t.Fatalf("kept %+v", c) - } - if strings.Contains(string(c.Args), "secret") { - t.Fatalf("a setting was kept: %s", c.Args) - } - abandoned, _, _ := second.Get(left.ID) - if abandoned.State != CallAbandoned || !strings.Contains(said.String(), left.ID) { - t.Fatalf("a call left running reads %q, and was said: %q", abandoned.State, said.String()) - } - listed, err := second.Recent() - if err != nil || len(listed) != 2 { - t.Fatalf("the next controller lists %d calls: %v", len(listed), err) - } - // Its own running calls are not its predecessor's: a controller starting again under the same - // name leaves them alone. - mine := second.begin("mesh-controller", "status", nil, "") - waitKept(t, keeper, mine.ID, CallRunning) - if err := second.Durably(t.Context(), keeper, "controller@two", nil); err != nil { - t.Fatal(err) - } - if c, _, _ := keeper.Kept(t.Context(), mine.ID); c.State != CallRunning { - t.Fatalf("a controller marked its own running call %q", c.State) - } -} // racingRecent wraps a CallKeeper so a finish lands right after its Recent() snapshot is taken, as // if that finish raced the moment the snapshot was read (novox/hq issue 711): the snapshot still @@ -125,58 +51,9 @@ func (r racingRecent) Recent(ctx context.Context) ([]Call, error) { // **A finish recorded before the sweep reaches a call stands, with its kept answer** (novox/hq issue // 711): a stale "running" snapshot is no excuse to overwrite a record that has since said how the // call ended. A caller reading `abandoned` for a call that succeeded would redo work already done. -func TestNatsAFinishRacingTheSweepsSnapshotReadsFinished(t *testing.T) { - keeper := callsBucket(t) - running := Call{ID: "call-711-1", Seat: "mesh-controller", Verb: "push", State: CallRunning, Holder: "controller@one"} - if err := keeper.Keep(t.Context(), running); err != nil { - t.Fatal(err) - } - waitKept(t, keeper, running.ID, CallRunning) - - finished := running - finished.State = CallAnswered - finished.Answer = json.RawMessage(`"anchor told"`) - wrapped := racingRecent{CallKeeper: keeper, thenLand: func() { - if err := keeper.Keep(context.Background(), finished); err != nil { - t.Error(err) - } - waitKept(t, keeper, running.ID, CallAnswered) - }} - - var said bytes.Buffer - second := NewCallLog() - if err := second.Durably(t.Context(), wrapped, "controller@two", log.New(&said, "", 0)); err != nil { - t.Fatal(err) - } - got, found, err := second.Get(running.ID) - if err != nil || !found || got.State != CallAnswered || !strings.Contains(string(got.Answer), "anchor told") { - t.Fatalf("its finish did not stand: %+v %v %v", got, found, err) - } - if strings.Contains(said.String(), running.ID) { - t.Errorf("a finished call was said abandoned: %q", said.String()) - } -} // A call left running with no finish recorded is still marked abandoned — unchanged (novox/hq issue // 711): only a finish actually recorded for it may stand instead. -func TestNatsARunningCallWithNoFinishIsStillAbandoned(t *testing.T) { - keeper := callsBucket(t) - running := Call{ID: "call-711-2", Seat: "mesh-controller", Verb: "plans", State: CallRunning, Holder: "controller@one"} - if err := keeper.Keep(t.Context(), running); err != nil { - t.Fatal(err) - } - waitKept(t, keeper, running.ID, CallRunning) - - var said bytes.Buffer - second := NewCallLog() - if err := second.Durably(t.Context(), keeper, "controller@two", log.New(&said, "", 0)); err != nil { - t.Fatal(err) - } - got, found, err := second.Get(running.ID) - if err != nil || !found || got.State != CallAbandoned || !strings.Contains(said.String(), running.ID) { - t.Fatalf("got %+v %v %v; said %q", got, found, err, said.String()) - } -} // delayedKV wraps a jetstream.KeyValue so a hook runs just before Update, to land a write between a // reader's Get and its own compare-and-set write. @@ -193,42 +70,6 @@ func (d delayedKV) Update(ctx context.Context, key string, value []byte, revisio // **The race where a finish lands after the sweep's own read but before its write: the finish // wins** (novox/hq issue 711). The sweep's write is a compare-and-set on the revision it read, so a // finish landing in between invalidates it; the sweep rereads, finds the finish, and leaves it alone. -func TestNatsAFinishRacingTheSweepsWriteWins(t *testing.T) { - keeper := callsBucket(t) - running := Call{ID: "call-711-3", Seat: "mesh-controller", Verb: "push", State: CallRunning, Holder: "controller@one"} - if err := keeper.Keep(t.Context(), running); err != nil { - t.Fatal(err) - } - waitKept(t, keeper, running.ID, CallRunning) - - finished := running - finished.State = CallFinishedAfter - finished.Answer = json.RawMessage(`"anchor told"`) - landed := false - racey := &BusCalls{kv: delayedKV{KeyValue: keeper.kv, before: func() { - if landed { - return - } - landed = true - if err := keeper.Keep(context.Background(), finished); err != nil { - t.Error(err) - } - waitKept(t, keeper, running.ID, CallFinishedAfter) - }}} - - var said bytes.Buffer - second := NewCallLog() - if err := second.Durably(t.Context(), racey, "controller@two", log.New(&said, "", 0)); err != nil { - t.Fatal(err) - } - got, found, err := second.Get(running.ID) - if err != nil || !found || got.State != CallFinishedAfter || !strings.Contains(string(got.Answer), "anchor told") { - t.Fatalf("the finish did not win the race: %+v %v %v", got, found, err) - } - if strings.Contains(said.String(), running.ID) { - t.Errorf("a finished call was said abandoned: %q", said.String()) - } -} // alwaysMismatchKV wraps a jetstream.KeyValue whose Update always fails with a revision mismatch, as // if some other writer kept winning the race forever (novox/hq issue 711 review round 1). @@ -245,55 +86,11 @@ func (a alwaysMismatchKV) Update(ctx context.Context, key string, value []byte, // **MarkAbandoned gives up after markAbandonedTries rather than loop forever** (novox/hq issue 711 // review round 1): a record whose compare-and-set keeps losing is left as it stands, said in an // error naming the call, not retried without end. -func TestNatsAMarkAbandonedGivesUpAfterTenTries(t *testing.T) { - keeper := callsBucket(t) - running := Call{ID: "call-711-4", Seat: "mesh-controller", Verb: "push", State: CallRunning, Holder: "controller@one"} - if err := keeper.Keep(t.Context(), running); err != nil { - t.Fatal(err) - } - waitKept(t, keeper, running.ID, CallRunning) - - tries := 0 - stuck := &BusCalls{kv: alwaysMismatchKV{KeyValue: keeper.kv, tries: &tries}} - _, err := stuck.MarkAbandoned(t.Context(), running.ID) - if err == nil || !strings.Contains(err.Error(), running.ID) { - t.Fatalf("expected an error naming %s, got %v", running.ID, err) - } - if tries != markAbandonedTries { - t.Fatalf("tried %d times, want %d", tries, markAbandonedTries) - } -} // The bucket holds the last thousand calls or fourteen days: a call is two keys, so the stream under // it holds twice as many messages, and asserting it again keeps the count. -func TestNatsTheCallsBucketIsBounded(t *testing.T) { - callsBucket(t) - js := aBus(t) - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - info, err := js.Context().StreamInfo("KV_" + broker.CallsBucket) - if err != nil { - t.Fatal(err) - } - if info.Config.MaxMsgs != 2*broker.KeptCallsDurably || info.Config.MaxAge != broker.CallsKeptFor { - t.Fatalf("kept %d messages for %s", info.Config.MaxMsgs, info.Config.MaxAge) - } -} // An answer larger than the bound is cut and says so, and the record is still written. -func TestNatsAnAnswerLargerThanTheBoundIsCut(t *testing.T) { - keeper := callsBucket(t) - big, _ := json.Marshal(map[string]string{"result": strings.Repeat("x", broker.CallAnswerBytes+10)}) - c := Call{ID: "call-1-1", Seat: "mesh-controller", Verb: "plan", State: CallAnswered, Answer: big} - if err := keeper.Keep(t.Context(), c); err != nil { - t.Fatal(err) - } - got, found, err := keeper.Kept(t.Context(), "call-1-1") - if err != nil || !found || !strings.Contains(string(got.Answer), "the first") { - t.Fatalf("%v %v %.200s", found, err, got.Answer) - } -} // Who asked is read from the inbox the answer goes to. func TestTheCallerIsTheInboxsPrincipal(t *testing.T) { diff --git a/internal/link/calls_test.go b/internal/link/calls_test.go index 419d6800..577f7c22 100644 --- a/internal/link/calls_test.go +++ b/internal/link/calls_test.go @@ -12,10 +12,6 @@ import ( "testing" "time" "unicode" - - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/testbus" ) // answers collects what a call answered, and fails a second answer: the bus permits one. @@ -280,26 +276,3 @@ func TestAJoinTokenIsShownToItsCallerAndNotKept(t *testing.T) { // A verb whose handler panics answers an error, and the controller serves the next call (review of // novox/hq issue 327: read verbs are answered in the serving process now). -func TestAHandlerThatPanicsAnswersAnError(t *testing.T) { - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - stop, err := OverNATS{Conn: conn}.ServeSeatTools("panicky", map[string]ToolHandler{ - "boom": func(context.Context, json.RawMessage) (any, error) { panic("nil map") }, - "fine": func(context.Context, json.RawMessage) (any, error) { return "ok", nil }, - }, nil) - if err != nil { - t.Fatal(err) - } - defer stop() - answer, err := AskMeshSeatTool(context.Background(), conn, "panicky", "boom", map[string]any{}, 5*time.Second) - if err != nil || !strings.Contains(answer.Error, "failed inside the controller") { - t.Fatalf("a panic answered %+v (%v)", answer, err) - } - answer, err = AskMeshSeatTool(context.Background(), conn, "panicky", "fine", map[string]any{}, 5*time.Second) - if err != nil || answer.Error != "" { - t.Fatalf("the call after a panic answered %+v (%v)", answer, err) - } -} diff --git a/internal/link/check_builds_integration_test.go b/internal/link/check_builds_integration_test.go new file mode 100644 index 00000000..47e22acf --- /dev/null +++ b/internal/link/check_builds_integration_test.go @@ -0,0 +1,67 @@ +//go:build integration + +package link + +import ( + "encoding/json" + "testing" + "time" +) + +func TestNatsTheBuildsAWalkMayTakeAreTheChecksOutcomesOfItsPullRequests(t *testing.T) { + js := aBus(t) + since := time.Now().Add(-time.Minute) + say := func(r BuildResult) { + body, _ := json.Marshal(r) + if _, err := js.Context().Publish(BuildOutcomeOf(TheBuildMachine), body); err != nil { + t.Fatal(err) + } + } + mine := &CheckRequest{Owner: "novox", Repo: "shop", Number: 7} + built := CheckBuilt{Module: "app", Key: "r1:1", Made: []MadeArtifact{{Name: "server", Reference: "s/app/server@sha256:a"}}} + say(BuildResult{ID: "build-1", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{built, + {Module: "web", NotBuilt: "it has no reuse key"}}}}) + say(BuildResult{ID: "build-2", Checked: &CheckRequest{Owner: "novox", Repo: "shop", Number: 8}, + Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{built}}}) + say(BuildResult{ID: "build-3", Repository: "novox/shop", Commit: "c0ffee"}) // a build, not a check + newer := built + newer.Key = "r1:2" + say(BuildResult{ID: "build-4", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{newer}}}) + + got, err := ReadCheckBuilds(t.Context(), js, TheBuildMachine, []PullRef{{Owner: "novox", Repo: "shop", Number: 7}}, since) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || got[0].Check != "build-4" || got[0].Built.Key != "r1:2" || got[1].Check != "build-1" || + got[1].Pull.Number != 7 { + t.Fatalf("read %+v", got) + } +} + +func TestNatsAKeptBuildIsTheBuildersCheckAndEveryBuildSaysItsHead(t *testing.T) { + js := aBus(t) + since := time.Now().Add(-time.Minute) + say := func(r BuildResult) { + body, _ := json.Marshal(r) + if _, err := js.Context().Publish(BuildOutcomeOf(TheBuildMachine), body); err != nil { + t.Fatal(err) + } + } + mine := &CheckRequest{Owner: "novox", Repo: "shop", Number: 7} + app := CheckBuilt{Module: "app", Key: "r1:1", Made: []MadeArtifact{{Name: "server", Reference: "s/app/server@sha256:a"}}} + say(BuildResult{ID: "build-1", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Head: "aaaaaaaaaaaa", + Built: []CheckBuilt{app}}}) + kept := app + kept.Kept = "build-1" + say(BuildResult{ID: "build-2", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Head: "aaaaaaaaaaaa", + Built: []CheckBuilt{kept}}}) + + got, err := ReadCheckBuilds(t.Context(), js, TheBuildMachine, []PullRef{{Owner: "novox", Repo: "shop", Number: 7}}, since) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || got[0].Check != "build-1" || got[0].Built.Kept != "build-1" || got[0].Head != "aaaaaaaaaaaa" || + got[1].Check != "build-1" || got[1].Head != "aaaaaaaaaaaa" { + t.Fatalf("read %+v", got) + } +} diff --git a/internal/link/check_builds_test.go b/internal/link/check_builds_test.go index ca938fe2..ed03bb5c 100644 --- a/internal/link/check_builds_test.go +++ b/internal/link/check_builds_test.go @@ -1,72 +1,14 @@ package link import ( - "encoding/json" "testing" - "time" ) // The builds a walk may take are read from the build seat's own outcomes of its checks (novox/hq ADR 0350 §2): only // checks of the walk's pull requests, only builds made, newest first. -func TestNatsTheBuildsAWalkMayTakeAreTheChecksOutcomesOfItsPullRequests(t *testing.T) { - js := aBus(t) - since := time.Now().Add(-time.Minute) - say := func(r BuildResult) { - body, _ := json.Marshal(r) - if _, err := js.Context().Publish(BuildOutcomeOf(TheBuildMachine), body); err != nil { - t.Fatal(err) - } - } - mine := &CheckRequest{Owner: "novox", Repo: "shop", Number: 7} - built := CheckBuilt{Module: "app", Key: "r1:1", Made: []MadeArtifact{{Name: "server", Reference: "s/app/server@sha256:a"}}} - say(BuildResult{ID: "build-1", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{built, - {Module: "web", NotBuilt: "it has no reuse key"}}}}) - say(BuildResult{ID: "build-2", Checked: &CheckRequest{Owner: "novox", Repo: "shop", Number: 8}, - Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{built}}}) - say(BuildResult{ID: "build-3", Repository: "novox/shop", Commit: "c0ffee"}) // a build, not a check - newer := built - newer.Key = "r1:2" - say(BuildResult{ID: "build-4", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{newer}}}) - - got, err := ReadCheckBuilds(t.Context(), js, TheBuildMachine, []PullRef{{Owner: "novox", Repo: "shop", Number: 7}}, since) - if err != nil { - t.Fatal(err) - } - if len(got) != 2 || got[0].Check != "build-4" || got[0].Built.Key != "r1:2" || got[1].Check != "build-1" || - got[1].Pull.Number != 7 { - t.Fatalf("read %+v", got) - } -} // A build is read with the head its check judged, and a kept build is named as the check's that built it (novox/hq // issue 703). -func TestNatsAKeptBuildIsTheBuildersCheckAndEveryBuildSaysItsHead(t *testing.T) { - js := aBus(t) - since := time.Now().Add(-time.Minute) - say := func(r BuildResult) { - body, _ := json.Marshal(r) - if _, err := js.Context().Publish(BuildOutcomeOf(TheBuildMachine), body); err != nil { - t.Fatal(err) - } - } - mine := &CheckRequest{Owner: "novox", Repo: "shop", Number: 7} - app := CheckBuilt{Module: "app", Key: "r1:1", Made: []MadeArtifact{{Name: "server", Reference: "s/app/server@sha256:a"}}} - say(BuildResult{ID: "build-1", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Head: "aaaaaaaaaaaa", - Built: []CheckBuilt{app}}}) - kept := app - kept.Kept = "build-1" - say(BuildResult{ID: "build-2", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Head: "aaaaaaaaaaaa", - Built: []CheckBuilt{kept}}}) - - got, err := ReadCheckBuilds(t.Context(), js, TheBuildMachine, []PullRef{{Owner: "novox", Repo: "shop", Number: 7}}, since) - if err != nil { - t.Fatal(err) - } - if len(got) != 2 || got[0].Check != "build-1" || got[0].Built.Kept != "build-1" || got[0].Head != "aaaaaaaaaaaa" || - got[1].Check != "build-1" || got[1].Head != "aaaaaaaaaaaa" { - t.Fatalf("read %+v", got) - } -} // Two spellings of a head are the same head only when both say at least twelve characters of it. func TestSameHeadNeedsTwelveCharactersOfBoth(t *testing.T) { diff --git a/internal/link/checks_nats_integration_test.go b/internal/link/checks_nats_integration_test.go new file mode 100644 index 00000000..71959b63 --- /dev/null +++ b/internal/link/checks_nats_integration_test.go @@ -0,0 +1,512 @@ +//go:build integration + +package link + +import ( + "context" + "encoding/json" + "reflect" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" +) + +func TestANewerCheckRequestReplacesOneStillWaiting(t *testing.T) { + js := aBusWithTheBuildRole(t) + ctx := t.Context() + asks := BuildsOn(js, TheBuildMachine) + for _, r := range []BuildRequest{ + aPullCheck("check-7-a", 7, "head-a-0000000", "trunk-0-00000", false), + aPullCheck("check-8-a", 8, "head-b-0000000", "trunk-0-00000", false), + aPullCheck("check-7-b", 7, "head-a-0000000", "trunk-1-00000", false), // a merge moved the trunk + aPullCheck("check-8-b", 8, "head-b-0000000", "trunk-1-00000", false), + aPullCheck("check-7-c", 7, "head-a-0000000", "trunk-2-00000", false), // and a second merge within the minute + } { + if err := asks.Ask(ctx, r); err != nil { + t.Fatal(err) + } + } + var got []string + for _, a := range queueOf(t, js).Asks { + if a.State != AskWaiting { + t.Errorf("%s is %s", a.ID, a.State) + } + got = append(got, a.ID) + } + if want := []string{"check-8-b", "check-7-c"}; !reflect.DeepEqual(got, want) { + t.Fatalf("queued %v; want one check a pull request, the newest: %v", got, want) + } +} + +func TestADeliveryBuildIsTakenBeforeAReadyCheckAndThatBeforeAnyOther(t *testing.T) { + js := aBusWithTheBuildRole(t) + ctx, stop := context.WithCancel(t.Context()) + defer stop() + asks := BuildsOn(js, TheBuildMachine) + for _, r := range []BuildRequest{ + aPullCheck("other-1", 1, "h1", "t", false), + aPullCheck("other-2", 2, "h2", "t", false), + aPullCheck("ready-3", 3, "h3", "t", true), + {ID: "delivery-4", Repository: "/d"}, + aPullCheck("other-5", 5, "h5", "t", false), + aPullCheck("ready-6", 6, "h6", "t", true), + } { + if err := asks.Ask(ctx, r); err != nil { + t.Fatal(err) + } + } + took := make(chan string, 8) + machine := MachineOverNATS(js, "anchor") + defer machine.Close() + go func() { + _ = machine.Take(ctx, func(ctx context.Context, work Build) { + took <- work.Request().ID + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) + _ = work.Done() + }) + }() + var order []string + for len(order) < 6 { + select { + case id := <-took: + order = append(order, id) + case <-time.After(15 * time.Second): + t.Fatalf("taken so far: %v", order) + } + } + if want := []string{"delivery-4", "ready-3", "ready-6", "other-1", "other-2", "other-5"}; !reflect.DeepEqual(order, want) { + t.Fatalf("taken as %v; want %v", order, want) + } +} + +func TestARequestLeftBehindForAJudgedHeadIsDroppedWhenTaken(t *testing.T) { + js := aBusWithTheBuildRole(t) + kv := aKeyBucket(t, js) + ctx, stop := context.WithCancel(t.Context()) + defer stop() + putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000", Verdict: "pass"}) + putKey(t, kv, 8, PullKey{Head: "head-8-000000000", Trunk: "trunk-1-000000"}) // asked, not judged + putKey(t, kv, 9, PullKey{Head: "head-9-000000000", Trunk: "trunk-2-000000", Verdict: "pass"}) // judged against another trunk + asks := BuildsOn(js, TheBuildMachine) + for _, r := range []BuildRequest{ + aPullCheck("left-7", 7, "head-7-000000000", "trunk-1-000000", false), + aPullCheck("fresh-8", 8, "head-8-000000000", "trunk-1-000000", false), + aPullCheck("moved-9", 9, "head-9-000000000", "trunk-1-000000", false), + } { + if err := asks.Ask(ctx, r); err != nil { + t.Fatal(err) + } + } + took := make(chan string, 8) + machine := MachineOverNATS(js, "anchor") + defer machine.Close() + go func() { + _ = machine.Take(ctx, func(ctx context.Context, work Build) { + took <- work.Request().ID + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) + _ = work.Done() + }) + }() + var ran []string + for len(ran) < 2 { + select { + case id := <-took: + ran = append(ran, id) + case <-time.After(15 * time.Second): + t.Fatalf("run so far: %v", ran) + } + } + if want := []string{"fresh-8", "moved-9"}; !reflect.DeepEqual(ran, want) { + t.Fatalf("run %v; want %v, the judged one dropped", ran, want) + } + time.Sleep(300 * time.Millisecond) + if n := len(queueOf(t, js).Asks); n != 0 { + t.Errorf("%d ask(s) are left in the queues; the dropped one was to be acknowledged", n) + } +} + +func TestARunningCheckStopsWhenItsKeysHeadMovesAndAGroupsDoesNot(t *testing.T) { + js := aBusWithTheBuildRole(t) + kv := aKeyBucket(t, js) + ctx, stop := context.WithCancel(t.Context()) + defer stop() + moved := make(chan PullKey, 4) + req := aPullCheck("check-7", 7, "head-7-000000000", "trunk-1-000000", false) + end, err := WatchPull(ctx, js.Conn(), req, func(k PullKey) { moved <- k }) + if err != nil { + t.Fatal(err) + } + defer end() + putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000"}) + putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000", Verdict: "pass"}) + select { + case k := <-moved: + t.Fatalf("stopped by a key that moved nothing: %+v", k) + case <-time.After(500 * time.Millisecond): + } + putKey(t, kv, 7, PullKey{Head: "head-7-111111111", Trunk: "trunk-1-000000"}) + select { + case k := <-moved: + if k.Head != "head-7-111111111" { + t.Errorf("stopped by %+v", k) + } + case <-time.After(5 * time.Second): + t.Fatal("a key whose head moved did not stop the check") + } + + // And the trunk commit. + req2 := aPullCheck("check-8", 8, "head-8-000000000", "trunk-1-000000", false) + end2, err := WatchPull(ctx, js.Conn(), req2, func(k PullKey) { moved <- k }) + if err != nil { + t.Fatal(err) + } + defer end2() + putKey(t, kv, 8, PullKey{Head: "head-8-000000000", Trunk: "trunk-2-000000"}) + select { + case <-moved: + case <-time.After(5 * time.Second): + t.Fatal("a key whose trunk commit moved did not stop the check") + } + + // A group's check watches nothing: a sibling's merge in the group's order writes nothing to a key, and a key it does + // have is the pull request's own. + group := BuildRequest{ID: "group-1", Ref: "head-7-000000000", + Check: &CheckRequest{Owner: "novox", Repo: "a", Number: 7, Group: "g1"}} + end3, err := WatchPull(ctx, js.Conn(), group, func(k PullKey) { moved <- k }) + if err != nil { + t.Fatal(err) + } + defer end3() + putKey(t, kv, 7, PullKey{Head: "head-7-222222222", Trunk: "trunk-9-000000"}) + select { + case k := <-moved: + t.Fatalf("a group's check was stopped by %+v", k) + case <-time.After(500 * time.Millisecond): + } +} + +func TestTheQueueIsReadAcrossTheThreeStreamsAndAnAskIsDeletedFromItsOwn(t *testing.T) { + js := aBusWithTheBuildRole(t) + ctx := t.Context() + asks := BuildsOn(js, TheBuildMachine) + for _, r := range []BuildRequest{ + aPullCheck("other-1", 1, "h1", "t", false), + aPullCheck("ready-2", 2, "h2", "t", true), + {ID: "delivery-3", Repository: "/d"}, + } { + if err := asks.Ask(ctx, r); err != nil { + t.Fatal(err) + } + } + q := queueOf(t, js) + var ids, streams []string + for _, a := range q.Asks { + ids, streams = append(ids, a.ID), append(streams, a.Stream) + } + if want := []string{"delivery-3", "ready-2", "other-1"}; !reflect.DeepEqual(ids, want) { + t.Fatalf("read %v; want the order they are handed out, %v", ids, want) + } + if want := []string{"SEAT_NODE_BUILD_AGENT", broker.ChecksReadyStream, broker.ChecksOtherStream}; !reflect.DeepEqual(streams, want) { + t.Fatalf("streams %v; want %v", streams, want) + } + ready, _ := q.Find("ready-2") + if w := WorkerOf(TheBuildMachine, ready); w.Name != broker.ChecksReadyWorker { + t.Errorf("the ready check is handed out by %s", w.Name) + } + if removed, err := DeleteAsk(ctx, js, TheBuildMachine, ready); err != nil || removed != ready.Seq { + t.Fatalf("deleted %d, %v", removed, err) + } + if removed, err := DeleteAsk(ctx, js, TheBuildMachine, ready); err != nil || removed != 0 { + t.Fatalf("deleted again: %d, %v", removed, err) + } + if n := len(queueOf(t, js).Asks); n != 2 { + t.Errorf("%d asks left; want 2", n) + } +} + +func TestARunningCheckOnOneBuilderMakesAnotherDropTheSameRequestWaiting(t *testing.T) { + js := aBusWithTheBuildRole(t) + aKeyBucket(t, js) + aRunningBucket(t, js, 2*time.Minute) + ctx, stop := context.WithCancel(t.Context()) + defer stop() + asks := BuildsOn(js, TheBuildMachine) + running := make(chan string, 4) + release := make(chan struct{}) + a := MachineOverNATS(js, "builder-a") + defer a.Close() + go func() { + _ = a.Take(ctx, func(ctx context.Context, work Build) { + running <- work.Request().ID + <-release + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"}) + _ = work.Done() + }) + }() + if err := asks.Ask(ctx, aPullCheck("first", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { + t.Fatal(err) + } + select { + case <-running: + case <-time.After(10 * time.Second): + t.Fatal("builder a took nothing") + } + // The same head and trunk commit asked again while it runs, and a pull request's other head asked beside it. + if err := asks.Ask(ctx, aPullCheck("again", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { + t.Fatal(err) + } + b := MachineOverNATS(js, "builder-b") + defer b.Close() + took := make(chan string, 4) + go func() { + _ = b.Take(ctx, func(ctx context.Context, work Build) { + took <- work.Request().ID + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"}) + _ = work.Done() + }) + }() + select { + case id := <-took: + t.Fatalf("builder b took %s, which builder a is running a check of", id) + case <-time.After(3 * time.Second): + } + if n := len(queueOf(t, js).Of(AskWaiting)); n != 0 { + t.Errorf("%d request(s) still wait; builder b was to drop the one for a head and trunk commit being checked", n) + } + // Another trunk commit is another check. + if err := asks.Ask(ctx, aPullCheck("moved", 7, "head-7-000000000", "trunk-2-000000", false)); err != nil { + t.Fatal(err) + } + select { + case id := <-took: + if id != "moved" { + t.Fatalf("builder b took %s", id) + } + case <-time.After(10 * time.Second): + t.Fatal("a check against another trunk commit was dropped too") + } + close(release) +} + +func TestAKeyNotRenewedAgesOutAndTheRequestIsThenTaken(t *testing.T) { + js := aBusWithTheBuildRole(t) + aKeyBucket(t, js) + kv := aRunningBucket(t, js, 2*time.Second) + ctx, stop := context.WithCancel(t.Context()) + defer stop() + body, _ := json.Marshal(CheckRunning{Head: "head-7-000000000", Trunk: "trunk-1-000000", Builder: "dead", Build: "gone"}) + if _, err := kv.Put(ctx, PullKeyOf("novox", "a", 7), body); err != nil { + t.Fatal(err) + } + asks := BuildsOn(js, TheBuildMachine) + took := make(chan string, 4) + b := MachineOverNATS(js, "builder-b") + defer b.Close() + go func() { + _ = b.Take(ctx, func(ctx context.Context, work Build) { + took <- work.Request().ID + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"}) + _ = work.Done() + }) + }() + if err := asks.Ask(ctx, aPullCheck("while-it-lives", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { + t.Fatal(err) + } + select { + case id := <-took: + t.Fatalf("%s was taken while the key was fresh", id) + case <-time.After(1200 * time.Millisecond): + } + time.Sleep(2500 * time.Millisecond) // the key was not renewed + if err := asks.Ask(ctx, aPullCheck("after-it-aged", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { + t.Fatal(err) + } + select { + case id := <-took: + if id != "after-it-aged" { + t.Fatalf("took %s", id) + } + case <-time.After(10 * time.Second): + t.Fatal("the request was not taken after the key aged out") + } +} + +func TestARunningCheckRenewsItsKeyAndDeletesItWhenItEnds(t *testing.T) { + js := aBusWithTheBuildRole(t) + kv := aRunningBucket(t, js, 2*time.Second) + was := heartbeatWhileWorking + heartbeatWhileWorking = 400 * time.Millisecond + t.Cleanup(func() { heartbeatWhileWorking = was }) + ctx, stop := context.WithCancel(t.Context()) + defer stop() + running := make(chan struct{}, 1) + release := make(chan struct{}) + m := MachineOverNATS(js, "builder-a") + defer m.Close() + go func() { + _ = m.Take(ctx, func(ctx context.Context, work Build) { + running <- struct{}{} + <-release + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"}) + _ = work.Done() + }) + }() + if err := BuildsOn(js, TheBuildMachine).Ask(ctx, aPullCheck("long", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { + t.Fatal(err) + } + <-running + key := PullKeyOf("novox", "a", 7) + e, err := kv.Get(ctx, key) + if err != nil { + t.Fatalf("no key while the check runs: %v", err) + } + var got CheckRunning + _ = json.Unmarshal(e.Value(), &got) + if got.Head != "head-7-000000000" || got.Trunk != "trunk-1-000000" || got.Builder != "builder-a" || got.Build != "long" { + t.Errorf("the key says %+v", got) + } + time.Sleep(3 * time.Second) // longer than the bucket's age + if _, err := kv.Get(ctx, key); err != nil { + t.Fatalf("the key aged out under a check still running: %v", err) + } + close(release) + for end := time.Now().Add(5 * time.Second); ; time.Sleep(50 * time.Millisecond) { + if _, err := kv.Get(ctx, key); err != nil { + break + } + if time.Now().After(end) { + t.Fatal("the key stays after the check ended") + } + } +} + +func TestACheckStoppedByATrunkMoveLeavesTheNewerRequestTakenAndRun(t *testing.T) { + js := aBusWithTheBuildRole(t) + kv := aKeyBucket(t, js) + aRunningBucket(t, js, 2*time.Minute) + ctx, stop := context.WithCancel(t.Context()) + defer stop() + asks := BuildsOn(js, TheBuildMachine) + outcomes := make(chan BuildResult, 8) + watching, err := js.Conn().Subscribe(BuildOutcome(), func(m *nats.Msg) { + var r BuildResult + if json.Unmarshal(m.Data, &r) == nil { + outcomes <- r + } + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = watching.Unsubscribe() }() + + ran := make(chan string, 4) + m := MachineOverNATS(js, "builder-a") + defer m.Close() + go func() { + _ = m.Take(ctx, func(ctx context.Context, work Build) { + building, cancel := context.WithCancel(ctx) + defer cancel() + ran <- work.Request().ID + stopped := make(chan struct{}, 1) + end, err := work.(interface { + WatchPull(context.Context, func(PullKey)) (func(), error) + }).WatchPull(building, func(PullKey) { stopped <- struct{}{}; cancel() }) + if err != nil { + t.Error(err) + return + } + defer end() + if work.Request().ID == "older" { + select { + case <-stopped: + _ = work.Announce(ctx, BuildResult{ID: "older", On: "builder-a", Failed: SupersededByKey}) + case <-time.After(15 * time.Second): + _ = work.Announce(ctx, BuildResult{ID: "older", On: "builder-a", Failed: "never stopped"}) + } + } else { + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Check: &CheckOutcome{Verdict: "pass"}}) + } + _ = work.Done() + }) + }() + if err := asks.Ask(ctx, aPullCheck("older", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { + t.Fatal(err) + } + if id := <-ran; id != "older" { + t.Fatalf("took %s", id) + } + time.Sleep(300 * time.Millisecond) // the watch is made + // The trunk moved; the head did not. mesh-delivery writes the key for the new request, and the request is asked. + putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-2-000000"}) + if err := asks.Ask(ctx, aPullCheck("newer", 7, "head-7-000000000", "trunk-2-000000", false)); err != nil { + t.Fatal(err) + } + select { + case id := <-ran: + if id != "newer" { + t.Fatalf("took %s", id) + } + case <-time.After(15 * time.Second): + t.Fatal("the newer request, for the same head on the new trunk commit, was not taken") + } + got := map[string]string{} + for len(got) < 2 { + select { + case r := <-outcomes: + got[r.ID] = r.Failed + case <-time.After(10 * time.Second): + t.Fatalf("outcomes so far: %v", got) + } + } + if got["older"] != SupersededByKey || got["newer"] != "" { + t.Errorf("outcomes %v; want the older stopped and the newer run to a verdict", got) + } +} + +func TestAnUnacknowledgedCheckGoesToASecondBuilder(t *testing.T) { + for _, ready := range []bool{false, true} { + js := aBusWithTheBuildRole(t) + name, worker := "other", broker.ChecksOtherWorker + if ready { + name, worker = "ready", broker.ChecksReadyWorker + } + // A worker whose wait is a second, as the mesh's is a minute. + for _, w := range broker.CheckWorkers() { + if w.Name == worker { + w.AckWaitSeconds = 1 + if err := js.EnsureConsumer(w); err != nil { + t.Fatal(err) + } + } + } + ctx, stop := context.WithCancel(t.Context()) + if err := BuildsOn(js, TheBuildMachine).Ask(ctx, aPullCheck("dies-"+name, 7, "head-7-000000000", "trunk-1-000000", ready)); err != nil { + t.Fatal(err) + } + // The first builder takes it and dies: its pull is fetched, never acknowledged. + if _, err := fetchOne(js, worker); err != nil { + t.Fatal(err) + } + took := make(chan BuildRequest, 2) + second := MachineOverNATS(js, "builder-b") + go func() { + _ = second.Take(ctx, func(ctx context.Context, work Build) { + took <- work.Request() + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"}) + _ = work.Done() + }) + }() + select { + case r := <-took: + if r.ID != "dies-"+name { + t.Errorf("%s: took %s", name, r.ID) + } + case <-time.After(15 * time.Second): + t.Errorf("%s: the unacknowledged check did not go to a second builder", name) + } + second.Close() + stop() + } +} diff --git a/internal/link/checks_nats_test.go b/internal/link/checks_nats_test.go index b5a2f51c..e4a7bc7a 100644 --- a/internal/link/checks_nats_test.go +++ b/internal/link/checks_nats_test.go @@ -3,7 +3,6 @@ package link import ( "context" "encoding/json" - "reflect" "testing" "time" @@ -56,75 +55,9 @@ func queueOf(t *testing.T, js *broker.JetStream) Queue { // A newer request for a pull request replaces the one still waiting, by the stream's rule, and each open pull request is // left with one queued check (acceptance 5). -func TestANewerCheckRequestReplacesOneStillWaiting(t *testing.T) { - js := aBusWithTheBuildRole(t) - ctx := t.Context() - asks := BuildsOn(js, TheBuildMachine) - for _, r := range []BuildRequest{ - aPullCheck("check-7-a", 7, "head-a-0000000", "trunk-0-00000", false), - aPullCheck("check-8-a", 8, "head-b-0000000", "trunk-0-00000", false), - aPullCheck("check-7-b", 7, "head-a-0000000", "trunk-1-00000", false), // a merge moved the trunk - aPullCheck("check-8-b", 8, "head-b-0000000", "trunk-1-00000", false), - aPullCheck("check-7-c", 7, "head-a-0000000", "trunk-2-00000", false), // and a second merge within the minute - } { - if err := asks.Ask(ctx, r); err != nil { - t.Fatal(err) - } - } - var got []string - for _, a := range queueOf(t, js).Asks { - if a.State != AskWaiting { - t.Errorf("%s is %s", a.ID, a.State) - } - got = append(got, a.ID) - } - if want := []string{"check-8-b", "check-7-c"}; !reflect.DeepEqual(got, want) { - t.Fatalf("queued %v; want one check a pull request, the newest: %v", got, want) - } -} // A delivery build is taken before a check of a ready pull request, that before any other check, and the others in the // order they were asked. -func TestADeliveryBuildIsTakenBeforeAReadyCheckAndThatBeforeAnyOther(t *testing.T) { - js := aBusWithTheBuildRole(t) - ctx, stop := context.WithCancel(t.Context()) - defer stop() - asks := BuildsOn(js, TheBuildMachine) - for _, r := range []BuildRequest{ - aPullCheck("other-1", 1, "h1", "t", false), - aPullCheck("other-2", 2, "h2", "t", false), - aPullCheck("ready-3", 3, "h3", "t", true), - {ID: "delivery-4", Repository: "/d"}, - aPullCheck("other-5", 5, "h5", "t", false), - aPullCheck("ready-6", 6, "h6", "t", true), - } { - if err := asks.Ask(ctx, r); err != nil { - t.Fatal(err) - } - } - took := make(chan string, 8) - machine := MachineOverNATS(js, "anchor") - defer machine.Close() - go func() { - _ = machine.Take(ctx, func(ctx context.Context, work Build) { - took <- work.Request().ID - _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) - _ = work.Done() - }) - }() - var order []string - for len(order) < 6 { - select { - case id := <-took: - order = append(order, id) - case <-time.After(15 * time.Second): - t.Fatalf("taken so far: %v", order) - } - } - if want := []string{"delivery-4", "ready-3", "ready-6", "other-1", "other-2", "other-5"}; !reflect.DeepEqual(order, want) { - t.Fatalf("taken as %v; want %v", order, want) - } -} // aKeyBucket makes mesh-delivery's bucket of pull requests' keys, which the bus here has not been given by that module. func aKeyBucket(t *testing.T, js *broker.JetStream) jetstream.KeyValue { @@ -150,113 +83,9 @@ func putKey(t *testing.T, kv jetstream.KeyValue, number int, k PullKey) { // A request left on the third queue for a head and a trunk commit its key says were judged is acknowledged and dropped when a // builder takes it; one for a head the key does not say judged is run. -func TestARequestLeftBehindForAJudgedHeadIsDroppedWhenTaken(t *testing.T) { - js := aBusWithTheBuildRole(t) - kv := aKeyBucket(t, js) - ctx, stop := context.WithCancel(t.Context()) - defer stop() - putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000", Verdict: "pass"}) - putKey(t, kv, 8, PullKey{Head: "head-8-000000000", Trunk: "trunk-1-000000"}) // asked, not judged - putKey(t, kv, 9, PullKey{Head: "head-9-000000000", Trunk: "trunk-2-000000", Verdict: "pass"}) // judged against another trunk - asks := BuildsOn(js, TheBuildMachine) - for _, r := range []BuildRequest{ - aPullCheck("left-7", 7, "head-7-000000000", "trunk-1-000000", false), - aPullCheck("fresh-8", 8, "head-8-000000000", "trunk-1-000000", false), - aPullCheck("moved-9", 9, "head-9-000000000", "trunk-1-000000", false), - } { - if err := asks.Ask(ctx, r); err != nil { - t.Fatal(err) - } - } - took := make(chan string, 8) - machine := MachineOverNATS(js, "anchor") - defer machine.Close() - go func() { - _ = machine.Take(ctx, func(ctx context.Context, work Build) { - took <- work.Request().ID - _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) - _ = work.Done() - }) - }() - var ran []string - for len(ran) < 2 { - select { - case id := <-took: - ran = append(ran, id) - case <-time.After(15 * time.Second): - t.Fatalf("run so far: %v", ran) - } - } - if want := []string{"fresh-8", "moved-9"}; !reflect.DeepEqual(ran, want) { - t.Fatalf("run %v; want %v, the judged one dropped", ran, want) - } - time.Sleep(300 * time.Millisecond) - if n := len(queueOf(t, js).Asks); n != 0 { - t.Errorf("%d ask(s) are left in the queues; the dropped one was to be acknowledged", n) - } -} // A running check stops when its key's head moves or its trunk commit does, not when the key only gains its verdict, and a // group's check, which a member's merge in the group's order writes nothing to the key of, is not stopped at all. -func TestARunningCheckStopsWhenItsKeysHeadMovesAndAGroupsDoesNot(t *testing.T) { - js := aBusWithTheBuildRole(t) - kv := aKeyBucket(t, js) - ctx, stop := context.WithCancel(t.Context()) - defer stop() - moved := make(chan PullKey, 4) - req := aPullCheck("check-7", 7, "head-7-000000000", "trunk-1-000000", false) - end, err := WatchPull(ctx, js.Conn(), req, func(k PullKey) { moved <- k }) - if err != nil { - t.Fatal(err) - } - defer end() - putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000"}) - putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000", Verdict: "pass"}) - select { - case k := <-moved: - t.Fatalf("stopped by a key that moved nothing: %+v", k) - case <-time.After(500 * time.Millisecond): - } - putKey(t, kv, 7, PullKey{Head: "head-7-111111111", Trunk: "trunk-1-000000"}) - select { - case k := <-moved: - if k.Head != "head-7-111111111" { - t.Errorf("stopped by %+v", k) - } - case <-time.After(5 * time.Second): - t.Fatal("a key whose head moved did not stop the check") - } - - // And the trunk commit. - req2 := aPullCheck("check-8", 8, "head-8-000000000", "trunk-1-000000", false) - end2, err := WatchPull(ctx, js.Conn(), req2, func(k PullKey) { moved <- k }) - if err != nil { - t.Fatal(err) - } - defer end2() - putKey(t, kv, 8, PullKey{Head: "head-8-000000000", Trunk: "trunk-2-000000"}) - select { - case <-moved: - case <-time.After(5 * time.Second): - t.Fatal("a key whose trunk commit moved did not stop the check") - } - - // A group's check watches nothing: a sibling's merge in the group's order writes nothing to a key, and a key it does - // have is the pull request's own. - group := BuildRequest{ID: "group-1", Ref: "head-7-000000000", - Check: &CheckRequest{Owner: "novox", Repo: "a", Number: 7, Group: "g1"}} - end3, err := WatchPull(ctx, js.Conn(), group, func(k PullKey) { moved <- k }) - if err != nil { - t.Fatal(err) - } - defer end3() - putKey(t, kv, 7, PullKey{Head: "head-7-222222222", Trunk: "trunk-9-000000"}) - select { - case k := <-moved: - t.Fatalf("a group's check was stopped by %+v", k) - case <-time.After(500 * time.Millisecond): - } -} func TestAPullKeyIsMovedByHeadOrTrunkAndJudgedByVerdict(t *testing.T) { k := PullKey{Head: "aaaaaaaaaaaa1111", Trunk: "tttttttttttt1111", Verdict: "pass"} @@ -270,44 +99,6 @@ func TestAPullKeyIsMovedByHeadOrTrunkAndJudgedByVerdict(t *testing.T) { // The queue is read across the three streams in the order they are handed out, each ask says the stream it waits in, and an // ask is removed from that stream. -func TestTheQueueIsReadAcrossTheThreeStreamsAndAnAskIsDeletedFromItsOwn(t *testing.T) { - js := aBusWithTheBuildRole(t) - ctx := t.Context() - asks := BuildsOn(js, TheBuildMachine) - for _, r := range []BuildRequest{ - aPullCheck("other-1", 1, "h1", "t", false), - aPullCheck("ready-2", 2, "h2", "t", true), - {ID: "delivery-3", Repository: "/d"}, - } { - if err := asks.Ask(ctx, r); err != nil { - t.Fatal(err) - } - } - q := queueOf(t, js) - var ids, streams []string - for _, a := range q.Asks { - ids, streams = append(ids, a.ID), append(streams, a.Stream) - } - if want := []string{"delivery-3", "ready-2", "other-1"}; !reflect.DeepEqual(ids, want) { - t.Fatalf("read %v; want the order they are handed out, %v", ids, want) - } - if want := []string{"SEAT_NODE_BUILD_AGENT", broker.ChecksReadyStream, broker.ChecksOtherStream}; !reflect.DeepEqual(streams, want) { - t.Fatalf("streams %v; want %v", streams, want) - } - ready, _ := q.Find("ready-2") - if w := WorkerOf(TheBuildMachine, ready); w.Name != broker.ChecksReadyWorker { - t.Errorf("the ready check is handed out by %s", w.Name) - } - if removed, err := DeleteAsk(ctx, js, TheBuildMachine, ready); err != nil || removed != ready.Seq { - t.Fatalf("deleted %d, %v", removed, err) - } - if removed, err := DeleteAsk(ctx, js, TheBuildMachine, ready); err != nil || removed != 0 { - t.Fatalf("deleted again: %d, %v", removed, err) - } - if n := len(queueOf(t, js).Asks); n != 2 { - t.Errorf("%d asks left; want 2", n) - } -} // aRunningBucket makes the bucket the builders say the checks they run in, with an age of its own short enough for a test // (the mesh's is two minutes): a key not renewed within it is gone. @@ -326,165 +117,12 @@ func aRunningBucket(t *testing.T, js *broker.JetStream, age time.Duration) jetst // A builder taking a request for a pull request's head and trunk commit that another builder is running a check of drops it: // the bucket of running checks names the head and the trunk, the builder and the build (issue 974). -func TestARunningCheckOnOneBuilderMakesAnotherDropTheSameRequestWaiting(t *testing.T) { - js := aBusWithTheBuildRole(t) - aKeyBucket(t, js) - aRunningBucket(t, js, 2*time.Minute) - ctx, stop := context.WithCancel(t.Context()) - defer stop() - asks := BuildsOn(js, TheBuildMachine) - running := make(chan string, 4) - release := make(chan struct{}) - a := MachineOverNATS(js, "builder-a") - defer a.Close() - go func() { - _ = a.Take(ctx, func(ctx context.Context, work Build) { - running <- work.Request().ID - <-release - _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"}) - _ = work.Done() - }) - }() - if err := asks.Ask(ctx, aPullCheck("first", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { - t.Fatal(err) - } - select { - case <-running: - case <-time.After(10 * time.Second): - t.Fatal("builder a took nothing") - } - // The same head and trunk commit asked again while it runs, and a pull request's other head asked beside it. - if err := asks.Ask(ctx, aPullCheck("again", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { - t.Fatal(err) - } - b := MachineOverNATS(js, "builder-b") - defer b.Close() - took := make(chan string, 4) - go func() { - _ = b.Take(ctx, func(ctx context.Context, work Build) { - took <- work.Request().ID - _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"}) - _ = work.Done() - }) - }() - select { - case id := <-took: - t.Fatalf("builder b took %s, which builder a is running a check of", id) - case <-time.After(3 * time.Second): - } - if n := len(queueOf(t, js).Of(AskWaiting)); n != 0 { - t.Errorf("%d request(s) still wait; builder b was to drop the one for a head and trunk commit being checked", n) - } - // Another trunk commit is another check. - if err := asks.Ask(ctx, aPullCheck("moved", 7, "head-7-000000000", "trunk-2-000000", false)); err != nil { - t.Fatal(err) - } - select { - case id := <-took: - if id != "moved" { - t.Fatalf("builder b took %s", id) - } - case <-time.After(10 * time.Second): - t.Fatal("a check against another trunk commit was dropped too") - } - close(release) -} // A key that is not renewed ages out, and the request for its head and trunk commit is then taken: a builder that died // running a check leaves nothing that holds the next one back for longer than the bucket's age. -func TestAKeyNotRenewedAgesOutAndTheRequestIsThenTaken(t *testing.T) { - js := aBusWithTheBuildRole(t) - aKeyBucket(t, js) - kv := aRunningBucket(t, js, 2*time.Second) - ctx, stop := context.WithCancel(t.Context()) - defer stop() - body, _ := json.Marshal(CheckRunning{Head: "head-7-000000000", Trunk: "trunk-1-000000", Builder: "dead", Build: "gone"}) - if _, err := kv.Put(ctx, PullKeyOf("novox", "a", 7), body); err != nil { - t.Fatal(err) - } - asks := BuildsOn(js, TheBuildMachine) - took := make(chan string, 4) - b := MachineOverNATS(js, "builder-b") - defer b.Close() - go func() { - _ = b.Take(ctx, func(ctx context.Context, work Build) { - took <- work.Request().ID - _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"}) - _ = work.Done() - }) - }() - if err := asks.Ask(ctx, aPullCheck("while-it-lives", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { - t.Fatal(err) - } - select { - case id := <-took: - t.Fatalf("%s was taken while the key was fresh", id) - case <-time.After(1200 * time.Millisecond): - } - time.Sleep(2500 * time.Millisecond) // the key was not renewed - if err := asks.Ask(ctx, aPullCheck("after-it-aged", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { - t.Fatal(err) - } - select { - case id := <-took: - if id != "after-it-aged" { - t.Fatalf("took %s", id) - } - case <-time.After(10 * time.Second): - t.Fatal("the request was not taken after the key aged out") - } -} // A builder running a check renews its key on each heartbeat, so it outlives the bucket's age, and deletes it when the check // ends, whatever the outcome. -func TestARunningCheckRenewsItsKeyAndDeletesItWhenItEnds(t *testing.T) { - js := aBusWithTheBuildRole(t) - kv := aRunningBucket(t, js, 2*time.Second) - was := heartbeatWhileWorking - heartbeatWhileWorking = 400 * time.Millisecond - t.Cleanup(func() { heartbeatWhileWorking = was }) - ctx, stop := context.WithCancel(t.Context()) - defer stop() - running := make(chan struct{}, 1) - release := make(chan struct{}) - m := MachineOverNATS(js, "builder-a") - defer m.Close() - go func() { - _ = m.Take(ctx, func(ctx context.Context, work Build) { - running <- struct{}{} - <-release - _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"}) - _ = work.Done() - }) - }() - if err := BuildsOn(js, TheBuildMachine).Ask(ctx, aPullCheck("long", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { - t.Fatal(err) - } - <-running - key := PullKeyOf("novox", "a", 7) - e, err := kv.Get(ctx, key) - if err != nil { - t.Fatalf("no key while the check runs: %v", err) - } - var got CheckRunning - _ = json.Unmarshal(e.Value(), &got) - if got.Head != "head-7-000000000" || got.Trunk != "trunk-1-000000" || got.Builder != "builder-a" || got.Build != "long" { - t.Errorf("the key says %+v", got) - } - time.Sleep(3 * time.Second) // longer than the bucket's age - if _, err := kv.Get(ctx, key); err != nil { - t.Fatalf("the key aged out under a check still running: %v", err) - } - close(release) - for end := time.Now().Add(5 * time.Second); ; time.Sleep(50 * time.Millisecond) { - if _, err := kv.Get(ctx, key); err != nil { - break - } - if time.Now().After(end) { - t.Fatal("the key stays after the check ended") - } - } -} // A key whose verdict is "error" — a check that could not run, or was stopped — judged nothing: a request for that head and // trunk commit must run (issue 974, review 2). @@ -499,136 +137,9 @@ func TestAnErrorVerdictJudgesNothing(t *testing.T) { // A check stopped because the trunk moved leaves the newer request, for the head against the new trunk commit, taken and // run: the stop is the stopped one's failure, not the key's verdict, and the newer request is not dropped by it. -func TestACheckStoppedByATrunkMoveLeavesTheNewerRequestTakenAndRun(t *testing.T) { - js := aBusWithTheBuildRole(t) - kv := aKeyBucket(t, js) - aRunningBucket(t, js, 2*time.Minute) - ctx, stop := context.WithCancel(t.Context()) - defer stop() - asks := BuildsOn(js, TheBuildMachine) - outcomes := make(chan BuildResult, 8) - watching, err := js.Conn().Subscribe(BuildOutcome(), func(m *nats.Msg) { - var r BuildResult - if json.Unmarshal(m.Data, &r) == nil { - outcomes <- r - } - }) - if err != nil { - t.Fatal(err) - } - defer func() { _ = watching.Unsubscribe() }() - - ran := make(chan string, 4) - m := MachineOverNATS(js, "builder-a") - defer m.Close() - go func() { - _ = m.Take(ctx, func(ctx context.Context, work Build) { - building, cancel := context.WithCancel(ctx) - defer cancel() - ran <- work.Request().ID - stopped := make(chan struct{}, 1) - end, err := work.(interface { - WatchPull(context.Context, func(PullKey)) (func(), error) - }).WatchPull(building, func(PullKey) { stopped <- struct{}{}; cancel() }) - if err != nil { - t.Error(err) - return - } - defer end() - if work.Request().ID == "older" { - select { - case <-stopped: - _ = work.Announce(ctx, BuildResult{ID: "older", On: "builder-a", Failed: SupersededByKey}) - case <-time.After(15 * time.Second): - _ = work.Announce(ctx, BuildResult{ID: "older", On: "builder-a", Failed: "never stopped"}) - } - } else { - _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Check: &CheckOutcome{Verdict: "pass"}}) - } - _ = work.Done() - }) - }() - if err := asks.Ask(ctx, aPullCheck("older", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { - t.Fatal(err) - } - if id := <-ran; id != "older" { - t.Fatalf("took %s", id) - } - time.Sleep(300 * time.Millisecond) // the watch is made - // The trunk moved; the head did not. mesh-delivery writes the key for the new request, and the request is asked. - putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-2-000000"}) - if err := asks.Ask(ctx, aPullCheck("newer", 7, "head-7-000000000", "trunk-2-000000", false)); err != nil { - t.Fatal(err) - } - select { - case id := <-ran: - if id != "newer" { - t.Fatalf("took %s", id) - } - case <-time.After(15 * time.Second): - t.Fatal("the newer request, for the same head on the new trunk commit, was not taken") - } - got := map[string]string{} - for len(got) < 2 { - select { - case r := <-outcomes: - got[r.ID] = r.Failed - case <-time.After(10 * time.Second): - t.Fatalf("outcomes so far: %v", got) - } - } - if got["older"] != SupersededByKey || got["newer"] != "" { - t.Errorf("outcomes %v; want the older stopped and the newer run to a verdict", got) - } -} // An unacknowledged check goes to a second builder: taken from a check worker and not acknowledged, it is handed out again // when the worker's wait is up. -func TestAnUnacknowledgedCheckGoesToASecondBuilder(t *testing.T) { - for _, ready := range []bool{false, true} { - js := aBusWithTheBuildRole(t) - name, worker := "other", broker.ChecksOtherWorker - if ready { - name, worker = "ready", broker.ChecksReadyWorker - } - // A worker whose wait is a second, as the mesh's is a minute. - for _, w := range broker.CheckWorkers() { - if w.Name == worker { - w.AckWaitSeconds = 1 - if err := js.EnsureConsumer(w); err != nil { - t.Fatal(err) - } - } - } - ctx, stop := context.WithCancel(t.Context()) - if err := BuildsOn(js, TheBuildMachine).Ask(ctx, aPullCheck("dies-"+name, 7, "head-7-000000000", "trunk-1-000000", ready)); err != nil { - t.Fatal(err) - } - // The first builder takes it and dies: its pull is fetched, never acknowledged. - if _, err := fetchOne(js, worker); err != nil { - t.Fatal(err) - } - took := make(chan BuildRequest, 2) - second := MachineOverNATS(js, "builder-b") - go func() { - _ = second.Take(ctx, func(ctx context.Context, work Build) { - took <- work.Request() - _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"}) - _ = work.Done() - }) - }() - select { - case r := <-took: - if r.ID != "dies-"+name { - t.Errorf("%s: took %s", name, r.ID) - } - case <-time.After(15 * time.Second): - t.Errorf("%s: the unacknowledged check did not go to a second builder", name) - } - second.Close() - stop() - } -} // fetchOne takes the next message of a worker and leaves it unacknowledged, as a builder that died would. func fetchOne(js *broker.JetStream, worker string) (*nats.Msg, error) { diff --git a/internal/link/conformance_integration_test.go b/internal/link/conformance_integration_test.go new file mode 100644 index 00000000..f7897c67 --- /dev/null +++ b/internal/link/conformance_integration_test.go @@ -0,0 +1,148 @@ +//go:build integration + +package link + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/beside" +) + +func loadFixture(t *testing.T, name string) fixture { + t.Helper() + raw, from, err := sdkFixture(os.Getenv(beside.Env), goModFile, capturedSDK, capturedLog, name) + if err != nil { + // Failed, never skipped: a skip here passed the suite with nothing judged. + t.Fatal(err) + } + t.Logf("%s: %s", name, from) + if strings.HasPrefix(from, notJudged) { + // Said on the check's own output too, where `go test` without -v prints no log of a passing test. + saidNotJudged.Do(func() { fmt.Fprintln(os.Stderr, "internal/link conformance: "+from) }) + } + var f fixture + if err := json.Unmarshal(raw, &f); err != nil { + t.Fatalf("%s: %v", name, err) + } + return f +} + +func sdkFixture(root, goMod, captured, capturedLog, name string) ([]byte, string, error) { + file := path.Join("conformance", name) + readCaptured := func() ([]byte, error) { + raw, err := os.ReadFile(filepath.Join(captured, filepath.FromSlash(file))) + if err != nil { + return nil, fmt.Errorf("the captured SDK's %s: %w", file, err) + } + return raw, nil + } + if root == "" { + raw, err := readCaptured() + return raw, "the copy captured in testdata/beside (see CAPTURED); a merge check reads the SDK's clone " + + "at this tree's pin", err + } + clone := filepath.Join(root, "mesh-sdk") + if _, err := os.Stat(clone); err != nil { + // A build seat asked by a controller from before mesh-controller #220 places no mesh-sdk beside the + // check, so #220 could never pass its own check if this failed. Until #220 has rolled out, the + // captured copy is read and the log says so loudly; the follow-up of novox/hq issue 449 makes this + // a failure again. + at, cerr := capturedCommit(capturedLog) + if cerr != nil { + return nil, "", cerr + } + raw, rerr := readCaptured() + return raw, fmt.Sprintf(notJudged+": the seat placed no mesh-sdk beside this check in "+ + "%s=%s (it does once mesh-controller #220 has rolled out); read the captured copy at %s", beside.Env, + root, at), rerr + } + ref, err := sdkRef(goMod) + if err != nil { + return nil, "", err + } + raw, err := gitShow(clone, ref, file) + if err != nil { + return nil, "", fmt.Errorf("the mesh-sdk clone beside this check has no %s at %s, the SDK this tree's "+ + "go.mod pins: %w", file, ref, err) + } + // The captured copy is the SDK's own, never edited by hand: the clone at the captured commit says so. + at, err := capturedCommit(capturedLog) + if err != nil { + return nil, "", err + } + theirs, err := gitShow(clone, at, file) + if err != nil { + return nil, "", fmt.Errorf("the mesh-sdk clone has no %s at %s, the commit CAPTURED names: %w", file, at, err) + } + ours, err := os.ReadFile(filepath.Join(captured, filepath.FromSlash(file))) + if err != nil { + return nil, "", fmt.Errorf("the captured SDK's %s: %w", file, err) + } + if !bytes.Equal(ours, theirs) { + return nil, "", fmt.Errorf("the captured %s is not mesh-sdk's at %s, the commit CAPTURED names: it was "+ + "edited or captured wrong; capture it again as CAPTURED says", file, at) + } + return raw, "mesh-sdk cloned beside this check, at " + ref + " (this tree's go.mod)", nil +} + +func gitShow(repository, ref, file string) ([]byte, error) { + cmd := exec.Command("git", "-c", "safe.directory=*", "-C", repository, "show", ref+":"+file) + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("git show %s:%s: %v: %s", ref, file, err, strings.TrimSpace(stderr.String())) + } + return out, nil +} + +func TestTheGoEmitterSetsEveryRequiredHeader(t *testing.T) { + f := loadFixture(t, "events/module-event.json") + sent := goEventHeaders(f.Given.Key, f.Given.Module, f.Given.Node) + for _, want := range f.Wire.RequiredHeaders { + if _, ok := sent[want]; !ok { + t.Errorf("the Go emitter does not set %q, which the fixture requires — an event it "+ + "emits is one a conforming consumer refuses", want) + } + } +} + +func TestTheGoEmittersHeaderFormatsMatch(t *testing.T) { + f := loadFixture(t, "events/module-event.json") + sent := goEventHeaders(f.Given.Key, f.Given.Module, f.Given.Node) + + if got := sent["content-type"]; got != f.Wire.HeaderFormats["content-type"] { + t.Errorf("content-type is %q, the fixture says %q", got, f.Wire.HeaderFormats["content-type"]) + } + if _, err := time.Parse(time.RFC3339, sent["x-time"]); err != nil { + t.Errorf("x-time %q is not RFC3339, which the fixture requires: %v", sent["x-time"], err) + } + if pattern := f.Wire.HeaderFormats["x-event-id"]; pattern != "" { + if !regexp.MustCompile(pattern).MatchString(sent["x-event-id"]) { + t.Errorf("x-event-id %q does not match %q", sent["x-event-id"], pattern) + } + } + // The origin the envelope claims is the one the bus enforces by namespace. A disagreement + // here means the envelope is lying about where it came from. + if sent["x-source"] != f.Given.Module { + t.Errorf("x-source is %q for module %q", sent["x-source"], f.Given.Module) + } +} + +func TestTheGoSubjectMatchesTheFixture(t *testing.T) { + f := loadFixture(t, "events/module-event.json") + got := "mesh.mod." + f.Given.Module + ".event." + f.Given.Key + if got != f.Wire.Subject { + t.Errorf("this implementation would publish on %q; the fixture says %q", got, f.Wire.Subject) + } +} diff --git a/internal/link/conformance_sources_test.go b/internal/link/conformance_sources_test.go index 06a0fcd7..1b58ba9c 100644 --- a/internal/link/conformance_sources_test.go +++ b/internal/link/conformance_sources_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/conformance_test.go b/internal/link/conformance_test.go index 048ba4f3..ed9cd038 100644 --- a/internal/link/conformance_test.go +++ b/internal/link/conformance_test.go @@ -1,12 +1,8 @@ package link import ( - "bytes" - "encoding/json" "fmt" "os" - "os/exec" - "path" "path/filepath" "regexp" "strings" @@ -61,25 +57,6 @@ var ( capturedLog = filepath.Join(beside.Captured(), "CAPTURED") ) -func loadFixture(t *testing.T, name string) fixture { - t.Helper() - raw, from, err := sdkFixture(os.Getenv(beside.Env), goModFile, capturedSDK, capturedLog, name) - if err != nil { - // Failed, never skipped: a skip here passed the suite with nothing judged. - t.Fatal(err) - } - t.Logf("%s: %s", name, from) - if strings.HasPrefix(from, notJudged) { - // Said on the check's own output too, where `go test` without -v prints no log of a passing test. - saidNotJudged.Do(func() { fmt.Fprintln(os.Stderr, "internal/link conformance: "+from) }) - } - var f fixture - if err := json.Unmarshal(raw, &f); err != nil { - t.Fatalf("%s: %v", name, err) - } - return f -} - // notJudged opens what a merge check without the SDK's clone says; saidNotJudged says it once on stderr. const notJudged = "NOT JUDGED AGAINST THE CLONE" @@ -87,76 +64,9 @@ var saidNotJudged sync.Once // sdkFixture is one conformance fixture and where it was read: from the mesh-sdk clone in root (a merge // check's MESH_CHECK_BESIDE) at the pin of goMod, or, with no root, from the captured copy. -func sdkFixture(root, goMod, captured, capturedLog, name string) ([]byte, string, error) { - file := path.Join("conformance", name) - readCaptured := func() ([]byte, error) { - raw, err := os.ReadFile(filepath.Join(captured, filepath.FromSlash(file))) - if err != nil { - return nil, fmt.Errorf("the captured SDK's %s: %w", file, err) - } - return raw, nil - } - if root == "" { - raw, err := readCaptured() - return raw, "the copy captured in testdata/beside (see CAPTURED); a merge check reads the SDK's clone " + - "at this tree's pin", err - } - clone := filepath.Join(root, "mesh-sdk") - if _, err := os.Stat(clone); err != nil { - // A build seat asked by a controller from before mesh-controller #220 places no mesh-sdk beside the - // check, so #220 could never pass its own check if this failed. Until #220 has rolled out, the - // captured copy is read and the log says so loudly; the follow-up of novox/hq issue 449 makes this - // a failure again. - at, cerr := capturedCommit(capturedLog) - if cerr != nil { - return nil, "", cerr - } - raw, rerr := readCaptured() - return raw, fmt.Sprintf(notJudged+": the seat placed no mesh-sdk beside this check in "+ - "%s=%s (it does once mesh-controller #220 has rolled out); read the captured copy at %s", beside.Env, - root, at), rerr - } - ref, err := sdkRef(goMod) - if err != nil { - return nil, "", err - } - raw, err := gitShow(clone, ref, file) - if err != nil { - return nil, "", fmt.Errorf("the mesh-sdk clone beside this check has no %s at %s, the SDK this tree's "+ - "go.mod pins: %w", file, ref, err) - } - // The captured copy is the SDK's own, never edited by hand: the clone at the captured commit says so. - at, err := capturedCommit(capturedLog) - if err != nil { - return nil, "", err - } - theirs, err := gitShow(clone, at, file) - if err != nil { - return nil, "", fmt.Errorf("the mesh-sdk clone has no %s at %s, the commit CAPTURED names: %w", file, at, err) - } - ours, err := os.ReadFile(filepath.Join(captured, filepath.FromSlash(file))) - if err != nil { - return nil, "", fmt.Errorf("the captured SDK's %s: %w", file, err) - } - if !bytes.Equal(ours, theirs) { - return nil, "", fmt.Errorf("the captured %s is not mesh-sdk's at %s, the commit CAPTURED names: it was "+ - "edited or captured wrong; capture it again as CAPTURED says", file, at) - } - return raw, "mesh-sdk cloned beside this check, at " + ref + " (this tree's go.mod)", nil -} // gitShow is one file of a repository at a ref. safe.directory, because the clone is the build seat's and // the test may run as another user. -func gitShow(repository, ref, file string) ([]byte, error) { - cmd := exec.Command("git", "-c", "safe.directory=*", "-C", repository, "show", ref+":"+file) - var stderr bytes.Buffer - cmd.Stderr = &stderr - out, err := cmd.Output() - if err != nil { - return nil, fmt.Errorf("git show %s:%s: %v: %s", ref, file, err, strings.TrimSpace(stderr.String())) - } - return out, nil -} // pseudoCommit is the commit a Go pseudo-version names: v0.1.11-0.20261009143344-f047d0a4a970 → f047d0a4a970. var pseudoCommit = regexp.MustCompile(`-([0-9a-f]{12})$`) @@ -216,50 +126,12 @@ func TestTheCapturedSDKIsTheOneGoModPins(t *testing.T) { } // Every header the fixture requires is one this implementation actually sets. -func TestTheGoEmitterSetsEveryRequiredHeader(t *testing.T) { - f := loadFixture(t, "events/module-event.json") - sent := goEventHeaders(f.Given.Key, f.Given.Module, f.Given.Node) - for _, want := range f.Wire.RequiredHeaders { - if _, ok := sent[want]; !ok { - t.Errorf("the Go emitter does not set %q, which the fixture requires — an event it "+ - "emits is one a conforming consumer refuses", want) - } - } -} // And each value is in the shape the fixture pins, because a header present but differently // formatted is the disagreement that does not announce itself. -func TestTheGoEmittersHeaderFormatsMatch(t *testing.T) { - f := loadFixture(t, "events/module-event.json") - sent := goEventHeaders(f.Given.Key, f.Given.Module, f.Given.Node) - - if got := sent["content-type"]; got != f.Wire.HeaderFormats["content-type"] { - t.Errorf("content-type is %q, the fixture says %q", got, f.Wire.HeaderFormats["content-type"]) - } - if _, err := time.Parse(time.RFC3339, sent["x-time"]); err != nil { - t.Errorf("x-time %q is not RFC3339, which the fixture requires: %v", sent["x-time"], err) - } - if pattern := f.Wire.HeaderFormats["x-event-id"]; pattern != "" { - if !regexp.MustCompile(pattern).MatchString(sent["x-event-id"]) { - t.Errorf("x-event-id %q does not match %q", sent["x-event-id"], pattern) - } - } - // The origin the envelope claims is the one the bus enforces by namespace. A disagreement - // here means the envelope is lying about where it came from. - if sent["x-source"] != f.Given.Module { - t.Errorf("x-source is %q for module %q", sent["x-source"], f.Given.Module) - } -} // The subject a module's event lands on is derived, not carried — so this implementation must // derive the same one the fixture names. -func TestTheGoSubjectMatchesTheFixture(t *testing.T) { - f := loadFixture(t, "events/module-event.json") - got := "mesh.mod." + f.Given.Module + ".event." + f.Given.Key - if got != f.Wire.Subject { - t.Errorf("this implementation would publish on %q; the fixture says %q", got, f.Wire.Subject) - } -} // goEventHeaders is the header set EmitEvent produces, factored so conformance can see it // without a broker. Kept beside the emitter so the two cannot drift apart silently. diff --git a/internal/link/deadletters_ask_test.go b/internal/link/deadletters_ask_test.go index 2def6f3e..f3533527 100644 --- a/internal/link/deadletters_ask_test.go +++ b/internal/link/deadletters_ask_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/deadletters_integration_test.go b/internal/link/deadletters_integration_test.go new file mode 100644 index 00000000..23d98fe2 --- /dev/null +++ b/internal/link/deadletters_integration_test.go @@ -0,0 +1,279 @@ +//go:build integration + +package link + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" +) + +func TestADeadLetterIsDeliveredAgainToItsConsumerAlone(t *testing.T) { + js := aBus(t) + keeping(t, js) + failing := aModuleConsumer(t, js, "media", "sonarr", "gitea.pull.merged") + handling := aModuleConsumer(t, js, "media", "radarr", "gitea.pull.merged") + + const subject = "mesh.mod.gitea.event.pull.merged" + msg := &nats.Msg{Subject: subject, Data: []byte(`{"n":1}`), Header: nats.Header{}} + msg.Header.Set("x-node", "forge") + msg.Header.Set(nats.MsgIdHdr, "event-1") + if _, err := js.Context().PublishMsg(msg); err != nil { + t.Fatal(err) + } + if m := next(t, handling); m == nil { + t.Fatal("the other module was not handed the event") + } else { + _ = m.Ack() + } + d := givenUp(t, js, failing) + if d.Consumer != "media_sonarr" || d.Stream != "EVENTS" || d.Subject != subject || d.Deliveries != 5 || + d.GaveUp.IsZero() || d.Published.IsZero() || d.Lost != "" { + t.Fatalf("kept as %+v", d) + } + held, err := HeldDeadLetters(js.Context()) + if err != nil || held["EVENTS.media_sonarr"] != 1 { + t.Fatalf("held %v (%v)", held, err) + } + + whole, err := DeadLetterNamed(js.Context(), d.ID) + if err != nil || whole.Body != `{"n":1}` || len(whole.Headers["x-node"]) != 1 || whole.Headers["x-node"][0] != "forge" { + t.Fatalf("one asked whole is %+v (%v)", whole, err) + } + + _, to, err := DeliverAgain(js.Context(), d.ID) + if err != nil { + t.Fatal(err) + } + if to != "mesh.again.media_sonarr.mod.gitea.event.pull.merged" { + t.Fatalf("delivered again on %s", to) + } + again := next(t, failing) + if again == nil { + t.Fatal("the consumer that gave it up was not handed it again") + } + if string(again.Data()) != `{"n":1}` || again.Headers().Get("x-node") != "forge" || + again.Headers().Get(AgainHeader) == "" { + t.Fatalf("handed again as %s %v", again.Data(), again.Headers()) + } + if original, ok := broker.OriginalOfAgain(again.Subject()); !ok || original != subject { + t.Fatalf("delivered again on %s, which does not say the event's own subject", again.Subject()) + } + _ = again.Ack() + if m := next(t, handling); m != nil { + t.Fatalf("the module that handled it was handed it twice: %s", m.Subject()) + } + if _, err := DeadLetterNamed(js.Context(), d.ID); !errors.Is(err, ErrNoDeadLetter) { + t.Fatalf("still kept after it was delivered again: %v", err) + } + held, _ = HeldDeadLetters(js.Context()) + if held["EVENTS.media_sonarr"] != 0 { + t.Fatalf("still held: %v", held) + } + // Asked twice, delivered once: the second finds nothing kept. + if _, _, err := DeliverAgain(js.Context(), d.ID); !errors.Is(err, ErrNoDeadLetter) { + t.Fatalf("a second delivery answered %v", err) + } +} + +func TestADeadLetterDroppedIsGone(t *testing.T) { + js := aBus(t) + keeping(t, js) + failing := aModuleConsumer(t, js, "media", "sonarr", "gitea.pull.merged") + if _, err := js.Context().Publish("mesh.mod.gitea.event.pull.merged", []byte(`{}`)); err != nil { + t.Fatal(err) + } + d := givenUp(t, js, failing) + if _, err := DropDeadLetter(js.Context(), d.ID); err != nil { + t.Fatal(err) + } + if left, total, err := DeadLetters(js.Context(), "", 0); err != nil || total != 0 || len(left) != 0 { + t.Fatalf("after the drop: %v %d %v", left, total, err) + } + if m := next(t, failing); m != nil { + t.Fatalf("a dropped event was handed over: %s", m.Subject()) + } +} + +func TestANoticeWhoseMessageIsGoneIsKeptAndSaysSo(t *testing.T) { + js := aBus(t) + d, err := KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":42,`+ + `"deliveries":5,"timestamp":"2026-10-06T15:51:00Z"}`)) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(d.Lost, "no longer held message 42") || d.Consumer != "media_sonarr" { + t.Fatalf("kept as %+v", d) + } + if _, _, err := DeliverAgain(js.Context(), d.ID); err == nil || !strings.Contains(err.Error(), "Drop it") { + t.Fatalf("a record without its message was delivered again: %v", err) + } + // The same notice taken twice is kept once. + if _, err := KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":42,`+ + `"deliveries":5}`)); err != nil { + t.Fatal(err) + } + if _, total, _ := DeadLetters(js.Context(), "media_sonarr", 0); total != 1 { + t.Fatalf("one notice taken twice is kept %d times", total) + } +} + +func TestADeadLetterIsDeliveredAgainOnlyWhereItIsReceived(t *testing.T) { + js := aBus(t) + keeping(t, js) + failing := aModuleConsumer(t, js, "media", "sonarr", "gitea.pull.merged") + if _, err := js.Context().Publish("mesh.mod.gitea.event.pull.merged", []byte(`{}`)); err != nil { + t.Fatal(err) + } + d := givenUp(t, js, failing) + + // A consumer as it was before this fix: its filters do not take what is delivered again. + c, _ := broker.ConsumerFor(broker.Principal{Kind: broker.KindModule, Node: "media", Module: "sonarr", + Consumes: []string{"gitea.pull.merged"}, PasswordHash: "x"}) + c.Filters = c.Filters[:len(c.Filters)-1] + if err := js.EnsureConsumer(c); err != nil { + t.Fatal(err) + } + if _, _, err := DeliverAgain(js.Context(), d.ID); err == nil || !strings.Contains(err.Error(), "does not yet take") { + t.Fatalf("delivered to a consumer that does not filter its again subject: %v", err) + } + if _, err := DeadLetterNamed(js.Context(), d.ID); err != nil { + t.Fatalf("a refused delivery let the dead letter go: %v", err) + } + + // The consumer gone: nothing would receive it. + if err := js.Context().DeleteConsumer(broker.EventsStream, c.Name); err != nil { + t.Fatal(err) + } + if _, _, err := DeliverAgain(js.Context(), d.ID); err == nil || !strings.Contains(err.Error(), "no longer on the bus") { + t.Fatalf("delivered to a consumer that is gone: %v", err) + } + if _, err := DeadLetterNamed(js.Context(), d.ID); err != nil { + t.Fatalf("a refused delivery let the dead letter go: %v", err) + } + + // Not an event: refused, whatever stream it is from. + for _, other := range []DeadLetter{ + {ID: 2, Stream: "SEAT_TELEGRAM_SENDER", Consumer: "SEAT_TELEGRAM_SENDER_worker", Subject: "mesh.seat.telegram-sender.accept.send"}, + {ID: 3, Stream: "KV_x", Consumer: "y", Subject: "$KV.x.k"}, + } { + if _, err := AgainTo(js.Context(), other); err == nil { + t.Errorf("%s was given a subject to be delivered again on", other.Stream) + } + } +} + +func TestTheListSaysTheTotalAndStopsAtItsLimit(t *testing.T) { + js := aBus(t) + for seq := 1; seq <= 5; seq++ { + if _, err := KeepDeadLetter(js.Context(), []byte(fmt.Sprintf(`{"stream":"EVENTS","consumer":"media_sonarr",`+ + `"stream_seq":%d,"deliveries":5}`, seq))); err != nil { + t.Fatal(err) + } + } + list, total, err := DeadLetters(js.Context(), "media_sonarr", 2) + if err != nil || total != 5 || len(list) != 2 || list[0].Sequence != 5 { + t.Fatalf("%d of %d (%v): %+v", len(list), total, err, list) + } + if _, total, _ := DeadLetters(js.Context(), "another_one", 2); total != 0 { + t.Fatalf("another consumer's total is %d", total) + } +} + +func TestTheControllerActsOnAnEventDeliveredAgain(t *testing.T) { + js := aBus(t) + told := &toldAbout{} + s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()} + if err := s.Follows(told); err != nil { + t.Fatal(err) + } + if err := s.Answers(replaysWith{}); err != nil { + t.Fatal(err) + } + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go func() { _ = s.Serve(ctx) }() + eventually(t, "the controller's event consumer being made", func() bool { + _, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName) + return err == nil + }) + moved, _ := json.Marshal(Upgraded{Module: "gitea", Commit: "abcdef0123"}) + if _, err := js.Context().Publish(broker.AgainSubject(broker.ControllerName, broker.ControllerFollows[0]), moved); err != nil { + t.Fatal(err) + } + eventually(t, "the upgrade delivered again reaching the controller", func() bool { return told.count() == 1 }) +} + +func TestNoticesThatCannotBeTakenAreSaidAndServingGoesOn(t *testing.T) { + js := aBus(t) + if err := js.Context().DeleteStream(broker.DeadLetterNoticesStream); err != nil { + t.Fatal(err) + } + stop := keepingGivenUp(context.Background(), js, quiet()) + defer stop() + eventually(t, "the failure said", func() bool { + for _, a := range Advisories.Since(time.Now().Add(-time.Minute)) { + if a.Token == AdvisoryNotKept && a.Stream == broker.DeadLetterNoticesStream { + return true + } + } + return false + }) + held := &counted{} + _, stopServing := servingOn(t, js, held) + defer stopServing() + if _, err := js.Context().Publish("mesh.control.anchor.report", []byte(`{"node":"anchor"}`)); err != nil { + t.Fatal(err) + } + eventually(t, "a report heard while the notices cannot be taken", func() bool { return held.count() == 1 }) +} + +func TestTheNewestHeldDeadLetterIsWhenItWasKept(t *testing.T) { + js := aBus(t) + gaveUp := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC) + keep := func(consumer string, seq int, at time.Time) { + t.Helper() + if _, err := KeepDeadLetter(js.Context(), []byte(fmt.Sprintf(`{"stream":"EVENTS","consumer":%q,`+ + `"stream_seq":%d,"deliveries":5,"timestamp":%q}`, consumer, seq, at.Format(time.RFC3339Nano)))); err != nil { + t.Fatal(err) + } + } + look := func() map[string]time.Time { + t.Helper() + held, err := HeldDeadLetters(js.Context()) + if err != nil { + t.Fatal(err) + } + newest, err := NewestDeadLetters(js.Context(), held) + if err != nil { + t.Fatal(err) + } + return newest + } + + // Sonarr gave up at 21:05 and that is kept first; radarr's letter is kept after it. + keep("media_sonarr", 2, gaveUp.Add(time.Minute)) + keep("media_radarr", 3, gaveUp) + before := look() + if len(before) != 2 || before["EVENTS.media_sonarr"].IsZero() || + !before["EVENTS.media_radarr"].After(before["EVENTS.media_sonarr"]) { + t.Fatalf("%v", before) + } + // Sonarr's give-up of 21:04, kept late, is newer than anything the condition has seen. + keep("media_sonarr", 1, gaveUp) + after := look() + if !after["EVENTS.media_sonarr"].After(before["EVENTS.media_sonarr"]) || + !after["EVENTS.media_sonarr"].After(before["EVENTS.media_radarr"]) { + t.Fatalf("a letter kept late does not read as newer: before %v, after %v", before, after) + } + if !after["EVENTS.media_radarr"].Equal(before["EVENTS.media_radarr"]) { + t.Fatalf("radarr's newest moved: before %v, after %v", before, after) + } +} diff --git a/internal/link/deadletters_module_stream_test.go b/internal/link/deadletters_module_stream_test.go index ce08a68c..ec075178 100644 --- a/internal/link/deadletters_module_stream_test.go +++ b/internal/link/deadletters_module_stream_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/deadletters_restricted_test.go b/internal/link/deadletters_restricted_test.go index c621bbc2..aefad688 100644 --- a/internal/link/deadletters_restricted_test.go +++ b/internal/link/deadletters_restricted_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/deadletters_test.go b/internal/link/deadletters_test.go index f575c4ce..3db61457 100644 --- a/internal/link/deadletters_test.go +++ b/internal/link/deadletters_test.go @@ -1,17 +1,12 @@ package link import ( - "context" - "encoding/json" - "errors" - "fmt" "io" "log" "strings" "testing" "time" - "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" "github.com/novox/mesh-controller/internal/broker" @@ -85,117 +80,9 @@ func givenUp(t *testing.T, js *broker.JetStream, c jetstream.Consumer) DeadLette // **Delivered again to the consumer that gave it up, and nobody else**: another module consuming the // same event handled it the first time and must not handle it twice. -func TestADeadLetterIsDeliveredAgainToItsConsumerAlone(t *testing.T) { - js := aBus(t) - keeping(t, js) - failing := aModuleConsumer(t, js, "media", "sonarr", "gitea.pull.merged") - handling := aModuleConsumer(t, js, "media", "radarr", "gitea.pull.merged") - - const subject = "mesh.mod.gitea.event.pull.merged" - msg := &nats.Msg{Subject: subject, Data: []byte(`{"n":1}`), Header: nats.Header{}} - msg.Header.Set("x-node", "forge") - msg.Header.Set(nats.MsgIdHdr, "event-1") - if _, err := js.Context().PublishMsg(msg); err != nil { - t.Fatal(err) - } - if m := next(t, handling); m == nil { - t.Fatal("the other module was not handed the event") - } else { - _ = m.Ack() - } - d := givenUp(t, js, failing) - if d.Consumer != "media_sonarr" || d.Stream != "EVENTS" || d.Subject != subject || d.Deliveries != 5 || - d.GaveUp.IsZero() || d.Published.IsZero() || d.Lost != "" { - t.Fatalf("kept as %+v", d) - } - held, err := HeldDeadLetters(js.Context()) - if err != nil || held["EVENTS.media_sonarr"] != 1 { - t.Fatalf("held %v (%v)", held, err) - } - - whole, err := DeadLetterNamed(js.Context(), d.ID) - if err != nil || whole.Body != `{"n":1}` || len(whole.Headers["x-node"]) != 1 || whole.Headers["x-node"][0] != "forge" { - t.Fatalf("one asked whole is %+v (%v)", whole, err) - } - - _, to, err := DeliverAgain(js.Context(), d.ID) - if err != nil { - t.Fatal(err) - } - if to != "mesh.again.media_sonarr.mod.gitea.event.pull.merged" { - t.Fatalf("delivered again on %s", to) - } - again := next(t, failing) - if again == nil { - t.Fatal("the consumer that gave it up was not handed it again") - } - if string(again.Data()) != `{"n":1}` || again.Headers().Get("x-node") != "forge" || - again.Headers().Get(AgainHeader) == "" { - t.Fatalf("handed again as %s %v", again.Data(), again.Headers()) - } - if original, ok := broker.OriginalOfAgain(again.Subject()); !ok || original != subject { - t.Fatalf("delivered again on %s, which does not say the event's own subject", again.Subject()) - } - _ = again.Ack() - if m := next(t, handling); m != nil { - t.Fatalf("the module that handled it was handed it twice: %s", m.Subject()) - } - if _, err := DeadLetterNamed(js.Context(), d.ID); !errors.Is(err, ErrNoDeadLetter) { - t.Fatalf("still kept after it was delivered again: %v", err) - } - held, _ = HeldDeadLetters(js.Context()) - if held["EVENTS.media_sonarr"] != 0 { - t.Fatalf("still held: %v", held) - } - // Asked twice, delivered once: the second finds nothing kept. - if _, _, err := DeliverAgain(js.Context(), d.ID); !errors.Is(err, ErrNoDeadLetter) { - t.Fatalf("a second delivery answered %v", err) - } -} - -func TestADeadLetterDroppedIsGone(t *testing.T) { - js := aBus(t) - keeping(t, js) - failing := aModuleConsumer(t, js, "media", "sonarr", "gitea.pull.merged") - if _, err := js.Context().Publish("mesh.mod.gitea.event.pull.merged", []byte(`{}`)); err != nil { - t.Fatal(err) - } - d := givenUp(t, js, failing) - if _, err := DropDeadLetter(js.Context(), d.ID); err != nil { - t.Fatal(err) - } - if left, total, err := DeadLetters(js.Context(), "", 0); err != nil || total != 0 || len(left) != 0 { - t.Fatalf("after the drop: %v %d %v", left, total, err) - } - if m := next(t, failing); m != nil { - t.Fatalf("a dropped event was handed over: %s", m.Subject()) - } -} // A notice whose message its stream no longer holds is kept as a record that says so — said, and // dropped by a person, never silently missing — and cannot be delivered again. -func TestANoticeWhoseMessageIsGoneIsKeptAndSaysSo(t *testing.T) { - js := aBus(t) - d, err := KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":42,`+ - `"deliveries":5,"timestamp":"2026-10-06T15:51:00Z"}`)) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(d.Lost, "no longer held message 42") || d.Consumer != "media_sonarr" { - t.Fatalf("kept as %+v", d) - } - if _, _, err := DeliverAgain(js.Context(), d.ID); err == nil || !strings.Contains(err.Error(), "Drop it") { - t.Fatalf("a record without its message was delivered again: %v", err) - } - // The same notice taken twice is kept once. - if _, err := KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":42,`+ - `"deliveries":5}`)); err != nil { - t.Fatal(err) - } - if _, total, _ := DeadLetters(js.Context(), "media_sonarr", 0); total != 1 { - t.Fatalf("one notice taken twice is kept %d times", total) - } -} // What only the giving-up consumer filters is its own: a subject delivered again reads back as the // event's, and a module's runtime reads the same key from it (the tokens around `.event.`). @@ -221,163 +108,15 @@ func TestASubjectDeliveredAgainSaysTheEventsOwn(t *testing.T) { // Only an event is delivered again, and only to a consumer that is on the bus and filters its again // subject: a dead letter is never let go as delivered while nobody receives it. -func TestADeadLetterIsDeliveredAgainOnlyWhereItIsReceived(t *testing.T) { - js := aBus(t) - keeping(t, js) - failing := aModuleConsumer(t, js, "media", "sonarr", "gitea.pull.merged") - if _, err := js.Context().Publish("mesh.mod.gitea.event.pull.merged", []byte(`{}`)); err != nil { - t.Fatal(err) - } - d := givenUp(t, js, failing) - - // A consumer as it was before this fix: its filters do not take what is delivered again. - c, _ := broker.ConsumerFor(broker.Principal{Kind: broker.KindModule, Node: "media", Module: "sonarr", - Consumes: []string{"gitea.pull.merged"}, PasswordHash: "x"}) - c.Filters = c.Filters[:len(c.Filters)-1] - if err := js.EnsureConsumer(c); err != nil { - t.Fatal(err) - } - if _, _, err := DeliverAgain(js.Context(), d.ID); err == nil || !strings.Contains(err.Error(), "does not yet take") { - t.Fatalf("delivered to a consumer that does not filter its again subject: %v", err) - } - if _, err := DeadLetterNamed(js.Context(), d.ID); err != nil { - t.Fatalf("a refused delivery let the dead letter go: %v", err) - } - - // The consumer gone: nothing would receive it. - if err := js.Context().DeleteConsumer(broker.EventsStream, c.Name); err != nil { - t.Fatal(err) - } - if _, _, err := DeliverAgain(js.Context(), d.ID); err == nil || !strings.Contains(err.Error(), "no longer on the bus") { - t.Fatalf("delivered to a consumer that is gone: %v", err) - } - if _, err := DeadLetterNamed(js.Context(), d.ID); err != nil { - t.Fatalf("a refused delivery let the dead letter go: %v", err) - } - - // Not an event: refused, whatever stream it is from. - for _, other := range []DeadLetter{ - {ID: 2, Stream: "SEAT_TELEGRAM_SENDER", Consumer: "SEAT_TELEGRAM_SENDER_worker", Subject: "mesh.seat.telegram-sender.accept.send"}, - {ID: 3, Stream: "KV_x", Consumer: "y", Subject: "$KV.x.k"}, - } { - if _, err := AgainTo(js.Context(), other); err == nil { - t.Errorf("%s was given a subject to be delivered again on", other.Stream) - } - } -} // A total says how many are held, however few the list carries. -func TestTheListSaysTheTotalAndStopsAtItsLimit(t *testing.T) { - js := aBus(t) - for seq := 1; seq <= 5; seq++ { - if _, err := KeepDeadLetter(js.Context(), []byte(fmt.Sprintf(`{"stream":"EVENTS","consumer":"media_sonarr",`+ - `"stream_seq":%d,"deliveries":5}`, seq))); err != nil { - t.Fatal(err) - } - } - list, total, err := DeadLetters(js.Context(), "media_sonarr", 2) - if err != nil || total != 5 || len(list) != 2 || list[0].Sequence != 5 { - t.Fatalf("%d of %d (%v): %+v", len(list), total, err, list) - } - if _, total, _ := DeadLetters(js.Context(), "another_one", 2); total != 0 { - t.Fatalf("another consumer's total is %d", total) - } -} // An event the controller itself gave up on, delivered again by a person, is acted on as the event it // was: it arrives under the controller's again subject, which its consumer filters. -func TestTheControllerActsOnAnEventDeliveredAgain(t *testing.T) { - js := aBus(t) - told := &toldAbout{} - s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()} - if err := s.Follows(told); err != nil { - t.Fatal(err) - } - if err := s.Answers(replaysWith{}); err != nil { - t.Fatal(err) - } - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go func() { _ = s.Serve(ctx) }() - eventually(t, "the controller's event consumer being made", func() bool { - _, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName) - return err == nil - }) - moved, _ := json.Marshal(Upgraded{Module: "gitea", Commit: "abcdef0123"}) - if _, err := js.Context().Publish(broker.AgainSubject(broker.ControllerName, broker.ControllerFollows[0]), moved); err != nil { - t.Fatal(err) - } - eventually(t, "the upgrade delivered again reaching the controller", func() bool { return told.count() == 1 }) -} // The notices cannot be taken (their stream is gone): said as a condition, and the controller serves on. -func TestNoticesThatCannotBeTakenAreSaidAndServingGoesOn(t *testing.T) { - js := aBus(t) - if err := js.Context().DeleteStream(broker.DeadLetterNoticesStream); err != nil { - t.Fatal(err) - } - stop := keepingGivenUp(context.Background(), js, quiet()) - defer stop() - eventually(t, "the failure said", func() bool { - for _, a := range Advisories.Since(time.Now().Add(-time.Minute)) { - if a.Token == AdvisoryNotKept && a.Stream == broker.DeadLetterNoticesStream { - return true - } - } - return false - }) - held := &counted{} - _, stopServing := servingOn(t, js, held) - defer stopServing() - if _, err := js.Context().Publish("mesh.control.anchor.report", []byte(`{"node":"anchor"}`)); err != nil { - t.Fatal(err) - } - eventually(t, "a report heard while the notices cannot be taken", func() bool { return held.count() == 1 }) -} // When DEAD_LETTERS kept each consumer's newest message, for the condition that counts what was given up // on rather than how often it was looked at (novox/hq issue 440). It rises with every message kept, even // one the consumer gave up on before the last: a notice that could not be kept is offered again a minute // later, so a later give-up can be kept first, and the one kept after it must still count. -func TestTheNewestHeldDeadLetterIsWhenItWasKept(t *testing.T) { - js := aBus(t) - gaveUp := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC) - keep := func(consumer string, seq int, at time.Time) { - t.Helper() - if _, err := KeepDeadLetter(js.Context(), []byte(fmt.Sprintf(`{"stream":"EVENTS","consumer":%q,`+ - `"stream_seq":%d,"deliveries":5,"timestamp":%q}`, consumer, seq, at.Format(time.RFC3339Nano)))); err != nil { - t.Fatal(err) - } - } - look := func() map[string]time.Time { - t.Helper() - held, err := HeldDeadLetters(js.Context()) - if err != nil { - t.Fatal(err) - } - newest, err := NewestDeadLetters(js.Context(), held) - if err != nil { - t.Fatal(err) - } - return newest - } - - // Sonarr gave up at 21:05 and that is kept first; radarr's letter is kept after it. - keep("media_sonarr", 2, gaveUp.Add(time.Minute)) - keep("media_radarr", 3, gaveUp) - before := look() - if len(before) != 2 || before["EVENTS.media_sonarr"].IsZero() || - !before["EVENTS.media_radarr"].After(before["EVENTS.media_sonarr"]) { - t.Fatalf("%v", before) - } - // Sonarr's give-up of 21:04, kept late, is newer than anything the condition has seen. - keep("media_sonarr", 1, gaveUp) - after := look() - if !after["EVENTS.media_sonarr"].After(before["EVENTS.media_sonarr"]) || - !after["EVENTS.media_sonarr"].After(before["EVENTS.media_radarr"]) { - t.Fatalf("a letter kept late does not read as newer: before %v, after %v", before, after) - } - if !after["EVENTS.media_radarr"].Equal(before["EVENTS.media_radarr"]) { - t.Fatalf("radarr's newest moved: before %v, after %v", before, after) - } -} diff --git a/internal/link/enrol_bus_credential_test.go b/internal/link/enrol_bus_credential_test.go index 9a35404e..6282d603 100644 --- a/internal/link/enrol_bus_credential_test.go +++ b/internal/link/enrol_bus_credential_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link_test import ( diff --git a/internal/link/enrol_claim_test.go b/internal/link/enrol_claim_test.go index 0d4eafd7..a48aadb8 100644 --- a/internal/link/enrol_claim_test.go +++ b/internal/link/enrol_claim_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link_test import ( diff --git a/internal/link/enrol_reply_probe_test.go b/internal/link/enrol_reply_probe_test.go index d9acfb4b..30597a73 100644 --- a/internal/link/enrol_reply_probe_test.go +++ b/internal/link/enrol_reply_probe_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/enrol_shape_test.go b/internal/link/enrol_shape_test.go index 54a1b54c..68cd60c4 100644 --- a/internal/link/enrol_shape_test.go +++ b/internal/link/enrol_shape_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link_test import ( diff --git a/internal/link/enrol_tunnel_key_test.go b/internal/link/enrol_tunnel_key_test.go index f5604f42..105fc356 100644 --- a/internal/link/enrol_tunnel_key_test.go +++ b/internal/link/enrol_tunnel_key_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link_test import ( diff --git a/internal/link/handacts_test.go b/internal/link/handacts_test.go index e670fe7c..c1f864c7 100644 --- a/internal/link/handacts_test.go +++ b/internal/link/handacts_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/handover_integration_test.go b/internal/link/handover_integration_test.go new file mode 100644 index 00000000..ec53ac50 --- /dev/null +++ b/internal/link/handover_integration_test.go @@ -0,0 +1,75 @@ +//go:build integration + +package link + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestAHandOverIsAskedOfTheMachineAndItsAnswerComesBack(t *testing.T) { + url := testbus.URL(t) + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(conn.Close) + + signer, public := testSigner(t) + var heard HandOverAsk + engine, err := conn.Subscribe(broker.AskHandOverSubject("laptop"), func(msg *nats.Msg) { + var signed SignedHandOver + _ = json.Unmarshal(msg.Data, &signed) + if !ed25519.Verify(public, append([]byte(HandOverContext), signed.Ask...), signed.Signature) { + _ = msg.Respond([]byte(`{"refused":"not the mesh's signature"}`)) + return + } + _ = json.Unmarshal(signed.Ask, &heard) + switch heard.Path { + case "/srv/notes": + body, _ := json.Marshal(HandOverAnswer{Said: "/srv/notes (notes.data) is handed to the mesh by " + heard.By}) + _ = msg.Respond(body) + case "/srv/empty": + _ = msg.Respond([]byte(`{}`)) + default: + body, _ := json.Marshal(HandOverAnswer{Refused: heard.Path + " is not a directory this machine uses as found; nothing was handed over"}) + _ = msg.Respond(body) + } + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = engine.Unsubscribe() }) + + ctx := context.Background() + a, err := AskHandOver(ctx, conn, signer, "laptop", "/srv/notes", "jo through mesh-cli on anchor", 5*time.Second) + if err != nil || a.Refused != "" || !strings.Contains(a.Said, "handed to the mesh by jo through mesh-cli on anchor") { + t.Fatalf("answered %+v, %v", a, err) + } + if heard.Node != "laptop" || heard.Path != "/srv/notes" || heard.By != "jo through mesh-cli on anchor" { + t.Fatalf("the engine heard %+v", heard) + } + a, err = AskHandOver(ctx, conn, signer, "laptop", "/srv/other", "jo", 5*time.Second) + if err != nil || a.Said != "" || !strings.Contains(a.Refused, "nothing was handed over") { + t.Fatalf("a refusal came back as %+v, %v", a, err) + } + if _, err := AskHandOver(ctx, conn, signer, "laptop", "/srv/empty", "jo", 5*time.Second); err == nil || + !strings.Contains(err.Error(), "neither") { + t.Fatalf("an empty answer was taken: %v", err) + } + if _, err := AskHandOver(ctx, conn, signer, "anchor", "/srv/notes", "jo", 5*time.Second); err == nil || + !strings.Contains(err.Error(), "node-engine") { + t.Fatalf("a machine with no engine listening: %v", err) + } + if _, err := AskHandOver(ctx, nil, signer, "anchor", "/srv/notes", "jo", time.Second); err == nil { + t.Fatal("asked with no bus") + } +} diff --git a/internal/link/handover_test.go b/internal/link/handover_test.go index bf97a68f..188af5e9 100644 --- a/internal/link/handover_test.go +++ b/internal/link/handover_test.go @@ -4,14 +4,10 @@ import ( "context" "crypto/ed25519" "encoding/json" - "strings" "testing" "time" - "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/testbus" ) // The hand-over's ask and answer hold these field names; the engine's side holds the same list (mesh-host @@ -99,64 +95,6 @@ func TestAHandOverIsSignedWithAContextANonceAndAnExpiry(t *testing.T) { // The ask reaches the machine's engine on its own subject and its answer comes back whole: what it recorded, or // its refusal as the engine worded it. A machine with no engine listening is said as not answering, and an // answer that is neither is refused rather than read as a record. -func TestAHandOverIsAskedOfTheMachineAndItsAnswerComesBack(t *testing.T) { - url := testbus.URL(t) - conn, err := nats.Connect(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - - signer, public := testSigner(t) - var heard HandOverAsk - engine, err := conn.Subscribe(broker.AskHandOverSubject("laptop"), func(msg *nats.Msg) { - var signed SignedHandOver - _ = json.Unmarshal(msg.Data, &signed) - if !ed25519.Verify(public, append([]byte(HandOverContext), signed.Ask...), signed.Signature) { - _ = msg.Respond([]byte(`{"refused":"not the mesh's signature"}`)) - return - } - _ = json.Unmarshal(signed.Ask, &heard) - switch heard.Path { - case "/srv/notes": - body, _ := json.Marshal(HandOverAnswer{Said: "/srv/notes (notes.data) is handed to the mesh by " + heard.By}) - _ = msg.Respond(body) - case "/srv/empty": - _ = msg.Respond([]byte(`{}`)) - default: - body, _ := json.Marshal(HandOverAnswer{Refused: heard.Path + " is not a directory this machine uses as found; nothing was handed over"}) - _ = msg.Respond(body) - } - }) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = engine.Unsubscribe() }) - - ctx := context.Background() - a, err := AskHandOver(ctx, conn, signer, "laptop", "/srv/notes", "jo through mesh-cli on anchor", 5*time.Second) - if err != nil || a.Refused != "" || !strings.Contains(a.Said, "handed to the mesh by jo through mesh-cli on anchor") { - t.Fatalf("answered %+v, %v", a, err) - } - if heard.Node != "laptop" || heard.Path != "/srv/notes" || heard.By != "jo through mesh-cli on anchor" { - t.Fatalf("the engine heard %+v", heard) - } - a, err = AskHandOver(ctx, conn, signer, "laptop", "/srv/other", "jo", 5*time.Second) - if err != nil || a.Said != "" || !strings.Contains(a.Refused, "nothing was handed over") { - t.Fatalf("a refusal came back as %+v, %v", a, err) - } - if _, err := AskHandOver(ctx, conn, signer, "laptop", "/srv/empty", "jo", 5*time.Second); err == nil || - !strings.Contains(err.Error(), "neither") { - t.Fatalf("an empty answer was taken: %v", err) - } - if _, err := AskHandOver(ctx, conn, signer, "anchor", "/srv/notes", "jo", 5*time.Second); err == nil || - !strings.Contains(err.Error(), "node-engine") { - t.Fatalf("a machine with no engine listening: %v", err) - } - if _, err := AskHandOver(ctx, nil, signer, "anchor", "/srv/notes", "jo", time.Second); err == nil { - t.Fatal("asked with no bus") - } -} // A machine's grant hears its own hand-over ask and nobody else's, and may answer it: the one request a node is // asked (novox/hq issue 356). diff --git a/internal/link/heard_order_integration_test.go b/internal/link/heard_order_integration_test.go new file mode 100644 index 00000000..5941b196 --- /dev/null +++ b/internal/link/heard_order_integration_test.go @@ -0,0 +1,118 @@ +//go:build integration + +package link_test + +import ( + "context" + "testing" + + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func anOrderedMachine(t *testing.T) (*inventory.Inventory, link.Enrolment, string) { + t.Helper() + inv := inventory.ForTest(t) + node, err := inv.AddNode(context.Background(), "home-server") + if err != nil { + t.Fatal(err) + } + return inv, link.Enrolment{Inventory: inv}, node.ID +} + +func TestAnAccountOfAnOlderDeclarationIsRefusedByItsSequence(t *testing.T) { + inv, heard, id := anOrderedMachine(t) + ctx := context.Background() + before := countFor(link.WriterNodeEngine("home-server")) + if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a", "b")); err != nil { + t.Fatal(err) + } + if _, err := heard.Heard(ctx, ordered("d11", 57, 11, 40, "a")); err != nil { + t.Fatal(err) + } + doing, _, err := inv.DoingOf(ctx, "home-server") + if err != nil || doing.Declared != "d12" || doing.Applied != 2 { + t.Fatalf("the older account replaced the newer: %+v (%v)", doing, err) + } + if got := countFor(link.WriterNodeEngine("home-server")) - before; got != 1 { + t.Fatalf("the refusal was counted %d times, want once", got) + } + kept, err := inv.KeptOrder(ctx, id) + if err != nil || kept != (inventory.ReportOrder{Epoch: 57, Sequence: 12, ReportSequence: 41}) { + t.Fatalf("the order kept is %+v (%v)", kept, err) + } +} + +func TestAnOlderReportOfTheSameDeclarationIsRefused(t *testing.T) { + inv, heard, _ := anOrderedMachine(t) + ctx := context.Background() + if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a", "b")); err != nil { + t.Fatal(err) + } + failed := ordered("d12", 57, 12, 40) + failed.Failed = map[string]string{"b": "it failed a moment before it applied"} + if _, err := heard.Heard(ctx, failed); err != nil { + t.Fatal(err) + } + doing, _, _ := inv.DoingOf(ctx, "home-server") + if doing.Outcome != inventory.OutcomeApplied { + t.Fatalf("an older report of the same declaration replaced the newer: %+v", doing) + } + // A newer report of it — the next reconcile — is kept. + again := ordered("d12", 57, 12, 42) + again.Failed = map[string]string{"b": "it failed since"} + if _, err := heard.Heard(ctx, again); err != nil { + t.Fatal(err) + } + if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Outcome != inventory.OutcomeFailed { + t.Fatalf("a newer report of the same declaration was not kept: %+v", doing) + } +} + +func TestAnOrderedAccountIsNotJudgedByTheDigestSent(t *testing.T) { + inv, heard, id := anOrderedMachine(t) + ctx := context.Background() + if err := inv.RecordSent(ctx, id, "d13", nil); err != nil { + t.Fatal(err) + } + if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a")); err != nil { + t.Fatal(err) + } + if doing, said, _ := inv.DoingOf(ctx, "home-server"); !said || doing.Declared != "d12" { + t.Fatalf("the newest account the machine gave was set aside by the digest: %+v", doing) + } + // And the machine reads an epoch: it orders its reports. + if reads, err := inv.ReadsEpoch(ctx, id); err != nil || !reads { + t.Fatalf("a machine whose reports carry a report sequence is not recorded as reading an epoch: %v", err) + } +} + +func TestAnUnorderedAccountClearsTheOrderAndTheEpoch(t *testing.T) { + inv, heard, id := anOrderedMachine(t) + ctx := context.Background() + if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a")); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, id, "d13", nil); err != nil { + t.Fatal(err) + } + if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d13", Applied: []string{"a", "b"}}); err != nil { + t.Fatal(err) + } + if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Declared != "d13" { + t.Fatalf("an older node-engine's account of what was sent was not kept: %+v", doing) + } + if kept, _ := inv.KeptOrder(ctx, id); kept != (inventory.ReportOrder{}) { + t.Fatalf("an unordered account left the order kept: %+v", kept) + } + if reads, _ := inv.ReadsEpoch(ctx, id); reads { + t.Fatal("a node-engine that orders nothing is still sent an epoch") + } + // The next ordered account, whatever its numbers, has nothing to be older than. + if _, err := heard.Heard(ctx, ordered("d14", 58, 14, 1, "a")); err != nil { + t.Fatal(err) + } + if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Declared != "d14" { + t.Fatalf("the first ordered account after an unordered one was refused: %+v", doing) + } +} diff --git a/internal/link/heard_order_test.go b/internal/link/heard_order_test.go index b00ab0c8..de59da59 100644 --- a/internal/link/heard_order_test.go +++ b/internal/link/heard_order_test.go @@ -1,11 +1,9 @@ package link_test import ( - "context" "testing" "time" - "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -14,122 +12,18 @@ import ( // Of the same declaration, report r then r−1: refused. The digest the mesh last sent decides nothing // for an ordered report; an unordered one, from an older node-engine, is judged by it as before. -func anOrderedMachine(t *testing.T) (*inventory.Inventory, link.Enrolment, string) { - t.Helper() - inv := inventory.ForTest(t) - node, err := inv.AddNode(context.Background(), "home-server") - if err != nil { - t.Fatal(err) - } - return inv, link.Enrolment{Inventory: inv}, node.ID -} - func ordered(declared string, epoch, sequence, report int64, applied ...string) link.Report { return link.Report{Node: "home-server", Declared: declared, Applied: applied, Order: link.Order{Epoch: epoch, Sequence: sequence}, ReportSequence: report} } // Issue 267, ordered: a reconcile's account of declaration 11 reaches the mesh after the apply's of 12. -func TestAnAccountOfAnOlderDeclarationIsRefusedByItsSequence(t *testing.T) { - inv, heard, id := anOrderedMachine(t) - ctx := context.Background() - before := countFor(link.WriterNodeEngine("home-server")) - if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a", "b")); err != nil { - t.Fatal(err) - } - if _, err := heard.Heard(ctx, ordered("d11", 57, 11, 40, "a")); err != nil { - t.Fatal(err) - } - doing, _, err := inv.DoingOf(ctx, "home-server") - if err != nil || doing.Declared != "d12" || doing.Applied != 2 { - t.Fatalf("the older account replaced the newer: %+v (%v)", doing, err) - } - if got := countFor(link.WriterNodeEngine("home-server")) - before; got != 1 { - t.Fatalf("the refusal was counted %d times, want once", got) - } - kept, err := inv.KeptOrder(ctx, id) - if err != nil || kept != (inventory.ReportOrder{Epoch: 57, Sequence: 12, ReportSequence: 41}) { - t.Fatalf("the order kept is %+v (%v)", kept, err) - } -} - -func TestAnOlderReportOfTheSameDeclarationIsRefused(t *testing.T) { - inv, heard, _ := anOrderedMachine(t) - ctx := context.Background() - if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a", "b")); err != nil { - t.Fatal(err) - } - failed := ordered("d12", 57, 12, 40) - failed.Failed = map[string]string{"b": "it failed a moment before it applied"} - if _, err := heard.Heard(ctx, failed); err != nil { - t.Fatal(err) - } - doing, _, _ := inv.DoingOf(ctx, "home-server") - if doing.Outcome != inventory.OutcomeApplied { - t.Fatalf("an older report of the same declaration replaced the newer: %+v", doing) - } - // A newer report of it — the next reconcile — is kept. - again := ordered("d12", 57, 12, 42) - again.Failed = map[string]string{"b": "it failed since"} - if _, err := heard.Heard(ctx, again); err != nil { - t.Fatal(err) - } - if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Outcome != inventory.OutcomeFailed { - t.Fatalf("a newer report of the same declaration was not kept: %+v", doing) - } -} // An ordered account is judged by its order, not by the digest the mesh last sent: the account of 12 // is kept although the mesh has sent 13 since — it is still the newest account of the machine. -func TestAnOrderedAccountIsNotJudgedByTheDigestSent(t *testing.T) { - inv, heard, id := anOrderedMachine(t) - ctx := context.Background() - if err := inv.RecordSent(ctx, id, "d13", nil); err != nil { - t.Fatal(err) - } - if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a")); err != nil { - t.Fatal(err) - } - if doing, said, _ := inv.DoingOf(ctx, "home-server"); !said || doing.Declared != "d12" { - t.Fatalf("the newest account the machine gave was set aside by the digest: %+v", doing) - } - // And the machine reads an epoch: it orders its reports. - if reads, err := inv.ReadsEpoch(ctx, id); err != nil || !reads { - t.Fatalf("a machine whose reports carry a report sequence is not recorded as reading an epoch: %v", err) - } -} // A node-engine rolled back to one that orders nothing: its account is taken by the digest rule, the // order kept is cleared, and it is no longer sent an epoch. -func TestAnUnorderedAccountClearsTheOrderAndTheEpoch(t *testing.T) { - inv, heard, id := anOrderedMachine(t) - ctx := context.Background() - if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a")); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, id, "d13", nil); err != nil { - t.Fatal(err) - } - if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d13", Applied: []string{"a", "b"}}); err != nil { - t.Fatal(err) - } - if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Declared != "d13" { - t.Fatalf("an older node-engine's account of what was sent was not kept: %+v", doing) - } - if kept, _ := inv.KeptOrder(ctx, id); kept != (inventory.ReportOrder{}) { - t.Fatalf("an unordered account left the order kept: %+v", kept) - } - if reads, _ := inv.ReadsEpoch(ctx, id); reads { - t.Fatal("a node-engine that orders nothing is still sent an epoch") - } - // The next ordered account, whatever its numbers, has nothing to be older than. - if _, err := heard.Heard(ctx, ordered("d14", 58, 14, 1, "a")); err != nil { - t.Fatal(err) - } - if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Declared != "d14" { - t.Fatalf("the first ordered account after an unordered one was refused: %+v", doing) - } -} // A stale refusal names its writer by the epoch the refused declaration claimed, and a refusal whose // own report was lost is still counted from the machine's own tally. diff --git a/internal/link/heard_test.go b/internal/link/heard_test.go index 22d97779..93a2dd0d 100644 --- a/internal/link/heard_test.go +++ b/internal/link/heard_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link_test import ( diff --git a/internal/link/judge_directories_test.go b/internal/link/judge_directories_test.go index 10cfdde1..315869b3 100644 --- a/internal/link/judge_directories_test.go +++ b/internal/link/judge_directories_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/meshcli_integration_test.go b/internal/link/meshcli_integration_test.go new file mode 100644 index 00000000..3da2b222 --- /dev/null +++ b/internal/link/meshcli_integration_test.go @@ -0,0 +1,96 @@ +//go:build integration + +package link + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/testbus" +) + +func cliBus(t *testing.T, l *CallLog, atOnce int, handle CLIHandler) *nats.Conn { + t.Helper() + conn, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(conn.Close) + stop, err := OverNATS{Conn: conn}.serveCLI(l, atOnce, handle, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(stop) + return conn +} + +func TestALongMeshCLILineIsFollowedToItsAnswer(t *testing.T) { + was := AnswerWithin + AnswerWithin = 50 * time.Millisecond + t.Cleanup(func() { AnswerWithin = was }) + release := make(chan struct{}) + conn := cliBus(t, NewCallLog(), 4, func(ctx context.Context, node string, asked CLIAsked) CLIAnswer { + <-release + return CLIAnswer{Stdout: []byte("done on " + node), Terminal: true} + }) + _, first, failed := askCLI(t, conn, "laptop", CLIAsked{Line: []string{"push", "laptop"}, Account: "op", UID: 1000}) + call, _ := first["call"].(string) + if failed != "" || first["running"] != true || call == "" { + t.Fatalf("a long line was not answered as running with its call: %v %q", first, failed) + } + _, still, _ := askCLI(t, conn, "laptop", CLIAsked{Follow: call, Account: "op", UID: 1000}) + if still["running"] != true { + t.Fatalf("following a running line answered %v", still) + } + close(release) + deadline := time.Now().Add(5 * time.Second) + for { + a, raw, failed := askCLI(t, conn, "laptop", CLIAsked{Follow: call, Account: "op", UID: 1000}) + if failed != "" { + t.Fatalf("following answered %q", failed) + } + if raw["running"] != true { + if string(a.Stdout) != "done on laptop" || !a.Terminal { + t.Fatalf("followed to %+v", a) + } + break + } + if time.Now().After(deadline) { + t.Fatal("the line never finished for its follower") + } + time.Sleep(20 * time.Millisecond) + } + if _, _, failed := askCLI(t, conn, "laptop", CLIAsked{Follow: call, Account: "agent", UID: 1001}); failed == "" { + t.Fatal("another account followed the operator's line") + } + if _, _, failed := askCLI(t, conn, "desktop", CLIAsked{Follow: call, Account: "op", UID: 1000}); failed == "" { + t.Fatal("another node followed the line") + } +} + +func TestMeshCLILinesAtOnceAreBounded(t *testing.T) { + was := AnswerWithin + AnswerWithin = 50 * time.Millisecond + t.Cleanup(func() { AnswerWithin = was }) + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + ran := make(chan struct{}, 4) + conn := cliBus(t, NewCallLog(), 1, func(context.Context, string, CLIAsked) CLIAnswer { + ran <- struct{}{} + <-release + return CLIAnswer{} + }) + if _, first, _ := askCLI(t, conn, "laptop", CLIAsked{Line: []string{"status"}, Account: "op"}); first["running"] != true { + t.Fatalf("the first line answered %v", first) + } + a, _, _ := askCLI(t, conn, "laptop", CLIAsked{Line: []string{"status"}, Account: "op"}) + if !strings.Contains(a.Refused, "busy") || a.Exit == 0 { + t.Fatalf("a line over the bound answered %+v", a) + } + if len(ran) != 1 { + t.Fatalf("%d lines ran, one was bound", len(ran)) + } +} diff --git a/internal/link/meshcli_test.go b/internal/link/meshcli_test.go index 1fc50085..7e5db365 100644 --- a/internal/link/meshcli_test.go +++ b/internal/link/meshcli_test.go @@ -10,8 +10,6 @@ import ( "time" "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/testbus" ) // The node-engine's request and the answer it hands mesh-cli hold these field names; the engine's side holds the @@ -94,20 +92,6 @@ func TestALargeMeshCLIAnswerIsCutAndSaysSo(t *testing.T) { } // cliBus serves mesh-cli on a bus of the test's own with a call log of its own, and returns a connection to ask on. -func cliBus(t *testing.T, l *CallLog, atOnce int, handle CLIHandler) *nats.Conn { - t.Helper() - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - stop, err := OverNATS{Conn: conn}.serveCLI(l, atOnce, handle, nil) - if err != nil { - t.Fatal(err) - } - t.Cleanup(stop) - return conn -} // askCLI asks one request on a node's subject and reads the envelope. func askCLI(t *testing.T, conn *nats.Conn, node string, asked CLIAsked) (CLIAnswer, map[string]any, string) { @@ -134,74 +118,8 @@ func askCLI(t *testing.T, conn *nats.Conn, node string, asked CLIAsked) (CLIAnsw // **A line that outlasts the bus's window for an answer is followed to its answer, never lost** (review of ADR // 0272): the first answer says it is running and names its call, and following the call by the same account on // the same node gives what the line printed once it ends. Another account, or another node, is told no such call. -func TestALongMeshCLILineIsFollowedToItsAnswer(t *testing.T) { - was := AnswerWithin - AnswerWithin = 50 * time.Millisecond - t.Cleanup(func() { AnswerWithin = was }) - release := make(chan struct{}) - conn := cliBus(t, NewCallLog(), 4, func(ctx context.Context, node string, asked CLIAsked) CLIAnswer { - <-release - return CLIAnswer{Stdout: []byte("done on " + node), Terminal: true} - }) - _, first, failed := askCLI(t, conn, "laptop", CLIAsked{Line: []string{"push", "laptop"}, Account: "op", UID: 1000}) - call, _ := first["call"].(string) - if failed != "" || first["running"] != true || call == "" { - t.Fatalf("a long line was not answered as running with its call: %v %q", first, failed) - } - _, still, _ := askCLI(t, conn, "laptop", CLIAsked{Follow: call, Account: "op", UID: 1000}) - if still["running"] != true { - t.Fatalf("following a running line answered %v", still) - } - close(release) - deadline := time.Now().Add(5 * time.Second) - for { - a, raw, failed := askCLI(t, conn, "laptop", CLIAsked{Follow: call, Account: "op", UID: 1000}) - if failed != "" { - t.Fatalf("following answered %q", failed) - } - if raw["running"] != true { - if string(a.Stdout) != "done on laptop" || !a.Terminal { - t.Fatalf("followed to %+v", a) - } - break - } - if time.Now().After(deadline) { - t.Fatal("the line never finished for its follower") - } - time.Sleep(20 * time.Millisecond) - } - if _, _, failed := askCLI(t, conn, "laptop", CLIAsked{Follow: call, Account: "agent", UID: 1001}); failed == "" { - t.Fatal("another account followed the operator's line") - } - if _, _, failed := askCLI(t, conn, "desktop", CLIAsked{Follow: call, Account: "op", UID: 1000}); failed == "" { - t.Fatal("another node followed the line") - } -} // Lines running at once are bounded: one over the bound is answered busy at once, and nothing ran. -func TestMeshCLILinesAtOnceAreBounded(t *testing.T) { - was := AnswerWithin - AnswerWithin = 50 * time.Millisecond - t.Cleanup(func() { AnswerWithin = was }) - release := make(chan struct{}) - t.Cleanup(func() { close(release) }) - ran := make(chan struct{}, 4) - conn := cliBus(t, NewCallLog(), 1, func(context.Context, string, CLIAsked) CLIAnswer { - ran <- struct{}{} - <-release - return CLIAnswer{} - }) - if _, first, _ := askCLI(t, conn, "laptop", CLIAsked{Line: []string{"status"}, Account: "op"}); first["running"] != true { - t.Fatalf("the first line answered %v", first) - } - a, _, _ := askCLI(t, conn, "laptop", CLIAsked{Line: []string{"status"}, Account: "op"}) - if !strings.Contains(a.Refused, "busy") || a.Exit == 0 { - t.Fatalf("a line over the bound answered %+v", a) - } - if len(ran) != 1 { - t.Fatalf("%d lines ran, one was bound", len(ran)) - } -} // A mesh-cli line's record keeps its first word, never the rest of its line, and its answer is never kept on the // bus, where `calls` answers anyone who may call the seat. diff --git a/internal/link/movesecret_integration_test.go b/internal/link/movesecret_integration_test.go new file mode 100644 index 00000000..848b5b10 --- /dev/null +++ b/internal/link/movesecret_integration_test.go @@ -0,0 +1,58 @@ +//go:build integration + +package link + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestAMoveIsAskedOfTheMachineSignedUnderItsOwnContext(t *testing.T) { + conn, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(conn.Close) + signer, public := testSigner(t) + var heard HandOverAsk + var row MoveSecretAsk + engine, err := conn.Subscribe(broker.AskMoveSecretSubject("laptop"), func(msg *nats.Msg) { + var signed SignedHandOver + _ = json.Unmarshal(msg.Data, &signed) + if ed25519.Verify(public, append([]byte(HandOverContext), signed.Ask...), signed.Signature) || + !ed25519.Verify(public, append([]byte(MoveSecretContext), signed.Ask...), signed.Signature) { + _ = msg.Respond([]byte(`{"refused":"not signed as a move"}`)) + return + } + _ = json.Unmarshal(signed.Ask, &heard) + _ = json.Unmarshal(signed.Ask, &row) + body, _ := json.Marshal(MoveSecretAnswer{Said: "moved", Secret: "own/laptop/" + row.Module + "/" + row.Name, + Generation: 1}) + _ = msg.Respond(body) + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = engine.Unsubscribe() }) + a, err := AskMoveSecret(context.Background(), conn, signer, "laptop", "the move", + MoveSecretAsk{Module: "postgres", Name: "superuser-password", Origin: "made", Sealed: "c2Vh"}, 5*time.Second) + if err != nil || a.Secret != "own/laptop/postgres/superuser-password" || a.Generation != 1 { + t.Fatalf("answered %+v, %v", a, err) + } + if heard.Node != "laptop" || heard.Path != "" || heard.Nonce == "" || heard.Expires.IsZero() || heard.By != "the move" || + row.Sealed != "c2Vh" || row.Origin != "made" { + t.Fatalf("the engine heard %+v %+v", heard, row) + } + if _, err := AskMoveSecret(context.Background(), conn, signer, "anchor", "the move", MoveSecretAsk{}, 2*time.Second); err == nil || + !strings.Contains(err.Error(), "nothing was moved") { + t.Fatalf("a machine with no engine listening: %v", err) + } +} diff --git a/internal/link/movesecret_test.go b/internal/link/movesecret_test.go index 4dc05905..cfad665f 100644 --- a/internal/link/movesecret_test.go +++ b/internal/link/movesecret_test.go @@ -1,18 +1,13 @@ package link import ( - "context" - "crypto/ed25519" "encoding/json" "sort" "strings" "testing" "time" - "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/testbus" ) // The move's ask and answer keep the engine's field names (mesh-host internal/link, @@ -47,44 +42,3 @@ func TestTheMovesAskAndAnswerKeepTheirFieldNames(t *testing.T) { // A move is asked of the machine on its own subject, signed under its own context — never a hand-over's — naming // no path and carrying the row beside the hand-over's fields; the engine's answer comes back whole. -func TestAMoveIsAskedOfTheMachineSignedUnderItsOwnContext(t *testing.T) { - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - signer, public := testSigner(t) - var heard HandOverAsk - var row MoveSecretAsk - engine, err := conn.Subscribe(broker.AskMoveSecretSubject("laptop"), func(msg *nats.Msg) { - var signed SignedHandOver - _ = json.Unmarshal(msg.Data, &signed) - if ed25519.Verify(public, append([]byte(HandOverContext), signed.Ask...), signed.Signature) || - !ed25519.Verify(public, append([]byte(MoveSecretContext), signed.Ask...), signed.Signature) { - _ = msg.Respond([]byte(`{"refused":"not signed as a move"}`)) - return - } - _ = json.Unmarshal(signed.Ask, &heard) - _ = json.Unmarshal(signed.Ask, &row) - body, _ := json.Marshal(MoveSecretAnswer{Said: "moved", Secret: "own/laptop/" + row.Module + "/" + row.Name, - Generation: 1}) - _ = msg.Respond(body) - }) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = engine.Unsubscribe() }) - a, err := AskMoveSecret(context.Background(), conn, signer, "laptop", "the move", - MoveSecretAsk{Module: "postgres", Name: "superuser-password", Origin: "made", Sealed: "c2Vh"}, 5*time.Second) - if err != nil || a.Secret != "own/laptop/postgres/superuser-password" || a.Generation != 1 { - t.Fatalf("answered %+v, %v", a, err) - } - if heard.Node != "laptop" || heard.Path != "" || heard.Nonce == "" || heard.Expires.IsZero() || heard.By != "the move" || - row.Sealed != "c2Vh" || row.Origin != "made" { - t.Fatalf("the engine heard %+v %+v", heard, row) - } - if _, err := AskMoveSecret(context.Background(), conn, signer, "anchor", "the move", MoveSecretAsk{}, 2*time.Second); err == nil || - !strings.Contains(err.Error(), "nothing was moved") { - t.Fatalf("a machine with no engine listening: %v", err) - } -} diff --git a/internal/link/pages_integration_test.go b/internal/link/pages_integration_test.go new file mode 100644 index 00000000..5467a161 --- /dev/null +++ b/internal/link/pages_integration_test.go @@ -0,0 +1,59 @@ +//go:build integration + +package link + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestAPartGoesOnlyToItsCaller(t *testing.T) { + conn, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + l := NewCallLog() + c := l.begin("mesh-controller", "status", nil, "_INBOX.operator.ABCDEFGHIJKLMNOPQRSTUV") + l.page(c, []byte(strings.Repeat("z", 5000)), 2048) + sub, err := conn.Subscribe("pages.test", func(m *nats.Msg) { l.servePage("mesh-controller", "status", m, nil) }) + if err != nil { + t.Fatal(err) + } + defer sub.Unsubscribe() + _, err = askPart(context.Background(), conn, "pages.test", Paged{Call: c.ID, Parts: 3}, 0) + if err == nil || !strings.Contains(err.Error(), "another caller") { + t.Fatalf("a part was handed to a caller it was not paged to: %v", err) + } + _, err = askPart(context.Background(), conn, "pages.test", Paged{Call: "call-0-0", Parts: 3}, 0) + if err == nil || !strings.Contains(err.Error(), "not held here") { + t.Fatalf("a part of nothing held: %v", err) + } +} + +func TestAMissingPartFailsTheWhole(t *testing.T) { + conn, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + sub, err := conn.Subscribe("pages.none", func(m *nats.Msg) { _ = m.Respond([]byte(`{"error":"gone"}`)) }) + if err != nil { + t.Fatal(err) + } + defer sub.Unsubscribe() + first, _ := json.Marshal(map[string]any{"error": "too large", "paged": Paged{Call: "call-1-1", Bytes: 10, Parts: 2}}) + _, err = Whole(context.Background(), conn, "pages.none", first) + if err == nil || !strings.Contains(err.Error(), "part 0") || !strings.Contains(err.Error(), "gone") { + t.Fatalf("a missing part read as %v", err) + } + if plain, err := Whole(context.Background(), conn, "pages.none", []byte(`{"result":1}`)); err != nil || + string(plain) != `{"result":1}` { + t.Fatalf("an answer that fits was changed: %s %v", plain, err) + } +} diff --git a/internal/link/pages_test.go b/internal/link/pages_test.go index 844fb099..08f28e03 100644 --- a/internal/link/pages_test.go +++ b/internal/link/pages_test.go @@ -7,8 +7,6 @@ import ( "testing" "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/testbus" ) // An answer too large for one message is recorded as paged, and a call's record says so: `calls` no @@ -53,49 +51,5 @@ func TestAnAnswerTheBusRefusesIsSaid(t *testing.T) { } // A part goes only to the caller its answer was paged to, on the verb that answered it. -func TestAPartGoesOnlyToItsCaller(t *testing.T) { - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - l := NewCallLog() - c := l.begin("mesh-controller", "status", nil, "_INBOX.operator.ABCDEFGHIJKLMNOPQRSTUV") - l.page(c, []byte(strings.Repeat("z", 5000)), 2048) - sub, err := conn.Subscribe("pages.test", func(m *nats.Msg) { l.servePage("mesh-controller", "status", m, nil) }) - if err != nil { - t.Fatal(err) - } - defer sub.Unsubscribe() - _, err = askPart(context.Background(), conn, "pages.test", Paged{Call: c.ID, Parts: 3}, 0) - if err == nil || !strings.Contains(err.Error(), "another caller") { - t.Fatalf("a part was handed to a caller it was not paged to: %v", err) - } - _, err = askPart(context.Background(), conn, "pages.test", Paged{Call: "call-0-0", Parts: 3}, 0) - if err == nil || !strings.Contains(err.Error(), "not held here") { - t.Fatalf("a part of nothing held: %v", err) - } -} // A part that cannot be had fails the whole answer, naming it — never a shorter answer. -func TestAMissingPartFailsTheWhole(t *testing.T) { - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - sub, err := conn.Subscribe("pages.none", func(m *nats.Msg) { _ = m.Respond([]byte(`{"error":"gone"}`)) }) - if err != nil { - t.Fatal(err) - } - defer sub.Unsubscribe() - first, _ := json.Marshal(map[string]any{"error": "too large", "paged": Paged{Call: "call-1-1", Bytes: 10, Parts: 2}}) - _, err = Whole(context.Background(), conn, "pages.none", first) - if err == nil || !strings.Contains(err.Error(), "part 0") || !strings.Contains(err.Error(), "gone") { - t.Fatalf("a missing part read as %v", err) - } - if plain, err := Whole(context.Background(), conn, "pages.none", []byte(`{"result":1}`)); err != nil || - string(plain) != `{"result":1}` { - t.Fatalf("an answer that fits was changed: %s %v", plain, err) - } -} diff --git a/internal/link/profile_report_test.go b/internal/link/profile_report_test.go index 5477f1f2..889fae48 100644 --- a/internal/link/profile_report_test.go +++ b/internal/link/profile_report_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link_test import ( diff --git a/internal/link/queue_integration_test.go b/internal/link/queue_integration_test.go new file mode 100644 index 00000000..d4684406 --- /dev/null +++ b/internal/link/queue_integration_test.go @@ -0,0 +1,220 @@ +//go:build integration + +package link + +import ( + "context" + "encoding/json" + "sync" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" +) + +func aBusWithACancelledSet(t *testing.T) *broker.JetStream { + t.Helper() + js := aBusWithTheBuildRole(t) + seat := broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}} + if err := broker.RaiseCancelledSets(js, []broker.DeclaredSeat{seat}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = js.Context().DeleteKeyValue(broker.CancelledSetName(TheBuildMachine)) }) + return js +} + +func TestNatsAnAskCancelledAsItWasTakenIsNotBuilt(t *testing.T) { + js := aBusWithACancelledSet(t) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + + if err := MarkCancelled(ctx, js, TheBuildMachine, "build-cancelled"); err != nil { + t.Fatal(err) + } + outcomes, err := js.Conn().SubscribeSync(BuildOutcome()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = outcomes.Unsubscribe() }() + _ = js.Conn().Flush() + for _, id := range []string{"build-cancelled", "build-kept"} { + body, _ := json.Marshal(BuildRequest{ID: id, Repository: "/r"}) + if _, err := js.Context().Publish(BuildWork(), body); err != nil { + t.Fatal(err) + } + } + + built := make(chan string, 2) + machine := MachineOverNATS(js, "anchor") + defer machine.Close() + go func() { + _ = machine.Take(ctx, func(ctx context.Context, work Build) { + built <- work.Request().ID + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) + _ = work.Done() + }) + }() + select { + case id := <-built: + if id != "build-kept" { + t.Fatalf("%s was built", id) + } + case <-time.After(10 * time.Second): + t.Fatal("the ask that was not cancelled was never built") + } + msg, err := outcomes.NextMsg(5 * time.Second) + if err != nil { + t.Fatal(err) + } + var said BuildResult + if err := json.Unmarshal(msg.Data, &said); err != nil || said.ID != "build-cancelled" || + said.Failed != CancelledByHand || said.On != "anchor" { + t.Fatalf("the cancelled ask's outcome is %+v (%v)", said, err) + } + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if info, err := js.Context().StreamInfo("SEAT_NODE_BUILD_AGENT"); err == nil && info.State.Msgs == 0 { + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatal("the cancelled ask is still queued: terminated, it would not be") +} + +func TestNatsAPausedHolderTakesNothingNew(t *testing.T) { + js := aBusWithACancelledSet(t) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + paused := make(chan bool, 1) + paused <- true + isPaused := func() bool { + p := <-paused + paused <- p + return p + } + took := make(chan string, 1) + machine := MachineOverNATSWith(js, "anchor", TheBuildMachine, MachineOptions{Paused: isPaused}) + defer machine.Close() + go func() { + _ = machine.Take(ctx, func(ctx context.Context, work Build) { + took <- work.Request().ID + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) + _ = work.Done() + }) + }() + body, _ := json.Marshal(BuildRequest{ID: "build-waits", Repository: "/r"}) + if _, err := js.Context().Publish(BuildWork(), body); err != nil { + t.Fatal(err) + } + select { + case id := <-took: + t.Fatalf("a paused holder took %s", id) + case <-time.After(3 * time.Second): + } + q, err := ReadQueue(ctx, js, TheBuildMachine) + if err != nil { + t.Fatal(err) + } + if len(q.Asks) != 1 || q.Asks[0].State != AskWaiting { + t.Fatalf("while paused the queue reads %+v", q.Asks) + } + <-paused + paused <- false + select { + case id := <-took: + if id != "build-waits" { + t.Fatalf("took %s", id) + } + case <-time.After(10 * time.Second): + t.Fatal("resumed, the holder never took what waited") + } +} + +func TestNatsAHoldersPausedStateIsReadBack(t *testing.T) { + js := aBusWithTheBuildRole(t) + for _, s := range []HolderState{{On: "ace", Paused: true}, {On: "g14", Paused: true}, {On: "g14", Paused: false}} { + body, _ := json.Marshal(s) + if _, err := js.Context().Publish(BuildPausedOf(TheBuildMachine, s.On), body); err != nil { + t.Fatal(err) + } + } + said, err := PausedSaid(js, TheBuildMachine, []string{"ace", "g14", "novox"}) + if err != nil { + t.Fatal(err) + } + if !said["ace"].Paused || said["g14"].Paused || len(said) != 2 { + t.Fatalf("read back %+v", said) + } +} + +func TestNatsAWaitingAskerHearsItsAskCancelledOrKilled(t *testing.T) { + js := aBusWithACancelledSet(t) + was := cancelLook + cancelLook = 200 * time.Millisecond + t.Cleanup(func() { cancelLook = was }) + ask := &natsBuilds{js: js, seat: TheBuildMachine} + + go func() { + time.Sleep(500 * time.Millisecond) + _ = MarkCancelled(context.Background(), js, TheBuildMachine, "build-waited-cancelled") + }() + result, err := ask.Submit(context.Background(), BuildRequest{ID: "build-waited-cancelled", Repository: "/r"}, 10*time.Second) + if err != nil || result.Failed != CancelledByHand || result.ID != "build-waited-cancelled" { + t.Fatalf("the waiter heard %+v (%v)", result, err) + } + + // Killed: the holder announces the failure as any outcome, and the waiter has it. + ctx, stop := context.WithCancel(context.Background()) + defer stop() + machine := MachineOverNATS(js, "ace") + defer machine.Close() + go func() { + _ = machine.Take(ctx, func(ctx context.Context, work Build) { + r := work.Request() + _ = work.Announce(ctx, BuildResult{ID: r.ID, Repository: r.Repository, On: "ace", Failed: KilledByHand}) + _ = work.Done() + }) + }() + result, err = ask.Submit(context.Background(), BuildRequest{ID: "build-waited-killed", Repository: "/r"}, 10*time.Second) + if err != nil || result.Failed != KilledByHand || result.On != "ace" { + t.Fatalf("the waiter heard %+v (%v)", result, err) + } +} + +func TestNatsAHolderPausedAsItsPullWasAnsweredHandsTheAskBack(t *testing.T) { + js := aBusWithACancelledSet(t) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + // Not paused when it asks; paused by the time the answer is read. + var looks int + var mu sync.Mutex + paused := func() bool { + mu.Lock() + defer mu.Unlock() + looks++ + return looks > 1 + } + took := make(chan string, 1) + machine := MachineOverNATSWith(js, "anchor", TheBuildMachine, MachineOptions{Paused: paused}) + defer machine.Close() + go func() { + _ = machine.Take(ctx, func(ctx context.Context, work Build) { took <- work.Request().ID }) + }() + time.Sleep(200 * time.Millisecond) + body, _ := json.Marshal(BuildRequest{ID: "build-handed-back", Repository: "/r"}) + if _, err := js.Context().Publish(BuildWork(), body); err != nil { + t.Fatal(err) + } + select { + case id := <-took: + t.Fatalf("a holder paused as its pull was answered built %s", id) + case <-time.After(2 * time.Second): + } + q, err := ReadQueue(ctx, js, TheBuildMachine) + if err != nil { + t.Fatal(err) + } + if len(q.Asks) != 1 || q.Asks[0].ID != "build-handed-back" { + t.Fatalf("the ask is not back in the queue: %+v", q.Asks) + } +} diff --git a/internal/link/queue_test.go b/internal/link/queue_test.go index 4ff3a74f..a3d7716f 100644 --- a/internal/link/queue_test.go +++ b/internal/link/queue_test.go @@ -1,15 +1,10 @@ package link import ( - "context" - "encoding/json" - "sync" "testing" "time" "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/broker" ) // The build queue read and changed by hand (novox/hq ADR 0219). @@ -121,216 +116,14 @@ func TestACancelledSetReadSaysWhatWasCancelled(t *testing.T) { // --- against a real server ---------------------------------------------------------------------- -func aBusWithACancelledSet(t *testing.T) *broker.JetStream { - t.Helper() - js := aBusWithTheBuildRole(t) - seat := broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}} - if err := broker.RaiseCancelledSets(js, []broker.DeclaredSeat{seat}); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = js.Context().DeleteKeyValue(broker.CancelledSetName(TheBuildMachine)) }) - return js -} - // **The race a delete alone leaves open**: an ask fetched in the moment it was cancelled is ended by // the holder that took it — terminated, never built, its outcome said as failed. -func TestNatsAnAskCancelledAsItWasTakenIsNotBuilt(t *testing.T) { - js := aBusWithACancelledSet(t) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - - if err := MarkCancelled(ctx, js, TheBuildMachine, "build-cancelled"); err != nil { - t.Fatal(err) - } - outcomes, err := js.Conn().SubscribeSync(BuildOutcome()) - if err != nil { - t.Fatal(err) - } - defer func() { _ = outcomes.Unsubscribe() }() - _ = js.Conn().Flush() - for _, id := range []string{"build-cancelled", "build-kept"} { - body, _ := json.Marshal(BuildRequest{ID: id, Repository: "/r"}) - if _, err := js.Context().Publish(BuildWork(), body); err != nil { - t.Fatal(err) - } - } - - built := make(chan string, 2) - machine := MachineOverNATS(js, "anchor") - defer machine.Close() - go func() { - _ = machine.Take(ctx, func(ctx context.Context, work Build) { - built <- work.Request().ID - _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) - _ = work.Done() - }) - }() - select { - case id := <-built: - if id != "build-kept" { - t.Fatalf("%s was built", id) - } - case <-time.After(10 * time.Second): - t.Fatal("the ask that was not cancelled was never built") - } - msg, err := outcomes.NextMsg(5 * time.Second) - if err != nil { - t.Fatal(err) - } - var said BuildResult - if err := json.Unmarshal(msg.Data, &said); err != nil || said.ID != "build-cancelled" || - said.Failed != CancelledByHand || said.On != "anchor" { - t.Fatalf("the cancelled ask's outcome is %+v (%v)", said, err) - } - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - if info, err := js.Context().StreamInfo("SEAT_NODE_BUILD_AGENT"); err == nil && info.State.Msgs == 0 { - return - } - time.Sleep(20 * time.Millisecond) - } - t.Fatal("the cancelled ask is still queued: terminated, it would not be") -} // A paused holder takes nothing; resumed, it takes what waited. -func TestNatsAPausedHolderTakesNothingNew(t *testing.T) { - js := aBusWithACancelledSet(t) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - paused := make(chan bool, 1) - paused <- true - isPaused := func() bool { - p := <-paused - paused <- p - return p - } - took := make(chan string, 1) - machine := MachineOverNATSWith(js, "anchor", TheBuildMachine, MachineOptions{Paused: isPaused}) - defer machine.Close() - go func() { - _ = machine.Take(ctx, func(ctx context.Context, work Build) { - took <- work.Request().ID - _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) - _ = work.Done() - }) - }() - body, _ := json.Marshal(BuildRequest{ID: "build-waits", Repository: "/r"}) - if _, err := js.Context().Publish(BuildWork(), body); err != nil { - t.Fatal(err) - } - select { - case id := <-took: - t.Fatalf("a paused holder took %s", id) - case <-time.After(3 * time.Second): - } - q, err := ReadQueue(ctx, js, TheBuildMachine) - if err != nil { - t.Fatal(err) - } - if len(q.Asks) != 1 || q.Asks[0].State != AskWaiting { - t.Fatalf("while paused the queue reads %+v", q.Asks) - } - <-paused - paused <- false - select { - case id := <-took: - if id != "build-waits" { - t.Fatalf("took %s", id) - } - case <-time.After(10 * time.Second): - t.Fatal("resumed, the holder never took what waited") - } -} // What a holder last said about taking work is read back per machine. -func TestNatsAHoldersPausedStateIsReadBack(t *testing.T) { - js := aBusWithTheBuildRole(t) - for _, s := range []HolderState{{On: "ace", Paused: true}, {On: "g14", Paused: true}, {On: "g14", Paused: false}} { - body, _ := json.Marshal(s) - if _, err := js.Context().Publish(BuildPausedOf(TheBuildMachine, s.On), body); err != nil { - t.Fatal(err) - } - } - said, err := PausedSaid(js, TheBuildMachine, []string{"ace", "g14", "novox"}) - if err != nil { - t.Fatal(err) - } - if !said["ace"].Paused || said["g14"].Paused || len(said) != 2 { - t.Fatalf("read back %+v", said) - } -} // A `build` waiting on its outcome hears a cancel — which publishes no outcome — from the cancelled // set, and a kill from the outcome the holder announces (novox/hq ADR 0219). -func TestNatsAWaitingAskerHearsItsAskCancelledOrKilled(t *testing.T) { - js := aBusWithACancelledSet(t) - was := cancelLook - cancelLook = 200 * time.Millisecond - t.Cleanup(func() { cancelLook = was }) - ask := &natsBuilds{js: js, seat: TheBuildMachine} - - go func() { - time.Sleep(500 * time.Millisecond) - _ = MarkCancelled(context.Background(), js, TheBuildMachine, "build-waited-cancelled") - }() - result, err := ask.Submit(context.Background(), BuildRequest{ID: "build-waited-cancelled", Repository: "/r"}, 10*time.Second) - if err != nil || result.Failed != CancelledByHand || result.ID != "build-waited-cancelled" { - t.Fatalf("the waiter heard %+v (%v)", result, err) - } - - // Killed: the holder announces the failure as any outcome, and the waiter has it. - ctx, stop := context.WithCancel(context.Background()) - defer stop() - machine := MachineOverNATS(js, "ace") - defer machine.Close() - go func() { - _ = machine.Take(ctx, func(ctx context.Context, work Build) { - r := work.Request() - _ = work.Announce(ctx, BuildResult{ID: r.ID, Repository: r.Repository, On: "ace", Failed: KilledByHand}) - _ = work.Done() - }) - }() - result, err = ask.Submit(context.Background(), BuildRequest{ID: "build-waited-killed", Repository: "/r"}, 10*time.Second) - if err != nil || result.Failed != KilledByHand || result.On != "ace" { - t.Fatalf("the waiter heard %+v (%v)", result, err) - } -} // Paused in the moment a pull was answered: the ask is handed back, not built (ADR 0219). -func TestNatsAHolderPausedAsItsPullWasAnsweredHandsTheAskBack(t *testing.T) { - js := aBusWithACancelledSet(t) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - // Not paused when it asks; paused by the time the answer is read. - var looks int - var mu sync.Mutex - paused := func() bool { - mu.Lock() - defer mu.Unlock() - looks++ - return looks > 1 - } - took := make(chan string, 1) - machine := MachineOverNATSWith(js, "anchor", TheBuildMachine, MachineOptions{Paused: paused}) - defer machine.Close() - go func() { - _ = machine.Take(ctx, func(ctx context.Context, work Build) { took <- work.Request().ID }) - }() - time.Sleep(200 * time.Millisecond) - body, _ := json.Marshal(BuildRequest{ID: "build-handed-back", Repository: "/r"}) - if _, err := js.Context().Publish(BuildWork(), body); err != nil { - t.Fatal(err) - } - select { - case id := <-took: - t.Fatalf("a holder paused as its pull was answered built %s", id) - case <-time.After(2 * time.Second): - } - q, err := ReadQueue(ctx, js, TheBuildMachine) - if err != nil { - t.Fatal(err) - } - if len(q.Asks) != 1 || q.Asks[0].ID != "build-handed-back" { - t.Fatalf("the ask is not back in the queue: %+v", q.Asks) - } -} diff --git a/internal/link/receive_nats_integration_test.go b/internal/link/receive_nats_integration_test.go new file mode 100644 index 00000000..7806ef70 --- /dev/null +++ b/internal/link/receive_nats_integration_test.go @@ -0,0 +1,440 @@ +//go:build integration + +package link + +import ( + "context" + "encoding/json" + "errors" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/testbus" +) + +func aBus(t *testing.T) *broker.JetStream { + t.Helper() + url := testbus.URL(t) + js, err := broker.Dial(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + + // **Streams purged, consumers removed.** Both halves, and each was learned by getting it wrong. + // + // The streams are emptied rather than deleted and recreated, because delete-then-add is not a + // reset: the server's teardown races the creation, and a test then inherits the previous one's + // messages — which reads as a redelivery bug in the code under test. + // + // The consumers are removed, because deleting a stream used to take them with it and purging + // does not. A durable *push* consumer that survives between tests keeps pushing to a delivery + // subject the previous test's subscription has gone from: the messages count as delivered, go + // nowhere, and the next test waits out its timeout for an announcement the server believes it + // already sent. The controller recreates what it needs on start, so leaving none is correct. + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + clean := func() { + for _, c := range broker.MeshConsumers() { + _ = js.Context().DeleteConsumer(c.Stream, c.Name) + } + for _, s := range broker.MeshStreams() { + _ = js.Context().PurgeStream(s.Name) + } + } + clean() + t.Cleanup(clean) + return js +} + +func TestNatsAReportIsHeardAndLeavesTheStream(t *testing.T) { + js := aBus(t) + store := &counted{} + _, stop := servingOn(t, js, store) + defer stop() + + body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) + if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { + t.Fatal(err) + } + eventually(t, "a report being recorded", func() bool { return store.count() == 1 }) + eventually(t, "the report leaving the work queue", func() bool { + info, err := js.Context().StreamInfo("CONTROL") + return err == nil && info.State.Msgs == 0 + }) +} + +func TestNatsAReportTheStoreCannotTakeIsHeldByTheServerAndComesBack(t *testing.T) { + js := aBus(t) + store := &counted{} + store.refusing(errors.Join(ErrTryAgain, errors.New("the database system is starting up"))) + _, stop := servingOn(t, js, store) + defer stop() + + body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) + if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { + t.Fatal(err) + } + + // Held: the message is the server's, unacknowledged, and still in the stream. + eventually(t, "the report being redelivered at least once", func() bool { + info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName) + return err == nil && info.NumRedelivered >= 1 + }) + info, err := js.Context().StreamInfo("CONTROL") + if err != nil || info.State.Msgs != 1 { + t.Fatalf("a held report did not stay on the server: %+v, %v", info, err) + } + if store.count() != 0 { + t.Fatalf("a report was recorded by a store that was refusing it") + } + + store.refusing(nil) + eventually(t, "the report being recorded once the store was back", + func() bool { return store.count() == 1 }) + eventually(t, "the recorded report leaving the work queue", func() bool { + info, err := js.Context().StreamInfo("CONTROL") + return err == nil && info.State.Msgs == 0 + }) +} + +func TestNatsASupersededReportIsSettledAndNotActedOn(t *testing.T) { + js := aBus(t) + store := &sentAndHeardSafely{sent: "d2"} + _, stop := servingOn(t, js, store) + defer stop() + + body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) + if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { + t.Fatal(err) + } + eventually(t, "the superseded report leaving the stream", func() bool { + info, err := js.Context().StreamInfo("CONTROL") + return err == nil && info.State.Msgs == 0 + }) + if store.count() != 0 { + t.Fatalf("a report about a superseded declaration was acted on") + } +} + +func TestNatsAnEnrolmentIsAnsweredOnTheAddressInItsPayload(t *testing.T) { + js := aBus(t) + s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, + enroller: enrolsAs{reply: EnrolReply{Accepted: true, Node: "anchor"}}, log: quiet()} + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go func() { _ = s.Serve(ctx) }() + + inbox := nats.NewInbox() + answers, err := js.Conn().SubscribeSync(inbox) + if err != nil { + t.Fatal(err) + } + body, _ := json.Marshal(EnrolRequest{Node: "anchor", Secret: "t", ReplyTo: inbox}) + if _, err := js.Context().Publish(EnrolSubject, body); err != nil { + t.Fatal(err) + } + + msg, err := answers.NextMsg(8 * time.Second) + if err != nil { + t.Fatalf("no answer reached the address the request named: %v", err) + } + var reply EnrolReply + if err := json.Unmarshal(msg.Data, &reply); err != nil { + t.Fatal(err) + } + if !reply.Accepted || reply.Node != "anchor" { + t.Fatalf("the answer was not the mesh's: %+v", reply) + } + // And the address really is not the one the transport carried: what the consumer saw was its + // own ack subject, which is why this had to travel in the payload. + if msg.Subject != inbox { + t.Fatalf("the answer arrived on %s, not the address the request named", msg.Subject) + } +} + +func TestNatsAHeartbeatIsHeardAndNothingIsKept(t *testing.T) { + js := aBus(t) + store := &counted{} + _, stop := servingOn(t, js, store) + defer stop() + + // Given time to subscribe: a core subscription that is not yet up misses what is published, + // which is the guarantee a heartbeat has and not a fault. + eventually(t, "the heartbeat subscription coming up", func() bool { + body, _ := json.Marshal(Alive{Node: "anchor"}) + _ = js.Conn().Publish(AliveSubject("anchor"), body) + _ = js.Conn().Flush() + return store.count() >= 1 + }) + info, err := js.Context().StreamInfo("CONTROL") + if err != nil || info.State.Msgs != 0 { + t.Fatalf("a heartbeat was persisted, and the mesh's least valuable message now competes "+ + "for retention with its most valuable: %+v, %v", info, err) + } +} + +func TestNatsTheEventsTheControllerFollowsArriveAndAreAcknowledged(t *testing.T) { + js := aBus(t) + told := &toldAbout{} + s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()} + if err := s.Follows(told); err != nil { + t.Fatal(err) + } + if err := s.Answers(replaysWith{}); err != nil { + t.Fatal(err) + } + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go func() { _ = s.Serve(ctx) }() + // The controller's event consumer is made from now (novox/hq issue 248): what was published before + // it existed is history it never replays. So the announcements are made once it is there. + eventually(t, "the controller's event consumer being made", func() bool { + _, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName) + return err == nil + }) + + moved, _ := json.Marshal(Upgraded{Module: "gitea", Commit: "abcdef0123"}) + if _, err := js.Context().Publish(broker.ControllerFollows[0], moved); err != nil { + t.Fatal(err) + } + if _, err := js.Context().Publish(broker.ControllerFollows[1], []byte(`{}`)); err != nil { + t.Fatal(err) + } + + eventually(t, "the catalogue's upgrade reaching the controller", + func() bool { return told.count() == 1 }) + eventually(t, "both announcements being acknowledged", func() bool { + info, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName) + return err == nil && info.NumAckPending == 0 && info.Delivered.Consumer == 2 + }) +} + +func TestNatsAReportIsLetGoOnceTheStoreHasBeenGoneTooLong(t *testing.T) { + js := aBus(t) + store := &counted{} + store.refusing(errors.Join(ErrTryAgain, errors.New("connection refused"))) + s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, + listener: store, log: quiet(), giveUp: 1500 * time.Millisecond} + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go func() { _ = s.Serve(ctx) }() + + body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) + if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { + t.Fatal(err) + } + eventually(t, "the report being let go once the bound passed", func() bool { + info, err := js.Context().StreamInfo("CONTROL") + return err == nil && info.State.Msgs == 0 + }) + if store.count() != 0 { + t.Fatalf("a report was recorded by a store that never came back") + } +} + +func TestNatsWorkSlowerThanTheWindowIsNotHandedOverAgain(t *testing.T) { + // Work that outlives a window of a second three times over, said while it runs four times a window (novox/hq + // issue 717: the test once set its window before the serving loop asserted the controller's own of thirty + // seconds, so it waited nine seconds and passed with the heartbeat removed). + before := heartbeatWhileWorking + heartbeatWhileWorking = 250 * time.Millisecond + t.Cleanup(func() { heartbeatWhileWorking = before }) + js := aBus(t) + var mu sync.Mutex + handled := 0 + slow := make(chan struct{}) + s, stop := servingOn(t, js, nil) + defer stop() + s.listener = slowly{func() { + mu.Lock() + handled++ + first := handled == 1 + mu.Unlock() + if first { + time.Sleep(3 * time.Second) + close(slow) + } + }} + // The controller's own consumer, given a window short enough to outlive in a test once the serving loop has + // asserted it and bound to it: asserted after this, it would be the controller's window of thirty seconds, + // which is how this test once waited out a window it never had. + for deadline := time.Now().Add(10 * time.Second); ; { + info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName) + if err == nil && info.PushBound { + break + } + if time.Now().After(deadline) { + t.Fatalf("the serving loop never bound the controller's consumer: %v", err) + } + time.Sleep(10 * time.Millisecond) + } + if err := js.EnsureConsumer(broker.Consumer{ + Name: broker.ControllerName, Stream: "CONTROL", Push: true, AckWaitSeconds: 1, + Why: "a window short enough to outlive in a test", + }); err != nil { + t.Fatal(err) + } + if info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName); err != nil || info.Config.AckWait != time.Second { + t.Fatalf("the controller's consumer does not have the test's window of a second: %v", err) + } + + body, err := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) + if err != nil { + t.Fatal(err) + } + if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { + t.Fatal(err) + } + select { + case <-slow: + case <-time.After(30 * time.Second): + t.Fatal("the slow work never finished") + } + // A moment for a redelivery to arrive, if the bus were going to send one. + time.Sleep(1500 * time.Millisecond) + mu.Lock() + defer mu.Unlock() + if handled != 1 { + t.Fatalf("one report was handled %d times, so slow work is run again while it is running", handled) + } + // Not handed over again at all: the loop drops a copy of what it is already doing, so the count above alone + // would pass on a bus that redelivered the work every window; the bus's own count says it did not. + info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName) + if err != nil { + t.Fatal(err) + } + if info.NumRedelivered != 0 { + t.Fatalf("the bus handed the report over again %d time(s) while it was being worked on", info.NumRedelivered) + } +} + +func TestNatsTheMeshSaysWhatAMachineApplied(t *testing.T) { + js := aBus(t) + heard := make(chan *nats.Msg, 4) + sub, err := js.Conn().Subscribe(SeatEventSubject(MeshControllerSeat, ">"), func(m *nats.Msg) { + heard <- m + }) + if err != nil { + t.Fatal(err) + } + defer sub.Unsubscribe() //nolint:errcheck // the subscription dies with the connection + + _, stop := servingOn(t, js, &counted{}) + defer stop() + + // A report that changed something: the store says it was news. + body, err := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store", "broker"}}) + if err != nil { + t.Fatal(err) + } + if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { + t.Fatal(err) + } + select { + case m := <-heard: + if m.Subject != SeatEventSubject(MeshControllerSeat, KeyApplied) { + t.Fatalf("the mesh stated %q", m.Subject) + } + var said Applied + if err := json.Unmarshal(m.Data, &said); err != nil { + t.Fatal(err) + } + if said.Node != "anchor" || said.Declared != "d1" || said.Resources != 2 { + t.Fatalf("it said %+v", said) + } + case <-time.After(10 * time.Second): + t.Fatal("the mesh said nothing about a machine that now runs something else") + } + + // A refusal is its own fact, with the reason in it rather than only in a log. + refusal, err := json.Marshal(Report{Node: "anchor", Declared: "d2", + Failed: map[string]string{"gitea.server": "no such image"}}) + if err != nil { + t.Fatal(err) + } + if _, err := js.Context().Publish(ReportSubject("anchor"), refusal); err != nil { + t.Fatal(err) + } + select { + case m := <-heard: + if m.Subject != SeatEventSubject(MeshControllerSeat, KeyRefused) { + t.Fatalf("a refusal was stated as %q", m.Subject) + } + var said Refused + if err := json.Unmarshal(m.Data, &said); err != nil { + t.Fatal(err) + } + if said.Failed["gitea.server"] == "" { + t.Fatalf("the refusal does not say which resource or why: %+v", said) + } + case <-time.After(10 * time.Second): + t.Fatal("the mesh said nothing about a machine that refused what it was sent") + } +} + +func TestNatsAReportThatIsNotNewsIsNotStated(t *testing.T) { + js := aBus(t) + heard := make(chan *nats.Msg, 4) + sub, err := js.Conn().Subscribe(SeatEventSubject(MeshControllerSeat, ">"), func(m *nats.Msg) { + heard <- m + }) + if err != nil { + t.Fatal(err) + } + defer sub.Unsubscribe() //nolint:errcheck // the subscription dies with the connection + + // A store that records the report and says it was nothing new — which is what the mesh's own + // store says about a machine repeating itself. + _, stop := servingOn(t, js, sameAgain{}) + defer stop() + + body, err := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) + if err != nil { + t.Fatal(err) + } + if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { + t.Fatal(err) + } + select { + case m := <-heard: + t.Fatalf("the mesh stated %q about a machine that changed nothing", m.Subject) + case <-time.After(3 * time.Second): + } +} + +func TestNatsAMergeHeldMoreThanFiveTimesIsStillHandedOverAndRecorded(t *testing.T) { + first, most := MergeHeldFirst, MergeHeldMost + MergeHeldFirst, MergeHeldMost = 5*time.Millisecond, 40*time.Millisecond + t.Cleanup(func() { MergeHeldFirst, MergeHeldMost = first, most }) + js := aBus(t) + up := &manyTimesBusy{busy: 8} + s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()} + if err := s.Follows(up); err != nil { + t.Fatal(err) + } + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go func() { _ = s.Serve(ctx) }() + eventually(t, "the merges consumer being made", func() bool { + _, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName) + return err == nil + }) + + body, _ := json.Marshal(SourceMoved{Owner: "novox", Repo: "one", Base: "main", Commit: "abcdef0123"}) + if _, err := js.Context().Publish(MergedSubject, body); err != nil { + t.Fatal(err) + } + eventually(t, "the ninth hand-over recording the merge", func() bool { return up.called() == 9 }) + eventually(t, "the merge being acknowledged", func() bool { + info, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName) + return err == nil && info.NumAckPending == 0 && info.NumPending == 0 + }) + if info, err := js.Context().StreamInfo(broker.DeadLettersStream); err == nil && info.State.Msgs != 0 { + t.Fatalf("%d message(s) in dead letters: a merge was given up on", info.State.Msgs) + } +} diff --git a/internal/link/receive_nats_test.go b/internal/link/receive_nats_test.go index b5940ca5..382bceac 100644 --- a/internal/link/receive_nats_test.go +++ b/internal/link/receive_nats_test.go @@ -2,7 +2,6 @@ package link import ( "context" - "encoding/json" "errors" "io" "log" @@ -10,11 +9,7 @@ import ( "testing" "time" - "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/broker" - - "github.com/novox/mesh-controller/internal/testbus" ) // The consume side against a real server, because what is being checked is what the server does. @@ -26,42 +21,6 @@ import ( // // go test ./internal/link/ -run TestNats (each test on a bus of its own: internal/testbus) -func aBus(t *testing.T) *broker.JetStream { - t.Helper() - url := testbus.URL(t) - js, err := broker.Dial(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - - // **Streams purged, consumers removed.** Both halves, and each was learned by getting it wrong. - // - // The streams are emptied rather than deleted and recreated, because delete-then-add is not a - // reset: the server's teardown races the creation, and a test then inherits the previous one's - // messages — which reads as a redelivery bug in the code under test. - // - // The consumers are removed, because deleting a stream used to take them with it and purging - // does not. A durable *push* consumer that survives between tests keeps pushing to a delivery - // subject the previous test's subscription has gone from: the messages count as delivered, go - // nowhere, and the next test waits out its timeout for an announcement the server believes it - // already sent. The controller recreates what it needs on start, so leaving none is correct. - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - clean := func() { - for _, c := range broker.MeshConsumers() { - _ = js.Context().DeleteConsumer(c.Stream, c.Name) - } - for _, s := range broker.MeshStreams() { - _ = js.Context().PurgeStream(s.Name) - } - } - clean() - t.Cleanup(clean) - return js -} - // serving1 is a controller reading from a real bus, and a way to stop it. func servingOn(t *testing.T, js *broker.JetStream, l Listener) (*Server, func()) { t.Helper() @@ -122,81 +81,13 @@ func eventually(t *testing.T, what string, is func() bool) { // stream does not hold it. A work queue is the check: what is acknowledged leaves it. func quiet() *log.Logger { return log.New(io.Discard, "", 0) } -func TestNatsAReportIsHeardAndLeavesTheStream(t *testing.T) { - js := aBus(t) - store := &counted{} - _, stop := servingOn(t, js, store) - defer stop() - - body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) - if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { - t.Fatal(err) - } - eventually(t, "a report being recorded", func() bool { return store.count() == 1 }) - eventually(t, "the report leaving the work queue", func() bool { - info, err := js.Context().StreamInfo("CONTROL") - return err == nil && info.State.Msgs == 0 - }) -} - // **The store window, in the server.** A report the store cannot take is naked with a delay and // comes back; once the store is there it is recorded and leaves the stream. The controller holds // nothing in the meantime — which is what the sequence check below is for: the message is still on // the server while it waits. -func TestNatsAReportTheStoreCannotTakeIsHeldByTheServerAndComesBack(t *testing.T) { - js := aBus(t) - store := &counted{} - store.refusing(errors.Join(ErrTryAgain, errors.New("the database system is starting up"))) - _, stop := servingOn(t, js, store) - defer stop() - - body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) - if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { - t.Fatal(err) - } - - // Held: the message is the server's, unacknowledged, and still in the stream. - eventually(t, "the report being redelivered at least once", func() bool { - info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName) - return err == nil && info.NumRedelivered >= 1 - }) - info, err := js.Context().StreamInfo("CONTROL") - if err != nil || info.State.Msgs != 1 { - t.Fatalf("a held report did not stay on the server: %+v, %v", info, err) - } - if store.count() != 0 { - t.Fatalf("a report was recorded by a store that was refusing it") - } - - store.refusing(nil) - eventually(t, "the report being recorded once the store was back", - func() bool { return store.count() == 1 }) - eventually(t, "the recorded report leaving the work queue", func() bool { - info, err := js.Context().StreamInfo("CONTROL") - return err == nil && info.State.Msgs == 0 - }) -} // A report about a declaration the mesh has moved past is settled without being acted on, and // leaves the stream rather than coming back for ever. -func TestNatsASupersededReportIsSettledAndNotActedOn(t *testing.T) { - js := aBus(t) - store := &sentAndHeardSafely{sent: "d2"} - _, stop := servingOn(t, js, store) - defer stop() - - body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) - if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { - t.Fatal(err) - } - eventually(t, "the superseded report leaving the stream", func() bool { - info, err := js.Context().StreamInfo("CONTROL") - return err == nil && info.State.Msgs == 0 - }) - if store.count() != 0 { - t.Fatalf("a report about a superseded declaration was acted on") - } -} // sentAndHeardSafely is sentAndHeard, read from two goroutines. type sentAndHeardSafely struct { @@ -228,41 +119,6 @@ func (s *sentAndHeardSafely) count() int { // that the answer arrives there — which is the whole reason the address is a field rather than the // transport's reply, and this is the test design 25 §2 asks for so the reason cannot quietly become // folklore. -func TestNatsAnEnrolmentIsAnsweredOnTheAddressInItsPayload(t *testing.T) { - js := aBus(t) - s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, - enroller: enrolsAs{reply: EnrolReply{Accepted: true, Node: "anchor"}}, log: quiet()} - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go func() { _ = s.Serve(ctx) }() - - inbox := nats.NewInbox() - answers, err := js.Conn().SubscribeSync(inbox) - if err != nil { - t.Fatal(err) - } - body, _ := json.Marshal(EnrolRequest{Node: "anchor", Secret: "t", ReplyTo: inbox}) - if _, err := js.Context().Publish(EnrolSubject, body); err != nil { - t.Fatal(err) - } - - msg, err := answers.NextMsg(8 * time.Second) - if err != nil { - t.Fatalf("no answer reached the address the request named: %v", err) - } - var reply EnrolReply - if err := json.Unmarshal(msg.Data, &reply); err != nil { - t.Fatal(err) - } - if !reply.Accepted || reply.Node != "anchor" { - t.Fatalf("the answer was not the mesh's: %+v", reply) - } - // And the address really is not the one the transport carried: what the consumer saw was its - // own ack subject, which is why this had to travel in the payload. - if msg.Subject != inbox { - t.Fatalf("the answer arrived on %s, not the address the request named", msg.Subject) - } -} // enrolsAs answers every request the same way. type enrolsAs struct{ reply EnrolReply } @@ -271,26 +127,6 @@ func (e enrolsAs) Enrol(context.Context, EnrolRequest) (EnrolReply, error) { ret // A heartbeat is core NATS: it reaches the controller and nothing is persisted, so the stream the // reports live in stays empty. -func TestNatsAHeartbeatIsHeardAndNothingIsKept(t *testing.T) { - js := aBus(t) - store := &counted{} - _, stop := servingOn(t, js, store) - defer stop() - - // Given time to subscribe: a core subscription that is not yet up misses what is published, - // which is the guarantee a heartbeat has and not a fault. - eventually(t, "the heartbeat subscription coming up", func() bool { - body, _ := json.Marshal(Alive{Node: "anchor"}) - _ = js.Conn().Publish(AliveSubject("anchor"), body) - _ = js.Conn().Flush() - return store.count() >= 1 - }) - info, err := js.Context().StreamInfo("CONTROL") - if err != nil || info.State.Msgs != 0 { - t.Fatalf("a heartbeat was persisted, and the mesh's least valuable message now competes "+ - "for retention with its most valuable: %+v, %v", info, err) - } -} // The two events the controller follows arrive over one durable consumer with two filters, and it // can acknowledge them. @@ -299,41 +135,6 @@ func TestNatsAHeartbeatIsHeardAndNothingIsKept(t *testing.T) { // is the first thing in the mesh to use one; and a delivery from the events stream is acknowledged // on a different ack subject from a delivery from the control stream, which the controller's own // permission list has to cover or every announcement is redelivered for ever. -func TestNatsTheEventsTheControllerFollowsArriveAndAreAcknowledged(t *testing.T) { - js := aBus(t) - told := &toldAbout{} - s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()} - if err := s.Follows(told); err != nil { - t.Fatal(err) - } - if err := s.Answers(replaysWith{}); err != nil { - t.Fatal(err) - } - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go func() { _ = s.Serve(ctx) }() - // The controller's event consumer is made from now (novox/hq issue 248): what was published before - // it existed is history it never replays. So the announcements are made once it is there. - eventually(t, "the controller's event consumer being made", func() bool { - _, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName) - return err == nil - }) - - moved, _ := json.Marshal(Upgraded{Module: "gitea", Commit: "abcdef0123"}) - if _, err := js.Context().Publish(broker.ControllerFollows[0], moved); err != nil { - t.Fatal(err) - } - if _, err := js.Context().Publish(broker.ControllerFollows[1], []byte(`{}`)); err != nil { - t.Fatal(err) - } - - eventually(t, "the catalogue's upgrade reaching the controller", - func() bool { return told.count() == 1 }) - eventually(t, "both announcements being acknowledged", func() bool { - info, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName) - return err == nil && info.NumAckPending == 0 && info.Delivered.Consumer == 2 - }) -} type toldAbout struct { mu sync.Mutex @@ -360,28 +161,6 @@ func (u *toldAbout) count() int { // A store that never comes back: the report is let go once the bound passes, and it leaves the // stream rather than being held for ever. The bound is the controller's, not the server's — nothing // here sets max-deliver, and that is deliberate (streams.go). -func TestNatsAReportIsLetGoOnceTheStoreHasBeenGoneTooLong(t *testing.T) { - js := aBus(t) - store := &counted{} - store.refusing(errors.Join(ErrTryAgain, errors.New("connection refused"))) - s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, - listener: store, log: quiet(), giveUp: 1500 * time.Millisecond} - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go func() { _ = s.Serve(ctx) }() - - body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) - if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { - t.Fatal(err) - } - eventually(t, "the report being let go once the bound passed", func() bool { - info, err := js.Context().StreamInfo("CONTROL") - return err == nil && info.State.Msgs == 0 - }) - if store.count() != 0 { - t.Fatalf("a report was recorded by a store that never came back") - } -} // A merge announcement is not what these tests are about; taken and forgotten. func (t *toldAbout) SourceMoved(context.Context, SourceMoved) error { return nil } @@ -406,81 +185,6 @@ func TestEverySubjectTheControllerFollowsDecodesToAKind(t *testing.T) { // a consumer that died and wrong for one that is busy. Acting on a merge builds modules — minutes // against a thirty-second window — and the same merge was handed over five times while the first // build was still running (2026-09-28). Here the window is a second and the work takes three. -func TestNatsWorkSlowerThanTheWindowIsNotHandedOverAgain(t *testing.T) { - // Work that outlives a window of a second three times over, said while it runs four times a window (novox/hq - // issue 717: the test once set its window before the serving loop asserted the controller's own of thirty - // seconds, so it waited nine seconds and passed with the heartbeat removed). - before := heartbeatWhileWorking - heartbeatWhileWorking = 250 * time.Millisecond - t.Cleanup(func() { heartbeatWhileWorking = before }) - js := aBus(t) - var mu sync.Mutex - handled := 0 - slow := make(chan struct{}) - s, stop := servingOn(t, js, nil) - defer stop() - s.listener = slowly{func() { - mu.Lock() - handled++ - first := handled == 1 - mu.Unlock() - if first { - time.Sleep(3 * time.Second) - close(slow) - } - }} - // The controller's own consumer, given a window short enough to outlive in a test once the serving loop has - // asserted it and bound to it: asserted after this, it would be the controller's window of thirty seconds, - // which is how this test once waited out a window it never had. - for deadline := time.Now().Add(10 * time.Second); ; { - info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName) - if err == nil && info.PushBound { - break - } - if time.Now().After(deadline) { - t.Fatalf("the serving loop never bound the controller's consumer: %v", err) - } - time.Sleep(10 * time.Millisecond) - } - if err := js.EnsureConsumer(broker.Consumer{ - Name: broker.ControllerName, Stream: "CONTROL", Push: true, AckWaitSeconds: 1, - Why: "a window short enough to outlive in a test", - }); err != nil { - t.Fatal(err) - } - if info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName); err != nil || info.Config.AckWait != time.Second { - t.Fatalf("the controller's consumer does not have the test's window of a second: %v", err) - } - - body, err := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) - if err != nil { - t.Fatal(err) - } - if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { - t.Fatal(err) - } - select { - case <-slow: - case <-time.After(30 * time.Second): - t.Fatal("the slow work never finished") - } - // A moment for a redelivery to arrive, if the bus were going to send one. - time.Sleep(1500 * time.Millisecond) - mu.Lock() - defer mu.Unlock() - if handled != 1 { - t.Fatalf("one report was handled %d times, so slow work is run again while it is running", handled) - } - // Not handed over again at all: the loop drops a copy of what it is already doing, so the count above alone - // would pass on a bus that redelivered the work every window; the bus's own count says it did not. - info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName) - if err != nil { - t.Fatal(err) - } - if info.NumRedelivered != 0 { - t.Fatalf("the bus handed the report over again %d time(s) while it was being worked on", info.NumRedelivered) - } -} // slowly is a listener that runs whatever it was given. type slowly struct{ work func() } @@ -490,101 +194,9 @@ func (s slowly) Heard(context.Context, Report) (bool, error) { s.work(); return // **The mesh says what it applied** (novox/hq ADR 0134), under the seat the control plane holds — and // says nothing when a report is the same state said again, which is what a machine reconciling every // minute sends. -func TestNatsTheMeshSaysWhatAMachineApplied(t *testing.T) { - js := aBus(t) - heard := make(chan *nats.Msg, 4) - sub, err := js.Conn().Subscribe(SeatEventSubject(MeshControllerSeat, ">"), func(m *nats.Msg) { - heard <- m - }) - if err != nil { - t.Fatal(err) - } - defer sub.Unsubscribe() //nolint:errcheck // the subscription dies with the connection - - _, stop := servingOn(t, js, &counted{}) - defer stop() - - // A report that changed something: the store says it was news. - body, err := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store", "broker"}}) - if err != nil { - t.Fatal(err) - } - if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { - t.Fatal(err) - } - select { - case m := <-heard: - if m.Subject != SeatEventSubject(MeshControllerSeat, KeyApplied) { - t.Fatalf("the mesh stated %q", m.Subject) - } - var said Applied - if err := json.Unmarshal(m.Data, &said); err != nil { - t.Fatal(err) - } - if said.Node != "anchor" || said.Declared != "d1" || said.Resources != 2 { - t.Fatalf("it said %+v", said) - } - case <-time.After(10 * time.Second): - t.Fatal("the mesh said nothing about a machine that now runs something else") - } - - // A refusal is its own fact, with the reason in it rather than only in a log. - refusal, err := json.Marshal(Report{Node: "anchor", Declared: "d2", - Failed: map[string]string{"gitea.server": "no such image"}}) - if err != nil { - t.Fatal(err) - } - if _, err := js.Context().Publish(ReportSubject("anchor"), refusal); err != nil { - t.Fatal(err) - } - select { - case m := <-heard: - if m.Subject != SeatEventSubject(MeshControllerSeat, KeyRefused) { - t.Fatalf("a refusal was stated as %q", m.Subject) - } - var said Refused - if err := json.Unmarshal(m.Data, &said); err != nil { - t.Fatal(err) - } - if said.Failed["gitea.server"] == "" { - t.Fatalf("the refusal does not say which resource or why: %+v", said) - } - case <-time.After(10 * time.Second): - t.Fatal("the mesh said nothing about a machine that refused what it was sent") - } -} // And a report that is not news is not a fact. A machine reconciles every minute; a fact per report // would be a fact per minute per machine, which is a stream nobody reads. -func TestNatsAReportThatIsNotNewsIsNotStated(t *testing.T) { - js := aBus(t) - heard := make(chan *nats.Msg, 4) - sub, err := js.Conn().Subscribe(SeatEventSubject(MeshControllerSeat, ">"), func(m *nats.Msg) { - heard <- m - }) - if err != nil { - t.Fatal(err) - } - defer sub.Unsubscribe() //nolint:errcheck // the subscription dies with the connection - - // A store that records the report and says it was nothing new — which is what the mesh's own - // store says about a machine repeating itself. - _, stop := servingOn(t, js, sameAgain{}) - defer stop() - - body, err := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) - if err != nil { - t.Fatal(err) - } - if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { - t.Fatal(err) - } - select { - case m := <-heard: - t.Fatalf("the mesh stated %q about a machine that changed nothing", m.Subject) - case <-time.After(3 * time.Second): - } -} // sameAgain records a report and says it was the same state said again. type sameAgain struct{} @@ -617,37 +229,6 @@ func (u *manyTimesBusy) called() int { // **A merge held more than five times is still handed over and recorded** (issue 973, novox/hq ADR 0407 decision // 3): the merges have a durable consumer of their own with no max-deliver, so they never go to DEAD_LETTERS to wait // for a person; the delay grows, and is capped. -func TestNatsAMergeHeldMoreThanFiveTimesIsStillHandedOverAndRecorded(t *testing.T) { - first, most := MergeHeldFirst, MergeHeldMost - MergeHeldFirst, MergeHeldMost = 5*time.Millisecond, 40*time.Millisecond - t.Cleanup(func() { MergeHeldFirst, MergeHeldMost = first, most }) - js := aBus(t) - up := &manyTimesBusy{busy: 8} - s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()} - if err := s.Follows(up); err != nil { - t.Fatal(err) - } - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go func() { _ = s.Serve(ctx) }() - eventually(t, "the merges consumer being made", func() bool { - _, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName) - return err == nil - }) - - body, _ := json.Marshal(SourceMoved{Owner: "novox", Repo: "one", Base: "main", Commit: "abcdef0123"}) - if _, err := js.Context().Publish(MergedSubject, body); err != nil { - t.Fatal(err) - } - eventually(t, "the ninth hand-over recording the merge", func() bool { return up.called() == 9 }) - eventually(t, "the merge being acknowledged", func() bool { - info, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName) - return err == nil && info.NumAckPending == 0 && info.NumPending == 0 - }) - if info, err := js.Context().StreamInfo(broker.DeadLettersStream); err == nil && info.State.Msgs != 0 { - t.Fatalf("%d message(s) in dead letters: a merge was given up on", info.State.Msgs) - } -} // The delay a held merge waits grows from the first to the most and stays there. func TestAHeldMergesDelayGrowsAndIsCapped(t *testing.T) { diff --git a/internal/link/rekey_test.go b/internal/link/rekey_test.go index fc7be801..c6f295ca 100644 --- a/internal/link/rekey_test.go +++ b/internal/link/rekey_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link_test import ( diff --git a/internal/link/replay314_test.go b/internal/link/replay314_test.go index f7d57965..6ef63ded 100644 --- a/internal/link/replay314_test.go +++ b/internal/link/replay314_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/replay330_test.go b/internal/link/replay330_test.go index ac25b362..7e56ce64 100644 --- a/internal/link/replay330_test.go +++ b/internal/link/replay330_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/seat_as_test.go b/internal/link/seat_as_test.go index c25ca8d0..a6f7f8b2 100644 --- a/internal/link/seat_as_test.go +++ b/internal/link/seat_as_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/setuid_search_test.go b/internal/link/setuid_search_test.go index 4a05aa57..7ced258b 100644 --- a/internal/link/setuid_search_test.go +++ b/internal/link/setuid_search_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/standby_nats_test.go b/internal/link/standby_nats_test.go index ac5940f0..31e5ba8d 100644 --- a/internal/link/standby_nats_test.go +++ b/internal/link/standby_nats_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/standing_integration_test.go b/internal/link/standing_integration_test.go new file mode 100644 index 00000000..68183867 --- /dev/null +++ b/internal/link/standing_integration_test.go @@ -0,0 +1,48 @@ +//go:build integration + +package link + +import ( + "context" + "testing" + + "github.com/novox/mesh-controller/internal/broker" +) + +func TestNatsAProvidersStandingReachesTheController(t *testing.T) { + js := aBus(t) + kept := &lockedStandings{} + s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()} + if err := s.Follows(&toldAbout{}); err != nil { + t.Fatal(err) + } + if err := s.Watches(kept); err != nil { + t.Fatal(err) + } + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go func() { _ = s.Serve(ctx) }() + eventually(t, "the controller's event consumer being made", func() bool { + _, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName) + return err == nil + }) + + for _, event := range []string{broker.ProvisionerFailing, broker.ProvisionerRecovered} { + if _, err := js.Context().Publish("mesh.mod.keycloak.event."+event, + []byte(`{"consumer":"mesh_home_grafana","provider-node":"anchor","class":"credentials-rejected"}`)); err != nil { + t.Fatal(err) + } + } + // Somebody else's event under the same prefix is not the controller's to hear. + if _, err := js.Context().Publish("mesh.mod.keycloak.event.client.created", []byte(`{}`)); err != nil { + t.Fatal(err) + } + eventually(t, "both standings being kept, in order, naming the emitter", func() bool { + got := kept.all() + return len(got) == 2 && got[0].Module == "keycloak" && got[0].Failing && !got[1].Failing + }) + eventually(t, "both being acknowledged and nothing else delivered", func() bool { + info, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName) + return err == nil && info.NumAckPending == 0 && info.Delivered.Consumer == 2 + }) +} diff --git a/internal/link/standing_test.go b/internal/link/standing_test.go index 4d7b3f34..1a4136af 100644 --- a/internal/link/standing_test.go +++ b/internal/link/standing_test.go @@ -150,43 +150,6 @@ func TestAStandingWithNothingKeepingItIsTaken(t *testing.T) { // Over a real bus: a provider's standing published under its own module's namespace reaches the // controller through the events consumer's filter — the one wildcard filter on it — names the emitter // from the subject, and is acknowledged. -func TestNatsAProvidersStandingReachesTheController(t *testing.T) { - js := aBus(t) - kept := &lockedStandings{} - s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()} - if err := s.Follows(&toldAbout{}); err != nil { - t.Fatal(err) - } - if err := s.Watches(kept); err != nil { - t.Fatal(err) - } - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go func() { _ = s.Serve(ctx) }() - eventually(t, "the controller's event consumer being made", func() bool { - _, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName) - return err == nil - }) - - for _, event := range []string{broker.ProvisionerFailing, broker.ProvisionerRecovered} { - if _, err := js.Context().Publish("mesh.mod.keycloak.event."+event, - []byte(`{"consumer":"mesh_home_grafana","provider-node":"anchor","class":"credentials-rejected"}`)); err != nil { - t.Fatal(err) - } - } - // Somebody else's event under the same prefix is not the controller's to hear. - if _, err := js.Context().Publish("mesh.mod.keycloak.event.client.created", []byte(`{}`)); err != nil { - t.Fatal(err) - } - eventually(t, "both standings being kept, in order, naming the emitter", func() bool { - got := kept.all() - return len(got) == 2 && got[0].Module == "keycloak" && got[0].Failing && !got[1].Failing - }) - eventually(t, "both being acknowledged and nothing else delivered", func() bool { - info, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName) - return err == nil && info.NumAckPending == 0 && info.Delivered.Consumer == 2 - }) -} type lockedStandings struct { mu sync.Mutex