diff --git a/bot_bottle/backend/consolidated_util.py b/bot_bottle/backend/consolidated_util.py index 1e77881..520ca0a 100644 --- a/bot_bottle/backend/consolidated_util.py +++ b/bot_bottle/backend/consolidated_util.py @@ -26,6 +26,7 @@ def provision_bottle( *, image_ref: str = "", tokens: dict[str, str] | None = None, + env_var_secret: str | None = None, ) -> RegisteredBottle: """Register the bottle and provision its git-gate state. Rolls back the registration if provisioning fails so no orphan is left. @@ -35,7 +36,7 @@ def provision_bottle( ``RegisteredBottle`` so callers can inject it into the agent container's environment.""" inputs = registration_inputs(egress_plan) - env_var_secret = new_env_var_secret() + env_var_secret = env_var_secret or new_env_var_secret() reg = client.register_bottle( source_ip, image_ref=image_ref, policy=inputs.policy, metadata=inputs.metadata, tokens=tokens, env_var_secret=env_var_secret, diff --git a/bot_bottle/backend/docker/consolidated_launch.py b/bot_bottle/backend/docker/consolidated_launch.py index e06e8f3..9f9768a 100644 --- a/bot_bottle/backend/docker/consolidated_launch.py +++ b/bot_bottle/backend/docker/consolidated_launch.py @@ -23,6 +23,7 @@ from ...orchestrator.client import OrchestratorClient from ...orchestrator.gateway import GATEWAY_NETWORK from ...orchestrator.lifecycle import INFRA_NAME, OrchestratorService from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME +from ...orchestrator.reprovision import reprovision_bottles from ..consolidated_util import provision_bottle from ..consolidated_util import teardown_consolidated as _teardown_util from .gateway_provision import DockerGatewayTransport @@ -102,17 +103,17 @@ def _reprovision_running_bottles( when the orchestrator already has all tokens loaded. Best-effort: a single container exec failure never blocks a new bottle launch.""" client = OrchestratorClient(orchestrator_url) - bottles = client.list_bottles() - if not bottles: - return - # Build {source_ip: container_name} from live containers on the gateway # network, excluding the infra container itself. - proc = run_docker([ - "docker", "network", "inspect", - "--format", "{{range .Containers}}{{.Name}} {{.IPv4Address}}\n{{end}}", - network, - ]) + try: + proc = run_docker([ + "docker", "network", "inspect", + "--format", "{{range .Containers}}{{.Name}} {{.IPv4Address}}\n{{end}}", + network, + ]) + except OSError as exc: + log.info(f"egress token reprovision skipped: {exc}") + return ip_to_container: dict[str, str] = {} for line in proc.stdout.splitlines(): parts = line.strip().split() @@ -121,26 +122,15 @@ def _reprovision_running_bottles( if ip: ip_to_container[ip] = parts[0] - reprovisioned = 0 - for bottle in bottles: - bottle_id = bottle.get("bottle_id") - source_ip = bottle.get("source_ip") - if not isinstance(bottle_id, str) or not isinstance(source_ip, str): - continue - container_name = ip_to_container.get(source_ip) - if not container_name: - continue + secrets_by_ip: dict[str, str] = {} + for source_ip, container_name in ip_to_container.items(): proc = run_docker( ["docker", "exec", container_name, "printenv", ENV_VAR_SECRET_NAME] ) - if proc.returncode != 0 or not proc.stdout.strip(): - continue - try: - if client.reprovision_gateway(bottle_id, proc.stdout.strip()): - reprovisioned += 1 - except Exception: # noqa: BLE001 — best-effort, never block a launch - pass + if proc.returncode == 0 and proc.stdout.strip(): + secrets_by_ip[source_ip] = proc.stdout.strip() + reprovisioned = reprovision_bottles(client, secrets_by_ip) if reprovisioned: log.info( "reprovisioned egress tokens", diff --git a/bot_bottle/backend/firecracker/bottle_plan.py b/bot_bottle/backend/firecracker/bottle_plan.py index 92c17c4..5a9603a 100644 --- a/bot_bottle/backend/firecracker/bottle_plan.py +++ b/bot_bottle/backend/firecracker/bottle_plan.py @@ -22,6 +22,9 @@ class FirecrackerBottlePlan(BottlePlan): # (egress proxy credentials, git-gate/supervise headers); set by launch # from the orchestrator registration. Empty pre-registration. identity_token: str = "" + # Applied to every agent SSH exec and mirrored into /run inside the VM so + # the host can recover it after the infra VM restarts. + env_var_secret: str = "" @property def container_name(self) -> str: diff --git a/bot_bottle/backend/firecracker/cleanup.py b/bot_bottle/backend/firecracker/cleanup.py index 6e9fe2d..bce6c9d 100644 --- a/bot_bottle/backend/firecracker/cleanup.py +++ b/bot_bottle/backend/firecracker/cleanup.py @@ -88,6 +88,12 @@ def _scan_processes(run_root: Path) -> tuple[set[str], list[int]]: return live, orphan_pids +def live_run_dirs() -> tuple[Path, ...]: + """Run directories backed by currently running agent microVMs.""" + live, _ = _scan_processes(_run_root()) + return tuple(Path(path) for path in sorted(live)) + + def _orphan_run_dirs(run_root: Path, live: set[str]) -> list[str]: """Run dirs with no live VM behind them — the leaked ones to remove.""" if not run_root.is_dir(): diff --git a/bot_bottle/backend/firecracker/consolidated_launch.py b/bot_bottle/backend/firecracker/consolidated_launch.py index cca473a..cb72690 100644 --- a/bot_bottle/backend/firecracker/consolidated_launch.py +++ b/bot_bottle/backend/firecracker/consolidated_launch.py @@ -25,16 +25,27 @@ The TAP slot allocation, rootfs build, and VM boot are the caller's job. from __future__ import annotations +import json +import subprocess from dataclasses import dataclass +from pathlib import Path from ...egress import EgressPlan from ...git_gate import GitGatePlan -from ...orchestrator.client import OrchestratorClient +from ...log import info +from ...orchestrator.client import OrchestratorClient, OrchestratorClientError from ...orchestrator.lifecycle import ( OrchestratorStartError, # re-exported so callers can catch it ) -from ..consolidated_util import provision_bottle, teardown_consolidated as _teardown_util -from . import infra_vm +from ...orchestrator.reprovision import reprovision_bottles +from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME +from ..consolidated_util import ( + provision_bottle, + teardown_consolidated as _teardown_util, +) +from . import cleanup, infra_vm, util + +_ENV_VAR_SECRET_PATH = "/run/bot-bottle/env-var-secret" class ConsolidatedLaunchError(RuntimeError): @@ -53,6 +64,54 @@ class LaunchContext: env_var_secret: str = "" # encryption key injected into the agent's env +def _guest_ip_from_config(config_path: Path) -> str: + """Read the kernel's configured guest IP from a Firecracker config.""" + try: + config = json.loads(config_path.read_text()) + args = config["boot-source"]["boot_args"] + ip_arg = next(part for part in args.split() if part.startswith("ip=")) + return ip_arg.removeprefix("ip=").split(":", 1)[0] + except (OSError, ValueError, KeyError, TypeError, StopIteration): + return "" + + +def persist_env_var_secret(private_key: Path, guest_ip: str, secret: str) -> None: + """Mirror the exec-time key into guest tmpfs for restart recovery.""" + proc = subprocess.run( + util.ssh_base_argv(private_key, guest_ip) + + [f"umask 077; mkdir -p /run/bot-bottle; cat > {_ENV_VAR_SECRET_PATH}"], + input=secret, capture_output=True, text=True, check=False, + ) + if proc.returncode != 0: + raise ConsolidatedLaunchError( + f"failed to persist {ENV_VAR_SECRET_NAME} in agent VM: " + f"{proc.stderr.strip() or ''}" + ) + + +def _reprovision_running_bottles(client: OrchestratorClient) -> None: + """Read keys from live agent VMs and restore the restarted gateway.""" + try: + secrets_by_ip: dict[str, str] = {} + for run_dir in cleanup.live_run_dirs(): + guest_ip = _guest_ip_from_config(run_dir / "config.json") + private_key = run_dir / "bottle_id_ed25519" + if not guest_ip or not private_key.is_file(): + continue + proc = subprocess.run( + util.ssh_base_argv(private_key, guest_ip) + + [f"cat {_ENV_VAR_SECRET_PATH}"], + capture_output=True, text=True, check=False, + ) + if proc.returncode == 0 and proc.stdout.strip(): + secrets_by_ip[guest_ip] = proc.stdout.strip() + count = reprovision_bottles(client, secrets_by_ip) + if count: + info(f"reprovisioned egress tokens for {count} Firecracker bottle(s)") + except (OSError, OrchestratorClientError) as exc: + info(f"egress token reprovision skipped: {exc}") + + def launch_consolidated( egress_plan: EgressPlan, git_gate_plan: GitGatePlan, @@ -67,6 +126,7 @@ def launch_consolidated( infra = infra_vm.ensure_running() url = infra.control_plane_url client = OrchestratorClient(url) + _reprovision_running_bottles(client) transport = infra_vm.gateway_transport() reg = provision_bottle( diff --git a/bot_bottle/backend/firecracker/launch.py b/bot_bottle/backend/firecracker/launch.py index a76f9ef..d72f40c 100644 --- a/bot_bottle/backend/firecracker/launch.py +++ b/bot_bottle/backend/firecracker/launch.py @@ -55,8 +55,10 @@ from . import firecracker_vm, image_builder, isolation_probe, netpool, util from .bottle import FirecrackerBottle from .bottle_plan import FirecrackerBottlePlan from ...orchestrator.config_store import resolve_teardown_timeout +from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME from .consolidated_launch import ( launch_consolidated, + persist_env_var_secret, teardown_consolidated, ) @@ -153,6 +155,7 @@ def launch( git_gate_plan=git_gate_plan, egress_plan=egress_plan, identity_token=ctx.identity_token, + env_var_secret=ctx.env_var_secret, # Deliver the identity token as egress proxy credentials — clients # honor `HTTPS_PROXY=http://id:token@gw` without app changes; the # gateway reads Proxy-Authorization, validates the (source_ip, @@ -187,6 +190,7 @@ def launch( ) stack.callback(vm.terminate) firecracker_vm.wait_for_ssh(vm, private_key) + persist_env_var_secret(private_key, slot.guest_ip, ctx.env_var_secret) # Authoritative fail-closed egress-boundary check, before the agent # runs: prove the VM cannot reach the host directly. @@ -281,6 +285,8 @@ def _agent_guest_env(plan: FirecrackerBottlePlan, host_ip: str) -> dict[str, str env["GIT_GATE_URL"] = plan.agent_git_gate_url if plan.agent_supervise_url: env["MCP_SUPERVISE_URL"] = plan.agent_supervise_url + if plan.env_var_secret: + env[ENV_VAR_SECRET_NAME] = plan.env_var_secret for entry in egress_agent_env_entries(plan.egress_plan): key, _, value = entry.partition("=") env[key] = value diff --git a/bot_bottle/backend/firecracker/util.py b/bot_bottle/backend/firecracker/util.py index 0d9aad8..465c348 100644 --- a/bot_bottle/backend/firecracker/util.py +++ b/bot_bottle/backend/firecracker/util.py @@ -399,6 +399,9 @@ fi chown -R 0:0 /root 2>/dev/null || true mkdir -p /etc/dropbear /run +# Keep restart-recovery key material memory-backed, separate from both the +# agent rootfs and the infra VM's persistent registry volume. +mount -t tmpfs -o mode=0755 tmpfs /run 2>/dev/null || true # -R: generate host keys on demand. -E: log auth failures to stderr, # captured in the host-side console.log for debugging. /bb-dropbear -R -E -p 22 & diff --git a/bot_bottle/backend/macos_container/bottle_plan.py b/bot_bottle/backend/macos_container/bottle_plan.py index f93a092..64ce323 100644 --- a/bot_bottle/backend/macos_container/bottle_plan.py +++ b/bot_bottle/backend/macos_container/bottle_plan.py @@ -20,6 +20,9 @@ class MacosContainerBottlePlan(BottlePlan): # bottle is registered. See launch.py's stamp for why it lives here and not # only in the exec-time proxy env. identity_token: str = "" + # Generated before `container run` so it becomes part of the container's + # configured environment and can be read back after an infra restart. + env_var_secret: str = "" @property def container_name(self) -> str: diff --git a/bot_bottle/backend/macos_container/consolidated_launch.py b/bot_bottle/backend/macos_container/consolidated_launch.py index 2ba7469..6c25f5e 100644 --- a/bot_bottle/backend/macos_container/consolidated_launch.py +++ b/bot_bottle/backend/macos_container/consolidated_launch.py @@ -38,7 +38,12 @@ from ...egress import EgressPlan from ...git_gate import GitGatePlan from ...log import info from ...orchestrator.client import OrchestratorClient, OrchestratorClientError -from ..consolidated_util import provision_bottle, teardown_consolidated as _teardown_util +from ...orchestrator.reprovision import reprovision_bottles +from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME +from ..consolidated_util import ( + provision_bottle, + teardown_consolidated as _teardown_util, +) from . import util as container_mod from .enumerate import CONTAINER_NAME_PREFIX, EnumerationError, enumerate_active from .gateway import GATEWAY_NETWORK @@ -84,12 +89,35 @@ def ensure_gateway( needs `gateway_ip` at run time.""" service = service or MacosInfraService() infra = service.ensure_running() - return GatewayEndpoint( + endpoint = GatewayEndpoint( orchestrator_url=infra.control_plane_url, gateway_ip=infra.gateway_ip, gateway_ca_pem=service.ca_cert_pem(), network=service.network, ) + _reprovision_running_bottles(endpoint) + return endpoint + + +def _reprovision_running_bottles(endpoint: GatewayEndpoint) -> None: + """Recover keys from live Apple containers and restore gateway tokens.""" + try: + secrets_by_ip: dict[str, str] = {} + for agent in enumerate_active(): + name = f"{CONTAINER_NAME_PREFIX}{agent.slug}" + source_ip = container_mod.inspect_container_network_ip(name, endpoint.network) + if not source_ip: + continue + secret = container_mod.read_container_env(name, ENV_VAR_SECRET_NAME) + if secret: + secrets_by_ip[source_ip] = secret + count = reprovision_bottles( + OrchestratorClient(endpoint.orchestrator_url), secrets_by_ip, + ) + if count: + info(f"reprovisioned egress tokens for {count} macOS bottle(s)") + except (OrchestratorClientError, EnumerationError, OSError) as exc: + info(f"egress token reprovision skipped: {exc}") def live_source_ips(network: str) -> list[str]: @@ -126,6 +154,7 @@ def register_agent( endpoint: GatewayEndpoint, image_ref: str = "", tokens: dict[str, str] | None = None, + env_var_secret: str | None = None, ) -> LaunchContext: """Register the (already running) agent by its address and provision its git-gate state into the gateway. `source_ip` must be read from the live @@ -145,6 +174,7 @@ def register_agent( reg = provision_bottle( client, source_ip, egress_plan, git_gate_plan, AppleGatewayTransport(), image_ref=image_ref, tokens=tokens, + env_var_secret=env_var_secret, ) return LaunchContext( bottle_id=reg.bottle_id, diff --git a/bot_bottle/backend/macos_container/launch.py b/bot_bottle/backend/macos_container/launch.py index 9a5ddae..73afc57 100644 --- a/bot_bottle/backend/macos_container/launch.py +++ b/bot_bottle/backend/macos_container/launch.py @@ -68,6 +68,7 @@ from .gateway_hosts import ( ) from .bottle_plan import MacosContainerBottlePlan from ...orchestrator.config_store import resolve_teardown_timeout +from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME, new_env_var_secret from .consolidated_launch import ( GatewayEndpoint, ensure_gateway, @@ -142,6 +143,7 @@ def launch( plan = _provision_git_gate_keys(plan) plan = _install_gateway_ca(plan, endpoint) plan = _stamp_agent_urls(plan, endpoint) + plan = dataclasses.replace(plan, env_var_secret=new_env_var_secret()) # Step 3: run the agent. It has no identity token yet — registration # needs the address this run assigns. @@ -176,6 +178,7 @@ def launch( endpoint=endpoint, image_ref=plan.image, tokens=token_values, + env_var_secret=plan.env_var_secret, ) stack.callback( teardown_consolidated, ctx.bottle_id, @@ -406,6 +409,8 @@ def _agent_env_entries( env.append(f"GIT_GATE_URL={plan.agent_git_gate_url}") if plan.agent_supervise_url: env.append(f"MCP_SUPERVISE_URL={plan.agent_supervise_url}") + if getattr(plan, "env_var_secret", ""): + env.append(f"{ENV_VAR_SECRET_NAME}={plan.env_var_secret}") for name, value in sorted(plan.agent_provision.guest_env.items()): env.append(f"{name}={value}") # Forwarded vars: bare name → inherits from the `container run` process env diff --git a/bot_bottle/backend/macos_container/util.py b/bot_bottle/backend/macos_container/util.py index 404d9c3..23a6403 100644 --- a/bot_bottle/backend/macos_container/util.py +++ b/bot_bottle/backend/macos_container/util.py @@ -361,6 +361,12 @@ def exec_container(name: str, argv: list[str]) -> None: ) +def read_container_env(name: str, env_name: str) -> str: + """Read one configured env value from a running container, or ``""``.""" + result = _run_container_op([_CONTAINER, "exec", name, "printenv", env_name]) + return result.stdout.strip() if result.returncode == 0 else "" + + def exec_container_as_root(name: str, argv: list[str]) -> None: """`exec_container`, but as uid 0 inside the container. diff --git a/bot_bottle/orchestrator/reprovision.py b/bot_bottle/orchestrator/reprovision.py new file mode 100644 index 0000000..e55ebea --- /dev/null +++ b/bot_bottle/orchestrator/reprovision.py @@ -0,0 +1,35 @@ +"""Shared host-side join for backend-discovered bottle encryption keys.""" + +from __future__ import annotations + +from .client import OrchestratorClient, OrchestratorClientError + + +def reprovision_bottles( + client: OrchestratorClient, + secrets_by_source_ip: dict[str, str], +) -> int: + """Restore tokens for registered bottles whose backend exposes a key. + + Backends own discovery because containers and microVMs have different + enumeration primitives. This helper owns the shared registry join and + intentionally tolerates one bad/missing key without blocking a launch. + """ + restored = 0 + for bottle in client.list_bottles(): + bottle_id = bottle.get("bottle_id") + source_ip = bottle.get("source_ip") + if not isinstance(bottle_id, str) or not isinstance(source_ip, str): + continue + secret = secrets_by_source_ip.get(source_ip, "").strip() + if not secret: + continue + try: + if client.reprovision_gateway(bottle_id, secret): + restored += 1 + except OrchestratorClientError: + continue + return restored + + +__all__ = ["reprovision_bottles"] diff --git a/bot_bottle/orchestrator/secret_store.py b/bot_bottle/orchestrator/secret_store.py index a7a6e51..d1924d9 100644 --- a/bot_bottle/orchestrator/secret_store.py +++ b/bot_bottle/orchestrator/secret_store.py @@ -12,12 +12,15 @@ reattachment path reads ENV_VAR_SECRET from the running agent container via ``POST /bottles//reprovision_gateway``; the orchestrator decrypts the stored rows and re-populates ``_tokens``. -Encryption scheme: HMAC-SHA256 used as a PRF in CTR mode (stdlib-only, -no external deps). Each value is encrypted independently. The output blob is -``nonce (16 bytes) || ciphertext`` encoded as URL-safe base64 (no padding). +Encryption scheme: HMAC-SHA256 used as a PRF in CTR mode plus an independent +HMAC-SHA256 authentication key (stdlib-only, no external deps). Each value is +encrypted independently. The output blob is ``nonce (16 bytes) || ciphertext +|| tag (32 bytes)`` encoded as URL-safe base64 (no padding). Both subkeys are +derived from ENV_VAR_SECRET with domain-separated HMAC labels. keystream_block_i = HMAC-SHA256(key, nonce || i.to_bytes(4, "big")) ciphertext_i = plaintext_i XOR keystream_block_i[:len(plaintext_i)] + tag = HMAC-SHA256(auth_key, nonce || ciphertext) """ from __future__ import annotations @@ -30,6 +33,7 @@ import secrets _KEY_BYTES = 32 # 256-bit key from ENV_VAR_SECRET _NONCE_BYTES = 16 # 128-bit random nonce per encrypt call _BLOCK = 32 # HMAC-SHA256 output width == one keystream block +_TAG_BYTES = 32 # full HMAC-SHA256 authentication tag # Env-var name the agent container receives at startup. ENV_VAR_SECRET_NAME = "ENV_VAR_SECRET" @@ -41,7 +45,9 @@ def new_env_var_secret() -> str: def _b64dec(s: str) -> bytes: - return base64.urlsafe_b64decode(s + "=" * (-len(s) % 4)) + return base64.b64decode( + s + "=" * (-len(s) % 4), altchars=b"-_", validate=True, + ) def _keystream(key: bytes, nonce: bytes, block_index: int) -> bytes: @@ -50,40 +56,54 @@ def _keystream(key: bytes, nonce: bytes, block_index: int) -> bytes: ).digest() +def _subkeys(key: bytes) -> tuple[bytes, bytes]: + """Derive independent encryption and authentication keys.""" + return ( + hmac.new(key, b"bot-bottle secret encryption v1", hashlib.sha256).digest(), + hmac.new(key, b"bot-bottle secret authentication v1", hashlib.sha256).digest(), + ) + + def encrypt_value(secret_b64: str, plaintext: str) -> str: """Encrypt a single string value with *secret_b64* (the ENV_VAR_SECRET). - Returns a URL-safe base64 blob ``nonce || ciphertext`` suitable for + Returns a URL-safe base64 blob ``nonce || ciphertext || tag`` suitable for the ``bottled_agent_secrets.value`` column.""" - key = _b64dec(secret_b64) + enc_key, auth_key = _subkeys(_b64dec(secret_b64)) pt = plaintext.encode() nonce = secrets.token_bytes(_NONCE_BYTES) ct = bytearray() for i in range(0, len(pt), _BLOCK): chunk = pt[i : i + _BLOCK] - ks = _keystream(key, nonce, i)[: len(chunk)] + ks = _keystream(enc_key, nonce, i)[: len(chunk)] ct.extend(p ^ k for p, k in zip(chunk, ks)) - return base64.urlsafe_b64encode(nonce + bytes(ct)).rstrip(b"=").decode() + authenticated = nonce + bytes(ct) + tag = hmac.new(auth_key, authenticated, hashlib.sha256).digest() + return base64.urlsafe_b64encode(authenticated + tag).rstrip(b"=").decode() def decrypt_value(secret_b64: str, blob_b64: str) -> str: """Decrypt a blob produced by :func:`encrypt_value`. Returns the original plaintext string. Raises ``ValueError`` for malformed - input or a key mismatch (wrong key produces garbage, not an error, unless - the plaintext is non-UTF-8 — treat all such failures as wrong key).""" - key = _b64dec(secret_b64) + input, tampering, or a key mismatch. Authentication is verified before any + plaintext is returned.""" + enc_key, auth_key = _subkeys(_b64dec(secret_b64)) try: blob = _b64dec(blob_b64) except Exception as exc: raise ValueError(f"invalid ciphertext blob: {exc}") from exc - if len(blob) < _NONCE_BYTES: + if len(blob) < _NONCE_BYTES + _TAG_BYTES: raise ValueError("ciphertext blob too short") - nonce, ciphertext = blob[:_NONCE_BYTES], blob[_NONCE_BYTES:] + authenticated, tag = blob[:-_TAG_BYTES], blob[-_TAG_BYTES:] + expected = hmac.new(auth_key, authenticated, hashlib.sha256).digest() + if not hmac.compare_digest(tag, expected): + raise ValueError("ciphertext authentication failed") + nonce, ciphertext = authenticated[:_NONCE_BYTES], authenticated[_NONCE_BYTES:] pt = bytearray() for i in range(0, len(ciphertext), _BLOCK): chunk = ciphertext[i : i + _BLOCK] - ks = _keystream(key, nonce, i)[: len(chunk)] + ks = _keystream(enc_key, nonce, i)[: len(chunk)] pt.extend(c ^ k for c, k in zip(chunk, ks)) try: return bytes(pt).decode() diff --git a/docs/prds/prd-new-secret-provider-encrypted-env.md b/docs/prds/prd-new-secret-provider-encrypted-env.md index 8ca5f9a..f01d451 100644 --- a/docs/prds/prd-new-secret-provider-encrypted-env.md +++ b/docs/prds/prd-new-secret-provider-encrypted-env.md @@ -79,8 +79,11 @@ credential wipe. upstream credentials. 4. The stored form is revocable and rotatable without relaunching bottles that are not affected. -5. Reap/teardown destroys a bottle's stored secrets along with its - registry row (no ciphertext outliving its bottle). +5. When the retained bottle record reaches the lifecycle status `removed`, + its stored secrets are destroyed. Status/event persistence and the removal + transition land separately; this interim slice intentionally retains the + ciphertext across today's teardown/reconcile calls until that lifecycle is + available. 6. Migration is transparent: existing bottles keep working, no manifest changes required. diff --git a/tests/unit/test_backend_secret_reprovision.py b/tests/unit/test_backend_secret_reprovision.py new file mode 100644 index 0000000..495b44c --- /dev/null +++ b/tests/unit/test_backend_secret_reprovision.py @@ -0,0 +1,160 @@ +"""Backend-agnostic and backend-specific encrypted-secret recovery tests.""" + +from __future__ import annotations + +import json +import subprocess +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock, call, patch + +from bot_bottle.orchestrator.reprovision import reprovision_bottles +from bot_bottle.backend.firecracker import consolidated_launch as fc +from bot_bottle.backend.macos_container import consolidated_launch as mac +from bot_bottle.backend.docker import consolidated_launch as docker +from bot_bottle.orchestrator.client import OrchestratorClientError + + +def _proc(returncode: int = 0, stdout: str = "", stderr: str = ""): + return subprocess.CompletedProcess([], returncode, stdout=stdout, stderr=stderr) + + +class TestSharedReprovision(unittest.TestCase): + def test_joins_registry_records_by_source_ip(self) -> None: + client = Mock() + client.list_bottles.return_value = [ + {"bottle_id": "b1", "source_ip": "10.0.0.1"}, + {"bottle_id": "b2", "source_ip": "10.0.0.2"}, + {"bottle_id": 3, "source_ip": "10.0.0.3"}, + ] + client.reprovision_gateway.side_effect = [True, False] + count = reprovision_bottles( + client, {"10.0.0.1": " key-1\n", "10.0.0.2": "key-2"}, + ) + self.assertEqual(1, count) + self.assertEqual( + [("b1", "key-1"), ("b2", "key-2")], + [call.args for call in client.reprovision_gateway.call_args_list], + ) + + def test_one_failure_does_not_block_other_bottles(self) -> None: + client = Mock() + client.list_bottles.return_value = [ + {"bottle_id": "b1", "source_ip": "10.0.0.1"}, + {"bottle_id": "b2", "source_ip": "10.0.0.2"}, + ] + client.reprovision_gateway.side_effect = [ + OrchestratorClientError("bad key"), True, + ] + self.assertEqual( + 1, + reprovision_bottles( + client, {"10.0.0.1": "key-1", "10.0.0.2": "key-2"}, + ), + ) + + +class TestMacosReprovision(unittest.TestCase): + def test_reads_configured_container_env_and_reprovisions(self) -> None: + endpoint = mac.GatewayEndpoint("http://orch", "10.0.0.9", "PEM", "net") + agent = SimpleNamespace(slug="demo") + client = Mock() + with patch.object(mac, "enumerate_active", return_value=[agent]), \ + patch.object(mac.container_mod, "inspect_container_network_ip", + return_value="10.0.0.1"), \ + patch.object(mac.container_mod, "read_container_env", return_value="key"), \ + patch.object(mac, "OrchestratorClient", return_value=client), \ + patch.object(mac, "reprovision_bottles", return_value=1) as restore, \ + patch.object(mac, "info"): + mac._reprovision_running_bottles(endpoint) + restore.assert_called_once_with(client, {"10.0.0.1": "key"}) + + def test_enumeration_failure_is_best_effort(self) -> None: + endpoint = mac.GatewayEndpoint("http://orch", "10.0.0.9", "PEM", "net") + with patch.object(mac, "enumerate_active", + side_effect=mac.EnumerationError("failed")), \ + patch.object(mac, "info") as info: + mac._reprovision_running_bottles(endpoint) + self.assertIn("skipped", info.call_args.args[0]) + + +class TestDockerReprovision(unittest.TestCase): + def test_maps_network_containers_to_keys(self) -> None: + inspect = _proc(stdout=( + "bot-bottle-infra 172.18.0.2/16\n" + "bot-bottle-a 172.18.0.3/16\n" + "malformed\n" + )) + key = _proc(stdout="secret\n") + client = Mock() + with patch.object(docker, "OrchestratorClient", return_value=client), \ + patch.object(docker, "run_docker", side_effect=[inspect, key]), \ + patch.object(docker, "reprovision_bottles", return_value=1) as restore, \ + patch.object(docker.log, "info"): + docker._reprovision_running_bottles("http://orch") + restore.assert_called_once_with(client, {"172.18.0.3": "secret"}) + + def test_missing_docker_is_best_effort(self) -> None: + with patch.object(docker, "run_docker", side_effect=FileNotFoundError("docker")), \ + patch.object(docker.log, "info") as info: + docker._reprovision_running_bottles("http://orch") + self.assertIn("skipped", info.call_args.args[0]) + + +class TestFirecrackerReprovision(unittest.TestCase): + def _run_dir(self, root: Path, ip: str = "10.243.0.3") -> Path: + run_dir = root / "demo" + run_dir.mkdir() + (run_dir / "bottle_id_ed25519").write_text("key") + (run_dir / "config.json").write_text(json.dumps({ + "boot-source": {"boot_args": f"root=/dev/vda ip={ip}::gw:mask::eth0:off"} + })) + return run_dir + + def test_extracts_guest_ip_from_config(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + run_dir = self._run_dir(Path(tmp)) + self.assertEqual("10.243.0.3", fc._guest_ip_from_config(run_dir / "config.json")) + self.assertEqual("", fc._guest_ip_from_config(run_dir / "missing.json")) + + def test_persists_key_over_stdin_not_argv(self) -> None: + with patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \ + patch.object(fc.subprocess, "run", return_value=_proc()) as run: + fc.persist_env_var_secret(Path("/key"), "10.0.0.1", "super-secret") + self.assertEqual("super-secret", run.call_args.kwargs["input"]) + self.assertNotIn("super-secret", " ".join(run.call_args.args[0])) + + def test_persist_failure_is_fatal_to_launch(self) -> None: + with patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \ + patch.object(fc.subprocess, "run", return_value=_proc(1, stderr="denied")): + with self.assertRaisesRegex(fc.ConsolidatedLaunchError, "denied"): + fc.persist_env_var_secret(Path("/key"), "10.0.0.1", "secret") + + def test_reads_live_vm_key_and_reprovisions(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + run_dir = self._run_dir(Path(tmp)) + client = Mock() + with patch.object(fc.cleanup, "live_run_dirs", return_value=(run_dir,)), \ + patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \ + patch.object(fc.subprocess, "run", return_value=_proc(stdout="secret\n")), \ + patch.object(fc, "reprovision_bottles", return_value=1) as restore, \ + patch.object(fc, "info"): + fc._reprovision_running_bottles(client) + restore.assert_called_once_with(client, {"10.243.0.3": "secret"}) + + def test_unreadable_vm_is_skipped(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + run_dir = self._run_dir(Path(tmp)) + client = Mock() + with patch.object(fc.cleanup, "live_run_dirs", return_value=(run_dir,)), \ + patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \ + patch.object(fc.subprocess, "run", return_value=_proc(1)), \ + patch.object(fc, "reprovision_bottles", return_value=0) as restore: + fc._reprovision_running_bottles(client) + restore.assert_called_once_with(client, {}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_consolidated_compose.py b/tests/unit/test_consolidated_compose.py index 33495e1..5bad1fa 100644 --- a/tests/unit/test_consolidated_compose.py +++ b/tests/unit/test_consolidated_compose.py @@ -46,6 +46,15 @@ class TestConsolidatedAgentCompose(unittest.TestCase): # forwarded secrets are bare names (value inherited from process env). self.assertIn("CLAUDE_CODE_OAUTH_TOKEN", env) + def test_env_var_secret_stays_a_bare_name(self) -> None: + plan = _plan(with_egress=True, supervise=True, with_git=True) + plan = type(plan)(**{**vars(plan), "env_var_secret": "secret-value"}) + env = consolidated_agent_compose( + plan, gateway_ip=_GW, source_ip=_IP, network=_NET, + )["services"]["agent"]["environment"] + self.assertIn("ENV_VAR_SECRET", env) + self.assertNotIn("ENV_VAR_SECRET=secret-value", env) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_docker_launch_teardown.py b/tests/unit/test_docker_launch_teardown.py index 55474a5..38b8a6e 100644 --- a/tests/unit/test_docker_launch_teardown.py +++ b/tests/unit/test_docker_launch_teardown.py @@ -91,6 +91,7 @@ class TestTeardownWarning(unittest.TestCase): bottle_id="b1", identity_token="t", source_ip="172.20.0.4", network="bot-bottle-gateway", gateway_ip="172.20.0.2", orchestrator_url="http://orch:8099", + env_var_secret="encryption-key", ) images = BottleImages(agent="bot-bottle-claude:latest", sidecar="bot-bottle-sidecars:latest") @@ -107,7 +108,7 @@ class TestTeardownWarning(unittest.TestCase): mock.patch.object( launch_mod, "write_compose_file", return_value=Path("/tmp/compose.yml"), ), \ - mock.patch.object(launch_mod, "compose_up"), \ + mock.patch.object(launch_mod, "compose_up") as compose_up, \ mock.patch.object(launch_mod, "compose_dump_logs"), \ mock.patch.object( launch_mod, "compose_down", @@ -122,6 +123,9 @@ class TestTeardownWarning(unittest.TestCase): self.assertIn("bot-bottle: warning:", output) self.assertIn("bot-bottle-test-teardown-abc", output) self.assertIn("compose-down", output) + self.assertEqual( + "encryption-key", compose_up.call_args.kwargs["env"]["ENV_VAR_SECRET"], + ) if __name__ == "__main__": diff --git a/tests/unit/test_firecracker_backend.py b/tests/unit/test_firecracker_backend.py index d72d795..3a1387b 100644 --- a/tests/unit/test_firecracker_backend.py +++ b/tests/unit/test_firecracker_backend.py @@ -66,6 +66,7 @@ class TestNetpoolRenderers(unittest.TestCase): self.assertIn("chown node:node /home/node", util._GUEST_INIT) self.assertIn("chmod 755 /home/node", util._GUEST_INIT) + self.assertIn("mount -t tmpfs -o mode=0755 tmpfs /run", util._GUEST_INIT) def test_nixos_module_is_non_invasive(self): # The NixOS module must NOT flip the host firewall backend or diff --git a/tests/unit/test_firecracker_cleanup.py b/tests/unit/test_firecracker_cleanup.py index 8670783..a66660d 100644 --- a/tests/unit/test_firecracker_cleanup.py +++ b/tests/unit/test_firecracker_cleanup.py @@ -62,6 +62,14 @@ class TestProcessScan(unittest.TestCase): with patch.object(fc_cleanup.subprocess, "run", return_value=_proc(returncode=1)): self.assertEqual((set(), []), fc_cleanup._scan_processes(Path("/x"))) + def test_live_run_dirs_returns_paths_in_stable_order(self): + with patch.object(fc_cleanup, "_run_root", return_value=Path("/run")), \ + patch.object(fc_cleanup, "_scan_processes", + return_value=({"/run/b", "/run/a"}, [])): + self.assertEqual( + (Path("/run/a"), Path("/run/b")), fc_cleanup.live_run_dirs(), + ) + def test_orphan_run_dirs_excludes_live_and_missing_root(self): with tempfile.TemporaryDirectory() as tmp: run_root = Path(tmp) diff --git a/tests/unit/test_macos_container_launch_wiring.py b/tests/unit/test_macos_container_launch_wiring.py index 94256a3..eedb729 100644 --- a/tests/unit/test_macos_container_launch_wiring.py +++ b/tests/unit/test_macos_container_launch_wiring.py @@ -49,6 +49,7 @@ def _plan( agent_git_gate_url: str = "", agent_supervise_url: str = "", image_policy: str = "fresh", + env_var_secret: str = "", ) -> MacosContainerBottlePlan: routes_path = stage_dir / "routes.yaml" routes_path.write_text("routes: []\n", encoding="utf-8") @@ -80,6 +81,7 @@ def _plan( ), agent_git_gate_url=agent_git_gate_url, agent_supervise_url=agent_supervise_url, + env_var_secret=env_var_secret, )) @@ -185,6 +187,12 @@ class TestAgentRunArgv(unittest.TestCase): "bot-bottle-mac-gateway", self.argv[self.argv.index("--network") + 1], ) + def test_env_var_secret_is_in_configured_container_environment(self) -> None: + argv = _agent_run_argv( + _plan(Path(self._tmp.name), env_var_secret="key-material"), _endpoint(), + ) + self.assertIn("ENV_VAR_SECRET=key-material", argv) + def test_never_pins_an_ip(self) -> None: """Apple Container 1.0.0 has no --ip: the address is DHCP-assigned and read back after start.""" diff --git a/tests/unit/test_macos_container_util.py b/tests/unit/test_macos_container_util.py index 5024fe4..3a419c8 100644 --- a/tests/unit/test_macos_container_util.py +++ b/tests/unit/test_macos_container_util.py @@ -28,6 +28,21 @@ class TestMacosContainerAvailability(unittest.TestCase): class TestMacosContainerCommands(unittest.TestCase): + def test_read_container_env(self): + completed = util.subprocess.CompletedProcess( + args=[], returncode=0, stdout="secret\n", stderr="", + ) + with patch.object(util, "_run_container_op", return_value=completed) as run: + self.assertEqual("secret", util.read_container_env("bottle", "KEY")) + run.assert_called_once_with(["container", "exec", "bottle", "printenv", "KEY"]) + + def test_read_container_env_returns_empty_on_failure(self): + completed = util.subprocess.CompletedProcess( + args=[], returncode=1, stdout="", stderr="missing", + ) + with patch.object(util, "_run_container_op", return_value=completed): + self.assertEqual("", util.read_container_env("bottle", "KEY")) + def test_dns_server_prefers_direct_host_ipv4_resolver(self): scutil = util.subprocess.CompletedProcess( args=[], diff --git a/tests/unit/test_orchestrator_client.py b/tests/unit/test_orchestrator_client.py index d6e07bb..88a060b 100644 --- a/tests/unit/test_orchestrator_client.py +++ b/tests/unit/test_orchestrator_client.py @@ -76,6 +76,27 @@ class TestTeardown(unittest.TestCase): self.assertEqual("DELETE", m.call_args.args[0].get_method()) +class TestReprovisionGateway(unittest.TestCase): + def setUp(self) -> None: + self.c = OrchestratorClient("http://orch:8080") + + def test_success_posts_key(self) -> None: + with patch(_URLOPEN, return_value=_resp(200, {"reprovisioned": True})) as opened: + self.assertTrue(self.c.reprovision_gateway("b1", "key")) + request = opened.call_args.args[0] + self.assertEqual("POST", request.get_method()) + self.assertEqual({"env_var_secret": "key"}, json.loads(request.data)) + + def test_missing_stored_secret_is_false(self) -> None: + with patch(_URLOPEN, side_effect=_http_error(404)): + self.assertFalse(self.c.reprovision_gateway("b1", "key")) + + def test_other_status_raises(self) -> None: + with patch(_URLOPEN, side_effect=_http_error(400)): + with self.assertRaises(OrchestratorClientError): + self.c.reprovision_gateway("b1", "key") + + class TestHealthAndPolicy(unittest.TestCase): def setUp(self) -> None: self.c = OrchestratorClient("http://orch:8080") diff --git a/tests/unit/test_orchestrator_control_plane.py b/tests/unit/test_orchestrator_control_plane.py index ea75d5c..3da0908 100644 --- a/tests/unit/test_orchestrator_control_plane.py +++ b/tests/unit/test_orchestrator_control_plane.py @@ -6,6 +6,7 @@ server tests), plus one real-socket round-trip to prove the handler wiring. from __future__ import annotations +import base64 import json import secrets import sqlite3 @@ -63,6 +64,42 @@ class TestDispatch(unittest.TestCase): self.assertTrue(payload["bottle_id"]) self.assertTrue(payload["identity_token"]) + def test_register_and_reprovision_encrypted_tokens(self) -> None: + key = base64.urlsafe_b64encode(b"unit-test-key").rstrip(b"=").decode() + status, payload = dispatch( + self.orch, "POST", "/bottles", _body({ + "source_ip": "10.243.0.11", + "tokens": {"EGRESS_TOKEN_0": "upstream-secret"}, + "env_var_secret": key, + }), + ) + self.assertEqual(201, status) + bottle_id = payload["bottle_id"] + self.orch._tokens.clear() + status, response = dispatch( + self.orch, "POST", f"/bottles/{bottle_id}/reprovision_gateway", + _body({"env_var_secret": key}), + ) + self.assertEqual((200, {"reprovisioned": True}), (status, response)) + self.assertEqual( + {"EGRESS_TOKEN_0": "upstream-secret"}, self.orch.tokens_for(bottle_id), + ) + + def test_reprovision_validates_request_and_missing_rows(self) -> None: + status, _ = dispatch( + self.orch, "POST", "/bottles/b1/reprovision_gateway", b"not-json", + ) + self.assertEqual(400, status) + status, _ = dispatch( + self.orch, "POST", "/bottles/b1/reprovision_gateway", _body({}), + ) + self.assertEqual(400, status) + status, _ = dispatch( + self.orch, "POST", "/bottles/b1/reprovision_gateway", + _body({"env_var_secret": "key"}), + ) + self.assertEqual(404, status) + def test_register_requires_source_ip(self) -> None: status, _ = dispatch(self.orch, "POST", "/bottles", _body({})) self.assertEqual(400, status) diff --git a/tests/unit/test_orchestrator_secret_store.py b/tests/unit/test_orchestrator_secret_store.py index 64dd453..4d6a1df 100644 --- a/tests/unit/test_orchestrator_secret_store.py +++ b/tests/unit/test_orchestrator_secret_store.py @@ -68,15 +68,21 @@ class TestDecryptErrors(unittest.TestCase): def test_wrong_key_raises_value_error(self) -> None: ct = encrypt_value(self.secret, "secret-token") other_key = new_env_var_secret() - # Wrong key produces garbage bytes; decrypt_value raises ValueError - # when the result is non-UTF-8 (which is very likely for 12-char data). - # We allow it to succeed only if garbage happens to be valid UTF-8, but - # the plaintext must not match. - try: - result = decrypt_value(other_key, ct) - self.assertNotEqual("secret-token", result) - except ValueError: - pass + with self.assertRaisesRegex(ValueError, "authentication failed"): + decrypt_value(other_key, ct) + + def test_modified_ciphertext_raises_value_error(self) -> None: + ct = encrypt_value(self.secret, "secret-token") + replacement = "A" if ct[-10] != "A" else "B" + tampered = ct[:-10] + replacement + ct[-9:] + with self.assertRaisesRegex(ValueError, "authentication failed"): + decrypt_value(self.secret, tampered) + + def test_modified_tag_raises_value_error(self) -> None: + ct = encrypt_value(self.secret, "secret-token") + replacement = "A" if ct[-1] != "A" else "B" + with self.assertRaisesRegex(ValueError, "authentication failed"): + decrypt_value(self.secret, ct[:-1] + replacement) def test_truncated_blob_raises_value_error(self) -> None: with self.assertRaises(ValueError): diff --git a/tests/unit/test_orchestrator_service.py b/tests/unit/test_orchestrator_service.py index efee477..61ed48d 100644 --- a/tests/unit/test_orchestrator_service.py +++ b/tests/unit/test_orchestrator_service.py @@ -15,6 +15,7 @@ from bot_bottle.orchestrator.broker import LaunchBroker, LaunchRequest, StubBrok from bot_bottle.orchestrator.registry import RegistryStore from bot_bottle.orchestrator.service import Orchestrator from bot_bottle.orchestrator.gateway import Gateway +from bot_bottle.orchestrator.secret_store import new_env_var_secret from bot_bottle.store_manager import StoreManager from bot_bottle.supervise import ( Proposal, @@ -117,6 +118,25 @@ class TestOrchestrator(unittest.TestCase): rec = self.orch.launch_bottle("10.243.0.6") self.assertEqual({}, self.orch.tokens_for(rec.bottle_id)) + def test_encrypted_tokens_can_be_reprovisioned_after_memory_loss(self) -> None: + key = new_env_var_secret() + rec = self.orch.launch_bottle( + "10.243.0.12", tokens={"EGRESS_TOKEN_0": "secret"}, + env_var_secret=key, + ) + self.assertNotEqual({}, self.store.get_agent_secrets(rec.bottle_id)) + self.orch._tokens.clear() + self.assertTrue(self.orch.reprovision_from_secret(rec.bottle_id, key)) + self.assertEqual({"EGRESS_TOKEN_0": "secret"}, self.orch.tokens_for(rec.bottle_id)) + + def test_reprovision_rejects_missing_rows_and_wrong_key(self) -> None: + self.assertFalse(self.orch.reprovision_from_secret("missing", new_env_var_secret())) + rec = self.orch.launch_bottle( + "10.243.0.13", tokens={"K": "value"}, + env_var_secret=new_env_var_secret(), + ) + self.assertFalse(self.orch.reprovision_from_secret(rec.bottle_id, new_env_var_secret())) + def test_set_policy_live_reload(self) -> None: rec = self.orch.launch_bottle("10.243.0.3") self.assertTrue(self.orch.set_policy(rec.bottle_id, '{"x":1}'))