Compare commits

..

2 Commits

Author SHA1 Message Date
didericis dff811f190 fix(gateway): persist git-gate state across gateway restarts
tracker-policy-pr / check-pr (pull_request) Successful in 12s
lint / lint (push) Successful in 59s
test / unit (pull_request) Successful in 53s
test / integration-docker (pull_request) Successful in 1m7s
test / coverage (pull_request) Successful in 19s
Per-bottle git-gate state (bare repos under /git/<id>, deploy creds under
/git-gate/creds/<id>) was provisioned once at bottle launch and lived only
in the gateway's ephemeral storage. A gateway rebuild/restart wiped it and
nothing re-provisioned already-running bottles, so their agents 404'd on
fetch/push. Same class of bug as the CA (#510); the orchestrator restores
only egress tokens, not git-gate declarations.

Persist the state on both backends, mirroring the CA-persistence approach:

- firecracker: attach a second persistent data drive (/dev/vdc) to the
  gateway VM and bind-mount its git/ + creds/ subdirs onto /git and
  /git-gate/creds in the gateway guest init, before the data plane starts.
  Generalize the VM config to a stable-ordered data_drives tuple (CA=vdb,
  git=vdc; orchestrator registry stays vdb).
- docker: bind-mount host dirs (host_gateway_git_dir / creds_dir, under the
  never-pruned app-data root) onto /git and /git-gate/creds, with
  BOT_BOTTLE_DOCKER_GIT_MOUNT / _CREDS_MOUNT env overrides so CI isolates
  them to per-run volumes it cleans up.

Teardown already rm -rf's /git/<id> + creds, so the persistent store
self-cleans over the normal lifecycle.

Closes #512

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-26 17:34:22 -04:00
didericis 0ddaa95c14 fix(firecracker): persist the gateway mitmproxy CA across rebuilds
prd-number-check / require-numbered-prds (pull_request) Successful in 6s
tracker-policy-pr / check-pr (pull_request) Successful in 7s
lint / lint (push) Successful in 55s
test / unit (pull_request) Successful in 47s
test / integration-docker (pull_request) Failing after 2m24s
test / coverage (pull_request) Has been skipped
The #450 CA-persistence fix was only implemented for the docker backend
(the host_gateway_ca_dir bind-mount). The firecracker gateway booted with
no persistent volume, so its mitmproxy CA lived only in the ephemeral
per-boot rootfs — every rebuild/restart minted a fresh CA that every
already-running bottle distrusted, failing egress TLS with "SSL
certificate verification failed".

Give the gateway VM a persistent CA volume, mirroring the orchestrator's
registry volume: boot with a small ext4 data_drive (_ensure_ca_volume)
and mount it at mitmproxy's confdir in the gateway guest init, before the
data plane starts, so mitmproxy reuses the on-disk CA instead of
regenerating it.

Closes #510

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-26 17:15:18 -04:00
54 changed files with 360 additions and 1050 deletions
+5 -1
View File
@@ -100,6 +100,8 @@ jobs:
export BOT_BOTTLE_DOCKER_CLIENT_NETWORK="$DOCKER_CLIENT_NETWORK"
export BOT_BOTTLE_DOCKER_ROOT_MOUNT="bot-bottle-ci-root-$RUN_KEY"
export BOT_BOTTLE_DOCKER_CA_MOUNT="bot-bottle-ci-ca-$RUN_KEY"
export BOT_BOTTLE_DOCKER_GIT_MOUNT="bot-bottle-ci-git-$RUN_KEY"
export BOT_BOTTLE_DOCKER_CREDS_MOUNT="bot-bottle-ci-creds-$RUN_KEY"
python3 -m coverage run -m scripts.unittest_gate \
-t . -s tests/integration -v \
--minimum-executed 22 --fail-on-skip
@@ -110,7 +112,9 @@ jobs:
RUN_KEY="${GITHUB_RUN_ID:-${GITHUB_RUN_NUMBER:-0}}"
docker volume rm --force \
"bot-bottle-ci-root-$RUN_KEY" \
"bot-bottle-ci-ca-$RUN_KEY" 2>/dev/null || true
"bot-bottle-ci-ca-$RUN_KEY" \
"bot-bottle-ci-git-$RUN_KEY" \
"bot-bottle-ci-creds-$RUN_KEY" 2>/dev/null || true
# Non-dot name so upload-artifact's dotfile-skipping glob picks it up.
- name: Stage docker coverage for upload
+5 -1
View File
@@ -116,6 +116,8 @@ jobs:
export BOT_BOTTLE_DOCKER_CLIENT_NETWORK="$DOCKER_CLIENT_NETWORK"
export BOT_BOTTLE_DOCKER_ROOT_MOUNT="bot-bottle-ci-root-$RUN_KEY"
export BOT_BOTTLE_DOCKER_CA_MOUNT="bot-bottle-ci-ca-$RUN_KEY"
export BOT_BOTTLE_DOCKER_GIT_MOUNT="bot-bottle-ci-git-$RUN_KEY"
export BOT_BOTTLE_DOCKER_CREDS_MOUNT="bot-bottle-ci-creds-$RUN_KEY"
python3 -m coverage run -m scripts.unittest_gate \
-t . -s tests/integration -v \
--minimum-executed 22 --fail-on-skip
@@ -126,7 +128,9 @@ jobs:
RUN_KEY="${GITHUB_RUN_ID:-${GITHUB_RUN_NUMBER:-0}}"
docker volume rm --force \
"bot-bottle-ci-root-$RUN_KEY" \
"bot-bottle-ci-ca-$RUN_KEY" 2>/dev/null || true
"bot-bottle-ci-ca-$RUN_KEY" \
"bot-bottle-ci-git-$RUN_KEY" \
"bot-bottle-ci-creds-$RUN_KEY" 2>/dev/null || true
- name: Stage docker coverage for upload
run: cp .coverage.docker coverage-docker.dat
+2 -41
View File
@@ -21,7 +21,7 @@ from abc import ABC, abstractmethod
from contextlib import AbstractContextManager, contextmanager
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING, Generator, Generic, Sequence, TypeVar
from typing import Generator, Generic, Sequence, TypeVar
from ..agent_provider import AgentProvisionPlan, get_provider
from ..egress import EgressPlan
@@ -35,9 +35,6 @@ from ..workspace import WorkspacePlan, workspace_plan
from .print_util import print_multi, visible_agent_env_names
from .util import host_skill_dir
if TYPE_CHECKING:
from .gateway_attach import GatewayAttachResources
class BackendStatus(enum.IntEnum):
"""Return codes for BottleBackend.status(). READY == 0 so callsites
@@ -265,17 +262,6 @@ class Bottle(ABC):
PlanT = TypeVar("PlanT", bound=BottlePlan)
CleanupT = TypeVar("CleanupT", bound=BottleCleanupPlan)
# The backend-specific handle for one running bottle the reconcile attaches to
# the gateway (a run dir for firecracker, a container name for docker/macOS).
# Registry-style callers that don't care about the handle bind it to `Any`
# (`BottleBackend[Any, Any, Any]`).
AttachTargetT = TypeVar("AttachTargetT")
class InfraLaunchError(RuntimeError):
"""A per-host infra (gateway + orchestrator) launch/reconcile step could not
complete. Shared across backends so the base backend can raise + catch one
type; each backend's ``infra_launch`` module re-exports it."""
@dataclass(frozen=True)
@@ -290,7 +276,7 @@ class BottleImages:
sidecar: str | Path = ""
class BottleBackend(ABC, Generic[PlanT, CleanupT, AttachTargetT]):
class BottleBackend(ABC, Generic[PlanT, CleanupT]):
"""Abstract base for selectable bottle backends. Concrete subclasses
(e.g. DockerBottleBackend) own their own prepare/launch impls.
Parameterized over the backend's concrete plan + cleanup-plan types
@@ -518,31 +504,6 @@ class BottleBackend(ABC, Generic[PlanT, CleanupT, AttachTargetT]):
compose ls`; firecracker cross-references its running gateway
containers against per-bottle metadata."""
def attach_bottled_agents_to_gateway(self) -> None:
"""Reconcile every running bottle against the freshly-(re)booted gateway
on the cold-boot bring-up path (PRD 0081). The shared flow + fail-hard
policy live in `gateway_attach`; backends override only the three
primitives below (ADR 0006)."""
from .gateway_attach import reconcile_running_bottles
reconcile_running_bottles(self)
@abstractmethod
def _gateway_attach_resources(self) -> "GatewayAttachResources":
"""Gather what every bottle needs to (re)attach to the current gateway
(the CA now). Raises if the gateway isn't reachable (aborts reconcile)."""
@abstractmethod
def _running_bottles(self) -> Sequence[AttachTargetT]:
"""The live bottles as backend handles for `_attach_bottle_to_gateway`.
Raises if the live set can't be determined (fail hard, no partial set)."""
@abstractmethod
def _attach_bottle_to_gateway(
self, bottle: AttachTargetT, resources: "GatewayAttachResources",
) -> None:
"""(Re)attach one bottle to the current gateway (SSH / `exec`+`cp`).
Raises `InfraLaunchError` naming the bottle on failure."""
@classmethod
@abstractmethod
def is_available(cls) -> bool:
+1 -17
View File
@@ -33,7 +33,6 @@ from ...git_gate import GitGatePlan
from ...supervisor.plan import SupervisePlan
from ...manifest import Manifest
from .. import ActiveAgent, BottleBackend, BottleImages, BottleSpec
from ..gateway_attach import GatewayAttachResources
from . import cleanup as _cleanup
from . import enumerate as _enumerate
from . import launch as _launch
@@ -41,7 +40,7 @@ from . import resolve_plan as _resolve_plan
from .bottle import DockerBottle
from .bottle_cleanup_plan import DockerBottleCleanupPlan
from .bottle_plan import DockerBottlePlan
class DockerBottleBackend(BottleBackend["DockerBottlePlan", "DockerBottleCleanupPlan", str]):
class DockerBottleBackend(BottleBackend["DockerBottlePlan", "DockerBottleCleanupPlan"]):
"""Docker backend implementation. Selected by BOT_BOTTLE_BACKEND
when set to `docker`; retained as a legacy/example backend."""
@@ -141,18 +140,3 @@ class DockerBottleBackend(BottleBackend["DockerBottlePlan", "DockerBottleCleanup
def enumerate_active(self) -> Sequence[ActiveAgent]:
return _enumerate.enumerate_active()
def _gateway_attach_resources(self) -> GatewayAttachResources:
from .infra import DockerInfraService
return GatewayAttachResources(
ca_pem=DockerInfraService().gateway().ca_cert_pem())
def _running_bottles(self) -> Sequence[str]:
from .infra_launch import running_agent_containers
return running_agent_containers()
def _attach_bottle_to_gateway(
self, bottle: str, resources: GatewayAttachResources,
) -> None:
from .infra_launch import push_ca_to_container
push_ca_to_container(bottle, resources.ca_pem)
@@ -13,7 +13,6 @@ orchestrator-facing wiring so that sequence stays testable in isolation.
from __future__ import annotations
import tempfile
from dataclasses import dataclass
from ... import log
@@ -25,13 +24,15 @@ from ...gateway import GATEWAY_NETWORK
from .infra import INFRA_NAME, DockerInfraService
from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME
from ...orchestrator.reprovision import reprovision_bottles
from ..base import InfraLaunchError
from ..provision_bottle import deprovision_bottle, provision_bottle
from ..util import AGENT_CA_PATH
from .gateway_transport import DockerGatewayTransport
from .gateway_net import next_free_ip
class ConsolidatedLaunchError(RuntimeError):
"""The consolidated register/provision sequence could not complete."""
@dataclass(frozen=True)
class LaunchContext:
"""What the agent container needs to join the shared gateway."""
@@ -53,7 +54,7 @@ def _network_cidr(network: str) -> str:
])
cidr = proc.stdout.strip()
if proc.returncode != 0 or not cidr:
raise InfraLaunchError(
raise ConsolidatedLaunchError(
f"gateway network {network} has no subnet: {proc.stderr.strip()}"
)
return cidr
@@ -73,59 +74,6 @@ def _network_container_ips(network: str) -> list[str]:
return ips
def running_agent_containers(
network: str = GATEWAY_NETWORK, gateway_name: str = INFRA_NAME,
) -> list[str]:
"""Every running agent container on the gateway `network` (the gateway
itself excluded) the bottles the bring-up reconcile attaches to the fresh
gateway (PRD 0081). Raises `OSError` if `docker` can't be run (fail hard: a
reconcile that can't list its bottles must not look like "no bottles")."""
proc = run_docker([
"docker", "network", "inspect", "--format",
"{{range .Containers}}{{.Name}}\n{{end}}", network,
])
return [
name for name in (line.strip() for line in proc.stdout.splitlines())
if name and name != gateway_name
]
def push_ca_to_container(container: str, ca_pem: str) -> None:
"""(Re)attach one running agent container to the current gateway: replace
its trusted gateway CA with `ca_pem` and rebuild its trust store via
`docker cp` + `docker exec` (PRD 0081). Unconditional install there is one
gateway, so no fingerprint match is needed.
Fail hard: raises `InfraLaunchError` (naming the container) on any failure
the base reconcile aggregates and surfaces it rather than leaving the agent
silently unable to reach the gateway."""
mkdir = run_docker(
["docker", "exec", container, "mkdir", "-p",
"/usr/local/share/ca-certificates"]
)
if mkdir.returncode != 0:
raise InfraLaunchError(
f"CA push to {container} failed (mkdir): {mkdir.stderr.strip()}"
)
with tempfile.NamedTemporaryFile("w", suffix=".crt") as tmp:
tmp.write(ca_pem)
tmp.flush()
cp = run_docker(["docker", "cp", tmp.name, f"{container}:{AGENT_CA_PATH}"])
if cp.returncode != 0:
raise InfraLaunchError(
f"CA push to {container} failed (cp): {cp.stderr.strip()}"
)
ex = run_docker(
["docker", "exec", container, "sh", "-c",
f"chmod 644 {AGENT_CA_PATH} && update-ca-certificates"]
)
if ex.returncode != 0:
raise InfraLaunchError(
f"CA push to {container} failed (update-ca-certificates): "
f"{ex.stderr.strip()}"
)
def _reprovision_running_bottles(
orchestrator_url: str,
network: str = GATEWAY_NETWORK,
@@ -232,8 +180,6 @@ def deprovision_consolidated(
__all__ = [
"LaunchContext",
"launch_consolidated",
"running_agent_containers",
"push_ca_to_container",
"deprovision_consolidated",
"InfraLaunchError",
"ConsolidatedLaunchError",
]
+23 -5
View File
@@ -9,6 +9,8 @@ from .gateway_transport import DockerGatewayTransport
from ...paths import (
ORCHESTRATOR_AUTH_JWT_ENV,
host_gateway_ca_dir,
host_gateway_git_dir,
host_gateway_creds_dir,
)
from ... import resources
from ...gateway import (
@@ -40,6 +42,8 @@ class DockerGateway(Gateway):
dockerfile: str | None = GATEWAY_DOCKERFILE,
host_port_bindings: tuple[int, ...] = (),
ca_mount_source: str | Path | None = None,
git_mount_source: str | Path | None = None,
creds_mount_source: str | Path | None = None,
subnet: str | None = None,
) -> None:
self.image_ref = image_ref
@@ -74,6 +78,17 @@ class DockerGateway(Gateway):
self._ca_mount_source = str(
ca_mount_source or configured_ca or host_gateway_ca_dir()
)
# The persistent git-gate mounts (/git bare repos, /git-gate/creds deploy
# creds) — same host-bind-mount rationale as the CA (issue #512). The env
# overrides let CI point them at per-run named volumes it cleans up.
configured_git = os.environ.get("BOT_BOTTLE_DOCKER_GIT_MOUNT", "").strip()
self._git_mount_source = str(
git_mount_source or configured_git or host_gateway_git_dir()
)
configured_creds = os.environ.get("BOT_BOTTLE_DOCKER_CREDS_MOUNT", "").strip()
self._creds_mount_source = str(
creds_mount_source or configured_creds or host_gateway_creds_dir()
)
def image_exists(self) -> bool:
return run_docker(["docker", "image", "inspect", self.image_ref]).returncode == 0
@@ -157,7 +172,7 @@ class DockerGateway(Gateway):
f"gateway network {self.network} failed to create: {proc.stderr.strip()}"
)
def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> bool:
def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> None:
# Bind to this orchestrator (PRD 0070): stash the URL its daemons resolve
# policy against + the pre-minted `gateway` token they present, then bring
# the container up. The gateway never mints, so it holds no signing key —
@@ -184,7 +199,7 @@ class DockerGateway(Gateway):
# just when the container is absent.
self._ensure_network()
if self.is_running() and self._running_image_is_current():
return False
return
# Clear any stale (stopped OR outdated-image) container holding the
# fixed name, then start fresh. `rm --force` on an absent name is a
# tolerated no-op.
@@ -198,6 +213,12 @@ class DockerGateway(Gateway):
# container recreation AND docker volume pruning (agents trust it)
# — see host_gateway_ca_dir / issue #450.
"--volume", f"{self._ca_mount_source}:{MITMPROXY_HOME}",
# Persist per-bottle git-gate state (bare repos + deploy creds) on
# the host so a gateway restart doesn't drop already-running bottles'
# repos — they would otherwise 404 on fetch/push (issue #512). Same
# host-bind-mount rationale as the CA.
"--volume", f"{self._git_mount_source}:/git",
"--volume", f"{self._creds_mount_source}:/git-gate/creds",
# No DB mount: the data plane (egress / supervise / git-gate) reaches
# the supervise queue over the control-plane RPC and never opens
# bot-bottle.db, so the gateway container gets no file handle on it
@@ -228,9 +249,6 @@ class DockerGateway(Gateway):
# pre-connect window (they retry /resolve per request).
if self._control_network:
self._connect_control_network()
# (Re)created a fresh container — signal a cold boot so the caller
# reconciles running bottles against it (PRD 0081).
return True
def _connect_control_network(self) -> None:
"""Ensure the `--internal` control network exists and attach the gateway
+1 -8
View File
@@ -142,16 +142,9 @@ class DockerInfraService(InfraService):
# mints the role-scoped `gateway` JWT here and hands it to the gateway,
# which never sees the key (#469). `connect_to_orchestrator` is
# idempotent.
cold_booted = gateway.connect_to_orchestrator(
gateway.connect_to_orchestrator(
orchestrator.gateway_url(), orchestrator.mint_gateway_token(),
)
# A (re)created gateway container minted/mounted its CA afresh and lost
# every bottle's per-bottle state; reconcile the already-running bottles
# against it (PRD 0081). Skipped when a healthy current gateway was left
# untouched — no cold boot, nothing to reconcile.
if cold_booted:
from .backend import DockerBottleBackend
DockerBottleBackend().attach_bottled_agents_to_gateway()
return orchestrator.url()
def stop(self) -> None:
+1 -1
View File
@@ -61,7 +61,7 @@ from .compose import (
)
from .consolidated_compose import consolidated_agent_compose
from ...orchestrator.store.config_store import resolve_teardown_timeout
from .infra_launch import launch_consolidated, deprovision_consolidated
from .consolidated_launch import launch_consolidated, deprovision_consolidated
from .infra import INFRA_NAME
from .gateway import DockerGateway
from ... import resources
+1 -15
View File
@@ -20,7 +20,6 @@ from ...git_gate import GitGatePlan
from ...manifest import Manifest
from ...supervisor.plan import SupervisePlan
from .. import ActiveAgent, BottleBackend, BottleImages, BottleSpec
from ..gateway_attach import GatewayAttachResources
from . import cleanup as _cleanup
from . import enumerate as _enumerate
from . import launch as _launch
@@ -32,7 +31,7 @@ from .bottle_plan import FirecrackerBottlePlan
class FirecrackerBottleBackend(
BottleBackend["FirecrackerBottlePlan", "FirecrackerBottleCleanupPlan", "Path"]
BottleBackend["FirecrackerBottlePlan", "FirecrackerBottleCleanupPlan"]
):
name = "firecracker"
@@ -120,19 +119,6 @@ class FirecrackerBottleBackend(
def enumerate_active(self) -> Sequence[ActiveAgent]:
return _enumerate.enumerate_active()
def _gateway_attach_resources(self) -> GatewayAttachResources:
from .gateway import FirecrackerGateway
return GatewayAttachResources(ca_pem=FirecrackerGateway().ca_cert_pem())
def _running_bottles(self) -> Sequence[Path]:
return _cleanup.live_run_dirs()
def _attach_bottle_to_gateway(
self, bottle: Path, resources: GatewayAttachResources,
) -> None:
from .infra_launch import attach_ca_to_agent_vm
attach_ca_to_agent_vm(bottle, resources.ca_pem)
def supervise_mcp_url(self, plan: FirecrackerBottlePlan) -> str:
return plan.agent_supervise_url
@@ -40,9 +40,7 @@ from ...orchestrator.lifecycle import (
)
from ...orchestrator.reprovision import reprovision_bottles
from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME
from ..base import InfraLaunchError
from ..provision_bottle import deprovision_bottle, provision_bottle
from ..util import AGENT_CA_PATH
from . import cleanup, util
from .gateway import FirecrackerGateway
from .infra import FirecrackerInfraService
@@ -50,6 +48,10 @@ from .infra import FirecrackerInfraService
_ENV_VAR_SECRET_PATH = "/run/bot-bottle/env-var-secret"
class ConsolidatedLaunchError(RuntimeError):
"""The consolidated register/provision sequence could not complete."""
@dataclass(frozen=True)
class LaunchContext:
"""What the Firecracker launch needs from the consolidated sequence."""
@@ -81,48 +83,12 @@ def persist_env_var_secret(private_key: Path, guest_ip: str, secret: str) -> Non
input=secret, capture_output=True, text=True, check=False,
)
if proc.returncode != 0:
raise InfraLaunchError(
raise ConsolidatedLaunchError(
f"failed to persist {ENV_VAR_SECRET_NAME} in agent VM: "
f"{proc.stderr.strip() or '<no stderr>'}"
)
def attach_ca_to_agent_vm(run_dir: Path, ca_pem: str) -> None:
"""(Re)attach one running agent VM (identified by its `run_dir`) to the
current gateway: replace its trusted gateway CA with `ca_pem` and rebuild
its trust store over SSH (PRD 0081). Unconditional install there is one
gateway, so no fingerprint match is needed. Bare-pipe input keeps the cert
off argv.
Fail hard: raises `InfraLaunchError` (naming the bottle) if the run dir is
malformed or the push fails the base reconcile aggregates and surfaces it
rather than leaving the agent silently unable to reach the gateway."""
guest_ip = _guest_ip_from_config(run_dir / "config.json")
private_key = run_dir / "bottle_id_ed25519"
if not guest_ip or not private_key.is_file():
raise InfraLaunchError(
f"{run_dir.name}: cannot resolve guest IP / SSH key to attach it "
f"to the gateway"
)
install = (
"umask 022; mkdir -p /usr/local/share/ca-certificates && "
f"cat > {AGENT_CA_PATH} && chmod 644 {AGENT_CA_PATH} && "
"update-ca-certificates"
)
try:
proc = subprocess.run(
util.ssh_base_argv(private_key, guest_ip) + [install],
input=ca_pem, capture_output=True, text=True, check=False,
)
except OSError as exc:
raise InfraLaunchError(f"{run_dir.name} ({guest_ip}): CA push failed: {exc}") from exc
if proc.returncode != 0:
raise InfraLaunchError(
f"{run_dir.name} ({guest_ip}): CA push failed: "
f"{proc.stderr.strip() or '<no stderr>'}"
)
def _reprovision_running_bottles(client: OrchestratorClient) -> None:
"""Read keys from live agent VMs and restore the restarted gateway."""
try:
@@ -195,8 +161,7 @@ def deprovision_consolidated(
__all__ = [
"LaunchContext",
"launch_consolidated",
"attach_ca_to_agent_vm",
"deprovision_consolidated",
"InfraLaunchError",
"ConsolidatedLaunchError",
"OrchestratorStartError",
]
@@ -84,7 +84,7 @@ def _config(
vcpus: int,
mem_mib: int,
guest_mac: str,
data_drive: Path | None = None,
data_drives: tuple[Path, ...] = (),
extra_boot_args: str = "",
) -> dict[str, object]:
drives: list[dict[str, object]] = [
@@ -95,12 +95,15 @@ def _config(
"is_read_only": False,
}
]
# A second virtio-block device (guest /dev/vdb) — the infra VM's
# persistent registry "volume", a host-side ext4 file that outlives the
# ephemeral rootfs across VM restarts.
if data_drive is not None:
# Extra virtio-block devices — the infra VMs' persistent "volumes",
# host-side ext4 files that outlive the ephemeral rootfs across VM restarts.
# They appear to the guest as /dev/vdb, /dev/vdc, ... in list order (after
# the root /dev/vda), so callers must keep the order stable: the orchestrator
# attaches its registry (vdb); the gateway attaches its CA (vdb) then its
# git-gate state (vdc).
for i, data_drive in enumerate(data_drives):
drives.append({
"drive_id": "data",
"drive_id": f"data{i}",
"path_on_host": str(data_drive),
"is_root_device": False,
"is_read_only": False,
@@ -138,7 +141,7 @@ def boot(
mem_mib: int = 2048,
guest_mac: str = "06:00:AC:10:00:02",
detached: bool = False,
data_drive: Path | None = None,
data_drives: tuple[Path, ...] = (),
extra_boot_args: str = "",
) -> VmHandle:
"""Write the config and launch the VMM. Returns once the process is
@@ -155,7 +158,7 @@ def boot(
_config(
rootfs=rootfs, tap=tap, guest_ip=guest_ip, host_ip=host_ip,
pubkey=pubkey, vcpus=vcpus, mem_mib=mem_mib, guest_mac=guest_mac,
data_drive=data_drive, extra_boot_args=extra_boot_args,
data_drives=data_drives, extra_boot_args=extra_boot_args,
),
indent=2,
))
+72 -8
View File
@@ -19,6 +19,7 @@ singleton flock) is `FirecrackerInfraService` (`infra.py`).
from __future__ import annotations
import subprocess
from pathlib import Path
from urllib.parse import urlparse
from ...gateway import (
@@ -27,6 +28,7 @@ from ...gateway import (
GatewayError,
GatewayTransport,
)
from ...log import die, info
from .. import util as backend_util
from . import infra_vm, netpool, util
from .gateway_transport import FirecrackerGatewayTransport
@@ -49,6 +51,26 @@ _GUEST_GATEWAY_JWT_PATH = infra_vm._GUEST_GATEWAY_JWT_PATH
# the gateway's TLS interception. Host-side only (SSH cat), so it lives here.
_GATEWAY_CA_PATH = "/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem"
# The gateway VM's persistent CA volume — a small ext4 file the gateway init
# mounts at mitmproxy's confdir (see `infra_vm._GATEWAY_CA_MOUNT`) so the
# self-generated CA SURVIVES a gateway-VM rebuild/restart. Without it the CA
# lives only in the ephemeral per-boot rootfs, so every rebuild mints a fresh CA
# that every already-running bottle distrusts, failing the TLS handshake (the
# firecracker analogue of the docker fix's persistent CA bind-mount — issue
# #450). Co-located with the orchestrator's registry volume under the infra dir,
# which outlives the ephemeral rootfs. mitmproxy is tiny; 16M is ample.
_CA_VOLUME_SIZE = "16M"
# The gateway VM's persistent git-gate volume — a (sparse) ext4 file the gateway
# init mounts, then bind-mounts onto /git + /git-gate/creds (see
# `infra_vm._GATEWAY_GIT_MOUNT`), so per-bottle bare repos and deploy creds
# SURVIVE a gateway-VM rebuild. Without it a restart drops every already-running
# bottle's git-gate state and its agent 404s on fetch/push (same class as the CA
# — issue #512). Bare repos hold upstream history, so this is sized far larger
# than the CA volume; mke2fs leaves the file sparse, so the host only stores
# blocks actually used.
_GIT_VOLUME_SIZE = "8G"
_CA_FETCH_TIMEOUT_SECONDS = 15.0
@@ -69,17 +91,13 @@ class FirecrackerGateway(Gateway):
self._orchestrator_url = ""
self._gateway_token = ""
def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> bool:
def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> None:
"""Boot the gateway VM on its link resolving policy against
`orchestrator_url`, then seed `gateway_token` over SSH. The orchestrator's
guest IP is parsed from the URL and passed on the cmdline (`bb_orch=`), so
the baked init stays IP-independent. Boot the orchestrator first the
gateway daemons reach it at startup. The gateway never mints, so it holds
no signing key, only this token (#469).
Always returns True: the firecracker gateway VM is booted fresh here
(this runs only on the infra cold-boot path), so it is always a cold
boot that minted a new CA the caller reconciles running bottles."""
no signing key, only this token (#469)."""
self._orchestrator_url = orchestrator_url
self._gateway_token = gateway_token
if not self._orchestrator_url:
@@ -98,18 +116,24 @@ class FirecrackerGateway(Gateway):
f"cannot resolve orchestrator guest IP from {self._orchestrator_url!r}"
)
# Boot on the gateway link from the gateway rootfs, then push the token
# the init waits for before starting the data plane.
# the init waits for before starting the data plane. Two persistent
# volumes ride along, attached in a FIXED order the gateway init depends
# on: the CA volume as /dev/vdb (mounted at mitmproxy's confdir) and the
# git-gate volume as /dev/vdc (bind-mounted onto /git + /git-gate/creds).
# Both keep gateway-side state STABLE across rebuilds so already-running
# bottles keep working — TLS interception (issue #450) and git-gate fetch
# /push (issue #512) respectively.
vm = infra_vm.boot_vm(
name=GATEWAY_NAME, slot=netpool.gw_slot(), run_dir=infra_vm._gw_dir(),
role="gateway", mem_mib=_GW_MEM_MIB,
extra_boot_args=f"bb_orch={orchestrator_guest_ip}",
data_drives=(self._ensure_ca_volume(), self._ensure_git_volume()),
)
infra_vm.push_secret(
vm, self._gateway_token, _GUEST_GATEWAY_JWT_PATH,
"the gateway JWT to the gateway VM (its data plane will not start)",
)
self._vm = vm
return True
def is_running(self) -> bool:
return infra_vm._pidfile_alive(infra_vm._gw_dir())
@@ -121,6 +145,46 @@ class FirecrackerGateway(Gateway):
infra_vm._kill_pidfile(infra_vm._gw_dir())
infra_vm._pid_file(infra_vm._gw_dir()).unlink(missing_ok=True)
def _ensure_ca_volume(self) -> Path:
"""Create the empty ext4 CA volume on first use; reuse it after.
A fresh (empty) volume makes mitmproxy generate a CA into it on first
boot; every later boot reuses the CA already on the volume which is
what keeps the CA stable across gateway-VM rebuilds. The mirror of
`FirecrackerOrchestrator._ensure_registry_volume`."""
vol = infra_vm._gw_dir() / "gateway-ca.ext4"
if vol.exists():
return vol
info(f"creating gateway CA volume {vol} ({_CA_VOLUME_SIZE})")
proc = subprocess.run(
["mke2fs", "-q", "-t", "ext4", "-F", str(vol), _CA_VOLUME_SIZE],
capture_output=True, text=True, check=False,
)
if proc.returncode != 0:
vol.unlink(missing_ok=True)
die(f"creating gateway CA volume failed: {proc.stderr.strip()}")
return vol
def _ensure_git_volume(self) -> Path:
"""Create the empty ext4 git-gate volume on first use; reuse it after.
Empty on first boot (the gateway init lays out `git/` + `creds/` subdirs
and bind-mounts them); every later boot reuses whatever repos + creds the
volume already holds which is what keeps git-gate state stable across a
gateway-VM rebuild (issue #512). Sibling of `_ensure_ca_volume`."""
vol = infra_vm._gw_dir() / "gateway-git.ext4"
if vol.exists():
return vol
info(f"creating gateway git-gate volume {vol} ({_GIT_VOLUME_SIZE})")
proc = subprocess.run(
["mke2fs", "-q", "-t", "ext4", "-F", str(vol), _GIT_VOLUME_SIZE],
capture_output=True, text=True, check=False,
)
if proc.returncode != 0:
vol.unlink(missing_ok=True)
die(f"creating gateway git-gate volume failed: {proc.stderr.strip()}")
return vol
def address(self) -> str:
"""The gateway VM's guest IP — the agent-facing target agent VMs'
gateway-port traffic is DNAT'd to."""
+1 -9
View File
@@ -67,17 +67,9 @@ class FirecrackerInfraService(InfraService):
# holds the signing key) mints the role-scoped `gateway` JWT for the
# gateway, which never sees the key (#469).
orchestrator.ensure_running(startup_timeout=startup_timeout)
cold_booted = self.gateway().connect_to_orchestrator(
self.gateway().connect_to_orchestrator(
orchestrator.gateway_url(), orchestrator.mint_gateway_token())
infra_vm.record_booted_version(want)
# The fresh gateway rootfs minted a new CA and lost every bottle's
# git-gate/token state; reconcile the already-running bottles against
# it so a gateway rebuild doesn't silently break their egress
# (PRD 0081). Cold-boot only — the adopt fast-paths above never reach
# here, so a healthy current gateway is left untouched.
if cold_booted:
from .backend import FirecrackerBottleBackend
FirecrackerBottleBackend().attach_bottled_agents_to_gateway()
return url
def stop(self) -> None:
+40 -6
View File
@@ -53,10 +53,25 @@ from . import firecracker_vm, infra_artifact, netpool, util
# tokens with the same key the host CLI signs with — the host token file stays
# the single source of truth, never clobbered per-backend (issue #469 review).
_GUEST_SIGNING_KEY_PATH = "/var/lib/bot-bottle/orchestrator-token"
# The gateway VM's pre-minted `gateway` JWT path (rootfs, not /dev/vdb — the
# data plane has no registry volume and never opens the DB). Pushed post-boot;
# the gateway daemons present it to the orchestrator, and never see the key.
# The gateway VM's pre-minted `gateway` JWT path (rootfs, not /dev/vdb — the JWT
# is re-pushed every boot, so it needn't persist). Pushed post-boot; the gateway
# daemons present it to the orchestrator, and never see the key.
_GUEST_GATEWAY_JWT_PATH = "/var/lib/bot-bottle/gateway-jwt"
# The gateway VM's persistent CA volume mount point — mitmproxy's confdir. The
# gateway boots with a persistent /dev/vdb CA volume (see
# `FirecrackerGateway._ensure_ca_volume`) mounted here so the self-generated CA
# survives a gateway-VM rebuild; without it every rebuild mints a fresh CA that
# already-running bottles distrust, breaking the TLS handshake (issue #450).
_GATEWAY_CA_MOUNT = "/home/mitmproxy/.mitmproxy"
# The gateway VM's persistent git-gate volume staging mount (/dev/vdc). The
# gateway boots with this volume (see `FirecrackerGateway._ensure_git_volume`)
# and the init bind-mounts its `git/` + `creds/` subdirs onto the load-bearing
# `/git` and `/git-gate/creds` paths, so per-bottle bare repos + deploy creds
# survive a gateway-VM rebuild; without it a restart drops every already-running
# bottle's git-gate state and its agent 404s on fetch/push (issue #512).
_GATEWAY_GIT_MOUNT = "/var/lib/bot-bottle-gitgate"
_GATEWAY_GIT_REPO_ROOT = "/git"
_GATEWAY_GIT_CREDS_DIR = "/git-gate/creds"
# The two per-plane rootfs source images. The orchestrator VM boots a control
# plane + buildah rootfs (Dockerfile.orchestrator.fc, FROM orchestrator); the
@@ -192,11 +207,12 @@ def boot_vm(
run_dir: Path,
role: str,
mem_mib: int,
data_drive: Path | None = None,
data_drives: tuple[Path, ...] = (),
extra_boot_args: str = "",
) -> InfraVm:
"""Boot the `role` infra VM from its per-plane rootfs on `slot`'s link.
Records the PID."""
Records the PID. `data_drives` are attached as /dev/vdb, /dev/vdc, ... in
order, so callers must keep the order stable (see `firecracker_vm._config`)."""
if not netpool.tap_present(slot.iface):
die(f"infra link {slot.iface} not present.\n"
f" ./cli.py backend setup --backend=firecracker")
@@ -218,7 +234,7 @@ def boot_vm(
name=name, rootfs=rootfs, tap=slot.iface,
guest_ip=slot.guest_ip, host_ip=slot.host_ip, pubkey=pubkey,
run_dir=run_dir, mem_mib=mem_mib, detached=True,
data_drive=data_drive, extra_boot_args=boot_args,
data_drives=data_drives, extra_boot_args=boot_args,
)
_pid_file(run_dir).write_text(str(vm.process.pid))
return InfraVm(guest_ip=slot.guest_ip, private_key=private_key, vm=vm)
@@ -448,6 +464,24 @@ def _gateway_init() -> str:
bot-bottle.db (PRD 0070 / #469). If the JWT never arrives, REFUSE to start
rather than run without auth."""
return _init_head() + f"""
# Persistent CA volume (second virtio-block device, /dev/vdb) mounted at
# mitmproxy's confdir, so the self-generated mitmproxy CA survives gateway-VM
# rebuilds (issue #450). On first boot the volume is empty and mitmproxy mints a
# CA into it; every later boot reuses it. Must mount BEFORE the data plane (hence
# mitmproxy) starts.
mkdir -p {_GATEWAY_CA_MOUNT}
mount -t ext4 /dev/vdb {_GATEWAY_CA_MOUNT} 2>/dev/null || true
# Persistent git-gate volume (/dev/vdc): its git/ + creds/ subdirs are
# bind-mounted onto the load-bearing /git and /git-gate/creds so per-bottle bare
# repos + deploy creds survive a gateway-VM rebuild (issue #512). On first boot
# the volume is empty; the subdirs are created here. Must mount BEFORE the data
# plane (hence git-http) starts, and before any per-bottle provisioning writes.
mkdir -p {_GATEWAY_GIT_MOUNT}
mount -t ext4 /dev/vdc {_GATEWAY_GIT_MOUNT} 2>/dev/null || true
mkdir -p {_GATEWAY_GIT_MOUNT}/git {_GATEWAY_GIT_MOUNT}/creds
mkdir -p {_GATEWAY_GIT_REPO_ROOT} {_GATEWAY_GIT_CREDS_DIR}
mount --bind {_GATEWAY_GIT_MOUNT}/git {_GATEWAY_GIT_REPO_ROOT} 2>/dev/null || true
mount --bind {_GATEWAY_GIT_MOUNT}/creds {_GATEWAY_GIT_CREDS_DIR} 2>/dev/null || true
ORCH=$(sed -n 's/.*bb_orch=\\([^ ]*\\).*/\\1/p' /proc/cmdline)
GW_JWT=""
i=0
+1 -1
View File
@@ -51,7 +51,7 @@ from .bottle import FirecrackerBottle
from .bottle_plan import FirecrackerBottlePlan
from ...orchestrator.store.config_store import resolve_teardown_timeout
from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME
from .infra_launch import (
from .consolidated_launch import (
launch_consolidated,
persist_env_var_secret,
deprovision_consolidated,
@@ -104,7 +104,7 @@ class FirecrackerOrchestrator(Orchestrator):
vm = infra_vm.boot_vm(
name=ORCHESTRATOR_NAME, slot=netpool.orch_slot(),
run_dir=infra_vm._orch_dir(), role="orchestrator", mem_mib=_ORCH_MEM_MIB,
data_drive=self._ensure_registry_volume(),
data_drives=(self._ensure_registry_volume(),),
)
# Push the host-canonical signing key (the init waits for it before
# starting the control plane). It comes through the shared provisioning
-54
View File
@@ -1,54 +0,0 @@
"""The shared gateway-attach reconcile flow (PRD 0081).
`BottleBackend.attach_bottled_agents_to_gateway()` delegates here so every
backend reconciles running bottles against a freshly-booted gateway *the same
way* the control flow + fail-hard policy live in one place and backends
override only the primitives (see ADR 0006). Kept out of `backend/base.py` so
the backend contract module stays lean (the base.py size guardrail).
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
from .base import InfraLaunchError
if TYPE_CHECKING:
from .base import BottleBackend
@dataclass(frozen=True)
class GatewayAttachResources:
"""Everything a running bottle needs to (re)attach to the current gateway on
a cold boot (PRD 0081). Gathered once per reconcile and handed to every
bottle. The CA now; egress secrets and git-gate state join as the reconcile
grows (#516)."""
ca_pem: str
def reconcile_running_bottles(backend: "BottleBackend[Any, Any, Any]") -> None:
"""Reconcile every already-running bottle against the freshly-(re)booted
gateway: gather the attach resources once, then attach every live bottle.
Fail hard, never skip: a bottle that silently can't reach the fresh gateway
(its egress just starts failing TLS) is worse than a loud bring-up failure,
so any attach failure aborts bring-up. Every bottle is attempted first and
the failures are raised together, so one bring-up surfaces the full blast
radius rather than one bottle at a time."""
resources = backend._gateway_attach_resources()
failures: list[str] = []
for bottle in backend._running_bottles():
try:
backend._attach_bottle_to_gateway(bottle, resources)
except InfraLaunchError as exc:
failures.append(str(exc))
if failures:
raise InfraLaunchError(
f"could not attach {len(failures)} running bottle(s) to the "
f"freshly-booted gateway:\n " + "\n ".join(failures)
)
__all__ = ["GatewayAttachResources", "reconcile_running_bottles"]
+1 -16
View File
@@ -14,7 +14,6 @@ from ...git_gate import GitGatePlan
from ...supervisor.plan import SupervisePlan
from ...manifest import Manifest
from .. import ActiveAgent, BottleBackend, BottleImages, BottleSpec
from ..gateway_attach import GatewayAttachResources
from . import cleanup as _cleanup
from . import enumerate as _enumerate
from . import launch as _launch
@@ -26,7 +25,7 @@ from .bottle_plan import MacosContainerBottlePlan
class MacosContainerBottleBackend(
BottleBackend["MacosContainerBottlePlan", "MacosContainerBottleCleanupPlan", str]
BottleBackend["MacosContainerBottlePlan", "MacosContainerBottleCleanupPlan"]
):
"""Apple Container backend. Selected by
`BOT_BOTTLE_BACKEND=macos-container` or
@@ -118,19 +117,5 @@ class MacosContainerBottleBackend(
def enumerate_active(self) -> Sequence[ActiveAgent]:
return _enumerate.enumerate_active()
def _gateway_attach_resources(self) -> GatewayAttachResources:
from .infra import MacosInfraService
return GatewayAttachResources(ca_pem=MacosInfraService().ca_cert_pem())
def _running_bottles(self) -> Sequence[str]:
from .infra_launch import running_agent_containers
return running_agent_containers()
def _attach_bottle_to_gateway(
self, bottle: str, resources: GatewayAttachResources,
) -> None:
from .infra_launch import push_ca_to_container
push_ca_to_container(bottle, resources.ca_pem)
def supervise_mcp_url(self, plan: MacosContainerBottlePlan) -> str:
return plan.agent_supervise_url
@@ -34,7 +34,6 @@ every real command arrives through `container exec`.
from __future__ import annotations
import tempfile
from dataclasses import dataclass
from ...egress import EgressPlan
@@ -43,9 +42,7 @@ from ...log import info
from ...orchestrator.client import OrchestratorClient, OrchestratorClientError
from ...orchestrator.reprovision import reprovision_bottles
from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME
from ..base import InfraLaunchError
from ..provision_bottle import deprovision_bottle, provision_bottle
from ..util import AGENT_CA_PATH
from . import util as container_mod
from .enumerate import CONTAINER_NAME_PREFIX, EnumerationError, enumerate_active
from .gateway import GATEWAY_NETWORK
@@ -53,6 +50,10 @@ from .gateway_transport import MacosGatewayTransport
from .infra import MacosInfraService, OrchestratorStartError
class ConsolidatedLaunchError(RuntimeError):
"""The consolidated register/provision sequence could not complete."""
@dataclass(frozen=True)
class GatewayEndpoint:
"""What the agent `container run` needs to reach the shared gateway.
@@ -97,52 +98,6 @@ def ensure_gateway(
return endpoint
def running_agent_containers() -> list[str]:
"""Every running agent container's name — the bottles the bring-up reconcile
attaches to the fresh gateway (PRD 0081). Raises `EnumerationError` if the
live set can't be determined (fail hard rather than reconcile a partial
set)."""
return [f"{CONTAINER_NAME_PREFIX}{agent.slug}" for agent in enumerate_active()]
def push_ca_to_container(name: str, ca_pem: str) -> None:
"""(Re)attach one running agent container to the current gateway: replace
its trusted gateway CA with `ca_pem` and rebuild its trust store via
`container cp` + `container exec` (PRD 0081). Unconditional install there
is one gateway, so no fingerprint match is needed.
Fail hard: raises `InfraLaunchError` (naming the container) on any failure
the base reconcile aggregates and surfaces it rather than leaving the agent
silently unable to reach the gateway."""
mkdir = container_mod.run_container_argv(
["container", "exec", name, "mkdir", "-p",
"/usr/local/share/ca-certificates"]
)
if mkdir.returncode != 0:
raise InfraLaunchError(
f"CA push to {name} failed (mkdir): {(mkdir.stderr or '').strip()}"
)
with tempfile.NamedTemporaryFile("w", suffix=".crt") as tmp:
tmp.write(ca_pem)
tmp.flush()
cp = container_mod.run_container_argv(
["container", "cp", tmp.name, f"{name}:{AGENT_CA_PATH}"]
)
if cp.returncode != 0:
raise InfraLaunchError(
f"CA push to {name} failed (cp): {(cp.stderr or '').strip()}"
)
ex = container_mod.run_container_argv(
["container", "exec", name, "sh", "-c",
f"chmod 644 {AGENT_CA_PATH} && update-ca-certificates"]
)
if ex.returncode != 0:
raise InfraLaunchError(
f"CA push to {name} failed (update-ca-certificates): "
f"{(ex.stderr or '').strip()}"
)
def _reprovision_running_bottles(endpoint: GatewayEndpoint) -> None:
"""Recover keys from live Apple containers and restore gateway tokens."""
try:
@@ -245,12 +200,10 @@ __all__ = [
"GatewayEndpoint",
"LaunchContext",
"ensure_gateway",
"running_agent_containers",
"push_ca_to_container",
"live_source_ips",
"register_agent",
"deprovision_consolidated",
"InfraLaunchError",
"ConsolidatedLaunchError",
"OrchestratorStartError",
"GATEWAY_NETWORK",
]
@@ -104,15 +104,11 @@ class MacosGateway(Gateway):
container_mod.build_image(
self.image_ref, str(self._repo_root), dockerfile="Dockerfile.gateway")
def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> bool:
def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> None:
"""Bind the gateway to this orchestrator and (re)start it, dual-homed on
the agent + control networks, resolving policy against `orchestrator_url`
(the orchestrator's control-network address — Apple has no container
DNS) and presenting `gateway_token`.
Always returns True: this recreates the gateway container unconditionally
(a cold boot), so the caller reconciles running bottles against it
(PRD 0081)."""
DNS) and presenting `gateway_token`."""
self._orchestrator_url = orchestrator_url
self._gateway_token = gateway_token
# Fail closed on a missing policy source or token: the resolver-only data
@@ -160,7 +156,6 @@ class MacosGateway(Gateway):
f"gateway container failed to start: "
f"{(result.stderr or '').strip() or '<no stderr>'}"
)
return True
def is_running(self) -> bool:
return container_mod.container_is_running(self.name)
+1 -9
View File
@@ -138,15 +138,7 @@ class MacosInfraService(InfraService):
# `gateway` JWT and hands it to the gateway, which never sees the key
# (#469).
url = orchestrator.url()
cold_booted = gateway.connect_to_orchestrator(
url, orchestrator.mint_gateway_token())
# A (re)created gateway container minted/mounted its CA afresh and lost
# every bottle's per-bottle state; reconcile the already-running bottles
# against it (PRD 0081). Skipped when a healthy current gateway was left
# untouched — no cold boot, nothing to reconcile.
if cold_booted:
from .backend import MacosContainerBottleBackend
MacosContainerBottleBackend().attach_bottled_agents_to_gateway()
gateway.connect_to_orchestrator(url, orchestrator.mint_gateway_token())
return url
def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
+2 -2
View File
@@ -5,7 +5,7 @@ proxies egress through the one per-host gateway, replacing the per-bottle
companion container removed in #385.
The order differs from docker's, forced by Apple Container 1.0.0 having no
`--ip` (see `infra_launch`): the agent is started *before* it is
`--ip` (see `consolidated_launch`): the agent is started *before* it is
registered, because its DHCP-assigned address the attribution key does not
exist until then.
@@ -64,7 +64,7 @@ from . import nested_containers as nested_containers_mod
from .bottle_plan import MacosContainerBottlePlan
from ...orchestrator.store.config_store import resolve_teardown_timeout
from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME, new_env_var_secret
from .infra_launch import (
from .consolidated_launch import (
GatewayEndpoint,
ensure_gateway,
register_agent,
+1 -1
View File
@@ -2,7 +2,7 @@
Register a bottle with the orchestrator and provision its git-gate state into
the shared gateway (`provision_bottle`), and the inverse teardown
(`deprovision_bottle`). Backend-neutral each backend's infra_launch
(`deprovision_bottle`). Backend-neutral each backend's consolidated_launch
drives these through its own `GatewayTransport` rather than re-implementing
them. The git-gate half of the work lives in `provision_gateway`.
"""
+3 -3
View File
@@ -26,10 +26,10 @@ from .base import ActiveAgent, BackendStatus, BottleBackend
# Tests may replace _backends with a {name: fake} dict via patch.object;
# _get_backends() returns the current module-level value as-is when it
# is not None, so test fakes take effect without triggering real imports.
_backends: dict[str, BottleBackend[Any, Any, Any]] | None = None
_backends: dict[str, BottleBackend[Any, Any]] | None = None
def _get_backends() -> dict[str, BottleBackend[Any, Any, Any]]:
def _get_backends() -> dict[str, BottleBackend[Any, Any]]:
"""Return the registry of all backend instances, loading lazily on first call."""
global _backends # pylint: disable=global-statement
if _backends is None:
@@ -48,7 +48,7 @@ def get_bottle_backend(
name: str | None = None,
*,
prompt: bool = True,
) -> BottleBackend[Any, Any, Any]:
) -> BottleBackend[Any, Any]:
"""Resolve the bottle backend.
`name` precedence:
+2 -8
View File
@@ -122,18 +122,12 @@ class Gateway(abc.ABC):
return
@abc.abstractmethod
def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> bool:
def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> None:
"""Bind the gateway to this orchestrator and bring it up: store the URL +
the pre-minted `gateway` token as instance state, then (re)start the
gateway unit carrying the mitmproxy CA + that token, resolving policy
against `orchestrator_url`. Idempotent a healthy, current gateway on
the same binding is left alone; a changed binding reconciles it.
Returns True when the gateway was actually (re)brought up (a cold boot,
so its mitmproxy minted a fresh CA and lost its per-bottle state), False
when a healthy current gateway was left untouched. The bring-up flow
uses this as the cold-boot signal that gates the running-bottle
reconcile (`attach_bottled_agents_to_gateway`, PRD 0081)."""
the same binding is left alone; a changed binding reconciles it."""
@abc.abstractmethod
def is_running(self) -> bool:
-21
View File
@@ -184,27 +184,6 @@ class OrchestratorClient:
)
return True
def update_agent_secret(
self, bottle_id: str, name: str, value: str, env_var_secret: str,
) -> bool:
"""Update ONE egress token for a running bottle in place
(`POST /bottles/<id>/secret`) the single-secret form of
`reprovision_gateway`, for pushing a freshly-refreshed host credential
into a bottle without a relaunch. Returns True on success, False when the
orchestrator doesn't know the bottle (404)."""
status, _ = self._request(
"POST",
f"/bottles/{bottle_id}/secret",
{"name": name, "value": value, "env_var_secret": env_var_secret},
)
if status == 404:
return False
if not 200 <= status < 300:
raise OrchestratorClientError(
f"update_agent_secret {bottle_id}: HTTP {status}"
)
return True
def teardown_bottle(self, bottle_id: str) -> bool:
"""Tear a bottle down (`DELETE /bottles/<id>`). False if the
orchestrator didn't know it (404) — idempotent for cleanup paths."""
-29
View File
@@ -200,35 +200,6 @@ def dispatch( # pylint: disable=too-many-return-statements,too-many-branches
return 200, {"reprovisioned": True}
return 404, {"error": "no stored secrets for this bottle"}
if (
method == "POST"
and route.startswith("/bottles/")
and route.endswith("/secret")
):
# Update ONE egress token for a running bottle in place — the
# single-secret form of reprovision, for refreshing a short-lived host
# credential (e.g. the Codex access token) without a relaunch. cli-only
# (not in _GATEWAY_ROUTES): a bottle must never set its own tokens.
bottle_id = route[len("/bottles/") : -len("/secret")]
try:
data = _parse_json_object(body)
except ValueError as e:
return 400, {"error": f"invalid JSON: {e}"}
name = data.get("name")
value = data.get("value")
env_var_secret = data.get("env_var_secret")
if (
not isinstance(name, str) or not name
or not isinstance(value, str) or not value
or not isinstance(env_var_secret, str) or not env_var_secret
):
return 400, {
"error": "name, value, and env_var_secret (non-empty strings) are required"
}
if orch.update_agent_secret(bottle_id, name, value, env_var_secret):
return 200, {"updated": True}
return 404, {"error": "no such bottle"}
if method == "DELETE" and route.startswith("/bottles/"):
bottle_id = route[len("/bottles/"):]
if orch.teardown_bottle(bottle_id):
-24
View File
@@ -377,30 +377,6 @@ class OrchestratorCore:
return False
return True
def update_agent_secret(
self, bottle_id: str, name: str, value: str, env_var_secret: str,
) -> bool:
"""Update ONE egress token for a known bottle in place — the
single-secret form of ``reprovision_from_secret``.
Sets the in-memory token AND upserts the single re-encrypted row under
*env_var_secret* (the same key the rest of the rows are encrypted with, so
the whole set stays decryptable by a later ``reprovision_from_secret``),
leaving every other token untouched. Returns False if the bottle is
unknown.
Unlike ``reprovision_from_secret`` (which restores the values captured at
launch), this pushes a *caller-supplied* value used to refresh a
short-lived host credential (e.g. the Codex access token) into a
still-running bottle without a relaunch."""
from .store.secret_store import encrypt_value
if self.registry.get(bottle_id) is None:
return False
self._tokens.setdefault(bottle_id, {})[name] = value
self.registry.store_agent_secret(
bottle_id, name, encrypt_value(env_var_secret, value))
return True
# --- consolidated gateway ----------------------------------------------
def gateway_status(self) -> dict[str, object]:
@@ -366,30 +366,6 @@ class RegistryStore(DbStore):
)
self._chmod()
def store_agent_secret(
self,
bottle_id: str,
key: str,
encrypted_value: str,
secret_type: str = "injected_env_var",
) -> None:
"""Upsert ONE encrypted secret row (env-var name → ciphertext) for
*bottle_id*, leaving the bottle's other secrets untouched — the per-key
counterpart of ``store_agent_secrets``' replace-all. Delete-then-insert
because the table carries no unique constraint to `ON CONFLICT` against."""
with self._connection() as conn:
conn.execute(
"DELETE FROM bottled_agent_secrets "
"WHERE bottled_agent_id = ? AND key = ? AND type = ?",
(bottle_id, key, secret_type),
)
conn.execute(
"INSERT INTO bottled_agent_secrets "
"(bottled_agent_id, key, value, type) VALUES (?, ?, ?, ?)",
(bottle_id, key, encrypted_value, secret_type),
)
self._chmod()
def get_agent_secrets(
self,
bottle_id: str,
+34
View File
@@ -53,6 +53,15 @@ ORCHESTRATOR_AUTH_JWT_ENV = "BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT"
# the shared gateway's TLS interception, so it must not rotate on restart. See
# host_gateway_ca_dir() for why this is a host bind-mount, not a named volume.
GATEWAY_CA_DIRNAME = "gateway-ca"
# The host directories holding the gateway's persistent git-gate state — the
# per-bottle bare repos (`gateway-git`) and deploy creds (`gateway-creds`).
# Bind-mounted into the gateway container at /git and /git-gate/creds so they
# survive container recreation; without it a gateway restart drops every
# already-running bottle's git-gate state and its agent 404s on fetch/push
# (issue #512). Host bind-mounts (not named volumes) for the same reason as the
# CA dir — see host_gateway_ca_dir().
GATEWAY_GIT_DIRNAME = "gateway-git"
GATEWAY_CREDS_DIRNAME = "gateway-creds"
def bot_bottle_root() -> Path:
@@ -97,6 +106,27 @@ def host_gateway_ca_dir() -> Path:
return ca_dir
def host_gateway_git_dir() -> Path:
"""The directory holding the gateway's persistent per-bottle bare repos,
created if missing. Bind-mounted into the gateway container at /git so the
repos survive container recreation (issue #512). A host bind-mount under the
app-data root, never pruned same rationale as host_gateway_ca_dir()."""
git_dir = bot_bottle_root() / GATEWAY_GIT_DIRNAME
git_dir.mkdir(parents=True, exist_ok=True)
return git_dir
def host_gateway_creds_dir() -> Path:
"""The directory holding the gateway's persistent per-bottle git-gate deploy
creds, created if missing. Bind-mounted into the gateway container at
/git-gate/creds so the creds survive container recreation (issue #512). A
host bind-mount under the app-data root same rationale as
host_gateway_ca_dir()."""
creds_dir = bot_bottle_root() / GATEWAY_CREDS_DIRNAME
creds_dir.mkdir(parents=True, exist_ok=True)
return creds_dir
def host_signing_key(filename: str) -> str:
"""A per-host signing key at `<root>/<filename>`, minted (256-bit, url-safe)
and persisted 0600 on first use, then reused.
@@ -143,10 +173,14 @@ __all__ = [
"ORCHESTRATOR_TOKEN_ENV",
"ORCHESTRATOR_AUTH_JWT_ENV",
"GATEWAY_CA_DIRNAME",
"GATEWAY_GIT_DIRNAME",
"GATEWAY_CREDS_DIRNAME",
"bot_bottle_root",
"host_db_path",
"host_db_dir",
"host_gateway_ca_dir",
"host_gateway_git_dir",
"host_gateway_creds_dir",
"host_signing_key",
"host_orchestrator_token",
]
@@ -1,65 +0,0 @@
# ADR 0006: Shared behaviour lives in the base backend
- **Status:** Accepted
- **Date:** 2026-07-27
- **Deciders:** didericis
## Context
`BottleBackend` has three concrete implementations (Firecracker, docker,
macOS-container). Cross-backend operations can be expressed two ways:
1. **Abstract method per backend** — the base declares `@abstractmethod foo()`
and each backend writes its own `foo()` end to end.
2. **Template method** — the base owns the *control flow* of `foo()` as a
concrete method and each backend overrides small **primitives** it calls.
Form 1 gives each backend total freedom, which is exactly the problem: the three
implementations drift. The bring-up reconcile (PRD 0081) first shipped as form 1
and immediately diverged — each backend re-implemented "gather CA → list running
bottles → push to each", and one backend's copy silently skipped failures while
another's aborted, an inconsistency caught only in review (#519). The behaviour
that must be identical (the loop, the error policy, the ordering) was duplicated
in the place least likely to stay in sync.
## Decision
Prefer the template-method form for cross-backend behaviour: **encode shared
control flow and policy in the base backend; backends override primitives, not
control flow.** Lean toward *less* freedom in the concrete backends and *more*
consistency in the base.
Concretely, a cross-backend operation on `BottleBackend` should be a concrete
method that:
- owns the sequencing, iteration, and error policy (e.g. fail-hard vs.
best-effort, fail-fast vs. attempt-all-then-aggregate); and
- delegates only the irreducibly backend-specific steps to `@abstractmethod`
primitives with narrow, well-typed signatures.
The first application is `attach_bottled_agents_to_gateway()` (PRD 0081): the
base gathers resources once, attempts every running bottle, and raises an
aggregate `InfraLaunchError` if any failed; backends supply only
`_gateway_attach_resources()`, `_running_bottles()`, and
`_attach_bottle_to_gateway()`.
A backend-specific `@abstractmethod` with no shared control flow (e.g.
`_launch_impl`, `enumerate_active`) stays as form 1 — this ADR is about
operations whose *shape* should be uniform, not about banning abstract methods.
## Consequences
- The error policy, ordering, and loop for a cross-backend operation have one
source of truth; a new backend cannot forget them or diverge.
- New backends implement smaller, more focused primitives instead of re-deriving
a whole flow.
- The base carries more logic and needs generic-enough primitive signatures
(e.g. a backend-specific attach-target type parameter on `BottleBackend`).
- Genuinely backend-specific one-offs still use a plain abstract method; the
guidance is a default, not an absolute.
## Links
- PRD 0081 (`docs/prds/0081-reprovision-gateway-state-on-bringup.md`).
- Review that prompted this: PR #519.
- `bot_bottle/backend/base.py` (`BottleBackend.attach_bottled_agents_to_gateway`).
-10
View File
@@ -55,13 +55,3 @@ agent has two observable states:
Shorthand for an active (running) Bottled Agent. Used in the supervisor TUI
and discovery layer to mean "a bottle whose agent runtime is currently up."
## Infra
The per-host pair of shared services every bottle attaches to: the
**orchestrator** (control plane — holds the signing key, owns the registry DB,
mints tokens) and the **gateway** (data plane — TLS-inspecting egress proxy,
git-gate, supervise). Brought up as an idempotent singleton pair by each
backend's `InfraService` (two containers on docker/macOS, two microVMs on
Firecracker). The per-backend `infra_launch` module composes the bring-up and
bottle register/provision sequence against this pair.
@@ -1,136 +0,0 @@
# PRD 0081: Reprovision gateway-dependent state on gateway bring-up
- **Status:** Active
- **Author:** claude
- **Created:** 2026-07-26
- **Issue:** #510, #512
## Summary
When the gateway is (re)built, reconcile every already-running bottle against the
fresh gateway instead of persisting the gateway's state. On a gateway cold boot,
the gateway reconciles all live bottles in one flow: **replace** each agent's
trusted CA with the freshly-minted gateway CA, **re-provision** each bottle's
git-gate repos + creds onto the gateway, and **restore** each bottle's egress
tokens. One mechanism across all three services; the CA rotates for free on every
bring-up.
## Problem
A gateway rebuild/restart silently breaks every already-running bottle:
- **CA (#510).** The gateway's mitmproxy mints a new CA on a fresh rootfs; agents
still trust the old one, so egress fails TLS verification (`SSL certificate
verification failed`).
- **git-gate (#512).** Per-bottle bare repos (`/git/<id>`) + deploy creds
(`/git-gate/creds/<id>`) live in the gateway's ephemeral rootfs; a rebuild wipes
them and the agent 404s on fetch/push.
Both are the same root cause: per-bottle gateway-dependent state is provisioned
**once, at bottle launch**, and nothing restores it for already-running bottles
when the gateway comes back. The existing launch-time reprovision
(`reprovision_bottles`) restores **only** egress tokens, and only as a side effect
of the *next* launch.
Persisting the state (a host bind-mount / a per-VM data volume per service) was
prototyped and rejected: it differs per service (a volume for the CA, another for
git-gate, reprovision for tokens), it pins the CA static forever (no rotation),
and it adds volume surface that `docker volume prune` / a wiped cache can silently
destroy (the original #450 failure mode).
## Goals / Success Criteria
- A gateway (re)boot restores **all** running bottles' gateway-dependent state
with no manual step and no relaunch — the agent's next egress / fetch / push
just works.
- **One** reconcile flow covering CA, git-gate, and egress tokens, rather than a
different mechanism per service.
- The CA **rotates** on every gateway bring-up (no long-lived CA), distributed to
running agents by the same reconcile.
- Reconcile failures are **loud, never hidden**: if a running bottle can't be
re-attached to the fresh gateway, the bring-up fails and names the bottle
rather than leaving it silently unable to reach the gateway (a bottle whose
egress just starts failing TLS is worse than a loud failure). Every bottle is
attempted first and the failures are raised together, so one bring-up surfaces
the full blast radius. (Reversed from an earlier "tolerate per-bottle
failures" draft after #519 review.)
- **All backends** (Firecracker, docker, macOS) reconcile through the *same*
contract — an abstract method on the backend base class, so a new backend
cannot forget to implement it and none drifts onto a bespoke mechanism.
## Non-goals
- Deliberate mid-session CA rotation *without* a gateway restart — `rotate_ca`
stays for that operator action.
- Changing source-IP attribution, `/resolve`, or the plane split (#469).
## Design
**The contract — `attach_bottled_agents_to_gateway()`, a concrete flow on the
backend base.** Reconciling running bottles against the current gateway is a
backend responsibility (only the backend can enumerate its agents and reach them
— firecracker over SSH, docker/macOS over `exec`/`cp`). To keep every backend
reconciling identically, the *control flow* lives in a concrete
`attach_bottled_agents_to_gateway()` on `BottleBackend` (`backend/base.py`) and
each backend overrides only three primitives:
1. `_gateway_attach_resources()` — gather what a bottle needs to attach (the
gateway CA now; egress secrets + git-gate state as this grows).
2. `_running_bottles()` — enumerate the live bottles as backend handles.
3. `_attach_bottle_to_gateway(bottle, resources)` — push the resources into one
bottle, raising `InfraLaunchError` (naming it) on failure.
The base gathers resources once, attempts every bottle, and — fail hard, never
skip — raises an aggregate `InfraLaunchError` if any failed, so a backend can't
drift onto a bespoke loop or a silent skip. (Encoding shared behaviour in the
base backend, with subclasses overriding primitives rather than control flow, is
[ADR 0006](../decisions/0006-shared-behavior-in-base-backend.md).) The host
calls it whenever the gateway is (re)brought up. It reprovisions **all**
registered bottles' gateway-dependent state: CA, git-gate, and egress tokens.
**Trigger — the gateway bring-up path.** The host calls
`attach_bottled_agents_to_gateway()` only when the gateway was actually
(re)brought up — the cold-boot branch of the infra bring-up (e.g.
`FirecrackerInfraService.ensure_running` after it boots a fresh pair), never on an
adopt of a healthy, current gateway (state intact). So it fires exactly when the
gateway was (re)booted — including orchestrator restarts, since the pair boots
together — and there is no bare-restart path that bypasses bring-up.
**Per-backend implementation.** Each backend's `attach_bottled_agents_to_gateway`
enumerates its live bottles and, for each, reconciles the three services against
the current gateway. The firecracker implementation, once the gateway VM is up
and its CA is available:
1. Map each live bottle's guest IP → `bottle_id` from the orchestrator registry
(`list_bottles`).
2. Install the **current** shared git-gate hooks (`git_gate_render_hook` /
`git_gate_render_access_hook`) into the fresh gateway once — rendered from
code, never from a bottle's possibly-stale state dir.
3. For each live agent VM (enumerated from its run dir):
- **CA:** SSH the current gateway CA into the agent's trust store and run
`update-ca-certificates` (unconditional replace — there is one gateway, so no
fingerprint match is needed).
- **git-gate:** rebuild the bottle's upstreams from its persisted git-gate
state dir (deploy key, known_hosts, upstream URL) and re-init its bare repos
+ per-repo creds under `/git/<bottle_id>`.
- **egress token:** read the agent's `ENV_VAR_SECRET` and feed
`reprovision_bottles`, restoring the orchestrator's in-memory tokens.
4. If any per-bottle step fails, the reconcile attempts the rest and then raises
an aggregate naming every bottle that failed — bring-up fails loudly rather
than leaving a bottle silently unable to reach the gateway.
**Retire the persistence prototype.** No CA data volume, no git-gate data volume
(the abandoned PRs #511 / #513). The launch-time `_reprovision_running_bottles`
call folds into this bring-up reconcile, so egress tokens are restored on the same
cold-boot trigger (an adopt needs no restore — the orchestrator never restarted).
**CA rotation.** Because the gateway rootfs is ephemeral, every cold boot mints a
fresh CA; reconcile is what distributes it, so a routine gateway rebuild doubles
as a CA rotation with zero extra machinery.
## Open questions
- Docker's existing host-bind-mounted CA (`host_gateway_ca_dir`): once docker's
`attach_bottled_agents_to_gateway` pushes the CA to running agents on bring-up,
the bind-mount is redundant — drop it (so docker rotates like firecracker) or
keep it as belt-and-suspenders? Leaning drop, for one behaviour across backends.
@@ -21,7 +21,7 @@ import subprocess
import time
import unittest
from bot_bottle.backend.docker.infra_launch import (
from bot_bottle.backend.docker.consolidated_launch import (
_network_cidr,
_network_container_ips,
)
+1 -6
View File
@@ -83,13 +83,8 @@ class TestRuntimeModuleSizes(unittest.TestCase):
self.assertEqual([], violations)
def test_backend_contract_does_not_absorb_preparation_logic(self) -> None:
# base.py holds the backend *contract*; implementation lives elsewhere.
# The cap is a tripwire against absorbing preparation/impl logic, not a
# ban on new contract surface — bumped 580->600 for the PRD 0081
# gateway-attach contract (delegator + 3 abstract primitives; the flow
# itself lives in backend/gateway_attach.py, not here).
caps = {
ROOT / "bot_bottle" / "backend" / "base.py": 600,
ROOT / "bot_bottle" / "backend" / "base.py": 580,
ROOT / "bot_bottle" / "backend" / "preparation.py": 160,
}
oversized = [
+4 -151
View File
@@ -11,11 +11,9 @@ from types import SimpleNamespace
from unittest.mock import Mock, patch
from bot_bottle.orchestrator.reprovision import reprovision_bottles
from bot_bottle.backend.base import BottleBackend, InfraLaunchError
from bot_bottle.backend.gateway_attach import GatewayAttachResources
from bot_bottle.backend.firecracker import infra_launch as fc
from bot_bottle.backend.macos_container import infra_launch as mac
from bot_bottle.backend.docker import infra_launch as docker
from bot_bottle.backend.firecracker import consolidated_launch as fc
from bot_bottle.backend.macos_container import consolidated_launch as mac
from bot_bottle.backend.docker import consolidated_launch as docker
from bot_bottle.orchestrator.client import OrchestratorClientError
@@ -134,7 +132,7 @@ class TestFirecrackerReprovision(unittest.TestCase):
def test_persist_failure_is_fatal_to_launch(self) -> None:
with patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \
patch.object(fc.subprocess, "run", return_value=_proc(1, stderr="denied")):
with self.assertRaisesRegex(fc.InfraLaunchError, "denied"):
with self.assertRaisesRegex(fc.ConsolidatedLaunchError, "denied"):
fc.persist_env_var_secret(Path("/key"), "10.0.0.1", "secret")
def test_reads_live_vm_key_and_reprovisions(self) -> None:
@@ -161,150 +159,5 @@ class TestFirecrackerReprovision(unittest.TestCase):
restore.assert_called_once_with(client, {})
class TestAttachFlow(unittest.TestCase):
"""PRD 0081 / #519 review: the base backend owns the reconcile flow and
fails hard gather resources, attempt every running bottle, then raise an
aggregate if any failed (never silently skip)."""
def _fake(self, **primitives: object) -> SimpleNamespace:
base: dict[str, object] = {
"_gateway_attach_resources": Mock(
return_value=GatewayAttachResources(ca_pem="PEM")),
"_running_bottles": Mock(return_value=["a", "b", "c"]),
"_attach_bottle_to_gateway": Mock(),
}
base.update(primitives)
return SimpleNamespace(**base)
def test_attaches_every_running_bottle_once(self) -> None:
fake = self._fake()
BottleBackend.attach_bottled_agents_to_gateway(fake) # type: ignore[arg-type]
self.assertEqual(3, fake._attach_bottle_to_gateway.call_count)
# Each bottle got the single gathered resource set.
res = fake._gateway_attach_resources.return_value
for call in fake._attach_bottle_to_gateway.call_args_list:
self.assertIs(res, call.args[1])
def test_attempts_all_then_raises_aggregate_on_failure(self) -> None:
def attach(bottle: str, _resources: object) -> None:
if bottle in ("b", "c"):
raise InfraLaunchError(f"{bottle} unreachable")
fake = self._fake(_attach_bottle_to_gateway=Mock(side_effect=attach))
with self.assertRaises(InfraLaunchError) as ctx:
BottleBackend.attach_bottled_agents_to_gateway(fake) # type: ignore[arg-type]
msg = str(ctx.exception)
# Every bottle was attempted (not fail-fast), and both failures surface.
self.assertEqual(3, fake._attach_bottle_to_gateway.call_count)
self.assertIn("b unreachable", msg)
self.assertIn("c unreachable", msg)
self.assertIn("2", msg)
def test_resource_gathering_failure_fails_hard_before_enumerating(self) -> None:
fake = self._fake(
_gateway_attach_resources=Mock(side_effect=RuntimeError("no CA yet")))
with self.assertRaises(RuntimeError):
BottleBackend.attach_bottled_agents_to_gateway(fake) # type: ignore[arg-type]
fake._running_bottles.assert_not_called()
class TestFirecrackerAttachPrimitive(unittest.TestCase):
"""PRD 0081: attach_ca_to_agent_vm pushes the current gateway CA into one
running agent VM over SSH, failing hard."""
def _run_dir(self, root: Path, ip: str = "10.243.0.3") -> Path:
run_dir = root / "demo"
run_dir.mkdir()
(run_dir / "bottle_id_ed25519").write_text("key")
(run_dir / "config.json").write_text(json.dumps({
"boot-source": {"boot_args": f"root=/dev/vda ip={ip}::gw:mask::eth0:off"}
}))
return run_dir
def test_pushes_ca_over_stdin_and_rebuilds_trust_store(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
run_dir = self._run_dir(Path(tmp))
with patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \
patch.object(fc.subprocess, "run", return_value=_proc()) as run:
fc.attach_ca_to_agent_vm(run_dir, "PEM-DATA")
# The cert goes over stdin (off argv), and the trust store is rebuilt.
self.assertEqual("PEM-DATA", run.call_args.kwargs["input"])
script = run.call_args.args[0][-1]
self.assertIn(fc.AGENT_CA_PATH, script)
self.assertIn("update-ca-certificates", script)
def test_malformed_run_dir_fails_hard(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
empty = Path(tmp) / "no-config"
empty.mkdir()
with self.assertRaises(fc.InfraLaunchError):
fc.attach_ca_to_agent_vm(empty, "PEM")
def test_push_failure_fails_hard(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
run_dir = self._run_dir(Path(tmp))
with patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \
patch.object(fc.subprocess, "run", return_value=_proc(1, stderr="down")):
with self.assertRaisesRegex(fc.InfraLaunchError, "down"):
fc.attach_ca_to_agent_vm(run_dir, "PEM")
class TestDockerAttachPrimitives(unittest.TestCase):
def test_running_agent_containers_excludes_gateway(self) -> None:
# network inspect lists the gateway (excluded) + one agent + a blank line.
inspect = _proc(stdout="bot-bottle-orch-gateway\nbot-bottle-a\n\n")
with patch.object(docker, "run_docker", return_value=inspect):
names = docker.running_agent_containers(
network="net", gateway_name="bot-bottle-orch-gateway")
self.assertEqual(["bot-bottle-a"], names)
def test_running_agent_containers_fails_hard_on_docker_error(self) -> None:
with patch.object(docker, "run_docker", side_effect=FileNotFoundError("docker")):
with self.assertRaises(FileNotFoundError):
docker.running_agent_containers()
def test_push_ca_cp_then_rebuilds_trust_store(self) -> None:
with patch.object(
docker, "run_docker", side_effect=[_proc(), _proc(), _proc()],
) as run:
docker.push_ca_to_container("bot-bottle-a", "PEM")
argvs = [c.args[0] for c in run.call_args_list]
cp = next(a for a in argvs if a[:2] == ["docker", "cp"])
self.assertEqual(f"bot-bottle-a:{docker.AGENT_CA_PATH}", cp[-1])
self.assertTrue(any("update-ca-certificates" in a[-1] for a in argvs))
def test_push_ca_fails_hard(self) -> None:
with patch.object(docker, "run_docker", return_value=_proc(1, stderr="gone")):
with self.assertRaisesRegex(docker.InfraLaunchError, "gone"):
docker.push_ca_to_container("bot-bottle-a", "PEM")
class TestMacosAttachPrimitives(unittest.TestCase):
def test_running_agent_containers_from_enumeration(self) -> None:
with patch.object(mac, "enumerate_active",
return_value=[SimpleNamespace(slug="demo")]):
names = mac.running_agent_containers()
self.assertEqual([f"{mac.CONTAINER_NAME_PREFIX}demo"], names)
def test_running_agent_containers_fails_hard(self) -> None:
with patch.object(mac, "enumerate_active",
side_effect=mac.EnumerationError("failed")):
with self.assertRaises(mac.EnumerationError):
mac.running_agent_containers()
def test_push_ca_cp_then_rebuilds_trust_store(self) -> None:
with patch.object(mac.container_mod, "run_container_argv",
return_value=_proc()) as run:
mac.push_ca_to_container(f"{mac.CONTAINER_NAME_PREFIX}demo", "PEM")
argvs = [c.args[0] for c in run.call_args_list]
self.assertTrue(any(mac.AGENT_CA_PATH in " ".join(a) for a in argvs))
self.assertTrue(any("update-ca-certificates" in a[-1] for a in argvs))
def test_push_ca_fails_hard(self) -> None:
with patch.object(mac.container_mod, "run_container_argv",
return_value=_proc(1, stderr="no")):
with self.assertRaisesRegex(mac.InfraLaunchError, "no"):
mac.push_ca_to_container("x", "PEM")
if __name__ == "__main__":
unittest.main()
@@ -6,7 +6,7 @@ import unittest
from pathlib import Path
from unittest.mock import MagicMock, Mock, patch
from bot_bottle.backend.docker.infra_launch import (
from bot_bottle.backend.docker.consolidated_launch import (
launch_consolidated,
deprovision_consolidated,
)
@@ -14,7 +14,7 @@ from bot_bottle.egress import EgressPlan, EgressRoute
from bot_bottle.git_gate import GitGatePlan
from bot_bottle.orchestrator.client import RegisteredBottle
_MOD = "bot_bottle.backend.docker.infra_launch"
_MOD = "bot_bottle.backend.docker.consolidated_launch"
_UTIL = "bot_bottle.backend.provision_bottle"
-24
View File
@@ -29,20 +29,10 @@ class TestDockerInfraService(unittest.TestCase):
self.orch.gateway_url.return_value = f"http://{ORCHESTRATOR_NAME}:8099"
self.orch.mint_gateway_token.return_value = "gw.jwt"
self.gw = MagicMock()
# Default to a cold boot (gateway (re)created) so the reconcile branch
# runs; the actual reconcile is stubbed so these composition tests stay
# isolated from docker.
self.gw.connect_to_orchestrator.return_value = True
for name, mock in (("orchestrator", self.orch), ("gateway", self.gw)):
p = patch.object(self.svc, name, return_value=mock)
p.start()
self.addCleanup(p.stop)
ap = patch(
"bot_bottle.backend.docker.backend.DockerBottleBackend"
".attach_bottled_agents_to_gateway"
)
self.attach = ap.start()
self.addCleanup(ap.stop)
def test_url_delegates_to_the_orchestrator(self) -> None:
self.assertEqual("http://127.0.0.1:8099", self.svc.url())
@@ -68,20 +58,6 @@ class TestDockerInfraService(unittest.TestCase):
f"http://{ORCHESTRATOR_NAME}:8099", "gw.jwt")
self.orch.mint_gateway_token.assert_called_once()
def test_cold_boot_reconciles_running_bottles(self) -> None:
# A (re)created gateway (connect returns True) triggers the bring-up
# reconcile so running bottles re-trust the fresh gateway (PRD 0081).
self.gw.connect_to_orchestrator.return_value = True
self.svc.ensure_running()
self.attach.assert_called_once_with()
def test_no_reconcile_when_gateway_left_untouched(self) -> None:
# A healthy current gateway (connect returns False) is not a cold boot,
# so there is nothing to reconcile.
self.gw.connect_to_orchestrator.return_value = False
self.svc.ensure_running()
self.attach.assert_not_called()
def test_stop_removes_both_containers(self) -> None:
with patch(_RUN) as run:
self.svc.stop()
@@ -16,7 +16,7 @@ from bot_bottle.agent_provider import AgentProvisionPlan
from bot_bottle.backend import BottleSpec
from bot_bottle.backend.docker import launch as launch_mod
from bot_bottle.backend.docker.bottle_plan import DockerBottlePlan
from bot_bottle.backend.docker.infra_launch import LaunchContext
from bot_bottle.backend.docker.consolidated_launch import LaunchContext
from bot_bottle.egress import EgressPlan
from bot_bottle.git_gate import GitGatePlan
from bot_bottle.log import Die
+1 -1
View File
@@ -19,7 +19,7 @@ from bot_bottle.agent_provider import AgentProvisionPlan
from bot_bottle.backend import BottleImages, BottleSpec
from bot_bottle.backend.docker import launch as launch_mod
from bot_bottle.backend.docker.bottle_plan import DockerBottlePlan
from bot_bottle.backend.docker.infra_launch import LaunchContext
from bot_bottle.backend.docker.consolidated_launch import LaunchContext
from bot_bottle.egress import EgressPlan
from bot_bottle.git_gate import GitGatePlan
from bot_bottle.manifest import ManifestIndex
+7 -4
View File
@@ -404,16 +404,19 @@ class TestBootArgs(unittest.TestCase):
self.assertEqual("bbfc0", cfg["network-interfaces"][0]["host_dev_name"])
self.assertEqual(1, len(cfg["drives"])) # no data drive by default
def test_config_adds_data_drive(self):
def test_config_adds_data_drives_in_order(self):
cfg = cast(Any, firecracker_vm._config(
rootfs=Path("/run/rootfs.ext4"), tap="bbfc0",
guest_ip="100.64.0.1", host_ip="100.64.0.0", pubkey="k",
vcpus=2, mem_mib=2048, guest_mac="06:00:AC:10:00:02",
data_drive=Path("/run/registry.ext4"),
data_drives=(Path("/run/ca.ext4"), Path("/run/git.ext4")),
))
self.assertEqual(2, len(cfg["drives"]))
# rootfs (vda) + two data drives, attached in list order so the guest
# sees them as /dev/vdb, /dev/vdc — an order callers depend on.
self.assertEqual(3, len(cfg["drives"]))
self.assertFalse(cfg["drives"][1]["is_root_device"])
self.assertEqual("/run/registry.ext4", cfg["drives"][1]["path_on_host"])
self.assertEqual("/run/ca.ext4", cfg["drives"][1]["path_on_host"])
self.assertEqual("/run/git.ext4", cfg["drives"][2]["path_on_host"])
class TestBottleExecClose(unittest.TestCase):
+67 -5
View File
@@ -59,23 +59,85 @@ class TestFirecrackerGatewayConnect(unittest.TestCase):
def test_boots_the_gateway_vm_and_seeds_the_token(self) -> None:
gw = FirecrackerGateway()
booted = infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k"))
ca_vol = Path("/gw/gateway-ca.ext4")
git_vol = Path("/gw/gateway-git.ext4")
with patch.object(infra_vm, "boot_vm", return_value=booted) as boot, \
patch.object(FirecrackerGateway, "_ensure_ca_volume",
return_value=ca_vol), \
patch.object(FirecrackerGateway, "_ensure_git_volume",
return_value=git_vol), \
patch.object(infra_vm, "push_secret") as push:
cold_booted = gw.connect_to_orchestrator(_ORCH_URL, _TOKEN)
# The VM is booted fresh here (cold-boot path only), so it always
# signals a cold boot → the caller reconciles running bottles (PRD 0081).
self.assertTrue(cold_booted)
gw.connect_to_orchestrator(_ORCH_URL, _TOKEN)
# Booted on the gateway link with the gateway role; the orchestrator's
# guest IP (parsed off the URL) rides the cmdline as bb_orch.
kw = boot.call_args.kwargs
self.assertEqual("gateway", kw["role"])
self.assertIn("bb_orch=10.243.255.1", kw["extra_boot_args"])
self.assertIsNone(kw.get("data_drive")) # data plane never opens the DB
# The persistent volumes ride in a FIXED order — CA as /dev/vdb, git-gate
# as /dev/vdc — so both survive a gateway-VM rebuild (issues #450, #512).
self.assertEqual((ca_vol, git_vol), kw.get("data_drives"))
# The host-minted token (never the key — #469) is pushed to the guest.
push.assert_called_once()
self.assertEqual(_TOKEN, push.call_args.args[1])
class TestGatewayCaVolume(unittest.TestCase):
"""The persistent CA volume that keeps the mitmproxy CA stable across a
gateway-VM rebuild (issue #450) — the mirror of the orchestrator's registry
volume."""
def test_reuses_existing_volume(self) -> None:
gw = FirecrackerGateway()
with tempfile.TemporaryDirectory() as td:
vol = Path(td) / "gateway-ca.ext4"
vol.write_bytes(b"") # already present
with patch.object(infra_vm, "_gw_dir", return_value=Path(td)), \
patch(f"{_GW}.subprocess.run") as run:
out = gw._ensure_ca_volume()
run.assert_not_called() # no mke2fs when it exists — the CA survives
self.assertEqual(vol, out)
def test_creates_volume_when_missing(self) -> None:
gw = FirecrackerGateway()
with tempfile.TemporaryDirectory() as td:
with patch.object(infra_vm, "_gw_dir", return_value=Path(td)), \
patch(f"{_GW}.subprocess.run",
return_value=CompletedProcess([], 0)) as run:
out = gw._ensure_ca_volume()
argv = run.call_args.args[0]
self.assertIn("mke2fs", argv)
self.assertEqual(str(Path(td) / "gateway-ca.ext4"), out.__fspath__())
self.assertIn(str(out), argv)
class TestGatewayGitVolume(unittest.TestCase):
"""The persistent git-gate volume that keeps per-bottle bare repos + creds
stable across a gateway-VM rebuild (issue #512)."""
def test_reuses_existing_volume(self) -> None:
gw = FirecrackerGateway()
with tempfile.TemporaryDirectory() as td:
vol = Path(td) / "gateway-git.ext4"
vol.write_bytes(b"") # already present
with patch.object(infra_vm, "_gw_dir", return_value=Path(td)), \
patch(f"{_GW}.subprocess.run") as run:
out = gw._ensure_git_volume()
run.assert_not_called() # no mke2fs when it exists — the repos survive
self.assertEqual(vol, out)
def test_creates_volume_when_missing(self) -> None:
gw = FirecrackerGateway()
with tempfile.TemporaryDirectory() as td:
with patch.object(infra_vm, "_gw_dir", return_value=Path(td)), \
patch(f"{_GW}.subprocess.run",
return_value=CompletedProcess([], 0)) as run:
out = gw._ensure_git_volume()
argv = run.call_args.args[0]
self.assertIn("mke2fs", argv)
self.assertEqual(str(Path(td) / "gateway-git.ext4"), out.__fspath__())
self.assertIn(str(out), argv)
class TestFirecrackerGatewaySurface(unittest.TestCase):
def test_is_running_reads_the_gateway_pidfile(self) -> None:
gw = FirecrackerGateway()
-18
View File
@@ -40,21 +40,7 @@ class TestAccessors(unittest.TestCase):
self.assertEqual(f"http://{ip}:{infra_vm.ORCHESTRATOR_PORT}", svc.url())
_ATTACH = (
"bot_bottle.backend.firecracker.backend.FirecrackerBottleBackend"
".attach_bottled_agents_to_gateway"
)
class TestEnsureRunning(unittest.TestCase):
def setUp(self) -> None:
# The cold-boot branch reconciles running bottles against the fresh
# gateway (PRD 0081). Stub it so these composition tests stay isolated
# from SSH; the wiring itself is asserted in test_cold_boot_reconciles*.
ap = patch(_ATTACH)
self.attach = ap.start()
self.addCleanup(ap.stop)
def test_adopts_when_healthy_alive_and_version_matches(self):
# Healthy control plane + live gateway + existing key + matching version
# marker -> adopt (no stop/build/boot); returns the orchestrator URL.
@@ -73,8 +59,6 @@ class TestEnsureRunning(unittest.TestCase):
built.assert_not_called()
ip = infra_vm.netpool.orch_slot().guest_ip
self.assertEqual(f"http://{ip}:{infra_vm.ORCHESTRATOR_PORT}", url)
# Adopt path — no cold boot, so no running-bottle reconcile.
self.attach.assert_not_called()
def test_reboots_both_when_version_stale(self):
# Healthy control plane but the running pair booted an OLDER image
@@ -136,8 +120,6 @@ class TestEnsureRunning(unittest.TestCase):
# reach the control plane at startup).
orch_cls.return_value.ensure_running.assert_called_once()
gw_cls.return_value.connect_to_orchestrator.assert_called_once()
# A cold boot minted a fresh gateway CA — reconcile running bottles.
self.attach.assert_called_once_with()
if __name__ == "__main__":
+17 -1
View File
@@ -61,7 +61,23 @@ class TestRoleInits(unittest.TestCase):
init = infra_vm.role_init("gateway")
self.assertNotIn("bb_role=", init)
self.assertNotIn("bot_bottle.orchestrator", init)
self.assertNotIn("/dev/vdb", init) # no registry volume on the data plane
# Persistent CA volume mounted at mitmproxy's confdir so the CA survives
# a gateway-VM rebuild (issue #450). Mounted BEFORE the data plane starts.
self.assertIn(f"mount -t ext4 /dev/vdb {infra_vm._GATEWAY_CA_MOUNT}", init)
self.assertLess(init.index("/dev/vdb"),
init.index("bot_bottle.gateway.bootstrap"))
# Persistent git-gate volume (/dev/vdc) bind-mounted onto /git and
# /git-gate/creds so per-bottle repos + creds survive a rebuild (#512),
# also before the data plane (and any provisioning writes).
self.assertIn(f"mount -t ext4 /dev/vdc {infra_vm._GATEWAY_GIT_MOUNT}", init)
self.assertIn(
f"mount --bind {infra_vm._GATEWAY_GIT_MOUNT}/git "
f"{infra_vm._GATEWAY_GIT_REPO_ROOT}", init)
self.assertIn(
f"mount --bind {infra_vm._GATEWAY_GIT_MOUNT}/creds "
f"{infra_vm._GATEWAY_GIT_CREDS_DIR}", init)
self.assertLess(init.index("/dev/vdc"),
init.index("bot_bottle.gateway.bootstrap"))
self.assertIn("export PATH=", init) # shared preamble
self.assertIn("BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise", init)
self.assertIn(f"cat {infra_vm._GUEST_GATEWAY_JWT_PATH}", init) # host-minted JWT
+1 -1
View File
@@ -52,7 +52,7 @@ class TestEnsureRunning(unittest.TestCase):
kw = boot.call_args.kwargs
self.assertEqual("orchestrator", kw["role"])
self.assertEqual(4096, kw["mem_mib"]) # orchestrator keeps build headroom
self.assertEqual(Path("/reg"), kw["data_drive"]) # DB volume on the CP
self.assertEqual((Path("/reg"),), kw["data_drives"]) # DB volume on the CP
# The host-canonical signing key is pushed to the guest signing-key path.
self.assertEqual("host-key", push.call_args.args[1])
self.assertEqual(infra_vm._GUEST_SIGNING_KEY_PATH, push.call_args.args[2])
@@ -6,7 +6,7 @@ import unittest
from pathlib import Path
from unittest.mock import MagicMock, Mock, patch
from bot_bottle.backend.macos_container.infra_launch import (
from bot_bottle.backend.macos_container.consolidated_launch import (
GatewayEndpoint,
ensure_gateway,
register_agent,
@@ -16,7 +16,7 @@ from bot_bottle.egress import EgressPlan, EgressRoute
from bot_bottle.git_gate import GitGatePlan
from bot_bottle.orchestrator.client import RegisteredBottle
_MOD = "bot_bottle.backend.macos_container.infra_launch"
_MOD = "bot_bottle.backend.macos_container.consolidated_launch"
_UTIL = "bot_bottle.backend.provision_bottle"
@@ -145,7 +145,7 @@ class TestLiveSourceIps(unittest.TestCase):
return [Mock(slug=s) for s in slugs]
def test_maps_slugs_to_container_addresses(self) -> None:
from bot_bottle.backend.macos_container.infra_launch import live_source_ips
from bot_bottle.backend.macos_container.consolidated_launch import live_source_ips
with patch(f"{_MOD}.enumerate_active", return_value=self._agents("a", "b")), \
patch(f"{_MOD}.container_mod.inspect_container_network_ip",
side_effect=["10.0.0.1", "10.0.0.2"]) as ip:
@@ -156,7 +156,7 @@ class TestLiveSourceIps(unittest.TestCase):
def test_containers_without_an_address_are_skipped(self) -> None:
"""A container that hasn't been given a DHCP address yet contributes
nothing the reap's grace window, not this list, protects it."""
from bot_bottle.backend.macos_container.infra_launch import live_source_ips
from bot_bottle.backend.macos_container.consolidated_launch import live_source_ips
with patch(f"{_MOD}.enumerate_active", return_value=self._agents("a", "b")), \
patch(f"{_MOD}.container_mod.inspect_container_network_ip",
side_effect=["", "10.0.0.2"]):
@@ -165,7 +165,7 @@ class TestLiveSourceIps(unittest.TestCase):
def test_container_list_failure_raises(self) -> None:
"""If container list fails, the live set is not authoritative and
reconciliation must be skipped."""
from bot_bottle.backend.macos_container.infra_launch import live_source_ips
from bot_bottle.backend.macos_container.consolidated_launch import live_source_ips
from bot_bottle.backend.macos_container.enumerate import EnumerationError
with patch(f"{_MOD}.enumerate_active",
side_effect=EnumerationError("container list failed")):
@@ -174,7 +174,7 @@ class TestLiveSourceIps(unittest.TestCase):
def test_per_container_inspect_failure_raises(self) -> None:
"""If any individual inspect fails, the live set is not authoritative."""
from bot_bottle.backend.macos_container.infra_launch import live_source_ips
from bot_bottle.backend.macos_container.consolidated_launch import live_source_ips
from bot_bottle.backend.macos_container.enumerate import EnumerationError
with patch(f"{_MOD}.enumerate_active", return_value=self._agents("a", "b")), \
patch(f"{_MOD}.container_mod.inspect_container_network_ip",
@@ -16,7 +16,7 @@ from unittest.mock import ANY, patch
from bot_bottle.backend.macos_container.bottle import MacosContainerBottle
from bot_bottle.backend.macos_container.bottle_plan import MacosContainerBottlePlan
from bot_bottle.backend.macos_container.infra_launch import GatewayEndpoint
from bot_bottle.backend.macos_container.consolidated_launch import GatewayEndpoint
from bot_bottle.backend.macos_container.gateway_hosts import GATEWAY_HOSTNAME
from bot_bottle.backend.macos_container.launch import (
_agent_run_argv,
-11
View File
@@ -44,17 +44,6 @@ class TestMacosGatewayConnect(unittest.TestCase):
def test_default_name(self) -> None:
self.assertEqual(GATEWAY_NAME, self.gw.name)
def test_connect_signals_cold_boot(self) -> None:
# The container is recreated unconditionally, so connect always signals
# a cold boot → the caller reconciles running bottles (PRD 0081).
run = Mock(return_value=_proc())
with patch(f"{_GW}.container_mod") as mod, \
patch(f"{_GW}.host_gateway_ca_dir", return_value=Path("/ca")):
mod.dns_server.return_value = "1.1.1.1"
mod.bind_mount_spec.side_effect = _spec
mod.run_container_argv = run
self.assertTrue(self.gw.connect_to_orchestrator(_ORCH_URL, _TOKEN))
def test_refuses_without_orchestrator_url(self) -> None:
with patch(f"{_GW}.container_mod") as mod:
with self.assertRaises(GatewayError):
-24
View File
@@ -24,20 +24,10 @@ class TestInfraEnsureRunning(unittest.TestCase):
self.orch.url.return_value = "http://192.168.128.2:8099"
self.orch.mint_gateway_token.return_value = "gw.jwt"
self.gw = MagicMock()
# Default to a cold boot (gateway (re)created) so the reconcile branch
# runs; the actual reconcile is stubbed so these composition tests stay
# isolated from the `container` CLI.
self.gw.connect_to_orchestrator.return_value = True
for name, mock in (("orchestrator", self.orch), ("gateway", self.gw)):
p = patch.object(self.svc, name, return_value=mock)
p.start()
self.addCleanup(p.stop)
ap = patch(
"bot_bottle.backend.macos_container.backend"
".MacosContainerBottleBackend.attach_bottled_agents_to_gateway"
)
self.attach = ap.start()
self.addCleanup(ap.stop)
def test_composes_networks_builds_and_brings_up_orchestrator_first(self) -> None:
with patch(f"{_INFRA}.ensure_networks") as nets:
@@ -56,20 +46,6 @@ class TestInfraEnsureRunning(unittest.TestCase):
"http://192.168.128.2:8099", "gw.jwt")
self.orch.mint_gateway_token.assert_called_once()
def test_cold_boot_reconciles_running_bottles(self) -> None:
# A (re)created gateway (connect returns True) triggers the bring-up
# reconcile so running bottles re-trust the fresh gateway (PRD 0081).
self.gw.connect_to_orchestrator.return_value = True
with patch(f"{_INFRA}.ensure_networks"):
self.svc.ensure_running()
self.attach.assert_called_once_with()
def test_no_reconcile_when_gateway_left_untouched(self) -> None:
self.gw.connect_to_orchestrator.return_value = False
with patch(f"{_INFRA}.ensure_networks"):
self.svc.ensure_running()
self.attach.assert_not_called()
def test_is_healthy_delegates_to_the_orchestrator(self) -> None:
self.orch.is_healthy.return_value = True
self.assertTrue(self.svc.is_healthy())
-25
View File
@@ -125,31 +125,6 @@ class TestReprovisionGateway(unittest.TestCase):
self.c.reprovision_gateway("b1", "key")
class TestUpdateAgentSecret(unittest.TestCase):
def setUp(self) -> None:
self.c = OrchestratorClient("http://orch:8080")
def test_success_posts_single_secret(self) -> None:
with patch(_URLOPEN, return_value=_resp(200, {"updated": True})) as opened:
self.assertTrue(self.c.update_agent_secret("b1", "A", "fresh", "key"))
request = opened.call_args.args[0]
self.assertEqual("POST", request.get_method())
self.assertTrue(request.full_url.endswith("/bottles/b1/secret"))
self.assertEqual(
{"name": "A", "value": "fresh", "env_var_secret": "key"},
json.loads(request.data),
)
def test_unknown_bottle_is_false(self) -> None:
with patch(_URLOPEN, side_effect=_http_error(404)):
self.assertFalse(self.c.update_agent_secret("b1", "A", "v", "key"))
def test_other_status_raises(self) -> None:
with patch(_URLOPEN, side_effect=_http_error(400)):
with self.assertRaises(OrchestratorClientError):
self.c.update_agent_secret("b1", "A", "v", "key")
class TestHealthAndPolicy(unittest.TestCase):
def setUp(self) -> None:
self.c = OrchestratorClient("http://orch:8080")
+23 -10
View File
@@ -86,12 +86,9 @@ class TestDockerGateway(unittest.TestCase):
return _proc()
with patch(_RUN_DOCKER, side_effect=fake):
cold_booted = self.sc.connect_to_orchestrator(_ORCH_URL, _TOKEN)
self.sc.connect_to_orchestrator(_ORCH_URL, _TOKEN)
self.assertEqual([], [c for c in calls if c[:2] == ["docker", "run"]])
self.assertEqual([], [c for c in calls if c[:2] == ["docker", "rm"]])
# A healthy current gateway left untouched is not a cold boot — the
# bring-up flow skips the running-bottle reconcile (PRD 0081).
self.assertFalse(cold_booted)
def test_ensure_running_recreates_when_image_is_stale(self) -> None:
# Running, but the container was built from an OLD image → recreate so
@@ -109,12 +106,9 @@ class TestDockerGateway(unittest.TestCase):
return _proc()
with patch(_RUN_DOCKER, side_effect=fake):
cold_booted = self.sc.connect_to_orchestrator(_ORCH_URL, _TOKEN)
self.sc.connect_to_orchestrator(_ORCH_URL, _TOKEN)
self.assertEqual(1, len([c for c in calls if c[:2] == ["docker", "run"]]))
self.assertTrue(any(c[:2] == ["docker", "rm"] for c in calls))
# A recreated container minted/mounted its CA afresh — a cold boot that
# triggers the running-bottle reconcile (PRD 0081).
self.assertTrue(cold_booted)
def test_ensure_running_starts_the_singleton_when_absent(self) -> None:
calls: list[list[str]] = []
@@ -124,8 +118,7 @@ class TestDockerGateway(unittest.TestCase):
return _proc(stdout="") if argv[:2] == ["docker", "ps"] else _proc()
with patch(_RUN_DOCKER, side_effect=fake):
cold_booted = self.sc.connect_to_orchestrator(_ORCH_URL, _TOKEN)
self.assertTrue(cold_booted) # started fresh → reconcile running bottles
self.sc.connect_to_orchestrator(_ORCH_URL, _TOKEN)
runs = [c for c in calls if c[:2] == ["docker", "run"]]
self.assertEqual(1, len(runs))
@@ -168,6 +161,26 @@ class TestDockerGateway(unittest.TestCase):
"ci-ca-volume:/home/mitmproxy/.mitmproxy", argv
)
def test_persists_git_gate_state_across_recreation(self) -> None:
# Per-bottle bare repos + deploy creds are bind-mounted from the host so
# a gateway restart doesn't drop already-running bottles' git-gate state
# (issue #512). Named sources here stand in for CI's per-run volumes.
sc = DockerGateway(
"bot-bottle-gateway:latest",
git_mount_source="ci-git-volume",
creds_mount_source="ci-creds-volume",
)
def fake(argv: list[str], **_kw: object) -> Mock:
return _proc(stdout="") if argv[:2] == ["docker", "ps"] else _proc()
with patch(_RUN_DOCKER, side_effect=fake) as run:
sc.connect_to_orchestrator(_ORCH_URL, _TOKEN)
argv = next(c.args[0] for c in run.call_args_list
if c.args[0][:2] == ["docker", "run"])
self.assertIn("ci-git-volume:/git", argv)
self.assertIn("ci-creds-volume:/git-gate/creds", argv)
def test_connect_injects_the_pre_minted_gateway_token(self) -> None:
# The gateway presents the token the orchestrator handed it — it never
# mints (holds no signing key). The value rides the env (bare `--env
@@ -199,20 +199,6 @@ class TestAgentSecrets(unittest.TestCase):
got = self.store.get_agent_secrets("bottle-1")
self.assertEqual({"K": "new", "K2": "v2"}, got)
def test_store_agent_secret_upserts_one_key_leaving_others(self) -> None:
self.store.store_agent_secrets("bottle-1", {"K": "v1", "K2": "v2"})
self.store.store_agent_secret("bottle-1", "K", "v1-new") # update existing
self.store.store_agent_secret("bottle-1", "K3", "v3") # insert new
self.assertEqual(
{"K": "v1-new", "K2": "v2", "K3": "v3"},
self.store.get_agent_secrets("bottle-1"),
)
def test_store_agent_secret_does_not_duplicate_rows(self) -> None:
self.store.store_agent_secret("bottle-1", "K", "v1")
self.store.store_agent_secret("bottle-1", "K", "v2")
self.assertEqual({"K": "v2"}, self.store.get_agent_secrets("bottle-1"))
def test_delete_removes_secrets(self) -> None:
self.store.store_agent_secrets("bottle-1", {"K": "v"})
self.store.delete_agent_secrets("bottle-1")
-41
View File
@@ -87,47 +87,6 @@ class TestDispatch(unittest.TestCase):
{"EGRESS_TOKEN_0": "upstream-secret"}, self.orch.tokens_for(bottle_id),
)
def test_update_agent_secret_in_place(self) -> None:
key = base64.urlsafe_b64encode(b"unit-test-key").rstrip(b"=").decode()
status, payload = dispatch(
self.orch, "POST", "/bottles", _body({
"source_ip": "10.243.0.21",
"tokens": {"A": "old", "B": "keep"},
"env_var_secret": key,
}),
)
self.assertEqual(201, status)
bottle_id = payload["bottle_id"]
assert isinstance(bottle_id, str)
status, response = dispatch(
self.orch, "POST", f"/bottles/{bottle_id}/secret",
_body({"name": "A", "value": "fresh", "env_var_secret": key}),
)
self.assertEqual((200, {"updated": True}), (status, response))
self.assertEqual({"A": "fresh", "B": "keep"}, self.orch.tokens_for(bottle_id))
def test_update_agent_secret_validates_and_unknown_bottle(self) -> None:
status, _ = dispatch(self.orch, "POST", "/bottles/b1/secret", b"not-json")
self.assertEqual(400, status)
status, _ = dispatch(
self.orch, "POST", "/bottles/b1/secret", _body({"name": "A"}),
)
self.assertEqual(400, status) # value + env_var_secret missing
status, _ = dispatch(
self.orch, "POST", "/bottles/ghost/secret",
_body({"name": "A", "value": "v", "env_var_secret": "k"}),
)
self.assertEqual(404, status)
def test_update_agent_secret_is_cli_only(self) -> None:
# A data-plane (`gateway`) caller must not set a bottle's tokens.
status, _ = dispatch(
self.orch, "POST", "/bottles/b1/secret",
_body({"name": "A", "value": "v", "env_var_secret": "k"}),
role=ROLE_GATEWAY,
)
self.assertEqual(403, status)
def test_reprovision_validates_request_and_missing_rows(self) -> None:
status, _ = dispatch(
self.orch, "POST", "/bottles/b1/reprovision_gateway", b"not-json",
-19
View File
@@ -103,25 +103,6 @@ class TestOrchestrator(unittest.TestCase):
self.assertTrue(self.orch.reprovision_from_secret(rec.bottle_id, key))
self.assertEqual({"EGRESS_TOKEN_0": "secret"}, self.orch.tokens_for(rec.bottle_id))
def test_update_agent_secret_refreshes_one_token_in_place(self) -> None:
key = new_env_var_secret()
rec = self.orch.launch_bottle(
"10.243.0.20", tokens={"A": "old-a", "B": "keep-b"}, env_var_secret=key,
)
self.assertTrue(self.orch.update_agent_secret(rec.bottle_id, "A", "new-a", key))
# In memory: A refreshed, B left untouched.
self.assertEqual({"A": "new-a", "B": "keep-b"}, self.orch.tokens_for(rec.bottle_id))
# At rest: the whole set stays decryptable with the SAME env_var_secret,
# so a later reprovision restores the refreshed value (not the launch one).
self.orch._tokens.clear()
self.assertTrue(self.orch.reprovision_from_secret(rec.bottle_id, key))
self.assertEqual({"A": "new-a", "B": "keep-b"}, self.orch.tokens_for(rec.bottle_id))
def test_update_agent_secret_unknown_bottle_is_false(self) -> None:
self.assertFalse(
self.orch.update_agent_secret("ghost", "A", "v", new_env_var_secret())
)
def test_reprovision_rejects_missing_rows_and_wrong_key(self) -> None:
self.assertFalse(self.orch.reprovision_from_secret("missing", new_env_var_secret()))
key = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE"