Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| dff811f190 | |||
| 0ddaa95c14 |
@@ -100,6 +100,8 @@ jobs:
|
||||
export BOT_BOTTLE_DOCKER_CLIENT_NETWORK="$DOCKER_CLIENT_NETWORK"
|
||||
export BOT_BOTTLE_DOCKER_ROOT_MOUNT="bot-bottle-ci-root-$RUN_KEY"
|
||||
export BOT_BOTTLE_DOCKER_CA_MOUNT="bot-bottle-ci-ca-$RUN_KEY"
|
||||
export BOT_BOTTLE_DOCKER_GIT_MOUNT="bot-bottle-ci-git-$RUN_KEY"
|
||||
export BOT_BOTTLE_DOCKER_CREDS_MOUNT="bot-bottle-ci-creds-$RUN_KEY"
|
||||
python3 -m coverage run -m scripts.unittest_gate \
|
||||
-t . -s tests/integration -v \
|
||||
--minimum-executed 22 --fail-on-skip
|
||||
@@ -110,7 +112,9 @@ jobs:
|
||||
RUN_KEY="${GITHUB_RUN_ID:-${GITHUB_RUN_NUMBER:-0}}"
|
||||
docker volume rm --force \
|
||||
"bot-bottle-ci-root-$RUN_KEY" \
|
||||
"bot-bottle-ci-ca-$RUN_KEY" 2>/dev/null || true
|
||||
"bot-bottle-ci-ca-$RUN_KEY" \
|
||||
"bot-bottle-ci-git-$RUN_KEY" \
|
||||
"bot-bottle-ci-creds-$RUN_KEY" 2>/dev/null || true
|
||||
|
||||
# Non-dot name so upload-artifact's dotfile-skipping glob picks it up.
|
||||
- name: Stage docker coverage for upload
|
||||
|
||||
@@ -116,6 +116,8 @@ jobs:
|
||||
export BOT_BOTTLE_DOCKER_CLIENT_NETWORK="$DOCKER_CLIENT_NETWORK"
|
||||
export BOT_BOTTLE_DOCKER_ROOT_MOUNT="bot-bottle-ci-root-$RUN_KEY"
|
||||
export BOT_BOTTLE_DOCKER_CA_MOUNT="bot-bottle-ci-ca-$RUN_KEY"
|
||||
export BOT_BOTTLE_DOCKER_GIT_MOUNT="bot-bottle-ci-git-$RUN_KEY"
|
||||
export BOT_BOTTLE_DOCKER_CREDS_MOUNT="bot-bottle-ci-creds-$RUN_KEY"
|
||||
python3 -m coverage run -m scripts.unittest_gate \
|
||||
-t . -s tests/integration -v \
|
||||
--minimum-executed 22 --fail-on-skip
|
||||
@@ -126,7 +128,9 @@ jobs:
|
||||
RUN_KEY="${GITHUB_RUN_ID:-${GITHUB_RUN_NUMBER:-0}}"
|
||||
docker volume rm --force \
|
||||
"bot-bottle-ci-root-$RUN_KEY" \
|
||||
"bot-bottle-ci-ca-$RUN_KEY" 2>/dev/null || true
|
||||
"bot-bottle-ci-ca-$RUN_KEY" \
|
||||
"bot-bottle-ci-git-$RUN_KEY" \
|
||||
"bot-bottle-ci-creds-$RUN_KEY" 2>/dev/null || true
|
||||
|
||||
- name: Stage docker coverage for upload
|
||||
run: cp .coverage.docker coverage-docker.dat
|
||||
|
||||
@@ -21,7 +21,7 @@ from abc import ABC, abstractmethod
|
||||
from contextlib import AbstractContextManager, contextmanager
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Generator, Generic, Sequence, TypeVar
|
||||
from typing import Generator, Generic, Sequence, TypeVar
|
||||
|
||||
from ..agent_provider import AgentProvisionPlan, get_provider
|
||||
from ..egress import EgressPlan
|
||||
@@ -35,9 +35,6 @@ from ..workspace import WorkspacePlan, workspace_plan
|
||||
from .print_util import print_multi, visible_agent_env_names
|
||||
from .util import host_skill_dir
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .gateway_attach import GatewayAttachResources
|
||||
|
||||
|
||||
class BackendStatus(enum.IntEnum):
|
||||
"""Return codes for BottleBackend.status(). READY == 0 so callsites
|
||||
@@ -265,17 +262,6 @@ class Bottle(ABC):
|
||||
|
||||
PlanT = TypeVar("PlanT", bound=BottlePlan)
|
||||
CleanupT = TypeVar("CleanupT", bound=BottleCleanupPlan)
|
||||
# The backend-specific handle for one running bottle the reconcile attaches to
|
||||
# the gateway (a run dir for firecracker, a container name for docker/macOS).
|
||||
# Registry-style callers that don't care about the handle bind it to `Any`
|
||||
# (`BottleBackend[Any, Any, Any]`).
|
||||
AttachTargetT = TypeVar("AttachTargetT")
|
||||
|
||||
|
||||
class InfraLaunchError(RuntimeError):
|
||||
"""A per-host infra (gateway + orchestrator) launch/reconcile step could not
|
||||
complete. Shared across backends so the base backend can raise + catch one
|
||||
type; each backend's ``infra_launch`` module re-exports it."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -290,7 +276,7 @@ class BottleImages:
|
||||
sidecar: str | Path = ""
|
||||
|
||||
|
||||
class BottleBackend(ABC, Generic[PlanT, CleanupT, AttachTargetT]):
|
||||
class BottleBackend(ABC, Generic[PlanT, CleanupT]):
|
||||
"""Abstract base for selectable bottle backends. Concrete subclasses
|
||||
(e.g. DockerBottleBackend) own their own prepare/launch impls.
|
||||
Parameterized over the backend's concrete plan + cleanup-plan types
|
||||
@@ -518,31 +504,6 @@ class BottleBackend(ABC, Generic[PlanT, CleanupT, AttachTargetT]):
|
||||
compose ls`; firecracker cross-references its running gateway
|
||||
containers against per-bottle metadata."""
|
||||
|
||||
def attach_bottled_agents_to_gateway(self) -> None:
|
||||
"""Reconcile every running bottle against the freshly-(re)booted gateway
|
||||
on the cold-boot bring-up path (PRD 0081). The shared flow + fail-hard
|
||||
policy live in `gateway_attach`; backends override only the three
|
||||
primitives below (ADR 0006)."""
|
||||
from .gateway_attach import reconcile_running_bottles
|
||||
reconcile_running_bottles(self)
|
||||
|
||||
@abstractmethod
|
||||
def _gateway_attach_resources(self) -> "GatewayAttachResources":
|
||||
"""Gather what every bottle needs to (re)attach to the current gateway
|
||||
(the CA now). Raises if the gateway isn't reachable (aborts reconcile)."""
|
||||
|
||||
@abstractmethod
|
||||
def _running_bottles(self) -> Sequence[AttachTargetT]:
|
||||
"""The live bottles as backend handles for `_attach_bottle_to_gateway`.
|
||||
Raises if the live set can't be determined (fail hard, no partial set)."""
|
||||
|
||||
@abstractmethod
|
||||
def _attach_bottle_to_gateway(
|
||||
self, bottle: AttachTargetT, resources: "GatewayAttachResources",
|
||||
) -> None:
|
||||
"""(Re)attach one bottle to the current gateway (SSH / `exec`+`cp`).
|
||||
Raises `InfraLaunchError` naming the bottle on failure."""
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def is_available(cls) -> bool:
|
||||
|
||||
@@ -33,7 +33,6 @@ from ...git_gate import GitGatePlan
|
||||
from ...supervisor.plan import SupervisePlan
|
||||
from ...manifest import Manifest
|
||||
from .. import ActiveAgent, BottleBackend, BottleImages, BottleSpec
|
||||
from ..gateway_attach import GatewayAttachResources
|
||||
from . import cleanup as _cleanup
|
||||
from . import enumerate as _enumerate
|
||||
from . import launch as _launch
|
||||
@@ -41,7 +40,7 @@ from . import resolve_plan as _resolve_plan
|
||||
from .bottle import DockerBottle
|
||||
from .bottle_cleanup_plan import DockerBottleCleanupPlan
|
||||
from .bottle_plan import DockerBottlePlan
|
||||
class DockerBottleBackend(BottleBackend["DockerBottlePlan", "DockerBottleCleanupPlan", str]):
|
||||
class DockerBottleBackend(BottleBackend["DockerBottlePlan", "DockerBottleCleanupPlan"]):
|
||||
"""Docker backend implementation. Selected by BOT_BOTTLE_BACKEND
|
||||
when set to `docker`; retained as a legacy/example backend."""
|
||||
|
||||
@@ -141,18 +140,3 @@ class DockerBottleBackend(BottleBackend["DockerBottlePlan", "DockerBottleCleanup
|
||||
|
||||
def enumerate_active(self) -> Sequence[ActiveAgent]:
|
||||
return _enumerate.enumerate_active()
|
||||
|
||||
def _gateway_attach_resources(self) -> GatewayAttachResources:
|
||||
from .infra import DockerInfraService
|
||||
return GatewayAttachResources(
|
||||
ca_pem=DockerInfraService().gateway().ca_cert_pem())
|
||||
|
||||
def _running_bottles(self) -> Sequence[str]:
|
||||
from .infra_launch import running_agent_containers
|
||||
return running_agent_containers()
|
||||
|
||||
def _attach_bottle_to_gateway(
|
||||
self, bottle: str, resources: GatewayAttachResources,
|
||||
) -> None:
|
||||
from .infra_launch import push_ca_to_container
|
||||
push_ca_to_container(bottle, resources.ca_pem)
|
||||
|
||||
+6
-60
@@ -13,7 +13,6 @@ orchestrator-facing wiring so that sequence stays testable in isolation.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
|
||||
from ... import log
|
||||
@@ -25,13 +24,15 @@ from ...gateway import GATEWAY_NETWORK
|
||||
from .infra import INFRA_NAME, DockerInfraService
|
||||
from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME
|
||||
from ...orchestrator.reprovision import reprovision_bottles
|
||||
from ..base import InfraLaunchError
|
||||
from ..provision_bottle import deprovision_bottle, provision_bottle
|
||||
from ..util import AGENT_CA_PATH
|
||||
from .gateway_transport import DockerGatewayTransport
|
||||
from .gateway_net import next_free_ip
|
||||
|
||||
|
||||
class ConsolidatedLaunchError(RuntimeError):
|
||||
"""The consolidated register/provision sequence could not complete."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LaunchContext:
|
||||
"""What the agent container needs to join the shared gateway."""
|
||||
@@ -53,7 +54,7 @@ def _network_cidr(network: str) -> str:
|
||||
])
|
||||
cidr = proc.stdout.strip()
|
||||
if proc.returncode != 0 or not cidr:
|
||||
raise InfraLaunchError(
|
||||
raise ConsolidatedLaunchError(
|
||||
f"gateway network {network} has no subnet: {proc.stderr.strip()}"
|
||||
)
|
||||
return cidr
|
||||
@@ -73,59 +74,6 @@ def _network_container_ips(network: str) -> list[str]:
|
||||
return ips
|
||||
|
||||
|
||||
def running_agent_containers(
|
||||
network: str = GATEWAY_NETWORK, gateway_name: str = INFRA_NAME,
|
||||
) -> list[str]:
|
||||
"""Every running agent container on the gateway `network` (the gateway
|
||||
itself excluded) — the bottles the bring-up reconcile attaches to the fresh
|
||||
gateway (PRD 0081). Raises `OSError` if `docker` can't be run (fail hard: a
|
||||
reconcile that can't list its bottles must not look like "no bottles")."""
|
||||
proc = run_docker([
|
||||
"docker", "network", "inspect", "--format",
|
||||
"{{range .Containers}}{{.Name}}\n{{end}}", network,
|
||||
])
|
||||
return [
|
||||
name for name in (line.strip() for line in proc.stdout.splitlines())
|
||||
if name and name != gateway_name
|
||||
]
|
||||
|
||||
|
||||
def push_ca_to_container(container: str, ca_pem: str) -> None:
|
||||
"""(Re)attach one running agent container to the current gateway: replace
|
||||
its trusted gateway CA with `ca_pem` and rebuild its trust store via
|
||||
`docker cp` + `docker exec` (PRD 0081). Unconditional install — there is one
|
||||
gateway, so no fingerprint match is needed.
|
||||
|
||||
Fail hard: raises `InfraLaunchError` (naming the container) on any failure —
|
||||
the base reconcile aggregates and surfaces it rather than leaving the agent
|
||||
silently unable to reach the gateway."""
|
||||
mkdir = run_docker(
|
||||
["docker", "exec", container, "mkdir", "-p",
|
||||
"/usr/local/share/ca-certificates"]
|
||||
)
|
||||
if mkdir.returncode != 0:
|
||||
raise InfraLaunchError(
|
||||
f"CA push to {container} failed (mkdir): {mkdir.stderr.strip()}"
|
||||
)
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".crt") as tmp:
|
||||
tmp.write(ca_pem)
|
||||
tmp.flush()
|
||||
cp = run_docker(["docker", "cp", tmp.name, f"{container}:{AGENT_CA_PATH}"])
|
||||
if cp.returncode != 0:
|
||||
raise InfraLaunchError(
|
||||
f"CA push to {container} failed (cp): {cp.stderr.strip()}"
|
||||
)
|
||||
ex = run_docker(
|
||||
["docker", "exec", container, "sh", "-c",
|
||||
f"chmod 644 {AGENT_CA_PATH} && update-ca-certificates"]
|
||||
)
|
||||
if ex.returncode != 0:
|
||||
raise InfraLaunchError(
|
||||
f"CA push to {container} failed (update-ca-certificates): "
|
||||
f"{ex.stderr.strip()}"
|
||||
)
|
||||
|
||||
|
||||
def _reprovision_running_bottles(
|
||||
orchestrator_url: str,
|
||||
network: str = GATEWAY_NETWORK,
|
||||
@@ -232,8 +180,6 @@ def deprovision_consolidated(
|
||||
__all__ = [
|
||||
"LaunchContext",
|
||||
"launch_consolidated",
|
||||
"running_agent_containers",
|
||||
"push_ca_to_container",
|
||||
"deprovision_consolidated",
|
||||
"InfraLaunchError",
|
||||
"ConsolidatedLaunchError",
|
||||
]
|
||||
@@ -9,6 +9,8 @@ from .gateway_transport import DockerGatewayTransport
|
||||
from ...paths import (
|
||||
ORCHESTRATOR_AUTH_JWT_ENV,
|
||||
host_gateway_ca_dir,
|
||||
host_gateway_git_dir,
|
||||
host_gateway_creds_dir,
|
||||
)
|
||||
from ... import resources
|
||||
from ...gateway import (
|
||||
@@ -40,6 +42,8 @@ class DockerGateway(Gateway):
|
||||
dockerfile: str | None = GATEWAY_DOCKERFILE,
|
||||
host_port_bindings: tuple[int, ...] = (),
|
||||
ca_mount_source: str | Path | None = None,
|
||||
git_mount_source: str | Path | None = None,
|
||||
creds_mount_source: str | Path | None = None,
|
||||
subnet: str | None = None,
|
||||
) -> None:
|
||||
self.image_ref = image_ref
|
||||
@@ -74,6 +78,17 @@ class DockerGateway(Gateway):
|
||||
self._ca_mount_source = str(
|
||||
ca_mount_source or configured_ca or host_gateway_ca_dir()
|
||||
)
|
||||
# The persistent git-gate mounts (/git bare repos, /git-gate/creds deploy
|
||||
# creds) — same host-bind-mount rationale as the CA (issue #512). The env
|
||||
# overrides let CI point them at per-run named volumes it cleans up.
|
||||
configured_git = os.environ.get("BOT_BOTTLE_DOCKER_GIT_MOUNT", "").strip()
|
||||
self._git_mount_source = str(
|
||||
git_mount_source or configured_git or host_gateway_git_dir()
|
||||
)
|
||||
configured_creds = os.environ.get("BOT_BOTTLE_DOCKER_CREDS_MOUNT", "").strip()
|
||||
self._creds_mount_source = str(
|
||||
creds_mount_source or configured_creds or host_gateway_creds_dir()
|
||||
)
|
||||
|
||||
def image_exists(self) -> bool:
|
||||
return run_docker(["docker", "image", "inspect", self.image_ref]).returncode == 0
|
||||
@@ -157,7 +172,7 @@ class DockerGateway(Gateway):
|
||||
f"gateway network {self.network} failed to create: {proc.stderr.strip()}"
|
||||
)
|
||||
|
||||
def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> bool:
|
||||
def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> None:
|
||||
# Bind to this orchestrator (PRD 0070): stash the URL its daemons resolve
|
||||
# policy against + the pre-minted `gateway` token they present, then bring
|
||||
# the container up. The gateway never mints, so it holds no signing key —
|
||||
@@ -184,7 +199,7 @@ class DockerGateway(Gateway):
|
||||
# just when the container is absent.
|
||||
self._ensure_network()
|
||||
if self.is_running() and self._running_image_is_current():
|
||||
return False
|
||||
return
|
||||
# Clear any stale (stopped OR outdated-image) container holding the
|
||||
# fixed name, then start fresh. `rm --force` on an absent name is a
|
||||
# tolerated no-op.
|
||||
@@ -198,6 +213,12 @@ class DockerGateway(Gateway):
|
||||
# container recreation AND docker volume pruning (agents trust it)
|
||||
# — see host_gateway_ca_dir / issue #450.
|
||||
"--volume", f"{self._ca_mount_source}:{MITMPROXY_HOME}",
|
||||
# Persist per-bottle git-gate state (bare repos + deploy creds) on
|
||||
# the host so a gateway restart doesn't drop already-running bottles'
|
||||
# repos — they would otherwise 404 on fetch/push (issue #512). Same
|
||||
# host-bind-mount rationale as the CA.
|
||||
"--volume", f"{self._git_mount_source}:/git",
|
||||
"--volume", f"{self._creds_mount_source}:/git-gate/creds",
|
||||
# No DB mount: the data plane (egress / supervise / git-gate) reaches
|
||||
# the supervise queue over the control-plane RPC and never opens
|
||||
# bot-bottle.db, so the gateway container gets no file handle on it
|
||||
@@ -228,9 +249,6 @@ class DockerGateway(Gateway):
|
||||
# pre-connect window (they retry /resolve per request).
|
||||
if self._control_network:
|
||||
self._connect_control_network()
|
||||
# (Re)created a fresh container — signal a cold boot so the caller
|
||||
# reconciles running bottles against it (PRD 0081).
|
||||
return True
|
||||
|
||||
def _connect_control_network(self) -> None:
|
||||
"""Ensure the `--internal` control network exists and attach the gateway
|
||||
|
||||
@@ -142,16 +142,9 @@ class DockerInfraService(InfraService):
|
||||
# mints the role-scoped `gateway` JWT here and hands it to the gateway,
|
||||
# which never sees the key (#469). `connect_to_orchestrator` is
|
||||
# idempotent.
|
||||
cold_booted = gateway.connect_to_orchestrator(
|
||||
gateway.connect_to_orchestrator(
|
||||
orchestrator.gateway_url(), orchestrator.mint_gateway_token(),
|
||||
)
|
||||
# A (re)created gateway container minted/mounted its CA afresh and lost
|
||||
# every bottle's per-bottle state; reconcile the already-running bottles
|
||||
# against it (PRD 0081). Skipped when a healthy current gateway was left
|
||||
# untouched — no cold boot, nothing to reconcile.
|
||||
if cold_booted:
|
||||
from .backend import DockerBottleBackend
|
||||
DockerBottleBackend().attach_bottled_agents_to_gateway()
|
||||
return orchestrator.url()
|
||||
|
||||
def stop(self) -> None:
|
||||
|
||||
@@ -61,7 +61,7 @@ from .compose import (
|
||||
)
|
||||
from .consolidated_compose import consolidated_agent_compose
|
||||
from ...orchestrator.store.config_store import resolve_teardown_timeout
|
||||
from .infra_launch import launch_consolidated, deprovision_consolidated
|
||||
from .consolidated_launch import launch_consolidated, deprovision_consolidated
|
||||
from .infra import INFRA_NAME
|
||||
from .gateway import DockerGateway
|
||||
from ... import resources
|
||||
|
||||
@@ -20,7 +20,6 @@ from ...git_gate import GitGatePlan
|
||||
from ...manifest import Manifest
|
||||
from ...supervisor.plan import SupervisePlan
|
||||
from .. import ActiveAgent, BottleBackend, BottleImages, BottleSpec
|
||||
from ..gateway_attach import GatewayAttachResources
|
||||
from . import cleanup as _cleanup
|
||||
from . import enumerate as _enumerate
|
||||
from . import launch as _launch
|
||||
@@ -32,7 +31,7 @@ from .bottle_plan import FirecrackerBottlePlan
|
||||
|
||||
|
||||
class FirecrackerBottleBackend(
|
||||
BottleBackend["FirecrackerBottlePlan", "FirecrackerBottleCleanupPlan", "Path"]
|
||||
BottleBackend["FirecrackerBottlePlan", "FirecrackerBottleCleanupPlan"]
|
||||
):
|
||||
name = "firecracker"
|
||||
|
||||
@@ -120,19 +119,6 @@ class FirecrackerBottleBackend(
|
||||
def enumerate_active(self) -> Sequence[ActiveAgent]:
|
||||
return _enumerate.enumerate_active()
|
||||
|
||||
def _gateway_attach_resources(self) -> GatewayAttachResources:
|
||||
from .gateway import FirecrackerGateway
|
||||
return GatewayAttachResources(ca_pem=FirecrackerGateway().ca_cert_pem())
|
||||
|
||||
def _running_bottles(self) -> Sequence[Path]:
|
||||
return _cleanup.live_run_dirs()
|
||||
|
||||
def _attach_bottle_to_gateway(
|
||||
self, bottle: Path, resources: GatewayAttachResources,
|
||||
) -> None:
|
||||
from .infra_launch import attach_ca_to_agent_vm
|
||||
attach_ca_to_agent_vm(bottle, resources.ca_pem)
|
||||
|
||||
def supervise_mcp_url(self, plan: FirecrackerBottlePlan) -> str:
|
||||
return plan.agent_supervise_url
|
||||
|
||||
|
||||
+6
-41
@@ -40,9 +40,7 @@ from ...orchestrator.lifecycle import (
|
||||
)
|
||||
from ...orchestrator.reprovision import reprovision_bottles
|
||||
from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME
|
||||
from ..base import InfraLaunchError
|
||||
from ..provision_bottle import deprovision_bottle, provision_bottle
|
||||
from ..util import AGENT_CA_PATH
|
||||
from . import cleanup, util
|
||||
from .gateway import FirecrackerGateway
|
||||
from .infra import FirecrackerInfraService
|
||||
@@ -50,6 +48,10 @@ from .infra import FirecrackerInfraService
|
||||
_ENV_VAR_SECRET_PATH = "/run/bot-bottle/env-var-secret"
|
||||
|
||||
|
||||
class ConsolidatedLaunchError(RuntimeError):
|
||||
"""The consolidated register/provision sequence could not complete."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LaunchContext:
|
||||
"""What the Firecracker launch needs from the consolidated sequence."""
|
||||
@@ -81,48 +83,12 @@ def persist_env_var_secret(private_key: Path, guest_ip: str, secret: str) -> Non
|
||||
input=secret, capture_output=True, text=True, check=False,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
raise InfraLaunchError(
|
||||
raise ConsolidatedLaunchError(
|
||||
f"failed to persist {ENV_VAR_SECRET_NAME} in agent VM: "
|
||||
f"{proc.stderr.strip() or '<no stderr>'}"
|
||||
)
|
||||
|
||||
|
||||
def attach_ca_to_agent_vm(run_dir: Path, ca_pem: str) -> None:
|
||||
"""(Re)attach one running agent VM (identified by its `run_dir`) to the
|
||||
current gateway: replace its trusted gateway CA with `ca_pem` and rebuild
|
||||
its trust store over SSH (PRD 0081). Unconditional install — there is one
|
||||
gateway, so no fingerprint match is needed. Bare-pipe input keeps the cert
|
||||
off argv.
|
||||
|
||||
Fail hard: raises `InfraLaunchError` (naming the bottle) if the run dir is
|
||||
malformed or the push fails — the base reconcile aggregates and surfaces it
|
||||
rather than leaving the agent silently unable to reach the gateway."""
|
||||
guest_ip = _guest_ip_from_config(run_dir / "config.json")
|
||||
private_key = run_dir / "bottle_id_ed25519"
|
||||
if not guest_ip or not private_key.is_file():
|
||||
raise InfraLaunchError(
|
||||
f"{run_dir.name}: cannot resolve guest IP / SSH key to attach it "
|
||||
f"to the gateway"
|
||||
)
|
||||
install = (
|
||||
"umask 022; mkdir -p /usr/local/share/ca-certificates && "
|
||||
f"cat > {AGENT_CA_PATH} && chmod 644 {AGENT_CA_PATH} && "
|
||||
"update-ca-certificates"
|
||||
)
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
util.ssh_base_argv(private_key, guest_ip) + [install],
|
||||
input=ca_pem, capture_output=True, text=True, check=False,
|
||||
)
|
||||
except OSError as exc:
|
||||
raise InfraLaunchError(f"{run_dir.name} ({guest_ip}): CA push failed: {exc}") from exc
|
||||
if proc.returncode != 0:
|
||||
raise InfraLaunchError(
|
||||
f"{run_dir.name} ({guest_ip}): CA push failed: "
|
||||
f"{proc.stderr.strip() or '<no stderr>'}"
|
||||
)
|
||||
|
||||
|
||||
def _reprovision_running_bottles(client: OrchestratorClient) -> None:
|
||||
"""Read keys from live agent VMs and restore the restarted gateway."""
|
||||
try:
|
||||
@@ -195,8 +161,7 @@ def deprovision_consolidated(
|
||||
__all__ = [
|
||||
"LaunchContext",
|
||||
"launch_consolidated",
|
||||
"attach_ca_to_agent_vm",
|
||||
"deprovision_consolidated",
|
||||
"InfraLaunchError",
|
||||
"ConsolidatedLaunchError",
|
||||
"OrchestratorStartError",
|
||||
]
|
||||
@@ -84,7 +84,7 @@ def _config(
|
||||
vcpus: int,
|
||||
mem_mib: int,
|
||||
guest_mac: str,
|
||||
data_drive: Path | None = None,
|
||||
data_drives: tuple[Path, ...] = (),
|
||||
extra_boot_args: str = "",
|
||||
) -> dict[str, object]:
|
||||
drives: list[dict[str, object]] = [
|
||||
@@ -95,12 +95,15 @@ def _config(
|
||||
"is_read_only": False,
|
||||
}
|
||||
]
|
||||
# A second virtio-block device (guest /dev/vdb) — the infra VM's
|
||||
# persistent registry "volume", a host-side ext4 file that outlives the
|
||||
# ephemeral rootfs across VM restarts.
|
||||
if data_drive is not None:
|
||||
# Extra virtio-block devices — the infra VMs' persistent "volumes",
|
||||
# host-side ext4 files that outlive the ephemeral rootfs across VM restarts.
|
||||
# They appear to the guest as /dev/vdb, /dev/vdc, ... in list order (after
|
||||
# the root /dev/vda), so callers must keep the order stable: the orchestrator
|
||||
# attaches its registry (vdb); the gateway attaches its CA (vdb) then its
|
||||
# git-gate state (vdc).
|
||||
for i, data_drive in enumerate(data_drives):
|
||||
drives.append({
|
||||
"drive_id": "data",
|
||||
"drive_id": f"data{i}",
|
||||
"path_on_host": str(data_drive),
|
||||
"is_root_device": False,
|
||||
"is_read_only": False,
|
||||
@@ -138,7 +141,7 @@ def boot(
|
||||
mem_mib: int = 2048,
|
||||
guest_mac: str = "06:00:AC:10:00:02",
|
||||
detached: bool = False,
|
||||
data_drive: Path | None = None,
|
||||
data_drives: tuple[Path, ...] = (),
|
||||
extra_boot_args: str = "",
|
||||
) -> VmHandle:
|
||||
"""Write the config and launch the VMM. Returns once the process is
|
||||
@@ -155,7 +158,7 @@ def boot(
|
||||
_config(
|
||||
rootfs=rootfs, tap=tap, guest_ip=guest_ip, host_ip=host_ip,
|
||||
pubkey=pubkey, vcpus=vcpus, mem_mib=mem_mib, guest_mac=guest_mac,
|
||||
data_drive=data_drive, extra_boot_args=extra_boot_args,
|
||||
data_drives=data_drives, extra_boot_args=extra_boot_args,
|
||||
),
|
||||
indent=2,
|
||||
))
|
||||
|
||||
@@ -19,6 +19,7 @@ singleton flock) is `FirecrackerInfraService` (`infra.py`).
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from ...gateway import (
|
||||
@@ -27,6 +28,7 @@ from ...gateway import (
|
||||
GatewayError,
|
||||
GatewayTransport,
|
||||
)
|
||||
from ...log import die, info
|
||||
from .. import util as backend_util
|
||||
from . import infra_vm, netpool, util
|
||||
from .gateway_transport import FirecrackerGatewayTransport
|
||||
@@ -49,6 +51,26 @@ _GUEST_GATEWAY_JWT_PATH = infra_vm._GUEST_GATEWAY_JWT_PATH
|
||||
# the gateway's TLS interception. Host-side only (SSH cat), so it lives here.
|
||||
_GATEWAY_CA_PATH = "/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem"
|
||||
|
||||
# The gateway VM's persistent CA volume — a small ext4 file the gateway init
|
||||
# mounts at mitmproxy's confdir (see `infra_vm._GATEWAY_CA_MOUNT`) so the
|
||||
# self-generated CA SURVIVES a gateway-VM rebuild/restart. Without it the CA
|
||||
# lives only in the ephemeral per-boot rootfs, so every rebuild mints a fresh CA
|
||||
# that every already-running bottle distrusts, failing the TLS handshake (the
|
||||
# firecracker analogue of the docker fix's persistent CA bind-mount — issue
|
||||
# #450). Co-located with the orchestrator's registry volume under the infra dir,
|
||||
# which outlives the ephemeral rootfs. mitmproxy is tiny; 16M is ample.
|
||||
_CA_VOLUME_SIZE = "16M"
|
||||
|
||||
# The gateway VM's persistent git-gate volume — a (sparse) ext4 file the gateway
|
||||
# init mounts, then bind-mounts onto /git + /git-gate/creds (see
|
||||
# `infra_vm._GATEWAY_GIT_MOUNT`), so per-bottle bare repos and deploy creds
|
||||
# SURVIVE a gateway-VM rebuild. Without it a restart drops every already-running
|
||||
# bottle's git-gate state and its agent 404s on fetch/push (same class as the CA
|
||||
# — issue #512). Bare repos hold upstream history, so this is sized far larger
|
||||
# than the CA volume; mke2fs leaves the file sparse, so the host only stores
|
||||
# blocks actually used.
|
||||
_GIT_VOLUME_SIZE = "8G"
|
||||
|
||||
_CA_FETCH_TIMEOUT_SECONDS = 15.0
|
||||
|
||||
|
||||
@@ -69,17 +91,13 @@ class FirecrackerGateway(Gateway):
|
||||
self._orchestrator_url = ""
|
||||
self._gateway_token = ""
|
||||
|
||||
def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> bool:
|
||||
def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> None:
|
||||
"""Boot the gateway VM on its link resolving policy against
|
||||
`orchestrator_url`, then seed `gateway_token` over SSH. The orchestrator's
|
||||
guest IP is parsed from the URL and passed on the cmdline (`bb_orch=`), so
|
||||
the baked init stays IP-independent. Boot the orchestrator first — the
|
||||
gateway daemons reach it at startup. The gateway never mints, so it holds
|
||||
no signing key, only this token (#469).
|
||||
|
||||
Always returns True: the firecracker gateway VM is booted fresh here
|
||||
(this runs only on the infra cold-boot path), so it is always a cold
|
||||
boot that minted a new CA — the caller reconciles running bottles."""
|
||||
no signing key, only this token (#469)."""
|
||||
self._orchestrator_url = orchestrator_url
|
||||
self._gateway_token = gateway_token
|
||||
if not self._orchestrator_url:
|
||||
@@ -98,18 +116,24 @@ class FirecrackerGateway(Gateway):
|
||||
f"cannot resolve orchestrator guest IP from {self._orchestrator_url!r}"
|
||||
)
|
||||
# Boot on the gateway link from the gateway rootfs, then push the token
|
||||
# the init waits for before starting the data plane.
|
||||
# the init waits for before starting the data plane. Two persistent
|
||||
# volumes ride along, attached in a FIXED order the gateway init depends
|
||||
# on: the CA volume as /dev/vdb (mounted at mitmproxy's confdir) and the
|
||||
# git-gate volume as /dev/vdc (bind-mounted onto /git + /git-gate/creds).
|
||||
# Both keep gateway-side state STABLE across rebuilds so already-running
|
||||
# bottles keep working — TLS interception (issue #450) and git-gate fetch
|
||||
# /push (issue #512) respectively.
|
||||
vm = infra_vm.boot_vm(
|
||||
name=GATEWAY_NAME, slot=netpool.gw_slot(), run_dir=infra_vm._gw_dir(),
|
||||
role="gateway", mem_mib=_GW_MEM_MIB,
|
||||
extra_boot_args=f"bb_orch={orchestrator_guest_ip}",
|
||||
data_drives=(self._ensure_ca_volume(), self._ensure_git_volume()),
|
||||
)
|
||||
infra_vm.push_secret(
|
||||
vm, self._gateway_token, _GUEST_GATEWAY_JWT_PATH,
|
||||
"the gateway JWT to the gateway VM (its data plane will not start)",
|
||||
)
|
||||
self._vm = vm
|
||||
return True
|
||||
|
||||
def is_running(self) -> bool:
|
||||
return infra_vm._pidfile_alive(infra_vm._gw_dir())
|
||||
@@ -121,6 +145,46 @@ class FirecrackerGateway(Gateway):
|
||||
infra_vm._kill_pidfile(infra_vm._gw_dir())
|
||||
infra_vm._pid_file(infra_vm._gw_dir()).unlink(missing_ok=True)
|
||||
|
||||
def _ensure_ca_volume(self) -> Path:
|
||||
"""Create the empty ext4 CA volume on first use; reuse it after.
|
||||
|
||||
A fresh (empty) volume makes mitmproxy generate a CA into it on first
|
||||
boot; every later boot reuses the CA already on the volume — which is
|
||||
what keeps the CA stable across gateway-VM rebuilds. The mirror of
|
||||
`FirecrackerOrchestrator._ensure_registry_volume`."""
|
||||
vol = infra_vm._gw_dir() / "gateway-ca.ext4"
|
||||
if vol.exists():
|
||||
return vol
|
||||
info(f"creating gateway CA volume {vol} ({_CA_VOLUME_SIZE})")
|
||||
proc = subprocess.run(
|
||||
["mke2fs", "-q", "-t", "ext4", "-F", str(vol), _CA_VOLUME_SIZE],
|
||||
capture_output=True, text=True, check=False,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
vol.unlink(missing_ok=True)
|
||||
die(f"creating gateway CA volume failed: {proc.stderr.strip()}")
|
||||
return vol
|
||||
|
||||
def _ensure_git_volume(self) -> Path:
|
||||
"""Create the empty ext4 git-gate volume on first use; reuse it after.
|
||||
|
||||
Empty on first boot (the gateway init lays out `git/` + `creds/` subdirs
|
||||
and bind-mounts them); every later boot reuses whatever repos + creds the
|
||||
volume already holds — which is what keeps git-gate state stable across a
|
||||
gateway-VM rebuild (issue #512). Sibling of `_ensure_ca_volume`."""
|
||||
vol = infra_vm._gw_dir() / "gateway-git.ext4"
|
||||
if vol.exists():
|
||||
return vol
|
||||
info(f"creating gateway git-gate volume {vol} ({_GIT_VOLUME_SIZE})")
|
||||
proc = subprocess.run(
|
||||
["mke2fs", "-q", "-t", "ext4", "-F", str(vol), _GIT_VOLUME_SIZE],
|
||||
capture_output=True, text=True, check=False,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
vol.unlink(missing_ok=True)
|
||||
die(f"creating gateway git-gate volume failed: {proc.stderr.strip()}")
|
||||
return vol
|
||||
|
||||
def address(self) -> str:
|
||||
"""The gateway VM's guest IP — the agent-facing target agent VMs'
|
||||
gateway-port traffic is DNAT'd to."""
|
||||
|
||||
@@ -67,17 +67,9 @@ class FirecrackerInfraService(InfraService):
|
||||
# holds the signing key) mints the role-scoped `gateway` JWT for the
|
||||
# gateway, which never sees the key (#469).
|
||||
orchestrator.ensure_running(startup_timeout=startup_timeout)
|
||||
cold_booted = self.gateway().connect_to_orchestrator(
|
||||
self.gateway().connect_to_orchestrator(
|
||||
orchestrator.gateway_url(), orchestrator.mint_gateway_token())
|
||||
infra_vm.record_booted_version(want)
|
||||
# The fresh gateway rootfs minted a new CA and lost every bottle's
|
||||
# git-gate/token state; reconcile the already-running bottles against
|
||||
# it so a gateway rebuild doesn't silently break their egress
|
||||
# (PRD 0081). Cold-boot only — the adopt fast-paths above never reach
|
||||
# here, so a healthy current gateway is left untouched.
|
||||
if cold_booted:
|
||||
from .backend import FirecrackerBottleBackend
|
||||
FirecrackerBottleBackend().attach_bottled_agents_to_gateway()
|
||||
return url
|
||||
|
||||
def stop(self) -> None:
|
||||
|
||||
@@ -53,10 +53,25 @@ from . import firecracker_vm, infra_artifact, netpool, util
|
||||
# tokens with the same key the host CLI signs with — the host token file stays
|
||||
# the single source of truth, never clobbered per-backend (issue #469 review).
|
||||
_GUEST_SIGNING_KEY_PATH = "/var/lib/bot-bottle/orchestrator-token"
|
||||
# The gateway VM's pre-minted `gateway` JWT path (rootfs, not /dev/vdb — the
|
||||
# data plane has no registry volume and never opens the DB). Pushed post-boot;
|
||||
# the gateway daemons present it to the orchestrator, and never see the key.
|
||||
# The gateway VM's pre-minted `gateway` JWT path (rootfs, not /dev/vdb — the JWT
|
||||
# is re-pushed every boot, so it needn't persist). Pushed post-boot; the gateway
|
||||
# daemons present it to the orchestrator, and never see the key.
|
||||
_GUEST_GATEWAY_JWT_PATH = "/var/lib/bot-bottle/gateway-jwt"
|
||||
# The gateway VM's persistent CA volume mount point — mitmproxy's confdir. The
|
||||
# gateway boots with a persistent /dev/vdb CA volume (see
|
||||
# `FirecrackerGateway._ensure_ca_volume`) mounted here so the self-generated CA
|
||||
# survives a gateway-VM rebuild; without it every rebuild mints a fresh CA that
|
||||
# already-running bottles distrust, breaking the TLS handshake (issue #450).
|
||||
_GATEWAY_CA_MOUNT = "/home/mitmproxy/.mitmproxy"
|
||||
# The gateway VM's persistent git-gate volume staging mount (/dev/vdc). The
|
||||
# gateway boots with this volume (see `FirecrackerGateway._ensure_git_volume`)
|
||||
# and the init bind-mounts its `git/` + `creds/` subdirs onto the load-bearing
|
||||
# `/git` and `/git-gate/creds` paths, so per-bottle bare repos + deploy creds
|
||||
# survive a gateway-VM rebuild; without it a restart drops every already-running
|
||||
# bottle's git-gate state and its agent 404s on fetch/push (issue #512).
|
||||
_GATEWAY_GIT_MOUNT = "/var/lib/bot-bottle-gitgate"
|
||||
_GATEWAY_GIT_REPO_ROOT = "/git"
|
||||
_GATEWAY_GIT_CREDS_DIR = "/git-gate/creds"
|
||||
|
||||
# The two per-plane rootfs source images. The orchestrator VM boots a control
|
||||
# plane + buildah rootfs (Dockerfile.orchestrator.fc, FROM orchestrator); the
|
||||
@@ -192,11 +207,12 @@ def boot_vm(
|
||||
run_dir: Path,
|
||||
role: str,
|
||||
mem_mib: int,
|
||||
data_drive: Path | None = None,
|
||||
data_drives: tuple[Path, ...] = (),
|
||||
extra_boot_args: str = "",
|
||||
) -> InfraVm:
|
||||
"""Boot the `role` infra VM from its per-plane rootfs on `slot`'s link.
|
||||
Records the PID."""
|
||||
Records the PID. `data_drives` are attached as /dev/vdb, /dev/vdc, ... in
|
||||
order, so callers must keep the order stable (see `firecracker_vm._config`)."""
|
||||
if not netpool.tap_present(slot.iface):
|
||||
die(f"infra link {slot.iface} not present.\n"
|
||||
f" ./cli.py backend setup --backend=firecracker")
|
||||
@@ -218,7 +234,7 @@ def boot_vm(
|
||||
name=name, rootfs=rootfs, tap=slot.iface,
|
||||
guest_ip=slot.guest_ip, host_ip=slot.host_ip, pubkey=pubkey,
|
||||
run_dir=run_dir, mem_mib=mem_mib, detached=True,
|
||||
data_drive=data_drive, extra_boot_args=boot_args,
|
||||
data_drives=data_drives, extra_boot_args=boot_args,
|
||||
)
|
||||
_pid_file(run_dir).write_text(str(vm.process.pid))
|
||||
return InfraVm(guest_ip=slot.guest_ip, private_key=private_key, vm=vm)
|
||||
@@ -448,6 +464,24 @@ def _gateway_init() -> str:
|
||||
bot-bottle.db (PRD 0070 / #469). If the JWT never arrives, REFUSE to start
|
||||
rather than run without auth."""
|
||||
return _init_head() + f"""
|
||||
# Persistent CA volume (second virtio-block device, /dev/vdb) mounted at
|
||||
# mitmproxy's confdir, so the self-generated mitmproxy CA survives gateway-VM
|
||||
# rebuilds (issue #450). On first boot the volume is empty and mitmproxy mints a
|
||||
# CA into it; every later boot reuses it. Must mount BEFORE the data plane (hence
|
||||
# mitmproxy) starts.
|
||||
mkdir -p {_GATEWAY_CA_MOUNT}
|
||||
mount -t ext4 /dev/vdb {_GATEWAY_CA_MOUNT} 2>/dev/null || true
|
||||
# Persistent git-gate volume (/dev/vdc): its git/ + creds/ subdirs are
|
||||
# bind-mounted onto the load-bearing /git and /git-gate/creds so per-bottle bare
|
||||
# repos + deploy creds survive a gateway-VM rebuild (issue #512). On first boot
|
||||
# the volume is empty; the subdirs are created here. Must mount BEFORE the data
|
||||
# plane (hence git-http) starts, and before any per-bottle provisioning writes.
|
||||
mkdir -p {_GATEWAY_GIT_MOUNT}
|
||||
mount -t ext4 /dev/vdc {_GATEWAY_GIT_MOUNT} 2>/dev/null || true
|
||||
mkdir -p {_GATEWAY_GIT_MOUNT}/git {_GATEWAY_GIT_MOUNT}/creds
|
||||
mkdir -p {_GATEWAY_GIT_REPO_ROOT} {_GATEWAY_GIT_CREDS_DIR}
|
||||
mount --bind {_GATEWAY_GIT_MOUNT}/git {_GATEWAY_GIT_REPO_ROOT} 2>/dev/null || true
|
||||
mount --bind {_GATEWAY_GIT_MOUNT}/creds {_GATEWAY_GIT_CREDS_DIR} 2>/dev/null || true
|
||||
ORCH=$(sed -n 's/.*bb_orch=\\([^ ]*\\).*/\\1/p' /proc/cmdline)
|
||||
GW_JWT=""
|
||||
i=0
|
||||
|
||||
@@ -51,7 +51,7 @@ from .bottle import FirecrackerBottle
|
||||
from .bottle_plan import FirecrackerBottlePlan
|
||||
from ...orchestrator.store.config_store import resolve_teardown_timeout
|
||||
from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME
|
||||
from .infra_launch import (
|
||||
from .consolidated_launch import (
|
||||
launch_consolidated,
|
||||
persist_env_var_secret,
|
||||
deprovision_consolidated,
|
||||
|
||||
@@ -104,7 +104,7 @@ class FirecrackerOrchestrator(Orchestrator):
|
||||
vm = infra_vm.boot_vm(
|
||||
name=ORCHESTRATOR_NAME, slot=netpool.orch_slot(),
|
||||
run_dir=infra_vm._orch_dir(), role="orchestrator", mem_mib=_ORCH_MEM_MIB,
|
||||
data_drive=self._ensure_registry_volume(),
|
||||
data_drives=(self._ensure_registry_volume(),),
|
||||
)
|
||||
# Push the host-canonical signing key (the init waits for it before
|
||||
# starting the control plane). It comes through the shared provisioning
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
"""The shared gateway-attach reconcile flow (PRD 0081).
|
||||
|
||||
`BottleBackend.attach_bottled_agents_to_gateway()` delegates here so every
|
||||
backend reconciles running bottles against a freshly-booted gateway *the same
|
||||
way* — the control flow + fail-hard policy live in one place and backends
|
||||
override only the primitives (see ADR 0006). Kept out of `backend/base.py` so
|
||||
the backend contract module stays lean (the base.py size guardrail).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from .base import InfraLaunchError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .base import BottleBackend
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GatewayAttachResources:
|
||||
"""Everything a running bottle needs to (re)attach to the current gateway on
|
||||
a cold boot (PRD 0081). Gathered once per reconcile and handed to every
|
||||
bottle. The CA now; egress secrets and git-gate state join as the reconcile
|
||||
grows (#516)."""
|
||||
|
||||
ca_pem: str
|
||||
|
||||
|
||||
def reconcile_running_bottles(backend: "BottleBackend[Any, Any, Any]") -> None:
|
||||
"""Reconcile every already-running bottle against the freshly-(re)booted
|
||||
gateway: gather the attach resources once, then attach every live bottle.
|
||||
|
||||
Fail hard, never skip: a bottle that silently can't reach the fresh gateway
|
||||
(its egress just starts failing TLS) is worse than a loud bring-up failure,
|
||||
so any attach failure aborts bring-up. Every bottle is attempted first and
|
||||
the failures are raised together, so one bring-up surfaces the full blast
|
||||
radius rather than one bottle at a time."""
|
||||
resources = backend._gateway_attach_resources()
|
||||
failures: list[str] = []
|
||||
for bottle in backend._running_bottles():
|
||||
try:
|
||||
backend._attach_bottle_to_gateway(bottle, resources)
|
||||
except InfraLaunchError as exc:
|
||||
failures.append(str(exc))
|
||||
if failures:
|
||||
raise InfraLaunchError(
|
||||
f"could not attach {len(failures)} running bottle(s) to the "
|
||||
f"freshly-booted gateway:\n " + "\n ".join(failures)
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["GatewayAttachResources", "reconcile_running_bottles"]
|
||||
@@ -14,7 +14,6 @@ from ...git_gate import GitGatePlan
|
||||
from ...supervisor.plan import SupervisePlan
|
||||
from ...manifest import Manifest
|
||||
from .. import ActiveAgent, BottleBackend, BottleImages, BottleSpec
|
||||
from ..gateway_attach import GatewayAttachResources
|
||||
from . import cleanup as _cleanup
|
||||
from . import enumerate as _enumerate
|
||||
from . import launch as _launch
|
||||
@@ -26,7 +25,7 @@ from .bottle_plan import MacosContainerBottlePlan
|
||||
|
||||
|
||||
class MacosContainerBottleBackend(
|
||||
BottleBackend["MacosContainerBottlePlan", "MacosContainerBottleCleanupPlan", str]
|
||||
BottleBackend["MacosContainerBottlePlan", "MacosContainerBottleCleanupPlan"]
|
||||
):
|
||||
"""Apple Container backend. Selected by
|
||||
`BOT_BOTTLE_BACKEND=macos-container` or
|
||||
@@ -118,19 +117,5 @@ class MacosContainerBottleBackend(
|
||||
def enumerate_active(self) -> Sequence[ActiveAgent]:
|
||||
return _enumerate.enumerate_active()
|
||||
|
||||
def _gateway_attach_resources(self) -> GatewayAttachResources:
|
||||
from .infra import MacosInfraService
|
||||
return GatewayAttachResources(ca_pem=MacosInfraService().ca_cert_pem())
|
||||
|
||||
def _running_bottles(self) -> Sequence[str]:
|
||||
from .infra_launch import running_agent_containers
|
||||
return running_agent_containers()
|
||||
|
||||
def _attach_bottle_to_gateway(
|
||||
self, bottle: str, resources: GatewayAttachResources,
|
||||
) -> None:
|
||||
from .infra_launch import push_ca_to_container
|
||||
push_ca_to_container(bottle, resources.ca_pem)
|
||||
|
||||
def supervise_mcp_url(self, plan: MacosContainerBottlePlan) -> str:
|
||||
return plan.agent_supervise_url
|
||||
|
||||
+5
-52
@@ -34,7 +34,6 @@ every real command arrives through `container exec`.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
|
||||
from ...egress import EgressPlan
|
||||
@@ -43,9 +42,7 @@ from ...log import info
|
||||
from ...orchestrator.client import OrchestratorClient, OrchestratorClientError
|
||||
from ...orchestrator.reprovision import reprovision_bottles
|
||||
from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME
|
||||
from ..base import InfraLaunchError
|
||||
from ..provision_bottle import deprovision_bottle, provision_bottle
|
||||
from ..util import AGENT_CA_PATH
|
||||
from . import util as container_mod
|
||||
from .enumerate import CONTAINER_NAME_PREFIX, EnumerationError, enumerate_active
|
||||
from .gateway import GATEWAY_NETWORK
|
||||
@@ -53,6 +50,10 @@ from .gateway_transport import MacosGatewayTransport
|
||||
from .infra import MacosInfraService, OrchestratorStartError
|
||||
|
||||
|
||||
class ConsolidatedLaunchError(RuntimeError):
|
||||
"""The consolidated register/provision sequence could not complete."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GatewayEndpoint:
|
||||
"""What the agent `container run` needs to reach the shared gateway.
|
||||
@@ -97,52 +98,6 @@ def ensure_gateway(
|
||||
return endpoint
|
||||
|
||||
|
||||
def running_agent_containers() -> list[str]:
|
||||
"""Every running agent container's name — the bottles the bring-up reconcile
|
||||
attaches to the fresh gateway (PRD 0081). Raises `EnumerationError` if the
|
||||
live set can't be determined (fail hard rather than reconcile a partial
|
||||
set)."""
|
||||
return [f"{CONTAINER_NAME_PREFIX}{agent.slug}" for agent in enumerate_active()]
|
||||
|
||||
|
||||
def push_ca_to_container(name: str, ca_pem: str) -> None:
|
||||
"""(Re)attach one running agent container to the current gateway: replace
|
||||
its trusted gateway CA with `ca_pem` and rebuild its trust store via
|
||||
`container cp` + `container exec` (PRD 0081). Unconditional install — there
|
||||
is one gateway, so no fingerprint match is needed.
|
||||
|
||||
Fail hard: raises `InfraLaunchError` (naming the container) on any failure —
|
||||
the base reconcile aggregates and surfaces it rather than leaving the agent
|
||||
silently unable to reach the gateway."""
|
||||
mkdir = container_mod.run_container_argv(
|
||||
["container", "exec", name, "mkdir", "-p",
|
||||
"/usr/local/share/ca-certificates"]
|
||||
)
|
||||
if mkdir.returncode != 0:
|
||||
raise InfraLaunchError(
|
||||
f"CA push to {name} failed (mkdir): {(mkdir.stderr or '').strip()}"
|
||||
)
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".crt") as tmp:
|
||||
tmp.write(ca_pem)
|
||||
tmp.flush()
|
||||
cp = container_mod.run_container_argv(
|
||||
["container", "cp", tmp.name, f"{name}:{AGENT_CA_PATH}"]
|
||||
)
|
||||
if cp.returncode != 0:
|
||||
raise InfraLaunchError(
|
||||
f"CA push to {name} failed (cp): {(cp.stderr or '').strip()}"
|
||||
)
|
||||
ex = container_mod.run_container_argv(
|
||||
["container", "exec", name, "sh", "-c",
|
||||
f"chmod 644 {AGENT_CA_PATH} && update-ca-certificates"]
|
||||
)
|
||||
if ex.returncode != 0:
|
||||
raise InfraLaunchError(
|
||||
f"CA push to {name} failed (update-ca-certificates): "
|
||||
f"{(ex.stderr or '').strip()}"
|
||||
)
|
||||
|
||||
|
||||
def _reprovision_running_bottles(endpoint: GatewayEndpoint) -> None:
|
||||
"""Recover keys from live Apple containers and restore gateway tokens."""
|
||||
try:
|
||||
@@ -245,12 +200,10 @@ __all__ = [
|
||||
"GatewayEndpoint",
|
||||
"LaunchContext",
|
||||
"ensure_gateway",
|
||||
"running_agent_containers",
|
||||
"push_ca_to_container",
|
||||
"live_source_ips",
|
||||
"register_agent",
|
||||
"deprovision_consolidated",
|
||||
"InfraLaunchError",
|
||||
"ConsolidatedLaunchError",
|
||||
"OrchestratorStartError",
|
||||
"GATEWAY_NETWORK",
|
||||
]
|
||||
@@ -104,15 +104,11 @@ class MacosGateway(Gateway):
|
||||
container_mod.build_image(
|
||||
self.image_ref, str(self._repo_root), dockerfile="Dockerfile.gateway")
|
||||
|
||||
def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> bool:
|
||||
def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> None:
|
||||
"""Bind the gateway to this orchestrator and (re)start it, dual-homed on
|
||||
the agent + control networks, resolving policy against `orchestrator_url`
|
||||
(the orchestrator's control-network address — Apple has no container
|
||||
DNS) and presenting `gateway_token`.
|
||||
|
||||
Always returns True: this recreates the gateway container unconditionally
|
||||
(a cold boot), so the caller reconciles running bottles against it
|
||||
(PRD 0081)."""
|
||||
DNS) and presenting `gateway_token`."""
|
||||
self._orchestrator_url = orchestrator_url
|
||||
self._gateway_token = gateway_token
|
||||
# Fail closed on a missing policy source or token: the resolver-only data
|
||||
@@ -160,7 +156,6 @@ class MacosGateway(Gateway):
|
||||
f"gateway container failed to start: "
|
||||
f"{(result.stderr or '').strip() or '<no stderr>'}"
|
||||
)
|
||||
return True
|
||||
|
||||
def is_running(self) -> bool:
|
||||
return container_mod.container_is_running(self.name)
|
||||
|
||||
@@ -138,15 +138,7 @@ class MacosInfraService(InfraService):
|
||||
# `gateway` JWT and hands it to the gateway, which never sees the key
|
||||
# (#469).
|
||||
url = orchestrator.url()
|
||||
cold_booted = gateway.connect_to_orchestrator(
|
||||
url, orchestrator.mint_gateway_token())
|
||||
# A (re)created gateway container minted/mounted its CA afresh and lost
|
||||
# every bottle's per-bottle state; reconcile the already-running bottles
|
||||
# against it (PRD 0081). Skipped when a healthy current gateway was left
|
||||
# untouched — no cold boot, nothing to reconcile.
|
||||
if cold_booted:
|
||||
from .backend import MacosContainerBottleBackend
|
||||
MacosContainerBottleBackend().attach_bottled_agents_to_gateway()
|
||||
gateway.connect_to_orchestrator(url, orchestrator.mint_gateway_token())
|
||||
return url
|
||||
|
||||
def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
|
||||
|
||||
@@ -5,7 +5,7 @@ proxies egress through the one per-host gateway, replacing the per-bottle
|
||||
companion container removed in #385.
|
||||
|
||||
The order differs from docker's, forced by Apple Container 1.0.0 having no
|
||||
`--ip` (see `infra_launch`): the agent is started *before* it is
|
||||
`--ip` (see `consolidated_launch`): the agent is started *before* it is
|
||||
registered, because its DHCP-assigned address — the attribution key — does not
|
||||
exist until then.
|
||||
|
||||
@@ -64,7 +64,7 @@ from . import nested_containers as nested_containers_mod
|
||||
from .bottle_plan import MacosContainerBottlePlan
|
||||
from ...orchestrator.store.config_store import resolve_teardown_timeout
|
||||
from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME, new_env_var_secret
|
||||
from .infra_launch import (
|
||||
from .consolidated_launch import (
|
||||
GatewayEndpoint,
|
||||
ensure_gateway,
|
||||
register_agent,
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
Register a bottle with the orchestrator and provision its git-gate state into
|
||||
the shared gateway (`provision_bottle`), and the inverse teardown
|
||||
(`deprovision_bottle`). Backend-neutral — each backend's infra_launch
|
||||
(`deprovision_bottle`). Backend-neutral — each backend's consolidated_launch
|
||||
drives these through its own `GatewayTransport` rather than re-implementing
|
||||
them. The git-gate half of the work lives in `provision_gateway`.
|
||||
"""
|
||||
|
||||
@@ -26,10 +26,10 @@ from .base import ActiveAgent, BackendStatus, BottleBackend
|
||||
# Tests may replace _backends with a {name: fake} dict via patch.object;
|
||||
# _get_backends() returns the current module-level value as-is when it
|
||||
# is not None, so test fakes take effect without triggering real imports.
|
||||
_backends: dict[str, BottleBackend[Any, Any, Any]] | None = None
|
||||
_backends: dict[str, BottleBackend[Any, Any]] | None = None
|
||||
|
||||
|
||||
def _get_backends() -> dict[str, BottleBackend[Any, Any, Any]]:
|
||||
def _get_backends() -> dict[str, BottleBackend[Any, Any]]:
|
||||
"""Return the registry of all backend instances, loading lazily on first call."""
|
||||
global _backends # pylint: disable=global-statement
|
||||
if _backends is None:
|
||||
@@ -48,7 +48,7 @@ def get_bottle_backend(
|
||||
name: str | None = None,
|
||||
*,
|
||||
prompt: bool = True,
|
||||
) -> BottleBackend[Any, Any, Any]:
|
||||
) -> BottleBackend[Any, Any]:
|
||||
"""Resolve the bottle backend.
|
||||
|
||||
`name` precedence:
|
||||
|
||||
@@ -122,18 +122,12 @@ class Gateway(abc.ABC):
|
||||
return
|
||||
|
||||
@abc.abstractmethod
|
||||
def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> bool:
|
||||
def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> None:
|
||||
"""Bind the gateway to this orchestrator and bring it up: store the URL +
|
||||
the pre-minted `gateway` token as instance state, then (re)start the
|
||||
gateway unit carrying the mitmproxy CA + that token, resolving policy
|
||||
against `orchestrator_url`. Idempotent — a healthy, current gateway on
|
||||
the same binding is left alone; a changed binding reconciles it.
|
||||
|
||||
Returns True when the gateway was actually (re)brought up (a cold boot,
|
||||
so its mitmproxy minted a fresh CA and lost its per-bottle state), False
|
||||
when a healthy current gateway was left untouched. The bring-up flow
|
||||
uses this as the cold-boot signal that gates the running-bottle
|
||||
reconcile (`attach_bottled_agents_to_gateway`, PRD 0081)."""
|
||||
the same binding is left alone; a changed binding reconciles it."""
|
||||
|
||||
@abc.abstractmethod
|
||||
def is_running(self) -> bool:
|
||||
|
||||
@@ -184,27 +184,6 @@ class OrchestratorClient:
|
||||
)
|
||||
return True
|
||||
|
||||
def update_agent_secret(
|
||||
self, bottle_id: str, name: str, value: str, env_var_secret: str,
|
||||
) -> bool:
|
||||
"""Update ONE egress token for a running bottle in place
|
||||
(`POST /bottles/<id>/secret`) — the single-secret form of
|
||||
`reprovision_gateway`, for pushing a freshly-refreshed host credential
|
||||
into a bottle without a relaunch. Returns True on success, False when the
|
||||
orchestrator doesn't know the bottle (404)."""
|
||||
status, _ = self._request(
|
||||
"POST",
|
||||
f"/bottles/{bottle_id}/secret",
|
||||
{"name": name, "value": value, "env_var_secret": env_var_secret},
|
||||
)
|
||||
if status == 404:
|
||||
return False
|
||||
if not 200 <= status < 300:
|
||||
raise OrchestratorClientError(
|
||||
f"update_agent_secret {bottle_id}: HTTP {status}"
|
||||
)
|
||||
return True
|
||||
|
||||
def teardown_bottle(self, bottle_id: str) -> bool:
|
||||
"""Tear a bottle down (`DELETE /bottles/<id>`). False if the
|
||||
orchestrator didn't know it (404) — idempotent for cleanup paths."""
|
||||
|
||||
@@ -200,35 +200,6 @@ def dispatch( # pylint: disable=too-many-return-statements,too-many-branches
|
||||
return 200, {"reprovisioned": True}
|
||||
return 404, {"error": "no stored secrets for this bottle"}
|
||||
|
||||
if (
|
||||
method == "POST"
|
||||
and route.startswith("/bottles/")
|
||||
and route.endswith("/secret")
|
||||
):
|
||||
# Update ONE egress token for a running bottle in place — the
|
||||
# single-secret form of reprovision, for refreshing a short-lived host
|
||||
# credential (e.g. the Codex access token) without a relaunch. cli-only
|
||||
# (not in _GATEWAY_ROUTES): a bottle must never set its own tokens.
|
||||
bottle_id = route[len("/bottles/") : -len("/secret")]
|
||||
try:
|
||||
data = _parse_json_object(body)
|
||||
except ValueError as e:
|
||||
return 400, {"error": f"invalid JSON: {e}"}
|
||||
name = data.get("name")
|
||||
value = data.get("value")
|
||||
env_var_secret = data.get("env_var_secret")
|
||||
if (
|
||||
not isinstance(name, str) or not name
|
||||
or not isinstance(value, str) or not value
|
||||
or not isinstance(env_var_secret, str) or not env_var_secret
|
||||
):
|
||||
return 400, {
|
||||
"error": "name, value, and env_var_secret (non-empty strings) are required"
|
||||
}
|
||||
if orch.update_agent_secret(bottle_id, name, value, env_var_secret):
|
||||
return 200, {"updated": True}
|
||||
return 404, {"error": "no such bottle"}
|
||||
|
||||
if method == "DELETE" and route.startswith("/bottles/"):
|
||||
bottle_id = route[len("/bottles/"):]
|
||||
if orch.teardown_bottle(bottle_id):
|
||||
|
||||
@@ -377,30 +377,6 @@ class OrchestratorCore:
|
||||
return False
|
||||
return True
|
||||
|
||||
def update_agent_secret(
|
||||
self, bottle_id: str, name: str, value: str, env_var_secret: str,
|
||||
) -> bool:
|
||||
"""Update ONE egress token for a known bottle in place — the
|
||||
single-secret form of ``reprovision_from_secret``.
|
||||
|
||||
Sets the in-memory token AND upserts the single re-encrypted row under
|
||||
*env_var_secret* (the same key the rest of the rows are encrypted with, so
|
||||
the whole set stays decryptable by a later ``reprovision_from_secret``),
|
||||
leaving every other token untouched. Returns False if the bottle is
|
||||
unknown.
|
||||
|
||||
Unlike ``reprovision_from_secret`` (which restores the values captured at
|
||||
launch), this pushes a *caller-supplied* value — used to refresh a
|
||||
short-lived host credential (e.g. the Codex access token) into a
|
||||
still-running bottle without a relaunch."""
|
||||
from .store.secret_store import encrypt_value
|
||||
if self.registry.get(bottle_id) is None:
|
||||
return False
|
||||
self._tokens.setdefault(bottle_id, {})[name] = value
|
||||
self.registry.store_agent_secret(
|
||||
bottle_id, name, encrypt_value(env_var_secret, value))
|
||||
return True
|
||||
|
||||
# --- consolidated gateway ----------------------------------------------
|
||||
|
||||
def gateway_status(self) -> dict[str, object]:
|
||||
|
||||
@@ -366,30 +366,6 @@ class RegistryStore(DbStore):
|
||||
)
|
||||
self._chmod()
|
||||
|
||||
def store_agent_secret(
|
||||
self,
|
||||
bottle_id: str,
|
||||
key: str,
|
||||
encrypted_value: str,
|
||||
secret_type: str = "injected_env_var",
|
||||
) -> None:
|
||||
"""Upsert ONE encrypted secret row (env-var name → ciphertext) for
|
||||
*bottle_id*, leaving the bottle's other secrets untouched — the per-key
|
||||
counterpart of ``store_agent_secrets``' replace-all. Delete-then-insert
|
||||
because the table carries no unique constraint to `ON CONFLICT` against."""
|
||||
with self._connection() as conn:
|
||||
conn.execute(
|
||||
"DELETE FROM bottled_agent_secrets "
|
||||
"WHERE bottled_agent_id = ? AND key = ? AND type = ?",
|
||||
(bottle_id, key, secret_type),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO bottled_agent_secrets "
|
||||
"(bottled_agent_id, key, value, type) VALUES (?, ?, ?, ?)",
|
||||
(bottle_id, key, encrypted_value, secret_type),
|
||||
)
|
||||
self._chmod()
|
||||
|
||||
def get_agent_secrets(
|
||||
self,
|
||||
bottle_id: str,
|
||||
|
||||
@@ -53,6 +53,15 @@ ORCHESTRATOR_AUTH_JWT_ENV = "BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT"
|
||||
# the shared gateway's TLS interception, so it must not rotate on restart. See
|
||||
# host_gateway_ca_dir() for why this is a host bind-mount, not a named volume.
|
||||
GATEWAY_CA_DIRNAME = "gateway-ca"
|
||||
# The host directories holding the gateway's persistent git-gate state — the
|
||||
# per-bottle bare repos (`gateway-git`) and deploy creds (`gateway-creds`).
|
||||
# Bind-mounted into the gateway container at /git and /git-gate/creds so they
|
||||
# survive container recreation; without it a gateway restart drops every
|
||||
# already-running bottle's git-gate state and its agent 404s on fetch/push
|
||||
# (issue #512). Host bind-mounts (not named volumes) for the same reason as the
|
||||
# CA dir — see host_gateway_ca_dir().
|
||||
GATEWAY_GIT_DIRNAME = "gateway-git"
|
||||
GATEWAY_CREDS_DIRNAME = "gateway-creds"
|
||||
|
||||
|
||||
def bot_bottle_root() -> Path:
|
||||
@@ -97,6 +106,27 @@ def host_gateway_ca_dir() -> Path:
|
||||
return ca_dir
|
||||
|
||||
|
||||
def host_gateway_git_dir() -> Path:
|
||||
"""The directory holding the gateway's persistent per-bottle bare repos,
|
||||
created if missing. Bind-mounted into the gateway container at /git so the
|
||||
repos survive container recreation (issue #512). A host bind-mount under the
|
||||
app-data root, never pruned — same rationale as host_gateway_ca_dir()."""
|
||||
git_dir = bot_bottle_root() / GATEWAY_GIT_DIRNAME
|
||||
git_dir.mkdir(parents=True, exist_ok=True)
|
||||
return git_dir
|
||||
|
||||
|
||||
def host_gateway_creds_dir() -> Path:
|
||||
"""The directory holding the gateway's persistent per-bottle git-gate deploy
|
||||
creds, created if missing. Bind-mounted into the gateway container at
|
||||
/git-gate/creds so the creds survive container recreation (issue #512). A
|
||||
host bind-mount under the app-data root — same rationale as
|
||||
host_gateway_ca_dir()."""
|
||||
creds_dir = bot_bottle_root() / GATEWAY_CREDS_DIRNAME
|
||||
creds_dir.mkdir(parents=True, exist_ok=True)
|
||||
return creds_dir
|
||||
|
||||
|
||||
def host_signing_key(filename: str) -> str:
|
||||
"""A per-host signing key at `<root>/<filename>`, minted (256-bit, url-safe)
|
||||
and persisted 0600 on first use, then reused.
|
||||
@@ -143,10 +173,14 @@ __all__ = [
|
||||
"ORCHESTRATOR_TOKEN_ENV",
|
||||
"ORCHESTRATOR_AUTH_JWT_ENV",
|
||||
"GATEWAY_CA_DIRNAME",
|
||||
"GATEWAY_GIT_DIRNAME",
|
||||
"GATEWAY_CREDS_DIRNAME",
|
||||
"bot_bottle_root",
|
||||
"host_db_path",
|
||||
"host_db_dir",
|
||||
"host_gateway_ca_dir",
|
||||
"host_gateway_git_dir",
|
||||
"host_gateway_creds_dir",
|
||||
"host_signing_key",
|
||||
"host_orchestrator_token",
|
||||
]
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
# ADR 0006: Shared behaviour lives in the base backend
|
||||
|
||||
- **Status:** Accepted
|
||||
- **Date:** 2026-07-27
|
||||
- **Deciders:** didericis
|
||||
|
||||
## Context
|
||||
|
||||
`BottleBackend` has three concrete implementations (Firecracker, docker,
|
||||
macOS-container). Cross-backend operations can be expressed two ways:
|
||||
|
||||
1. **Abstract method per backend** — the base declares `@abstractmethod foo()`
|
||||
and each backend writes its own `foo()` end to end.
|
||||
2. **Template method** — the base owns the *control flow* of `foo()` as a
|
||||
concrete method and each backend overrides small **primitives** it calls.
|
||||
|
||||
Form 1 gives each backend total freedom, which is exactly the problem: the three
|
||||
implementations drift. The bring-up reconcile (PRD 0081) first shipped as form 1
|
||||
and immediately diverged — each backend re-implemented "gather CA → list running
|
||||
bottles → push to each", and one backend's copy silently skipped failures while
|
||||
another's aborted, an inconsistency caught only in review (#519). The behaviour
|
||||
that must be identical (the loop, the error policy, the ordering) was duplicated
|
||||
in the place least likely to stay in sync.
|
||||
|
||||
## Decision
|
||||
|
||||
Prefer the template-method form for cross-backend behaviour: **encode shared
|
||||
control flow and policy in the base backend; backends override primitives, not
|
||||
control flow.** Lean toward *less* freedom in the concrete backends and *more*
|
||||
consistency in the base.
|
||||
|
||||
Concretely, a cross-backend operation on `BottleBackend` should be a concrete
|
||||
method that:
|
||||
|
||||
- owns the sequencing, iteration, and error policy (e.g. fail-hard vs.
|
||||
best-effort, fail-fast vs. attempt-all-then-aggregate); and
|
||||
- delegates only the irreducibly backend-specific steps to `@abstractmethod`
|
||||
primitives with narrow, well-typed signatures.
|
||||
|
||||
The first application is `attach_bottled_agents_to_gateway()` (PRD 0081): the
|
||||
base gathers resources once, attempts every running bottle, and raises an
|
||||
aggregate `InfraLaunchError` if any failed; backends supply only
|
||||
`_gateway_attach_resources()`, `_running_bottles()`, and
|
||||
`_attach_bottle_to_gateway()`.
|
||||
|
||||
A backend-specific `@abstractmethod` with no shared control flow (e.g.
|
||||
`_launch_impl`, `enumerate_active`) stays as form 1 — this ADR is about
|
||||
operations whose *shape* should be uniform, not about banning abstract methods.
|
||||
|
||||
## Consequences
|
||||
|
||||
- The error policy, ordering, and loop for a cross-backend operation have one
|
||||
source of truth; a new backend cannot forget them or diverge.
|
||||
- New backends implement smaller, more focused primitives instead of re-deriving
|
||||
a whole flow.
|
||||
- The base carries more logic and needs generic-enough primitive signatures
|
||||
(e.g. a backend-specific attach-target type parameter on `BottleBackend`).
|
||||
- Genuinely backend-specific one-offs still use a plain abstract method; the
|
||||
guidance is a default, not an absolute.
|
||||
|
||||
## Links
|
||||
|
||||
- PRD 0081 (`docs/prds/0081-reprovision-gateway-state-on-bringup.md`).
|
||||
- Review that prompted this: PR #519.
|
||||
- `bot_bottle/backend/base.py` (`BottleBackend.attach_bottled_agents_to_gateway`).
|
||||
@@ -55,13 +55,3 @@ agent has two observable states:
|
||||
|
||||
Shorthand for an active (running) Bottled Agent. Used in the supervisor TUI
|
||||
and discovery layer to mean "a bottle whose agent runtime is currently up."
|
||||
|
||||
## Infra
|
||||
|
||||
The per-host pair of shared services every bottle attaches to: the
|
||||
**orchestrator** (control plane — holds the signing key, owns the registry DB,
|
||||
mints tokens) and the **gateway** (data plane — TLS-inspecting egress proxy,
|
||||
git-gate, supervise). Brought up as an idempotent singleton pair by each
|
||||
backend's `InfraService` (two containers on docker/macOS, two microVMs on
|
||||
Firecracker). The per-backend `infra_launch` module composes the bring-up and
|
||||
bottle register/provision sequence against this pair.
|
||||
|
||||
@@ -1,136 +0,0 @@
|
||||
# PRD 0081: Reprovision gateway-dependent state on gateway bring-up
|
||||
|
||||
- **Status:** Active
|
||||
- **Author:** claude
|
||||
- **Created:** 2026-07-26
|
||||
- **Issue:** #510, #512
|
||||
|
||||
## Summary
|
||||
|
||||
When the gateway is (re)built, reconcile every already-running bottle against the
|
||||
fresh gateway instead of persisting the gateway's state. On a gateway cold boot,
|
||||
the gateway reconciles all live bottles in one flow: **replace** each agent's
|
||||
trusted CA with the freshly-minted gateway CA, **re-provision** each bottle's
|
||||
git-gate repos + creds onto the gateway, and **restore** each bottle's egress
|
||||
tokens. One mechanism across all three services; the CA rotates for free on every
|
||||
bring-up.
|
||||
|
||||
## Problem
|
||||
|
||||
A gateway rebuild/restart silently breaks every already-running bottle:
|
||||
|
||||
- **CA (#510).** The gateway's mitmproxy mints a new CA on a fresh rootfs; agents
|
||||
still trust the old one, so egress fails TLS verification (`SSL certificate
|
||||
verification failed`).
|
||||
- **git-gate (#512).** Per-bottle bare repos (`/git/<id>`) + deploy creds
|
||||
(`/git-gate/creds/<id>`) live in the gateway's ephemeral rootfs; a rebuild wipes
|
||||
them and the agent 404s on fetch/push.
|
||||
|
||||
Both are the same root cause: per-bottle gateway-dependent state is provisioned
|
||||
**once, at bottle launch**, and nothing restores it for already-running bottles
|
||||
when the gateway comes back. The existing launch-time reprovision
|
||||
(`reprovision_bottles`) restores **only** egress tokens, and only as a side effect
|
||||
of the *next* launch.
|
||||
|
||||
Persisting the state (a host bind-mount / a per-VM data volume per service) was
|
||||
prototyped and rejected: it differs per service (a volume for the CA, another for
|
||||
git-gate, reprovision for tokens), it pins the CA static forever (no rotation),
|
||||
and it adds volume surface that `docker volume prune` / a wiped cache can silently
|
||||
destroy (the original #450 failure mode).
|
||||
|
||||
## Goals / Success Criteria
|
||||
|
||||
- A gateway (re)boot restores **all** running bottles' gateway-dependent state
|
||||
with no manual step and no relaunch — the agent's next egress / fetch / push
|
||||
just works.
|
||||
- **One** reconcile flow covering CA, git-gate, and egress tokens, rather than a
|
||||
different mechanism per service.
|
||||
- The CA **rotates** on every gateway bring-up (no long-lived CA), distributed to
|
||||
running agents by the same reconcile.
|
||||
- Reconcile failures are **loud, never hidden**: if a running bottle can't be
|
||||
re-attached to the fresh gateway, the bring-up fails and names the bottle
|
||||
rather than leaving it silently unable to reach the gateway (a bottle whose
|
||||
egress just starts failing TLS is worse than a loud failure). Every bottle is
|
||||
attempted first and the failures are raised together, so one bring-up surfaces
|
||||
the full blast radius. (Reversed from an earlier "tolerate per-bottle
|
||||
failures" draft after #519 review.)
|
||||
- **All backends** (Firecracker, docker, macOS) reconcile through the *same*
|
||||
contract — an abstract method on the backend base class, so a new backend
|
||||
cannot forget to implement it and none drifts onto a bespoke mechanism.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Deliberate mid-session CA rotation *without* a gateway restart — `rotate_ca`
|
||||
stays for that operator action.
|
||||
- Changing source-IP attribution, `/resolve`, or the plane split (#469).
|
||||
|
||||
## Design
|
||||
|
||||
**The contract — `attach_bottled_agents_to_gateway()`, a concrete flow on the
|
||||
backend base.** Reconciling running bottles against the current gateway is a
|
||||
backend responsibility (only the backend can enumerate its agents and reach them
|
||||
— firecracker over SSH, docker/macOS over `exec`/`cp`). To keep every backend
|
||||
reconciling identically, the *control flow* lives in a concrete
|
||||
`attach_bottled_agents_to_gateway()` on `BottleBackend` (`backend/base.py`) and
|
||||
each backend overrides only three primitives:
|
||||
|
||||
1. `_gateway_attach_resources()` — gather what a bottle needs to attach (the
|
||||
gateway CA now; egress secrets + git-gate state as this grows).
|
||||
2. `_running_bottles()` — enumerate the live bottles as backend handles.
|
||||
3. `_attach_bottle_to_gateway(bottle, resources)` — push the resources into one
|
||||
bottle, raising `InfraLaunchError` (naming it) on failure.
|
||||
|
||||
The base gathers resources once, attempts every bottle, and — fail hard, never
|
||||
skip — raises an aggregate `InfraLaunchError` if any failed, so a backend can't
|
||||
drift onto a bespoke loop or a silent skip. (Encoding shared behaviour in the
|
||||
base backend, with subclasses overriding primitives rather than control flow, is
|
||||
[ADR 0006](../decisions/0006-shared-behavior-in-base-backend.md).) The host
|
||||
calls it whenever the gateway is (re)brought up. It reprovisions **all**
|
||||
registered bottles' gateway-dependent state: CA, git-gate, and egress tokens.
|
||||
|
||||
**Trigger — the gateway bring-up path.** The host calls
|
||||
`attach_bottled_agents_to_gateway()` only when the gateway was actually
|
||||
(re)brought up — the cold-boot branch of the infra bring-up (e.g.
|
||||
`FirecrackerInfraService.ensure_running` after it boots a fresh pair), never on an
|
||||
adopt of a healthy, current gateway (state intact). So it fires exactly when the
|
||||
gateway was (re)booted — including orchestrator restarts, since the pair boots
|
||||
together — and there is no bare-restart path that bypasses bring-up.
|
||||
|
||||
**Per-backend implementation.** Each backend's `attach_bottled_agents_to_gateway`
|
||||
enumerates its live bottles and, for each, reconciles the three services against
|
||||
the current gateway. The firecracker implementation, once the gateway VM is up
|
||||
and its CA is available:
|
||||
|
||||
1. Map each live bottle's guest IP → `bottle_id` from the orchestrator registry
|
||||
(`list_bottles`).
|
||||
2. Install the **current** shared git-gate hooks (`git_gate_render_hook` /
|
||||
`git_gate_render_access_hook`) into the fresh gateway once — rendered from
|
||||
code, never from a bottle's possibly-stale state dir.
|
||||
3. For each live agent VM (enumerated from its run dir):
|
||||
- **CA:** SSH the current gateway CA into the agent's trust store and run
|
||||
`update-ca-certificates` (unconditional replace — there is one gateway, so no
|
||||
fingerprint match is needed).
|
||||
- **git-gate:** rebuild the bottle's upstreams from its persisted git-gate
|
||||
state dir (deploy key, known_hosts, upstream URL) and re-init its bare repos
|
||||
+ per-repo creds under `/git/<bottle_id>`.
|
||||
- **egress token:** read the agent's `ENV_VAR_SECRET` and feed
|
||||
`reprovision_bottles`, restoring the orchestrator's in-memory tokens.
|
||||
4. If any per-bottle step fails, the reconcile attempts the rest and then raises
|
||||
an aggregate naming every bottle that failed — bring-up fails loudly rather
|
||||
than leaving a bottle silently unable to reach the gateway.
|
||||
|
||||
**Retire the persistence prototype.** No CA data volume, no git-gate data volume
|
||||
(the abandoned PRs #511 / #513). The launch-time `_reprovision_running_bottles`
|
||||
call folds into this bring-up reconcile, so egress tokens are restored on the same
|
||||
cold-boot trigger (an adopt needs no restore — the orchestrator never restarted).
|
||||
|
||||
**CA rotation.** Because the gateway rootfs is ephemeral, every cold boot mints a
|
||||
fresh CA; reconcile is what distributes it, so a routine gateway rebuild doubles
|
||||
as a CA rotation with zero extra machinery.
|
||||
|
||||
## Open questions
|
||||
|
||||
- Docker's existing host-bind-mounted CA (`host_gateway_ca_dir`): once docker's
|
||||
`attach_bottled_agents_to_gateway` pushes the CA to running agents on bring-up,
|
||||
the bind-mount is redundant — drop it (so docker rotates like firecracker) or
|
||||
keep it as belt-and-suspenders? Leaning drop, for one behaviour across backends.
|
||||
@@ -21,7 +21,7 @@ import subprocess
|
||||
import time
|
||||
import unittest
|
||||
|
||||
from bot_bottle.backend.docker.infra_launch import (
|
||||
from bot_bottle.backend.docker.consolidated_launch import (
|
||||
_network_cidr,
|
||||
_network_container_ips,
|
||||
)
|
||||
|
||||
@@ -83,13 +83,8 @@ class TestRuntimeModuleSizes(unittest.TestCase):
|
||||
self.assertEqual([], violations)
|
||||
|
||||
def test_backend_contract_does_not_absorb_preparation_logic(self) -> None:
|
||||
# base.py holds the backend *contract*; implementation lives elsewhere.
|
||||
# The cap is a tripwire against absorbing preparation/impl logic, not a
|
||||
# ban on new contract surface — bumped 580->600 for the PRD 0081
|
||||
# gateway-attach contract (delegator + 3 abstract primitives; the flow
|
||||
# itself lives in backend/gateway_attach.py, not here).
|
||||
caps = {
|
||||
ROOT / "bot_bottle" / "backend" / "base.py": 600,
|
||||
ROOT / "bot_bottle" / "backend" / "base.py": 580,
|
||||
ROOT / "bot_bottle" / "backend" / "preparation.py": 160,
|
||||
}
|
||||
oversized = [
|
||||
|
||||
@@ -11,11 +11,9 @@ from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from bot_bottle.orchestrator.reprovision import reprovision_bottles
|
||||
from bot_bottle.backend.base import BottleBackend, InfraLaunchError
|
||||
from bot_bottle.backend.gateway_attach import GatewayAttachResources
|
||||
from bot_bottle.backend.firecracker import infra_launch as fc
|
||||
from bot_bottle.backend.macos_container import infra_launch as mac
|
||||
from bot_bottle.backend.docker import infra_launch as docker
|
||||
from bot_bottle.backend.firecracker import consolidated_launch as fc
|
||||
from bot_bottle.backend.macos_container import consolidated_launch as mac
|
||||
from bot_bottle.backend.docker import consolidated_launch as docker
|
||||
from bot_bottle.orchestrator.client import OrchestratorClientError
|
||||
|
||||
|
||||
@@ -134,7 +132,7 @@ class TestFirecrackerReprovision(unittest.TestCase):
|
||||
def test_persist_failure_is_fatal_to_launch(self) -> None:
|
||||
with patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \
|
||||
patch.object(fc.subprocess, "run", return_value=_proc(1, stderr="denied")):
|
||||
with self.assertRaisesRegex(fc.InfraLaunchError, "denied"):
|
||||
with self.assertRaisesRegex(fc.ConsolidatedLaunchError, "denied"):
|
||||
fc.persist_env_var_secret(Path("/key"), "10.0.0.1", "secret")
|
||||
|
||||
def test_reads_live_vm_key_and_reprovisions(self) -> None:
|
||||
@@ -161,150 +159,5 @@ class TestFirecrackerReprovision(unittest.TestCase):
|
||||
restore.assert_called_once_with(client, {})
|
||||
|
||||
|
||||
class TestAttachFlow(unittest.TestCase):
|
||||
"""PRD 0081 / #519 review: the base backend owns the reconcile flow and
|
||||
fails hard — gather resources, attempt every running bottle, then raise an
|
||||
aggregate if any failed (never silently skip)."""
|
||||
|
||||
def _fake(self, **primitives: object) -> SimpleNamespace:
|
||||
base: dict[str, object] = {
|
||||
"_gateway_attach_resources": Mock(
|
||||
return_value=GatewayAttachResources(ca_pem="PEM")),
|
||||
"_running_bottles": Mock(return_value=["a", "b", "c"]),
|
||||
"_attach_bottle_to_gateway": Mock(),
|
||||
}
|
||||
base.update(primitives)
|
||||
return SimpleNamespace(**base)
|
||||
|
||||
def test_attaches_every_running_bottle_once(self) -> None:
|
||||
fake = self._fake()
|
||||
BottleBackend.attach_bottled_agents_to_gateway(fake) # type: ignore[arg-type]
|
||||
self.assertEqual(3, fake._attach_bottle_to_gateway.call_count)
|
||||
# Each bottle got the single gathered resource set.
|
||||
res = fake._gateway_attach_resources.return_value
|
||||
for call in fake._attach_bottle_to_gateway.call_args_list:
|
||||
self.assertIs(res, call.args[1])
|
||||
|
||||
def test_attempts_all_then_raises_aggregate_on_failure(self) -> None:
|
||||
def attach(bottle: str, _resources: object) -> None:
|
||||
if bottle in ("b", "c"):
|
||||
raise InfraLaunchError(f"{bottle} unreachable")
|
||||
fake = self._fake(_attach_bottle_to_gateway=Mock(side_effect=attach))
|
||||
with self.assertRaises(InfraLaunchError) as ctx:
|
||||
BottleBackend.attach_bottled_agents_to_gateway(fake) # type: ignore[arg-type]
|
||||
msg = str(ctx.exception)
|
||||
# Every bottle was attempted (not fail-fast), and both failures surface.
|
||||
self.assertEqual(3, fake._attach_bottle_to_gateway.call_count)
|
||||
self.assertIn("b unreachable", msg)
|
||||
self.assertIn("c unreachable", msg)
|
||||
self.assertIn("2", msg)
|
||||
|
||||
def test_resource_gathering_failure_fails_hard_before_enumerating(self) -> None:
|
||||
fake = self._fake(
|
||||
_gateway_attach_resources=Mock(side_effect=RuntimeError("no CA yet")))
|
||||
with self.assertRaises(RuntimeError):
|
||||
BottleBackend.attach_bottled_agents_to_gateway(fake) # type: ignore[arg-type]
|
||||
fake._running_bottles.assert_not_called()
|
||||
|
||||
|
||||
class TestFirecrackerAttachPrimitive(unittest.TestCase):
|
||||
"""PRD 0081: attach_ca_to_agent_vm pushes the current gateway CA into one
|
||||
running agent VM over SSH, failing hard."""
|
||||
|
||||
def _run_dir(self, root: Path, ip: str = "10.243.0.3") -> Path:
|
||||
run_dir = root / "demo"
|
||||
run_dir.mkdir()
|
||||
(run_dir / "bottle_id_ed25519").write_text("key")
|
||||
(run_dir / "config.json").write_text(json.dumps({
|
||||
"boot-source": {"boot_args": f"root=/dev/vda ip={ip}::gw:mask::eth0:off"}
|
||||
}))
|
||||
return run_dir
|
||||
|
||||
def test_pushes_ca_over_stdin_and_rebuilds_trust_store(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
run_dir = self._run_dir(Path(tmp))
|
||||
with patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \
|
||||
patch.object(fc.subprocess, "run", return_value=_proc()) as run:
|
||||
fc.attach_ca_to_agent_vm(run_dir, "PEM-DATA")
|
||||
# The cert goes over stdin (off argv), and the trust store is rebuilt.
|
||||
self.assertEqual("PEM-DATA", run.call_args.kwargs["input"])
|
||||
script = run.call_args.args[0][-1]
|
||||
self.assertIn(fc.AGENT_CA_PATH, script)
|
||||
self.assertIn("update-ca-certificates", script)
|
||||
|
||||
def test_malformed_run_dir_fails_hard(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
empty = Path(tmp) / "no-config"
|
||||
empty.mkdir()
|
||||
with self.assertRaises(fc.InfraLaunchError):
|
||||
fc.attach_ca_to_agent_vm(empty, "PEM")
|
||||
|
||||
def test_push_failure_fails_hard(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
run_dir = self._run_dir(Path(tmp))
|
||||
with patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \
|
||||
patch.object(fc.subprocess, "run", return_value=_proc(1, stderr="down")):
|
||||
with self.assertRaisesRegex(fc.InfraLaunchError, "down"):
|
||||
fc.attach_ca_to_agent_vm(run_dir, "PEM")
|
||||
|
||||
|
||||
class TestDockerAttachPrimitives(unittest.TestCase):
|
||||
def test_running_agent_containers_excludes_gateway(self) -> None:
|
||||
# network inspect lists the gateway (excluded) + one agent + a blank line.
|
||||
inspect = _proc(stdout="bot-bottle-orch-gateway\nbot-bottle-a\n\n")
|
||||
with patch.object(docker, "run_docker", return_value=inspect):
|
||||
names = docker.running_agent_containers(
|
||||
network="net", gateway_name="bot-bottle-orch-gateway")
|
||||
self.assertEqual(["bot-bottle-a"], names)
|
||||
|
||||
def test_running_agent_containers_fails_hard_on_docker_error(self) -> None:
|
||||
with patch.object(docker, "run_docker", side_effect=FileNotFoundError("docker")):
|
||||
with self.assertRaises(FileNotFoundError):
|
||||
docker.running_agent_containers()
|
||||
|
||||
def test_push_ca_cp_then_rebuilds_trust_store(self) -> None:
|
||||
with patch.object(
|
||||
docker, "run_docker", side_effect=[_proc(), _proc(), _proc()],
|
||||
) as run:
|
||||
docker.push_ca_to_container("bot-bottle-a", "PEM")
|
||||
argvs = [c.args[0] for c in run.call_args_list]
|
||||
cp = next(a for a in argvs if a[:2] == ["docker", "cp"])
|
||||
self.assertEqual(f"bot-bottle-a:{docker.AGENT_CA_PATH}", cp[-1])
|
||||
self.assertTrue(any("update-ca-certificates" in a[-1] for a in argvs))
|
||||
|
||||
def test_push_ca_fails_hard(self) -> None:
|
||||
with patch.object(docker, "run_docker", return_value=_proc(1, stderr="gone")):
|
||||
with self.assertRaisesRegex(docker.InfraLaunchError, "gone"):
|
||||
docker.push_ca_to_container("bot-bottle-a", "PEM")
|
||||
|
||||
|
||||
class TestMacosAttachPrimitives(unittest.TestCase):
|
||||
def test_running_agent_containers_from_enumeration(self) -> None:
|
||||
with patch.object(mac, "enumerate_active",
|
||||
return_value=[SimpleNamespace(slug="demo")]):
|
||||
names = mac.running_agent_containers()
|
||||
self.assertEqual([f"{mac.CONTAINER_NAME_PREFIX}demo"], names)
|
||||
|
||||
def test_running_agent_containers_fails_hard(self) -> None:
|
||||
with patch.object(mac, "enumerate_active",
|
||||
side_effect=mac.EnumerationError("failed")):
|
||||
with self.assertRaises(mac.EnumerationError):
|
||||
mac.running_agent_containers()
|
||||
|
||||
def test_push_ca_cp_then_rebuilds_trust_store(self) -> None:
|
||||
with patch.object(mac.container_mod, "run_container_argv",
|
||||
return_value=_proc()) as run:
|
||||
mac.push_ca_to_container(f"{mac.CONTAINER_NAME_PREFIX}demo", "PEM")
|
||||
argvs = [c.args[0] for c in run.call_args_list]
|
||||
self.assertTrue(any(mac.AGENT_CA_PATH in " ".join(a) for a in argvs))
|
||||
self.assertTrue(any("update-ca-certificates" in a[-1] for a in argvs))
|
||||
|
||||
def test_push_ca_fails_hard(self) -> None:
|
||||
with patch.object(mac.container_mod, "run_container_argv",
|
||||
return_value=_proc(1, stderr="no")):
|
||||
with self.assertRaisesRegex(mac.InfraLaunchError, "no"):
|
||||
mac.push_ca_to_container("x", "PEM")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -6,7 +6,7 @@ import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
from bot_bottle.backend.docker.infra_launch import (
|
||||
from bot_bottle.backend.docker.consolidated_launch import (
|
||||
launch_consolidated,
|
||||
deprovision_consolidated,
|
||||
)
|
||||
@@ -14,7 +14,7 @@ from bot_bottle.egress import EgressPlan, EgressRoute
|
||||
from bot_bottle.git_gate import GitGatePlan
|
||||
from bot_bottle.orchestrator.client import RegisteredBottle
|
||||
|
||||
_MOD = "bot_bottle.backend.docker.infra_launch"
|
||||
_MOD = "bot_bottle.backend.docker.consolidated_launch"
|
||||
_UTIL = "bot_bottle.backend.provision_bottle"
|
||||
|
||||
|
||||
@@ -29,20 +29,10 @@ class TestDockerInfraService(unittest.TestCase):
|
||||
self.orch.gateway_url.return_value = f"http://{ORCHESTRATOR_NAME}:8099"
|
||||
self.orch.mint_gateway_token.return_value = "gw.jwt"
|
||||
self.gw = MagicMock()
|
||||
# Default to a cold boot (gateway (re)created) so the reconcile branch
|
||||
# runs; the actual reconcile is stubbed so these composition tests stay
|
||||
# isolated from docker.
|
||||
self.gw.connect_to_orchestrator.return_value = True
|
||||
for name, mock in (("orchestrator", self.orch), ("gateway", self.gw)):
|
||||
p = patch.object(self.svc, name, return_value=mock)
|
||||
p.start()
|
||||
self.addCleanup(p.stop)
|
||||
ap = patch(
|
||||
"bot_bottle.backend.docker.backend.DockerBottleBackend"
|
||||
".attach_bottled_agents_to_gateway"
|
||||
)
|
||||
self.attach = ap.start()
|
||||
self.addCleanup(ap.stop)
|
||||
|
||||
def test_url_delegates_to_the_orchestrator(self) -> None:
|
||||
self.assertEqual("http://127.0.0.1:8099", self.svc.url())
|
||||
@@ -68,20 +58,6 @@ class TestDockerInfraService(unittest.TestCase):
|
||||
f"http://{ORCHESTRATOR_NAME}:8099", "gw.jwt")
|
||||
self.orch.mint_gateway_token.assert_called_once()
|
||||
|
||||
def test_cold_boot_reconciles_running_bottles(self) -> None:
|
||||
# A (re)created gateway (connect returns True) triggers the bring-up
|
||||
# reconcile so running bottles re-trust the fresh gateway (PRD 0081).
|
||||
self.gw.connect_to_orchestrator.return_value = True
|
||||
self.svc.ensure_running()
|
||||
self.attach.assert_called_once_with()
|
||||
|
||||
def test_no_reconcile_when_gateway_left_untouched(self) -> None:
|
||||
# A healthy current gateway (connect returns False) is not a cold boot,
|
||||
# so there is nothing to reconcile.
|
||||
self.gw.connect_to_orchestrator.return_value = False
|
||||
self.svc.ensure_running()
|
||||
self.attach.assert_not_called()
|
||||
|
||||
def test_stop_removes_both_containers(self) -> None:
|
||||
with patch(_RUN) as run:
|
||||
self.svc.stop()
|
||||
|
||||
@@ -16,7 +16,7 @@ from bot_bottle.agent_provider import AgentProvisionPlan
|
||||
from bot_bottle.backend import BottleSpec
|
||||
from bot_bottle.backend.docker import launch as launch_mod
|
||||
from bot_bottle.backend.docker.bottle_plan import DockerBottlePlan
|
||||
from bot_bottle.backend.docker.infra_launch import LaunchContext
|
||||
from bot_bottle.backend.docker.consolidated_launch import LaunchContext
|
||||
from bot_bottle.egress import EgressPlan
|
||||
from bot_bottle.git_gate import GitGatePlan
|
||||
from bot_bottle.log import Die
|
||||
|
||||
@@ -19,7 +19,7 @@ from bot_bottle.agent_provider import AgentProvisionPlan
|
||||
from bot_bottle.backend import BottleImages, BottleSpec
|
||||
from bot_bottle.backend.docker import launch as launch_mod
|
||||
from bot_bottle.backend.docker.bottle_plan import DockerBottlePlan
|
||||
from bot_bottle.backend.docker.infra_launch import LaunchContext
|
||||
from bot_bottle.backend.docker.consolidated_launch import LaunchContext
|
||||
from bot_bottle.egress import EgressPlan
|
||||
from bot_bottle.git_gate import GitGatePlan
|
||||
from bot_bottle.manifest import ManifestIndex
|
||||
|
||||
@@ -404,16 +404,19 @@ class TestBootArgs(unittest.TestCase):
|
||||
self.assertEqual("bbfc0", cfg["network-interfaces"][0]["host_dev_name"])
|
||||
self.assertEqual(1, len(cfg["drives"])) # no data drive by default
|
||||
|
||||
def test_config_adds_data_drive(self):
|
||||
def test_config_adds_data_drives_in_order(self):
|
||||
cfg = cast(Any, firecracker_vm._config(
|
||||
rootfs=Path("/run/rootfs.ext4"), tap="bbfc0",
|
||||
guest_ip="100.64.0.1", host_ip="100.64.0.0", pubkey="k",
|
||||
vcpus=2, mem_mib=2048, guest_mac="06:00:AC:10:00:02",
|
||||
data_drive=Path("/run/registry.ext4"),
|
||||
data_drives=(Path("/run/ca.ext4"), Path("/run/git.ext4")),
|
||||
))
|
||||
self.assertEqual(2, len(cfg["drives"]))
|
||||
# rootfs (vda) + two data drives, attached in list order so the guest
|
||||
# sees them as /dev/vdb, /dev/vdc — an order callers depend on.
|
||||
self.assertEqual(3, len(cfg["drives"]))
|
||||
self.assertFalse(cfg["drives"][1]["is_root_device"])
|
||||
self.assertEqual("/run/registry.ext4", cfg["drives"][1]["path_on_host"])
|
||||
self.assertEqual("/run/ca.ext4", cfg["drives"][1]["path_on_host"])
|
||||
self.assertEqual("/run/git.ext4", cfg["drives"][2]["path_on_host"])
|
||||
|
||||
|
||||
class TestBottleExecClose(unittest.TestCase):
|
||||
|
||||
@@ -59,23 +59,85 @@ class TestFirecrackerGatewayConnect(unittest.TestCase):
|
||||
def test_boots_the_gateway_vm_and_seeds_the_token(self) -> None:
|
||||
gw = FirecrackerGateway()
|
||||
booted = infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k"))
|
||||
ca_vol = Path("/gw/gateway-ca.ext4")
|
||||
git_vol = Path("/gw/gateway-git.ext4")
|
||||
with patch.object(infra_vm, "boot_vm", return_value=booted) as boot, \
|
||||
patch.object(FirecrackerGateway, "_ensure_ca_volume",
|
||||
return_value=ca_vol), \
|
||||
patch.object(FirecrackerGateway, "_ensure_git_volume",
|
||||
return_value=git_vol), \
|
||||
patch.object(infra_vm, "push_secret") as push:
|
||||
cold_booted = gw.connect_to_orchestrator(_ORCH_URL, _TOKEN)
|
||||
# The VM is booted fresh here (cold-boot path only), so it always
|
||||
# signals a cold boot → the caller reconciles running bottles (PRD 0081).
|
||||
self.assertTrue(cold_booted)
|
||||
gw.connect_to_orchestrator(_ORCH_URL, _TOKEN)
|
||||
# Booted on the gateway link with the gateway role; the orchestrator's
|
||||
# guest IP (parsed off the URL) rides the cmdline as bb_orch.
|
||||
kw = boot.call_args.kwargs
|
||||
self.assertEqual("gateway", kw["role"])
|
||||
self.assertIn("bb_orch=10.243.255.1", kw["extra_boot_args"])
|
||||
self.assertIsNone(kw.get("data_drive")) # data plane never opens the DB
|
||||
# The persistent volumes ride in a FIXED order — CA as /dev/vdb, git-gate
|
||||
# as /dev/vdc — so both survive a gateway-VM rebuild (issues #450, #512).
|
||||
self.assertEqual((ca_vol, git_vol), kw.get("data_drives"))
|
||||
# The host-minted token (never the key — #469) is pushed to the guest.
|
||||
push.assert_called_once()
|
||||
self.assertEqual(_TOKEN, push.call_args.args[1])
|
||||
|
||||
|
||||
class TestGatewayCaVolume(unittest.TestCase):
|
||||
"""The persistent CA volume that keeps the mitmproxy CA stable across a
|
||||
gateway-VM rebuild (issue #450) — the mirror of the orchestrator's registry
|
||||
volume."""
|
||||
|
||||
def test_reuses_existing_volume(self) -> None:
|
||||
gw = FirecrackerGateway()
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
vol = Path(td) / "gateway-ca.ext4"
|
||||
vol.write_bytes(b"") # already present
|
||||
with patch.object(infra_vm, "_gw_dir", return_value=Path(td)), \
|
||||
patch(f"{_GW}.subprocess.run") as run:
|
||||
out = gw._ensure_ca_volume()
|
||||
run.assert_not_called() # no mke2fs when it exists — the CA survives
|
||||
self.assertEqual(vol, out)
|
||||
|
||||
def test_creates_volume_when_missing(self) -> None:
|
||||
gw = FirecrackerGateway()
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
with patch.object(infra_vm, "_gw_dir", return_value=Path(td)), \
|
||||
patch(f"{_GW}.subprocess.run",
|
||||
return_value=CompletedProcess([], 0)) as run:
|
||||
out = gw._ensure_ca_volume()
|
||||
argv = run.call_args.args[0]
|
||||
self.assertIn("mke2fs", argv)
|
||||
self.assertEqual(str(Path(td) / "gateway-ca.ext4"), out.__fspath__())
|
||||
self.assertIn(str(out), argv)
|
||||
|
||||
|
||||
class TestGatewayGitVolume(unittest.TestCase):
|
||||
"""The persistent git-gate volume that keeps per-bottle bare repos + creds
|
||||
stable across a gateway-VM rebuild (issue #512)."""
|
||||
|
||||
def test_reuses_existing_volume(self) -> None:
|
||||
gw = FirecrackerGateway()
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
vol = Path(td) / "gateway-git.ext4"
|
||||
vol.write_bytes(b"") # already present
|
||||
with patch.object(infra_vm, "_gw_dir", return_value=Path(td)), \
|
||||
patch(f"{_GW}.subprocess.run") as run:
|
||||
out = gw._ensure_git_volume()
|
||||
run.assert_not_called() # no mke2fs when it exists — the repos survive
|
||||
self.assertEqual(vol, out)
|
||||
|
||||
def test_creates_volume_when_missing(self) -> None:
|
||||
gw = FirecrackerGateway()
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
with patch.object(infra_vm, "_gw_dir", return_value=Path(td)), \
|
||||
patch(f"{_GW}.subprocess.run",
|
||||
return_value=CompletedProcess([], 0)) as run:
|
||||
out = gw._ensure_git_volume()
|
||||
argv = run.call_args.args[0]
|
||||
self.assertIn("mke2fs", argv)
|
||||
self.assertEqual(str(Path(td) / "gateway-git.ext4"), out.__fspath__())
|
||||
self.assertIn(str(out), argv)
|
||||
|
||||
|
||||
class TestFirecrackerGatewaySurface(unittest.TestCase):
|
||||
def test_is_running_reads_the_gateway_pidfile(self) -> None:
|
||||
gw = FirecrackerGateway()
|
||||
|
||||
@@ -40,21 +40,7 @@ class TestAccessors(unittest.TestCase):
|
||||
self.assertEqual(f"http://{ip}:{infra_vm.ORCHESTRATOR_PORT}", svc.url())
|
||||
|
||||
|
||||
_ATTACH = (
|
||||
"bot_bottle.backend.firecracker.backend.FirecrackerBottleBackend"
|
||||
".attach_bottled_agents_to_gateway"
|
||||
)
|
||||
|
||||
|
||||
class TestEnsureRunning(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
# The cold-boot branch reconciles running bottles against the fresh
|
||||
# gateway (PRD 0081). Stub it so these composition tests stay isolated
|
||||
# from SSH; the wiring itself is asserted in test_cold_boot_reconciles*.
|
||||
ap = patch(_ATTACH)
|
||||
self.attach = ap.start()
|
||||
self.addCleanup(ap.stop)
|
||||
|
||||
def test_adopts_when_healthy_alive_and_version_matches(self):
|
||||
# Healthy control plane + live gateway + existing key + matching version
|
||||
# marker -> adopt (no stop/build/boot); returns the orchestrator URL.
|
||||
@@ -73,8 +59,6 @@ class TestEnsureRunning(unittest.TestCase):
|
||||
built.assert_not_called()
|
||||
ip = infra_vm.netpool.orch_slot().guest_ip
|
||||
self.assertEqual(f"http://{ip}:{infra_vm.ORCHESTRATOR_PORT}", url)
|
||||
# Adopt path — no cold boot, so no running-bottle reconcile.
|
||||
self.attach.assert_not_called()
|
||||
|
||||
def test_reboots_both_when_version_stale(self):
|
||||
# Healthy control plane but the running pair booted an OLDER image
|
||||
@@ -136,8 +120,6 @@ class TestEnsureRunning(unittest.TestCase):
|
||||
# reach the control plane at startup).
|
||||
orch_cls.return_value.ensure_running.assert_called_once()
|
||||
gw_cls.return_value.connect_to_orchestrator.assert_called_once()
|
||||
# A cold boot minted a fresh gateway CA — reconcile running bottles.
|
||||
self.attach.assert_called_once_with()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -61,7 +61,23 @@ class TestRoleInits(unittest.TestCase):
|
||||
init = infra_vm.role_init("gateway")
|
||||
self.assertNotIn("bb_role=", init)
|
||||
self.assertNotIn("bot_bottle.orchestrator", init)
|
||||
self.assertNotIn("/dev/vdb", init) # no registry volume on the data plane
|
||||
# Persistent CA volume mounted at mitmproxy's confdir so the CA survives
|
||||
# a gateway-VM rebuild (issue #450). Mounted BEFORE the data plane starts.
|
||||
self.assertIn(f"mount -t ext4 /dev/vdb {infra_vm._GATEWAY_CA_MOUNT}", init)
|
||||
self.assertLess(init.index("/dev/vdb"),
|
||||
init.index("bot_bottle.gateway.bootstrap"))
|
||||
# Persistent git-gate volume (/dev/vdc) bind-mounted onto /git and
|
||||
# /git-gate/creds so per-bottle repos + creds survive a rebuild (#512),
|
||||
# also before the data plane (and any provisioning writes).
|
||||
self.assertIn(f"mount -t ext4 /dev/vdc {infra_vm._GATEWAY_GIT_MOUNT}", init)
|
||||
self.assertIn(
|
||||
f"mount --bind {infra_vm._GATEWAY_GIT_MOUNT}/git "
|
||||
f"{infra_vm._GATEWAY_GIT_REPO_ROOT}", init)
|
||||
self.assertIn(
|
||||
f"mount --bind {infra_vm._GATEWAY_GIT_MOUNT}/creds "
|
||||
f"{infra_vm._GATEWAY_GIT_CREDS_DIR}", init)
|
||||
self.assertLess(init.index("/dev/vdc"),
|
||||
init.index("bot_bottle.gateway.bootstrap"))
|
||||
self.assertIn("export PATH=", init) # shared preamble
|
||||
self.assertIn("BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise", init)
|
||||
self.assertIn(f"cat {infra_vm._GUEST_GATEWAY_JWT_PATH}", init) # host-minted JWT
|
||||
|
||||
@@ -52,7 +52,7 @@ class TestEnsureRunning(unittest.TestCase):
|
||||
kw = boot.call_args.kwargs
|
||||
self.assertEqual("orchestrator", kw["role"])
|
||||
self.assertEqual(4096, kw["mem_mib"]) # orchestrator keeps build headroom
|
||||
self.assertEqual(Path("/reg"), kw["data_drive"]) # DB volume on the CP
|
||||
self.assertEqual((Path("/reg"),), kw["data_drives"]) # DB volume on the CP
|
||||
# The host-canonical signing key is pushed to the guest signing-key path.
|
||||
self.assertEqual("host-key", push.call_args.args[1])
|
||||
self.assertEqual(infra_vm._GUEST_SIGNING_KEY_PATH, push.call_args.args[2])
|
||||
|
||||
+6
-6
@@ -6,7 +6,7 @@ import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
from bot_bottle.backend.macos_container.infra_launch import (
|
||||
from bot_bottle.backend.macos_container.consolidated_launch import (
|
||||
GatewayEndpoint,
|
||||
ensure_gateway,
|
||||
register_agent,
|
||||
@@ -16,7 +16,7 @@ from bot_bottle.egress import EgressPlan, EgressRoute
|
||||
from bot_bottle.git_gate import GitGatePlan
|
||||
from bot_bottle.orchestrator.client import RegisteredBottle
|
||||
|
||||
_MOD = "bot_bottle.backend.macos_container.infra_launch"
|
||||
_MOD = "bot_bottle.backend.macos_container.consolidated_launch"
|
||||
_UTIL = "bot_bottle.backend.provision_bottle"
|
||||
|
||||
|
||||
@@ -145,7 +145,7 @@ class TestLiveSourceIps(unittest.TestCase):
|
||||
return [Mock(slug=s) for s in slugs]
|
||||
|
||||
def test_maps_slugs_to_container_addresses(self) -> None:
|
||||
from bot_bottle.backend.macos_container.infra_launch import live_source_ips
|
||||
from bot_bottle.backend.macos_container.consolidated_launch import live_source_ips
|
||||
with patch(f"{_MOD}.enumerate_active", return_value=self._agents("a", "b")), \
|
||||
patch(f"{_MOD}.container_mod.inspect_container_network_ip",
|
||||
side_effect=["10.0.0.1", "10.0.0.2"]) as ip:
|
||||
@@ -156,7 +156,7 @@ class TestLiveSourceIps(unittest.TestCase):
|
||||
def test_containers_without_an_address_are_skipped(self) -> None:
|
||||
"""A container that hasn't been given a DHCP address yet contributes
|
||||
nothing — the reap's grace window, not this list, protects it."""
|
||||
from bot_bottle.backend.macos_container.infra_launch import live_source_ips
|
||||
from bot_bottle.backend.macos_container.consolidated_launch import live_source_ips
|
||||
with patch(f"{_MOD}.enumerate_active", return_value=self._agents("a", "b")), \
|
||||
patch(f"{_MOD}.container_mod.inspect_container_network_ip",
|
||||
side_effect=["", "10.0.0.2"]):
|
||||
@@ -165,7 +165,7 @@ class TestLiveSourceIps(unittest.TestCase):
|
||||
def test_container_list_failure_raises(self) -> None:
|
||||
"""If container list fails, the live set is not authoritative and
|
||||
reconciliation must be skipped."""
|
||||
from bot_bottle.backend.macos_container.infra_launch import live_source_ips
|
||||
from bot_bottle.backend.macos_container.consolidated_launch import live_source_ips
|
||||
from bot_bottle.backend.macos_container.enumerate import EnumerationError
|
||||
with patch(f"{_MOD}.enumerate_active",
|
||||
side_effect=EnumerationError("container list failed")):
|
||||
@@ -174,7 +174,7 @@ class TestLiveSourceIps(unittest.TestCase):
|
||||
|
||||
def test_per_container_inspect_failure_raises(self) -> None:
|
||||
"""If any individual inspect fails, the live set is not authoritative."""
|
||||
from bot_bottle.backend.macos_container.infra_launch import live_source_ips
|
||||
from bot_bottle.backend.macos_container.consolidated_launch import live_source_ips
|
||||
from bot_bottle.backend.macos_container.enumerate import EnumerationError
|
||||
with patch(f"{_MOD}.enumerate_active", return_value=self._agents("a", "b")), \
|
||||
patch(f"{_MOD}.container_mod.inspect_container_network_ip",
|
||||
@@ -16,7 +16,7 @@ from unittest.mock import ANY, patch
|
||||
|
||||
from bot_bottle.backend.macos_container.bottle import MacosContainerBottle
|
||||
from bot_bottle.backend.macos_container.bottle_plan import MacosContainerBottlePlan
|
||||
from bot_bottle.backend.macos_container.infra_launch import GatewayEndpoint
|
||||
from bot_bottle.backend.macos_container.consolidated_launch import GatewayEndpoint
|
||||
from bot_bottle.backend.macos_container.gateway_hosts import GATEWAY_HOSTNAME
|
||||
from bot_bottle.backend.macos_container.launch import (
|
||||
_agent_run_argv,
|
||||
|
||||
@@ -44,17 +44,6 @@ class TestMacosGatewayConnect(unittest.TestCase):
|
||||
def test_default_name(self) -> None:
|
||||
self.assertEqual(GATEWAY_NAME, self.gw.name)
|
||||
|
||||
def test_connect_signals_cold_boot(self) -> None:
|
||||
# The container is recreated unconditionally, so connect always signals
|
||||
# a cold boot → the caller reconciles running bottles (PRD 0081).
|
||||
run = Mock(return_value=_proc())
|
||||
with patch(f"{_GW}.container_mod") as mod, \
|
||||
patch(f"{_GW}.host_gateway_ca_dir", return_value=Path("/ca")):
|
||||
mod.dns_server.return_value = "1.1.1.1"
|
||||
mod.bind_mount_spec.side_effect = _spec
|
||||
mod.run_container_argv = run
|
||||
self.assertTrue(self.gw.connect_to_orchestrator(_ORCH_URL, _TOKEN))
|
||||
|
||||
def test_refuses_without_orchestrator_url(self) -> None:
|
||||
with patch(f"{_GW}.container_mod") as mod:
|
||||
with self.assertRaises(GatewayError):
|
||||
|
||||
@@ -24,20 +24,10 @@ class TestInfraEnsureRunning(unittest.TestCase):
|
||||
self.orch.url.return_value = "http://192.168.128.2:8099"
|
||||
self.orch.mint_gateway_token.return_value = "gw.jwt"
|
||||
self.gw = MagicMock()
|
||||
# Default to a cold boot (gateway (re)created) so the reconcile branch
|
||||
# runs; the actual reconcile is stubbed so these composition tests stay
|
||||
# isolated from the `container` CLI.
|
||||
self.gw.connect_to_orchestrator.return_value = True
|
||||
for name, mock in (("orchestrator", self.orch), ("gateway", self.gw)):
|
||||
p = patch.object(self.svc, name, return_value=mock)
|
||||
p.start()
|
||||
self.addCleanup(p.stop)
|
||||
ap = patch(
|
||||
"bot_bottle.backend.macos_container.backend"
|
||||
".MacosContainerBottleBackend.attach_bottled_agents_to_gateway"
|
||||
)
|
||||
self.attach = ap.start()
|
||||
self.addCleanup(ap.stop)
|
||||
|
||||
def test_composes_networks_builds_and_brings_up_orchestrator_first(self) -> None:
|
||||
with patch(f"{_INFRA}.ensure_networks") as nets:
|
||||
@@ -56,20 +46,6 @@ class TestInfraEnsureRunning(unittest.TestCase):
|
||||
"http://192.168.128.2:8099", "gw.jwt")
|
||||
self.orch.mint_gateway_token.assert_called_once()
|
||||
|
||||
def test_cold_boot_reconciles_running_bottles(self) -> None:
|
||||
# A (re)created gateway (connect returns True) triggers the bring-up
|
||||
# reconcile so running bottles re-trust the fresh gateway (PRD 0081).
|
||||
self.gw.connect_to_orchestrator.return_value = True
|
||||
with patch(f"{_INFRA}.ensure_networks"):
|
||||
self.svc.ensure_running()
|
||||
self.attach.assert_called_once_with()
|
||||
|
||||
def test_no_reconcile_when_gateway_left_untouched(self) -> None:
|
||||
self.gw.connect_to_orchestrator.return_value = False
|
||||
with patch(f"{_INFRA}.ensure_networks"):
|
||||
self.svc.ensure_running()
|
||||
self.attach.assert_not_called()
|
||||
|
||||
def test_is_healthy_delegates_to_the_orchestrator(self) -> None:
|
||||
self.orch.is_healthy.return_value = True
|
||||
self.assertTrue(self.svc.is_healthy())
|
||||
|
||||
@@ -125,31 +125,6 @@ class TestReprovisionGateway(unittest.TestCase):
|
||||
self.c.reprovision_gateway("b1", "key")
|
||||
|
||||
|
||||
class TestUpdateAgentSecret(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.c = OrchestratorClient("http://orch:8080")
|
||||
|
||||
def test_success_posts_single_secret(self) -> None:
|
||||
with patch(_URLOPEN, return_value=_resp(200, {"updated": True})) as opened:
|
||||
self.assertTrue(self.c.update_agent_secret("b1", "A", "fresh", "key"))
|
||||
request = opened.call_args.args[0]
|
||||
self.assertEqual("POST", request.get_method())
|
||||
self.assertTrue(request.full_url.endswith("/bottles/b1/secret"))
|
||||
self.assertEqual(
|
||||
{"name": "A", "value": "fresh", "env_var_secret": "key"},
|
||||
json.loads(request.data),
|
||||
)
|
||||
|
||||
def test_unknown_bottle_is_false(self) -> None:
|
||||
with patch(_URLOPEN, side_effect=_http_error(404)):
|
||||
self.assertFalse(self.c.update_agent_secret("b1", "A", "v", "key"))
|
||||
|
||||
def test_other_status_raises(self) -> None:
|
||||
with patch(_URLOPEN, side_effect=_http_error(400)):
|
||||
with self.assertRaises(OrchestratorClientError):
|
||||
self.c.update_agent_secret("b1", "A", "v", "key")
|
||||
|
||||
|
||||
class TestHealthAndPolicy(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.c = OrchestratorClient("http://orch:8080")
|
||||
|
||||
@@ -86,12 +86,9 @@ class TestDockerGateway(unittest.TestCase):
|
||||
return _proc()
|
||||
|
||||
with patch(_RUN_DOCKER, side_effect=fake):
|
||||
cold_booted = self.sc.connect_to_orchestrator(_ORCH_URL, _TOKEN)
|
||||
self.sc.connect_to_orchestrator(_ORCH_URL, _TOKEN)
|
||||
self.assertEqual([], [c for c in calls if c[:2] == ["docker", "run"]])
|
||||
self.assertEqual([], [c for c in calls if c[:2] == ["docker", "rm"]])
|
||||
# A healthy current gateway left untouched is not a cold boot — the
|
||||
# bring-up flow skips the running-bottle reconcile (PRD 0081).
|
||||
self.assertFalse(cold_booted)
|
||||
|
||||
def test_ensure_running_recreates_when_image_is_stale(self) -> None:
|
||||
# Running, but the container was built from an OLD image → recreate so
|
||||
@@ -109,12 +106,9 @@ class TestDockerGateway(unittest.TestCase):
|
||||
return _proc()
|
||||
|
||||
with patch(_RUN_DOCKER, side_effect=fake):
|
||||
cold_booted = self.sc.connect_to_orchestrator(_ORCH_URL, _TOKEN)
|
||||
self.sc.connect_to_orchestrator(_ORCH_URL, _TOKEN)
|
||||
self.assertEqual(1, len([c for c in calls if c[:2] == ["docker", "run"]]))
|
||||
self.assertTrue(any(c[:2] == ["docker", "rm"] for c in calls))
|
||||
# A recreated container minted/mounted its CA afresh — a cold boot that
|
||||
# triggers the running-bottle reconcile (PRD 0081).
|
||||
self.assertTrue(cold_booted)
|
||||
|
||||
def test_ensure_running_starts_the_singleton_when_absent(self) -> None:
|
||||
calls: list[list[str]] = []
|
||||
@@ -124,8 +118,7 @@ class TestDockerGateway(unittest.TestCase):
|
||||
return _proc(stdout="") if argv[:2] == ["docker", "ps"] else _proc()
|
||||
|
||||
with patch(_RUN_DOCKER, side_effect=fake):
|
||||
cold_booted = self.sc.connect_to_orchestrator(_ORCH_URL, _TOKEN)
|
||||
self.assertTrue(cold_booted) # started fresh → reconcile running bottles
|
||||
self.sc.connect_to_orchestrator(_ORCH_URL, _TOKEN)
|
||||
|
||||
runs = [c for c in calls if c[:2] == ["docker", "run"]]
|
||||
self.assertEqual(1, len(runs))
|
||||
@@ -168,6 +161,26 @@ class TestDockerGateway(unittest.TestCase):
|
||||
"ci-ca-volume:/home/mitmproxy/.mitmproxy", argv
|
||||
)
|
||||
|
||||
def test_persists_git_gate_state_across_recreation(self) -> None:
|
||||
# Per-bottle bare repos + deploy creds are bind-mounted from the host so
|
||||
# a gateway restart doesn't drop already-running bottles' git-gate state
|
||||
# (issue #512). Named sources here stand in for CI's per-run volumes.
|
||||
sc = DockerGateway(
|
||||
"bot-bottle-gateway:latest",
|
||||
git_mount_source="ci-git-volume",
|
||||
creds_mount_source="ci-creds-volume",
|
||||
)
|
||||
|
||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
||||
return _proc(stdout="") if argv[:2] == ["docker", "ps"] else _proc()
|
||||
|
||||
with patch(_RUN_DOCKER, side_effect=fake) as run:
|
||||
sc.connect_to_orchestrator(_ORCH_URL, _TOKEN)
|
||||
argv = next(c.args[0] for c in run.call_args_list
|
||||
if c.args[0][:2] == ["docker", "run"])
|
||||
self.assertIn("ci-git-volume:/git", argv)
|
||||
self.assertIn("ci-creds-volume:/git-gate/creds", argv)
|
||||
|
||||
def test_connect_injects_the_pre_minted_gateway_token(self) -> None:
|
||||
# The gateway presents the token the orchestrator handed it — it never
|
||||
# mints (holds no signing key). The value rides the env (bare `--env
|
||||
|
||||
@@ -199,20 +199,6 @@ class TestAgentSecrets(unittest.TestCase):
|
||||
got = self.store.get_agent_secrets("bottle-1")
|
||||
self.assertEqual({"K": "new", "K2": "v2"}, got)
|
||||
|
||||
def test_store_agent_secret_upserts_one_key_leaving_others(self) -> None:
|
||||
self.store.store_agent_secrets("bottle-1", {"K": "v1", "K2": "v2"})
|
||||
self.store.store_agent_secret("bottle-1", "K", "v1-new") # update existing
|
||||
self.store.store_agent_secret("bottle-1", "K3", "v3") # insert new
|
||||
self.assertEqual(
|
||||
{"K": "v1-new", "K2": "v2", "K3": "v3"},
|
||||
self.store.get_agent_secrets("bottle-1"),
|
||||
)
|
||||
|
||||
def test_store_agent_secret_does_not_duplicate_rows(self) -> None:
|
||||
self.store.store_agent_secret("bottle-1", "K", "v1")
|
||||
self.store.store_agent_secret("bottle-1", "K", "v2")
|
||||
self.assertEqual({"K": "v2"}, self.store.get_agent_secrets("bottle-1"))
|
||||
|
||||
def test_delete_removes_secrets(self) -> None:
|
||||
self.store.store_agent_secrets("bottle-1", {"K": "v"})
|
||||
self.store.delete_agent_secrets("bottle-1")
|
||||
|
||||
@@ -87,47 +87,6 @@ class TestDispatch(unittest.TestCase):
|
||||
{"EGRESS_TOKEN_0": "upstream-secret"}, self.orch.tokens_for(bottle_id),
|
||||
)
|
||||
|
||||
def test_update_agent_secret_in_place(self) -> None:
|
||||
key = base64.urlsafe_b64encode(b"unit-test-key").rstrip(b"=").decode()
|
||||
status, payload = dispatch(
|
||||
self.orch, "POST", "/bottles", _body({
|
||||
"source_ip": "10.243.0.21",
|
||||
"tokens": {"A": "old", "B": "keep"},
|
||||
"env_var_secret": key,
|
||||
}),
|
||||
)
|
||||
self.assertEqual(201, status)
|
||||
bottle_id = payload["bottle_id"]
|
||||
assert isinstance(bottle_id, str)
|
||||
status, response = dispatch(
|
||||
self.orch, "POST", f"/bottles/{bottle_id}/secret",
|
||||
_body({"name": "A", "value": "fresh", "env_var_secret": key}),
|
||||
)
|
||||
self.assertEqual((200, {"updated": True}), (status, response))
|
||||
self.assertEqual({"A": "fresh", "B": "keep"}, self.orch.tokens_for(bottle_id))
|
||||
|
||||
def test_update_agent_secret_validates_and_unknown_bottle(self) -> None:
|
||||
status, _ = dispatch(self.orch, "POST", "/bottles/b1/secret", b"not-json")
|
||||
self.assertEqual(400, status)
|
||||
status, _ = dispatch(
|
||||
self.orch, "POST", "/bottles/b1/secret", _body({"name": "A"}),
|
||||
)
|
||||
self.assertEqual(400, status) # value + env_var_secret missing
|
||||
status, _ = dispatch(
|
||||
self.orch, "POST", "/bottles/ghost/secret",
|
||||
_body({"name": "A", "value": "v", "env_var_secret": "k"}),
|
||||
)
|
||||
self.assertEqual(404, status)
|
||||
|
||||
def test_update_agent_secret_is_cli_only(self) -> None:
|
||||
# A data-plane (`gateway`) caller must not set a bottle's tokens.
|
||||
status, _ = dispatch(
|
||||
self.orch, "POST", "/bottles/b1/secret",
|
||||
_body({"name": "A", "value": "v", "env_var_secret": "k"}),
|
||||
role=ROLE_GATEWAY,
|
||||
)
|
||||
self.assertEqual(403, status)
|
||||
|
||||
def test_reprovision_validates_request_and_missing_rows(self) -> None:
|
||||
status, _ = dispatch(
|
||||
self.orch, "POST", "/bottles/b1/reprovision_gateway", b"not-json",
|
||||
|
||||
@@ -103,25 +103,6 @@ class TestOrchestrator(unittest.TestCase):
|
||||
self.assertTrue(self.orch.reprovision_from_secret(rec.bottle_id, key))
|
||||
self.assertEqual({"EGRESS_TOKEN_0": "secret"}, self.orch.tokens_for(rec.bottle_id))
|
||||
|
||||
def test_update_agent_secret_refreshes_one_token_in_place(self) -> None:
|
||||
key = new_env_var_secret()
|
||||
rec = self.orch.launch_bottle(
|
||||
"10.243.0.20", tokens={"A": "old-a", "B": "keep-b"}, env_var_secret=key,
|
||||
)
|
||||
self.assertTrue(self.orch.update_agent_secret(rec.bottle_id, "A", "new-a", key))
|
||||
# In memory: A refreshed, B left untouched.
|
||||
self.assertEqual({"A": "new-a", "B": "keep-b"}, self.orch.tokens_for(rec.bottle_id))
|
||||
# At rest: the whole set stays decryptable with the SAME env_var_secret,
|
||||
# so a later reprovision restores the refreshed value (not the launch one).
|
||||
self.orch._tokens.clear()
|
||||
self.assertTrue(self.orch.reprovision_from_secret(rec.bottle_id, key))
|
||||
self.assertEqual({"A": "new-a", "B": "keep-b"}, self.orch.tokens_for(rec.bottle_id))
|
||||
|
||||
def test_update_agent_secret_unknown_bottle_is_false(self) -> None:
|
||||
self.assertFalse(
|
||||
self.orch.update_agent_secret("ghost", "A", "v", new_env_var_secret())
|
||||
)
|
||||
|
||||
def test_reprovision_rejects_missing_rows_and_wrong_key(self) -> None:
|
||||
self.assertFalse(self.orch.reprovision_from_secret("missing", new_env_var_secret()))
|
||||
key = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE"
|
||||
|
||||
Reference in New Issue
Block a user