diff --git a/bot_bottle/backend/docker/consolidated_launch.py b/bot_bottle/backend/docker/consolidated_launch.py index 2d7c230..383f99d 100644 --- a/bot_bottle/backend/docker/consolidated_launch.py +++ b/bot_bottle/backend/docker/consolidated_launch.py @@ -142,11 +142,19 @@ def launch_consolidated( def teardown_consolidated( - bottle_id: str, *, orchestrator_url: str, gateway_name: str = GATEWAY_NAME, + bottle_id: str, + *, + orchestrator_url: str, + gateway_name: str = GATEWAY_NAME, + timeout: float | None = None, ) -> None: """Deregister the bottle and remove its git-gate state from the gateway. Both steps are idempotent so this is safe from a cleanup trap.""" - OrchestratorClient(orchestrator_url).teardown_bottle(bottle_id) + from ...orchestrator.config_store import DEFAULT_TEARDOWN_TIMEOUT_SECONDS + OrchestratorClient( + orchestrator_url, + timeout=timeout if timeout is not None else DEFAULT_TEARDOWN_TIMEOUT_SECONDS, + ).teardown_bottle(bottle_id) deprovision_git_gate(DockerGatewayTransport(gateway_name), bottle_id) diff --git a/bot_bottle/backend/docker/launch.py b/bot_bottle/backend/docker/launch.py index 432b142..f4a3043 100644 --- a/bot_bottle/backend/docker/launch.py +++ b/bot_bottle/backend/docker/launch.py @@ -62,6 +62,7 @@ from .compose import ( write_compose_file, ) from .consolidated_compose import consolidated_agent_compose +from ...orchestrator.config_store import resolve_teardown_timeout from .consolidated_launch import launch_consolidated, teardown_consolidated from ...orchestrator.gateway import DockerGateway @@ -133,11 +134,14 @@ def launch( token_values = egress_resolve_token_values( plan.egress_plan.token_env_map, effective_env, ) + teardown_timeout = resolve_teardown_timeout() ctx = launch_consolidated( plan.egress_plan, git_gate_plan, image_ref=plan.image, tokens=token_values, ) stack.callback( - teardown_consolidated, ctx.bottle_id, orchestrator_url=ctx.orchestrator_url, + teardown_consolidated, ctx.bottle_id, + orchestrator_url=ctx.orchestrator_url, + timeout=teardown_timeout, ) # Step 4: install the SHARED gateway CA into the agent (replaces the diff --git a/bot_bottle/backend/firecracker/consolidated_launch.py b/bot_bottle/backend/firecracker/consolidated_launch.py index 899676b..f9c9db4 100644 --- a/bot_bottle/backend/firecracker/consolidated_launch.py +++ b/bot_bottle/backend/firecracker/consolidated_launch.py @@ -91,12 +91,18 @@ def launch_consolidated( ) -def teardown_consolidated(bottle_id: str, *, orchestrator_url: str) -> None: +def teardown_consolidated( + bottle_id: str, *, orchestrator_url: str, timeout: float | None = None, +) -> None: """Deregister the bottle and remove its git-gate state from the gateway VM. Both steps are idempotent so this is safe from a cleanup trap. Does NOT stop the infra VM — it's a persistent per-host singleton shared by every bottle.""" - OrchestratorClient(orchestrator_url).teardown_bottle(bottle_id) + from ...orchestrator.config_store import DEFAULT_TEARDOWN_TIMEOUT_SECONDS + OrchestratorClient( + orchestrator_url, + timeout=timeout if timeout is not None else DEFAULT_TEARDOWN_TIMEOUT_SECONDS, + ).teardown_bottle(bottle_id) deprovision_git_gate(infra_vm.gateway_transport(), bottle_id) diff --git a/bot_bottle/backend/firecracker/launch.py b/bot_bottle/backend/firecracker/launch.py index 26d3779..a5efc00 100644 --- a/bot_bottle/backend/firecracker/launch.py +++ b/bot_bottle/backend/firecracker/launch.py @@ -52,6 +52,7 @@ from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH from . import firecracker_vm, image_builder, isolation_probe, netpool, util from .bottle import FirecrackerBottle from .bottle_plan import FirecrackerBottlePlan +from ...orchestrator.config_store import resolve_teardown_timeout from .consolidated_launch import ( launch_consolidated, teardown_consolidated, @@ -112,6 +113,7 @@ def launch( token_values = egress_resolve_token_values( plan.egress_plan.token_env_map, effective_env, ) + teardown_timeout = resolve_teardown_timeout() ctx = launch_consolidated( plan.egress_plan, git_gate_plan, guest_ip=slot.guest_ip, @@ -121,6 +123,7 @@ def launch( stack.callback( teardown_consolidated, ctx.bottle_id, orchestrator_url=ctx.orchestrator_url, + timeout=teardown_timeout, ) # Step 5: install the SHARED gateway CA (replaces the per-bottle CA). diff --git a/bot_bottle/backend/macos_container/consolidated_launch.py b/bot_bottle/backend/macos_container/consolidated_launch.py index 5b7efc1..0b712fb 100644 --- a/bot_bottle/backend/macos_container/consolidated_launch.py +++ b/bot_bottle/backend/macos_container/consolidated_launch.py @@ -124,11 +124,17 @@ def register_agent( ) -def teardown_consolidated(bottle_id: str, *, orchestrator_url: str) -> None: +def teardown_consolidated( + bottle_id: str, *, orchestrator_url: str, timeout: float | None = None, +) -> None: """Deregister the bottle and remove its git-gate state from the gateway. Both steps are idempotent so this is safe from a cleanup trap. Does NOT stop the gateway — it's a persistent per-host singleton.""" - OrchestratorClient(orchestrator_url).teardown_bottle(bottle_id) + from ...orchestrator.config_store import DEFAULT_TEARDOWN_TIMEOUT_SECONDS + OrchestratorClient( + orchestrator_url, + timeout=timeout if timeout is not None else DEFAULT_TEARDOWN_TIMEOUT_SECONDS, + ).teardown_bottle(bottle_id) deprovision_git_gate(AppleGatewayTransport(), bottle_id) diff --git a/bot_bottle/backend/macos_container/launch.py b/bot_bottle/backend/macos_container/launch.py index 97bef37..5bfec1f 100644 --- a/bot_bottle/backend/macos_container/launch.py +++ b/bot_bottle/backend/macos_container/launch.py @@ -60,6 +60,7 @@ from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH from . import util as container_mod from .bottle import MacosContainerBottle from .bottle_plan import MacosContainerBottlePlan +from ...orchestrator.config_store import resolve_teardown_timeout from .consolidated_launch import ( GatewayEndpoint, ensure_gateway, @@ -129,6 +130,7 @@ def launch( token_values = egress_resolve_token_values( plan.egress_plan.token_env_map, effective_env, ) + teardown_timeout = resolve_teardown_timeout() ctx = register_agent( plan.egress_plan, plan.git_gate_plan, @@ -140,6 +142,7 @@ def launch( stack.callback( teardown_consolidated, ctx.bottle_id, orchestrator_url=ctx.orchestrator_url, + timeout=teardown_timeout, ) info( f"agent {plan.container_name} registered " diff --git a/bot_bottle/orchestrator/config_store.py b/bot_bottle/orchestrator/config_store.py new file mode 100644 index 0000000..2cd6f85 --- /dev/null +++ b/bot_bottle/orchestrator/config_store.py @@ -0,0 +1,107 @@ +"""Per-host orchestrator configuration store (settings in bot-bottle.db). + +Co-tenants the shared `bot-bottle.db` via the `DbStore` framework. Settings +are readable by the host launch path directly (no HTTP round-trip to the +orchestrator), so they take effect even before the orchestrator is reachable. +""" + +from __future__ import annotations + +import os +import sqlite3 +from pathlib import Path + +from ..db_store import DbStore +from ..migrations import TableMigrations +from ..paths import host_db_path + +TEARDOWN_TIMEOUT_ENV = "BOT_BOTTLE_ORCHESTRATOR_TEARDOWN_TIMEOUT_SECONDS" +DEFAULT_TEARDOWN_TIMEOUT_SECONDS = 30.0 + +_MIGRATIONS = TableMigrations( + "orchestrator_config", + [ + """ + CREATE TABLE IF NOT EXISTS orchestrator_config ( + id INTEGER PRIMARY KEY CHECK (id = 1), + teardown_timeout_seconds REAL + ) + """, + ], +) + + +class OrchestratorConfigStore(DbStore): + """Orchestrator settings in the shared host DB.""" + + def __init__(self, db_path: Path | None = None) -> None: + super().__init__(db_path or host_db_path(), _MIGRATIONS) + + def _connect(self) -> sqlite3.Connection: + conn = super()._connect() + conn.execute("PRAGMA busy_timeout=5000") + return conn + + def get_teardown_timeout_seconds(self) -> float | None: + """Return the configured teardown timeout, or None if not set.""" + try: + with self._connection() as conn: + row = conn.execute( + "SELECT teardown_timeout_seconds FROM orchestrator_config WHERE id = 1" + ).fetchone() + except sqlite3.OperationalError: + return None + return row["teardown_timeout_seconds"] if row else None + + def set_teardown_timeout_seconds(self, value: float) -> None: + """Persist the teardown timeout.""" + with self._connection() as conn: + conn.execute( + "INSERT OR REPLACE INTO orchestrator_config" + " (id, teardown_timeout_seconds) VALUES (1, ?)", + (value,), + ) + self._chmod() + + def delete_teardown_timeout_seconds(self) -> bool: + """Clear the stored teardown timeout. Returns True if a value existed.""" + with self._connection() as conn: + cur = conn.execute( + "UPDATE orchestrator_config SET teardown_timeout_seconds = NULL" + " WHERE id = 1 AND teardown_timeout_seconds IS NOT NULL" + ) + return cur.rowcount > 0 + + +def resolve_teardown_timeout(db_path: Path | None = None) -> float: + """Return the teardown timeout to use, in priority order: + + 1. ``BOT_BOTTLE_ORCHESTRATOR_TEARDOWN_TIMEOUT_SECONDS`` env var + 2. ``teardown_timeout_seconds`` in the orchestrator config DB + 3. ``DEFAULT_TEARDOWN_TIMEOUT_SECONDS`` (30 s) + """ + raw = os.environ.get(TEARDOWN_TIMEOUT_ENV, "").strip() + if raw: + try: + value = float(raw) + if value > 0: + return value + except ValueError: + pass + + store = OrchestratorConfigStore(db_path) + if not store.is_migrated(): + store.migrate() + db_value = store.get_teardown_timeout_seconds() + if db_value is not None and db_value > 0: + return db_value + + return DEFAULT_TEARDOWN_TIMEOUT_SECONDS + + +__all__ = [ + "OrchestratorConfigStore", + "resolve_teardown_timeout", + "TEARDOWN_TIMEOUT_ENV", + "DEFAULT_TEARDOWN_TIMEOUT_SECONDS", +] diff --git a/tests/unit/test_orchestrator_config_store.py b/tests/unit/test_orchestrator_config_store.py new file mode 100644 index 0000000..ff43987 --- /dev/null +++ b/tests/unit/test_orchestrator_config_store.py @@ -0,0 +1,147 @@ +"""Unit: OrchestratorConfigStore and resolve_teardown_timeout. + +Also verifies the lifecycle ordering invariant: resolve_teardown_timeout() +must be called before launch_consolidated() / register_agent() so that a +resolver failure cannot leave an orphaned registration with no teardown +callback. +""" + +from __future__ import annotations + +import inspect +import os +import tempfile +import unittest +from pathlib import Path +from types import ModuleType + +from bot_bottle.orchestrator.config_store import ( + DEFAULT_TEARDOWN_TIMEOUT_SECONDS, + TEARDOWN_TIMEOUT_ENV, + OrchestratorConfigStore, + resolve_teardown_timeout, +) + + +class TestOrchestratorConfigStore(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.db = Path(self._tmp.name) / "test.db" + self.store = OrchestratorConfigStore(self.db) + self.store.migrate() + + def tearDown(self) -> None: + self._tmp.cleanup() + + def test_get_returns_none_when_not_set(self) -> None: + self.assertIsNone(self.store.get_teardown_timeout_seconds()) + + def test_set_and_get_roundtrip(self) -> None: + self.store.set_teardown_timeout_seconds(42.5) + self.assertEqual(42.5, self.store.get_teardown_timeout_seconds()) + + def test_set_overwrites_existing_value(self) -> None: + self.store.set_teardown_timeout_seconds(10.0) + self.store.set_teardown_timeout_seconds(20.0) + self.assertEqual(20.0, self.store.get_teardown_timeout_seconds()) + + def test_delete_clears_value_and_returns_true(self) -> None: + self.store.set_teardown_timeout_seconds(30.0) + deleted = self.store.delete_teardown_timeout_seconds() + self.assertTrue(deleted) + self.assertIsNone(self.store.get_teardown_timeout_seconds()) + + def test_delete_absent_returns_false(self) -> None: + self.assertFalse(self.store.delete_teardown_timeout_seconds()) + + def test_is_migrated_true_after_migrate(self) -> None: + self.assertTrue(self.store.is_migrated()) + + def test_is_migrated_false_before_migrate(self) -> None: + store = OrchestratorConfigStore(Path(self._tmp.name) / "new.db") + self.assertFalse(store.is_migrated()) + + +class TestResolveTeardownTimeout(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.db = Path(self._tmp.name) / "cfg.db" + + def tearDown(self) -> None: + self._tmp.cleanup() + os.environ.pop(TEARDOWN_TIMEOUT_ENV, None) + + def test_returns_default_when_nothing_configured(self) -> None: + self.assertEqual( + DEFAULT_TEARDOWN_TIMEOUT_SECONDS, + resolve_teardown_timeout(self.db), + ) + + def test_env_var_overrides_default(self) -> None: + os.environ[TEARDOWN_TIMEOUT_ENV] = "99" + self.assertEqual(99.0, resolve_teardown_timeout(self.db)) + + def test_env_var_overrides_db_value(self) -> None: + store = OrchestratorConfigStore(self.db) + store.migrate() + store.set_teardown_timeout_seconds(55.0) + os.environ[TEARDOWN_TIMEOUT_ENV] = "77" + self.assertEqual(77.0, resolve_teardown_timeout(self.db)) + + def test_db_value_overrides_default(self) -> None: + store = OrchestratorConfigStore(self.db) + store.migrate() + store.set_teardown_timeout_seconds(42.0) + self.assertEqual(42.0, resolve_teardown_timeout(self.db)) + + def test_invalid_env_var_falls_through_to_default(self) -> None: + os.environ[TEARDOWN_TIMEOUT_ENV] = "not-a-number" + self.assertEqual(DEFAULT_TEARDOWN_TIMEOUT_SECONDS, resolve_teardown_timeout(self.db)) + + def test_non_positive_env_var_falls_through_to_default(self) -> None: + os.environ[TEARDOWN_TIMEOUT_ENV] = "0" + self.assertEqual(DEFAULT_TEARDOWN_TIMEOUT_SECONDS, resolve_teardown_timeout(self.db)) + + def test_non_positive_db_value_falls_through_to_default(self) -> None: + store = OrchestratorConfigStore(self.db) + store.migrate() + store.set_teardown_timeout_seconds(0.0) + self.assertEqual(DEFAULT_TEARDOWN_TIMEOUT_SECONDS, resolve_teardown_timeout(self.db)) + + def test_migrates_db_on_first_call(self) -> None: + result = resolve_teardown_timeout(self.db) + self.assertEqual(DEFAULT_TEARDOWN_TIMEOUT_SECONDS, result) + self.assertTrue(OrchestratorConfigStore(self.db).is_migrated()) + + +class TestTeardownTimeoutResolvedBeforeRegistration(unittest.TestCase): + """Ordering invariant: if resolve_teardown_timeout() raises, the bottle + must not yet be registered — no orphaned state can result.""" + + def _src(self, module: ModuleType) -> str: + return inspect.getsource(module) + + def test_docker_resolves_timeout_before_launch_consolidated(self) -> None: + from bot_bottle.backend.docker import launch + src = self._src(launch) + resolve_at = src.index("teardown_timeout = resolve_teardown_timeout()") + launch_at = src.index("ctx = launch_consolidated(") + self.assertLess(resolve_at, launch_at) + + def test_firecracker_resolves_timeout_before_launch_consolidated(self) -> None: + from bot_bottle.backend.firecracker import launch + src = self._src(launch) + resolve_at = src.index("teardown_timeout = resolve_teardown_timeout()") + launch_at = src.index("ctx = launch_consolidated(") + self.assertLess(resolve_at, launch_at) + + def test_macos_resolves_timeout_before_register_agent(self) -> None: + from bot_bottle.backend.macos_container import launch + src = self._src(launch) + resolve_at = src.index("teardown_timeout = resolve_teardown_timeout()") + register_at = src.index("ctx = register_agent(") + self.assertLess(resolve_at, register_at) + + +if __name__ == "__main__": + unittest.main()