fix: make orchestrator teardown timeout configurable #436

Merged
didericis merged 4 commits from fix/orchestrator-teardown-timeout into main 2026-07-21 00:41:01 -04:00
8 changed files with 291 additions and 7 deletions
@@ -142,11 +142,19 @@ def launch_consolidated(
def teardown_consolidated(
bottle_id: str, *, orchestrator_url: str, gateway_name: str = GATEWAY_NAME,
bottle_id: str,
*,
orchestrator_url: str,
gateway_name: str = GATEWAY_NAME,
timeout: float | None = None,
) -> None:
"""Deregister the bottle and remove its git-gate state from the gateway.
Both steps are idempotent so this is safe from a cleanup trap."""
OrchestratorClient(orchestrator_url).teardown_bottle(bottle_id)
from ...orchestrator.config_store import DEFAULT_TEARDOWN_TIMEOUT_SECONDS
OrchestratorClient(
orchestrator_url,
timeout=timeout if timeout is not None else DEFAULT_TEARDOWN_TIMEOUT_SECONDS,
).teardown_bottle(bottle_id)
deprovision_git_gate(DockerGatewayTransport(gateway_name), bottle_id)
+5 -1
View File
@@ -62,6 +62,7 @@ from .compose import (
write_compose_file,
)
from .consolidated_compose import consolidated_agent_compose
from ...orchestrator.config_store import resolve_teardown_timeout
from .consolidated_launch import launch_consolidated, teardown_consolidated
from ...orchestrator.gateway import DockerGateway
@@ -133,11 +134,14 @@ def launch(
token_values = egress_resolve_token_values(
plan.egress_plan.token_env_map, effective_env,
)
teardown_timeout = resolve_teardown_timeout()
ctx = launch_consolidated(
plan.egress_plan, git_gate_plan, image_ref=plan.image, tokens=token_values,
)
stack.callback(
teardown_consolidated, ctx.bottle_id, orchestrator_url=ctx.orchestrator_url,
teardown_consolidated, ctx.bottle_id,
orchestrator_url=ctx.orchestrator_url,
timeout=teardown_timeout,
)
# Step 4: install the SHARED gateway CA into the agent (replaces the
@@ -91,12 +91,18 @@ def launch_consolidated(
)
def teardown_consolidated(bottle_id: str, *, orchestrator_url: str) -> None:
def teardown_consolidated(
bottle_id: str, *, orchestrator_url: str, timeout: float | None = None,
) -> None:
"""Deregister the bottle and remove its git-gate state from the gateway
VM. Both steps are idempotent so this is safe from a cleanup trap. Does
NOT stop the infra VM — it's a persistent per-host singleton shared by
every bottle."""
OrchestratorClient(orchestrator_url).teardown_bottle(bottle_id)
from ...orchestrator.config_store import DEFAULT_TEARDOWN_TIMEOUT_SECONDS
OrchestratorClient(
orchestrator_url,
timeout=timeout if timeout is not None else DEFAULT_TEARDOWN_TIMEOUT_SECONDS,
).teardown_bottle(bottle_id)
deprovision_git_gate(infra_vm.gateway_transport(), bottle_id)
+3
View File
@@ -52,6 +52,7 @@ from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
from . import firecracker_vm, image_builder, isolation_probe, netpool, util
from .bottle import FirecrackerBottle
from .bottle_plan import FirecrackerBottlePlan
from ...orchestrator.config_store import resolve_teardown_timeout
from .consolidated_launch import (
launch_consolidated,
teardown_consolidated,
@@ -112,6 +113,7 @@ def launch(
token_values = egress_resolve_token_values(
plan.egress_plan.token_env_map, effective_env,
)
teardown_timeout = resolve_teardown_timeout()
ctx = launch_consolidated(
plan.egress_plan, git_gate_plan,
guest_ip=slot.guest_ip,
@@ -121,6 +123,7 @@ def launch(
stack.callback(
teardown_consolidated, ctx.bottle_id,
orchestrator_url=ctx.orchestrator_url,
timeout=teardown_timeout,
)
# Step 5: install the SHARED gateway CA (replaces the per-bottle CA).
@@ -124,11 +124,17 @@ def register_agent(
)
def teardown_consolidated(bottle_id: str, *, orchestrator_url: str) -> None:
def teardown_consolidated(
bottle_id: str, *, orchestrator_url: str, timeout: float | None = None,
) -> None:
"""Deregister the bottle and remove its git-gate state from the gateway.
Both steps are idempotent so this is safe from a cleanup trap. Does NOT
stop the gateway — it's a persistent per-host singleton."""
OrchestratorClient(orchestrator_url).teardown_bottle(bottle_id)
from ...orchestrator.config_store import DEFAULT_TEARDOWN_TIMEOUT_SECONDS
OrchestratorClient(
orchestrator_url,
timeout=timeout if timeout is not None else DEFAULT_TEARDOWN_TIMEOUT_SECONDS,
).teardown_bottle(bottle_id)
deprovision_git_gate(AppleGatewayTransport(), bottle_id)
@@ -60,6 +60,7 @@ from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
from . import util as container_mod
from .bottle import MacosContainerBottle
from .bottle_plan import MacosContainerBottlePlan
from ...orchestrator.config_store import resolve_teardown_timeout
from .consolidated_launch import (
GatewayEndpoint,
ensure_gateway,
@@ -129,6 +130,7 @@ def launch(
token_values = egress_resolve_token_values(
plan.egress_plan.token_env_map, effective_env,
)
teardown_timeout = resolve_teardown_timeout()
ctx = register_agent(
plan.egress_plan,
plan.git_gate_plan,
@@ -140,6 +142,7 @@ def launch(
stack.callback(
teardown_consolidated, ctx.bottle_id,
orchestrator_url=ctx.orchestrator_url,
timeout=teardown_timeout,
)
info(
f"agent {plan.container_name} registered "
+107
View File
@@ -0,0 +1,107 @@
"""Per-host orchestrator configuration store (settings in bot-bottle.db).
Co-tenants the shared `bot-bottle.db` via the `DbStore` framework. Settings
are readable by the host launch path directly (no HTTP round-trip to the
orchestrator), so they take effect even before the orchestrator is reachable.
"""
from __future__ import annotations
import os
import sqlite3
from pathlib import Path
from ..db_store import DbStore
from ..migrations import TableMigrations
from ..paths import host_db_path
TEARDOWN_TIMEOUT_ENV = "BOT_BOTTLE_ORCHESTRATOR_TEARDOWN_TIMEOUT_SECONDS"
DEFAULT_TEARDOWN_TIMEOUT_SECONDS = 30.0
_MIGRATIONS = TableMigrations(
"orchestrator_config",
[
"""
CREATE TABLE IF NOT EXISTS orchestrator_config (
id INTEGER PRIMARY KEY CHECK (id = 1),
Review

Done in 256a4b2. The generic key/value table is replaced with an explicit single-row schema:

CREATE TABLE IF NOT EXISTS orchestrator_config (
    id                       INTEGER PRIMARY KEY CHECK (id = 1),
    teardown_timeout_seconds REAL
)

OrchestratorConfigStore now exposes get/set/delete_teardown_timeout_seconds() instead of the generic get/set/delete(key). TEARDOWN_TIMEOUT_KEY is removed. The REAL column means resolve_teardown_timeout() no longer needs a float() conversion.

Done in 256a4b2. The generic `key/value` table is replaced with an explicit single-row schema: ```sql CREATE TABLE IF NOT EXISTS orchestrator_config ( id INTEGER PRIMARY KEY CHECK (id = 1), teardown_timeout_seconds REAL ) ``` `OrchestratorConfigStore` now exposes `get/set/delete_teardown_timeout_seconds()` instead of the generic `get/set/delete(key)`. `TEARDOWN_TIMEOUT_KEY` is removed. The REAL column means `resolve_teardown_timeout()` no longer needs a `float()` conversion.
teardown_timeout_seconds REAL
)
Review

Shouldn't be a generic key value store, should have an explicit schema (want a teardown_timeout_seconds column)

Shouldn't be a generic key value store, should have an explicit schema (want a `teardown_timeout_seconds` column)
""",
],
)
class OrchestratorConfigStore(DbStore):
"""Orchestrator settings in the shared host DB."""
def __init__(self, db_path: Path | None = None) -> None:
super().__init__(db_path or host_db_path(), _MIGRATIONS)
def _connect(self) -> sqlite3.Connection:
conn = super()._connect()
conn.execute("PRAGMA busy_timeout=5000")
return conn
def get_teardown_timeout_seconds(self) -> float | None:
"""Return the configured teardown timeout, or None if not set."""
try:
with self._connection() as conn:
row = conn.execute(
"SELECT teardown_timeout_seconds FROM orchestrator_config WHERE id = 1"
).fetchone()
except sqlite3.OperationalError:
return None
return row["teardown_timeout_seconds"] if row else None
def set_teardown_timeout_seconds(self, value: float) -> None:
"""Persist the teardown timeout."""
with self._connection() as conn:
conn.execute(
"INSERT OR REPLACE INTO orchestrator_config"
" (id, teardown_timeout_seconds) VALUES (1, ?)",
(value,),
)
self._chmod()
def delete_teardown_timeout_seconds(self) -> bool:
"""Clear the stored teardown timeout. Returns True if a value existed."""
with self._connection() as conn:
cur = conn.execute(
"UPDATE orchestrator_config SET teardown_timeout_seconds = NULL"
" WHERE id = 1 AND teardown_timeout_seconds IS NOT NULL"
)
return cur.rowcount > 0
def resolve_teardown_timeout(db_path: Path | None = None) -> float:
"""Return the teardown timeout to use, in priority order:
1. ``BOT_BOTTLE_ORCHESTRATOR_TEARDOWN_TIMEOUT_SECONDS`` env var
2. ``teardown_timeout_seconds`` in the orchestrator config DB
3. ``DEFAULT_TEARDOWN_TIMEOUT_SECONDS`` (30 s)
"""
raw = os.environ.get(TEARDOWN_TIMEOUT_ENV, "").strip()
if raw:
try:
value = float(raw)
if value > 0:
return value
except ValueError:
pass
store = OrchestratorConfigStore(db_path)
if not store.is_migrated():
store.migrate()
db_value = store.get_teardown_timeout_seconds()
if db_value is not None and db_value > 0:
return db_value
return DEFAULT_TEARDOWN_TIMEOUT_SECONDS
__all__ = [
"OrchestratorConfigStore",
"resolve_teardown_timeout",
"TEARDOWN_TIMEOUT_ENV",
"DEFAULT_TEARDOWN_TIMEOUT_SECONDS",
]
@@ -0,0 +1,147 @@
"""Unit: OrchestratorConfigStore and resolve_teardown_timeout.
Also verifies the lifecycle ordering invariant: resolve_teardown_timeout()
must be called before launch_consolidated() / register_agent() so that a
resolver failure cannot leave an orphaned registration with no teardown
callback.
"""
from __future__ import annotations
import inspect
import os
import tempfile
import unittest
from pathlib import Path
from types import ModuleType
from bot_bottle.orchestrator.config_store import (
DEFAULT_TEARDOWN_TIMEOUT_SECONDS,
TEARDOWN_TIMEOUT_ENV,
OrchestratorConfigStore,
resolve_teardown_timeout,
)
class TestOrchestratorConfigStore(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.db = Path(self._tmp.name) / "test.db"
self.store = OrchestratorConfigStore(self.db)
self.store.migrate()
def tearDown(self) -> None:
self._tmp.cleanup()
def test_get_returns_none_when_not_set(self) -> None:
self.assertIsNone(self.store.get_teardown_timeout_seconds())
def test_set_and_get_roundtrip(self) -> None:
self.store.set_teardown_timeout_seconds(42.5)
self.assertEqual(42.5, self.store.get_teardown_timeout_seconds())
def test_set_overwrites_existing_value(self) -> None:
self.store.set_teardown_timeout_seconds(10.0)
self.store.set_teardown_timeout_seconds(20.0)
self.assertEqual(20.0, self.store.get_teardown_timeout_seconds())
def test_delete_clears_value_and_returns_true(self) -> None:
self.store.set_teardown_timeout_seconds(30.0)
deleted = self.store.delete_teardown_timeout_seconds()
self.assertTrue(deleted)
self.assertIsNone(self.store.get_teardown_timeout_seconds())
def test_delete_absent_returns_false(self) -> None:
self.assertFalse(self.store.delete_teardown_timeout_seconds())
def test_is_migrated_true_after_migrate(self) -> None:
self.assertTrue(self.store.is_migrated())
def test_is_migrated_false_before_migrate(self) -> None:
store = OrchestratorConfigStore(Path(self._tmp.name) / "new.db")
self.assertFalse(store.is_migrated())
class TestResolveTeardownTimeout(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.db = Path(self._tmp.name) / "cfg.db"
def tearDown(self) -> None:
self._tmp.cleanup()
os.environ.pop(TEARDOWN_TIMEOUT_ENV, None)
def test_returns_default_when_nothing_configured(self) -> None:
self.assertEqual(
DEFAULT_TEARDOWN_TIMEOUT_SECONDS,
resolve_teardown_timeout(self.db),
)
def test_env_var_overrides_default(self) -> None:
os.environ[TEARDOWN_TIMEOUT_ENV] = "99"
self.assertEqual(99.0, resolve_teardown_timeout(self.db))
def test_env_var_overrides_db_value(self) -> None:
store = OrchestratorConfigStore(self.db)
store.migrate()
store.set_teardown_timeout_seconds(55.0)
os.environ[TEARDOWN_TIMEOUT_ENV] = "77"
self.assertEqual(77.0, resolve_teardown_timeout(self.db))
def test_db_value_overrides_default(self) -> None:
store = OrchestratorConfigStore(self.db)
store.migrate()
store.set_teardown_timeout_seconds(42.0)
self.assertEqual(42.0, resolve_teardown_timeout(self.db))
def test_invalid_env_var_falls_through_to_default(self) -> None:
os.environ[TEARDOWN_TIMEOUT_ENV] = "not-a-number"
self.assertEqual(DEFAULT_TEARDOWN_TIMEOUT_SECONDS, resolve_teardown_timeout(self.db))
def test_non_positive_env_var_falls_through_to_default(self) -> None:
os.environ[TEARDOWN_TIMEOUT_ENV] = "0"
self.assertEqual(DEFAULT_TEARDOWN_TIMEOUT_SECONDS, resolve_teardown_timeout(self.db))
def test_non_positive_db_value_falls_through_to_default(self) -> None:
store = OrchestratorConfigStore(self.db)
store.migrate()
store.set_teardown_timeout_seconds(0.0)
self.assertEqual(DEFAULT_TEARDOWN_TIMEOUT_SECONDS, resolve_teardown_timeout(self.db))
def test_migrates_db_on_first_call(self) -> None:
result = resolve_teardown_timeout(self.db)
self.assertEqual(DEFAULT_TEARDOWN_TIMEOUT_SECONDS, result)
self.assertTrue(OrchestratorConfigStore(self.db).is_migrated())
class TestTeardownTimeoutResolvedBeforeRegistration(unittest.TestCase):
"""Ordering invariant: if resolve_teardown_timeout() raises, the bottle
must not yet be registered no orphaned state can result."""
def _src(self, module: ModuleType) -> str:
return inspect.getsource(module)
def test_docker_resolves_timeout_before_launch_consolidated(self) -> None:
from bot_bottle.backend.docker import launch
src = self._src(launch)
resolve_at = src.index("teardown_timeout = resolve_teardown_timeout()")
launch_at = src.index("ctx = launch_consolidated(")
self.assertLess(resolve_at, launch_at)
def test_firecracker_resolves_timeout_before_launch_consolidated(self) -> None:
from bot_bottle.backend.firecracker import launch
src = self._src(launch)
resolve_at = src.index("teardown_timeout = resolve_teardown_timeout()")
launch_at = src.index("ctx = launch_consolidated(")
self.assertLess(resolve_at, launch_at)
def test_macos_resolves_timeout_before_register_agent(self) -> None:
from bot_bottle.backend.macos_container import launch
src = self._src(launch)
resolve_at = src.index("teardown_timeout = resolve_teardown_timeout()")
register_at = src.index("ctx = register_agent(")
self.assertLess(resolve_at, register_at)
if __name__ == "__main__":
unittest.main()