fix: make orchestrator teardown timeout configurable #436
@@ -142,11 +142,19 @@ def launch_consolidated(
|
||||
|
||||
|
||||
def teardown_consolidated(
|
||||
bottle_id: str, *, orchestrator_url: str, gateway_name: str = GATEWAY_NAME,
|
||||
bottle_id: str,
|
||||
*,
|
||||
orchestrator_url: str,
|
||||
gateway_name: str = GATEWAY_NAME,
|
||||
timeout: float | None = None,
|
||||
) -> None:
|
||||
"""Deregister the bottle and remove its git-gate state from the gateway.
|
||||
Both steps are idempotent so this is safe from a cleanup trap."""
|
||||
OrchestratorClient(orchestrator_url).teardown_bottle(bottle_id)
|
||||
from ...orchestrator.config_store import DEFAULT_TEARDOWN_TIMEOUT_SECONDS
|
||||
OrchestratorClient(
|
||||
orchestrator_url,
|
||||
timeout=timeout if timeout is not None else DEFAULT_TEARDOWN_TIMEOUT_SECONDS,
|
||||
).teardown_bottle(bottle_id)
|
||||
deprovision_git_gate(DockerGatewayTransport(gateway_name), bottle_id)
|
||||
|
||||
|
||||
|
||||
@@ -62,6 +62,7 @@ from .compose import (
|
||||
write_compose_file,
|
||||
)
|
||||
from .consolidated_compose import consolidated_agent_compose
|
||||
from ...orchestrator.config_store import resolve_teardown_timeout
|
||||
from .consolidated_launch import launch_consolidated, teardown_consolidated
|
||||
from ...orchestrator.gateway import DockerGateway
|
||||
|
||||
@@ -133,11 +134,14 @@ def launch(
|
||||
token_values = egress_resolve_token_values(
|
||||
plan.egress_plan.token_env_map, effective_env,
|
||||
)
|
||||
teardown_timeout = resolve_teardown_timeout()
|
||||
ctx = launch_consolidated(
|
||||
plan.egress_plan, git_gate_plan, image_ref=plan.image, tokens=token_values,
|
||||
)
|
||||
stack.callback(
|
||||
teardown_consolidated, ctx.bottle_id, orchestrator_url=ctx.orchestrator_url,
|
||||
teardown_consolidated, ctx.bottle_id,
|
||||
orchestrator_url=ctx.orchestrator_url,
|
||||
timeout=teardown_timeout,
|
||||
)
|
||||
|
||||
# Step 4: install the SHARED gateway CA into the agent (replaces the
|
||||
|
||||
@@ -91,12 +91,18 @@ def launch_consolidated(
|
||||
)
|
||||
|
||||
|
||||
def teardown_consolidated(bottle_id: str, *, orchestrator_url: str) -> None:
|
||||
def teardown_consolidated(
|
||||
bottle_id: str, *, orchestrator_url: str, timeout: float | None = None,
|
||||
) -> None:
|
||||
"""Deregister the bottle and remove its git-gate state from the gateway
|
||||
VM. Both steps are idempotent so this is safe from a cleanup trap. Does
|
||||
NOT stop the infra VM — it's a persistent per-host singleton shared by
|
||||
every bottle."""
|
||||
OrchestratorClient(orchestrator_url).teardown_bottle(bottle_id)
|
||||
from ...orchestrator.config_store import DEFAULT_TEARDOWN_TIMEOUT_SECONDS
|
||||
OrchestratorClient(
|
||||
orchestrator_url,
|
||||
timeout=timeout if timeout is not None else DEFAULT_TEARDOWN_TIMEOUT_SECONDS,
|
||||
).teardown_bottle(bottle_id)
|
||||
deprovision_git_gate(infra_vm.gateway_transport(), bottle_id)
|
||||
|
||||
|
||||
|
||||
@@ -52,6 +52,7 @@ from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
|
||||
from . import firecracker_vm, image_builder, isolation_probe, netpool, util
|
||||
from .bottle import FirecrackerBottle
|
||||
from .bottle_plan import FirecrackerBottlePlan
|
||||
from ...orchestrator.config_store import resolve_teardown_timeout
|
||||
from .consolidated_launch import (
|
||||
launch_consolidated,
|
||||
teardown_consolidated,
|
||||
@@ -112,6 +113,7 @@ def launch(
|
||||
token_values = egress_resolve_token_values(
|
||||
plan.egress_plan.token_env_map, effective_env,
|
||||
)
|
||||
teardown_timeout = resolve_teardown_timeout()
|
||||
ctx = launch_consolidated(
|
||||
plan.egress_plan, git_gate_plan,
|
||||
guest_ip=slot.guest_ip,
|
||||
@@ -121,6 +123,7 @@ def launch(
|
||||
stack.callback(
|
||||
teardown_consolidated, ctx.bottle_id,
|
||||
orchestrator_url=ctx.orchestrator_url,
|
||||
timeout=teardown_timeout,
|
||||
)
|
||||
|
||||
# Step 5: install the SHARED gateway CA (replaces the per-bottle CA).
|
||||
|
||||
@@ -124,11 +124,17 @@ def register_agent(
|
||||
)
|
||||
|
||||
|
||||
def teardown_consolidated(bottle_id: str, *, orchestrator_url: str) -> None:
|
||||
def teardown_consolidated(
|
||||
bottle_id: str, *, orchestrator_url: str, timeout: float | None = None,
|
||||
) -> None:
|
||||
"""Deregister the bottle and remove its git-gate state from the gateway.
|
||||
Both steps are idempotent so this is safe from a cleanup trap. Does NOT
|
||||
stop the gateway — it's a persistent per-host singleton."""
|
||||
OrchestratorClient(orchestrator_url).teardown_bottle(bottle_id)
|
||||
from ...orchestrator.config_store import DEFAULT_TEARDOWN_TIMEOUT_SECONDS
|
||||
OrchestratorClient(
|
||||
orchestrator_url,
|
||||
timeout=timeout if timeout is not None else DEFAULT_TEARDOWN_TIMEOUT_SECONDS,
|
||||
).teardown_bottle(bottle_id)
|
||||
deprovision_git_gate(AppleGatewayTransport(), bottle_id)
|
||||
|
||||
|
||||
|
||||
@@ -60,6 +60,7 @@ from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
|
||||
from . import util as container_mod
|
||||
from .bottle import MacosContainerBottle
|
||||
from .bottle_plan import MacosContainerBottlePlan
|
||||
from ...orchestrator.config_store import resolve_teardown_timeout
|
||||
from .consolidated_launch import (
|
||||
GatewayEndpoint,
|
||||
ensure_gateway,
|
||||
@@ -129,6 +130,7 @@ def launch(
|
||||
token_values = egress_resolve_token_values(
|
||||
plan.egress_plan.token_env_map, effective_env,
|
||||
)
|
||||
teardown_timeout = resolve_teardown_timeout()
|
||||
ctx = register_agent(
|
||||
plan.egress_plan,
|
||||
plan.git_gate_plan,
|
||||
@@ -140,6 +142,7 @@ def launch(
|
||||
stack.callback(
|
||||
teardown_consolidated, ctx.bottle_id,
|
||||
orchestrator_url=ctx.orchestrator_url,
|
||||
timeout=teardown_timeout,
|
||||
)
|
||||
info(
|
||||
f"agent {plan.container_name} registered "
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Per-host orchestrator configuration store (settings in bot-bottle.db).
|
||||
|
||||
Co-tenants the shared `bot-bottle.db` via the `DbStore` framework. Settings
|
||||
are readable by the host launch path directly (no HTTP round-trip to the
|
||||
orchestrator), so they take effect even before the orchestrator is reachable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
from ..db_store import DbStore
|
||||
from ..migrations import TableMigrations
|
||||
from ..paths import host_db_path
|
||||
|
||||
TEARDOWN_TIMEOUT_ENV = "BOT_BOTTLE_ORCHESTRATOR_TEARDOWN_TIMEOUT_SECONDS"
|
||||
DEFAULT_TEARDOWN_TIMEOUT_SECONDS = 30.0
|
||||
|
||||
_MIGRATIONS = TableMigrations(
|
||||
"orchestrator_config",
|
||||
[
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS orchestrator_config (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
|
|
||||
teardown_timeout_seconds REAL
|
||||
)
|
||||
|
didericis
commented
Shouldn't be a generic key value store, should have an explicit schema (want a Shouldn't be a generic key value store, should have an explicit schema (want a `teardown_timeout_seconds` column)
|
||||
""",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class OrchestratorConfigStore(DbStore):
|
||||
"""Orchestrator settings in the shared host DB."""
|
||||
|
||||
def __init__(self, db_path: Path | None = None) -> None:
|
||||
super().__init__(db_path or host_db_path(), _MIGRATIONS)
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
conn = super()._connect()
|
||||
conn.execute("PRAGMA busy_timeout=5000")
|
||||
return conn
|
||||
|
||||
def get_teardown_timeout_seconds(self) -> float | None:
|
||||
"""Return the configured teardown timeout, or None if not set."""
|
||||
try:
|
||||
with self._connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT teardown_timeout_seconds FROM orchestrator_config WHERE id = 1"
|
||||
).fetchone()
|
||||
except sqlite3.OperationalError:
|
||||
return None
|
||||
return row["teardown_timeout_seconds"] if row else None
|
||||
|
||||
def set_teardown_timeout_seconds(self, value: float) -> None:
|
||||
"""Persist the teardown timeout."""
|
||||
with self._connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO orchestrator_config"
|
||||
" (id, teardown_timeout_seconds) VALUES (1, ?)",
|
||||
(value,),
|
||||
)
|
||||
self._chmod()
|
||||
|
||||
def delete_teardown_timeout_seconds(self) -> bool:
|
||||
"""Clear the stored teardown timeout. Returns True if a value existed."""
|
||||
with self._connection() as conn:
|
||||
cur = conn.execute(
|
||||
"UPDATE orchestrator_config SET teardown_timeout_seconds = NULL"
|
||||
" WHERE id = 1 AND teardown_timeout_seconds IS NOT NULL"
|
||||
)
|
||||
return cur.rowcount > 0
|
||||
|
||||
|
||||
def resolve_teardown_timeout(db_path: Path | None = None) -> float:
|
||||
"""Return the teardown timeout to use, in priority order:
|
||||
|
||||
1. ``BOT_BOTTLE_ORCHESTRATOR_TEARDOWN_TIMEOUT_SECONDS`` env var
|
||||
2. ``teardown_timeout_seconds`` in the orchestrator config DB
|
||||
3. ``DEFAULT_TEARDOWN_TIMEOUT_SECONDS`` (30 s)
|
||||
"""
|
||||
raw = os.environ.get(TEARDOWN_TIMEOUT_ENV, "").strip()
|
||||
if raw:
|
||||
try:
|
||||
value = float(raw)
|
||||
if value > 0:
|
||||
return value
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
store = OrchestratorConfigStore(db_path)
|
||||
if not store.is_migrated():
|
||||
store.migrate()
|
||||
db_value = store.get_teardown_timeout_seconds()
|
||||
if db_value is not None and db_value > 0:
|
||||
return db_value
|
||||
|
||||
return DEFAULT_TEARDOWN_TIMEOUT_SECONDS
|
||||
|
||||
|
||||
__all__ = [
|
||||
"OrchestratorConfigStore",
|
||||
"resolve_teardown_timeout",
|
||||
"TEARDOWN_TIMEOUT_ENV",
|
||||
"DEFAULT_TEARDOWN_TIMEOUT_SECONDS",
|
||||
]
|
||||
@@ -0,0 +1,147 @@
|
||||
"""Unit: OrchestratorConfigStore and resolve_teardown_timeout.
|
||||
|
||||
Also verifies the lifecycle ordering invariant: resolve_teardown_timeout()
|
||||
must be called before launch_consolidated() / register_agent() so that a
|
||||
resolver failure cannot leave an orphaned registration with no teardown
|
||||
callback.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
|
||||
from bot_bottle.orchestrator.config_store import (
|
||||
DEFAULT_TEARDOWN_TIMEOUT_SECONDS,
|
||||
TEARDOWN_TIMEOUT_ENV,
|
||||
OrchestratorConfigStore,
|
||||
resolve_teardown_timeout,
|
||||
)
|
||||
|
||||
|
||||
class TestOrchestratorConfigStore(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.db = Path(self._tmp.name) / "test.db"
|
||||
self.store = OrchestratorConfigStore(self.db)
|
||||
self.store.migrate()
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
def test_get_returns_none_when_not_set(self) -> None:
|
||||
self.assertIsNone(self.store.get_teardown_timeout_seconds())
|
||||
|
||||
def test_set_and_get_roundtrip(self) -> None:
|
||||
self.store.set_teardown_timeout_seconds(42.5)
|
||||
self.assertEqual(42.5, self.store.get_teardown_timeout_seconds())
|
||||
|
||||
def test_set_overwrites_existing_value(self) -> None:
|
||||
self.store.set_teardown_timeout_seconds(10.0)
|
||||
self.store.set_teardown_timeout_seconds(20.0)
|
||||
self.assertEqual(20.0, self.store.get_teardown_timeout_seconds())
|
||||
|
||||
def test_delete_clears_value_and_returns_true(self) -> None:
|
||||
self.store.set_teardown_timeout_seconds(30.0)
|
||||
deleted = self.store.delete_teardown_timeout_seconds()
|
||||
self.assertTrue(deleted)
|
||||
self.assertIsNone(self.store.get_teardown_timeout_seconds())
|
||||
|
||||
def test_delete_absent_returns_false(self) -> None:
|
||||
self.assertFalse(self.store.delete_teardown_timeout_seconds())
|
||||
|
||||
def test_is_migrated_true_after_migrate(self) -> None:
|
||||
self.assertTrue(self.store.is_migrated())
|
||||
|
||||
def test_is_migrated_false_before_migrate(self) -> None:
|
||||
store = OrchestratorConfigStore(Path(self._tmp.name) / "new.db")
|
||||
self.assertFalse(store.is_migrated())
|
||||
|
||||
|
||||
class TestResolveTeardownTimeout(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.db = Path(self._tmp.name) / "cfg.db"
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
os.environ.pop(TEARDOWN_TIMEOUT_ENV, None)
|
||||
|
||||
def test_returns_default_when_nothing_configured(self) -> None:
|
||||
self.assertEqual(
|
||||
DEFAULT_TEARDOWN_TIMEOUT_SECONDS,
|
||||
resolve_teardown_timeout(self.db),
|
||||
)
|
||||
|
||||
def test_env_var_overrides_default(self) -> None:
|
||||
os.environ[TEARDOWN_TIMEOUT_ENV] = "99"
|
||||
self.assertEqual(99.0, resolve_teardown_timeout(self.db))
|
||||
|
||||
def test_env_var_overrides_db_value(self) -> None:
|
||||
store = OrchestratorConfigStore(self.db)
|
||||
store.migrate()
|
||||
store.set_teardown_timeout_seconds(55.0)
|
||||
os.environ[TEARDOWN_TIMEOUT_ENV] = "77"
|
||||
self.assertEqual(77.0, resolve_teardown_timeout(self.db))
|
||||
|
||||
def test_db_value_overrides_default(self) -> None:
|
||||
store = OrchestratorConfigStore(self.db)
|
||||
store.migrate()
|
||||
store.set_teardown_timeout_seconds(42.0)
|
||||
self.assertEqual(42.0, resolve_teardown_timeout(self.db))
|
||||
|
||||
def test_invalid_env_var_falls_through_to_default(self) -> None:
|
||||
os.environ[TEARDOWN_TIMEOUT_ENV] = "not-a-number"
|
||||
self.assertEqual(DEFAULT_TEARDOWN_TIMEOUT_SECONDS, resolve_teardown_timeout(self.db))
|
||||
|
||||
def test_non_positive_env_var_falls_through_to_default(self) -> None:
|
||||
os.environ[TEARDOWN_TIMEOUT_ENV] = "0"
|
||||
self.assertEqual(DEFAULT_TEARDOWN_TIMEOUT_SECONDS, resolve_teardown_timeout(self.db))
|
||||
|
||||
def test_non_positive_db_value_falls_through_to_default(self) -> None:
|
||||
store = OrchestratorConfigStore(self.db)
|
||||
store.migrate()
|
||||
store.set_teardown_timeout_seconds(0.0)
|
||||
self.assertEqual(DEFAULT_TEARDOWN_TIMEOUT_SECONDS, resolve_teardown_timeout(self.db))
|
||||
|
||||
def test_migrates_db_on_first_call(self) -> None:
|
||||
result = resolve_teardown_timeout(self.db)
|
||||
self.assertEqual(DEFAULT_TEARDOWN_TIMEOUT_SECONDS, result)
|
||||
self.assertTrue(OrchestratorConfigStore(self.db).is_migrated())
|
||||
|
||||
|
||||
class TestTeardownTimeoutResolvedBeforeRegistration(unittest.TestCase):
|
||||
"""Ordering invariant: if resolve_teardown_timeout() raises, the bottle
|
||||
must not yet be registered — no orphaned state can result."""
|
||||
|
||||
def _src(self, module: ModuleType) -> str:
|
||||
return inspect.getsource(module)
|
||||
|
||||
def test_docker_resolves_timeout_before_launch_consolidated(self) -> None:
|
||||
from bot_bottle.backend.docker import launch
|
||||
src = self._src(launch)
|
||||
resolve_at = src.index("teardown_timeout = resolve_teardown_timeout()")
|
||||
launch_at = src.index("ctx = launch_consolidated(")
|
||||
self.assertLess(resolve_at, launch_at)
|
||||
|
||||
def test_firecracker_resolves_timeout_before_launch_consolidated(self) -> None:
|
||||
from bot_bottle.backend.firecracker import launch
|
||||
src = self._src(launch)
|
||||
resolve_at = src.index("teardown_timeout = resolve_teardown_timeout()")
|
||||
launch_at = src.index("ctx = launch_consolidated(")
|
||||
self.assertLess(resolve_at, launch_at)
|
||||
|
||||
def test_macos_resolves_timeout_before_register_agent(self) -> None:
|
||||
from bot_bottle.backend.macos_container import launch
|
||||
src = self._src(launch)
|
||||
resolve_at = src.index("teardown_timeout = resolve_teardown_timeout()")
|
||||
register_at = src.index("ctx = register_agent(")
|
||||
self.assertLess(resolve_at, register_at)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user
Done in
256a4b2. The generickey/valuetable is replaced with an explicit single-row schema:OrchestratorConfigStorenow exposesget/set/delete_teardown_timeout_seconds()instead of the genericget/set/delete(key).TEARDOWN_TIMEOUT_KEYis removed. The REAL column meansresolve_teardown_timeout()no longer needs afloat()conversion.