fix: make orchestrator teardown timeout configurable (closes #435)
test / stage-firecracker-inputs (pull_request) Successful in 3s
test / integration-docker (pull_request) Successful in 8s
tracker-policy-pr / check-pr (pull_request) Successful in 8s
test / unit (pull_request) Successful in 30s
lint / lint (push) Successful in 44s
test / build-infra (pull_request) Successful in 9m58s
test / integration-firecracker (pull_request) Successful in 1m32s
test / coverage (pull_request) Failing after 2m7s
test / publish-infra (pull_request) Has been skipped

Resolves via ENV VAR -> orchestrator DB config -> default (30 s, up from 5 s):
  BOT_BOTTLE_ORCHESTRATOR_TEARDOWN_TIMEOUT_SECONDS
  teardown_timeout_seconds key in new orchestrator_config table (bot-bottle.db)

New OrchestratorConfigStore (same DbStore/TableMigrations pattern as the
registry) stores the DB-level setting. resolve_teardown_timeout() implements
the priority chain and is called at stack.callback registration time in all
three backends (macos_container, docker, firecracker).
This commit is contained in:
2026-07-20 19:36:41 +00:00
committed by didericis
parent c7375051fd
commit 3674d7780b
7 changed files with 146 additions and 7 deletions
@@ -142,11 +142,19 @@ def launch_consolidated(
def teardown_consolidated(
bottle_id: str, *, orchestrator_url: str, gateway_name: str = GATEWAY_NAME,
bottle_id: str,
*,
orchestrator_url: str,
gateway_name: str = GATEWAY_NAME,
timeout: float | None = None,
) -> None:
"""Deregister the bottle and remove its git-gate state from the gateway.
Both steps are idempotent so this is safe from a cleanup trap."""
OrchestratorClient(orchestrator_url).teardown_bottle(bottle_id)
from ...orchestrator.config_store import DEFAULT_TEARDOWN_TIMEOUT_SECONDS
OrchestratorClient(
orchestrator_url,
timeout=timeout if timeout is not None else DEFAULT_TEARDOWN_TIMEOUT_SECONDS,
).teardown_bottle(bottle_id)
deprovision_git_gate(DockerGatewayTransport(gateway_name), bottle_id)
+4 -1
View File
@@ -62,6 +62,7 @@ from .compose import (
write_compose_file,
)
from .consolidated_compose import consolidated_agent_compose
from ...orchestrator.config_store import resolve_teardown_timeout
from .consolidated_launch import launch_consolidated, teardown_consolidated
from ...orchestrator.gateway import DockerGateway
@@ -137,7 +138,9 @@ def launch(
plan.egress_plan, git_gate_plan, image_ref=plan.image, tokens=token_values,
)
stack.callback(
teardown_consolidated, ctx.bottle_id, orchestrator_url=ctx.orchestrator_url,
teardown_consolidated, ctx.bottle_id,
orchestrator_url=ctx.orchestrator_url,
timeout=resolve_teardown_timeout(),
)
# Step 4: install the SHARED gateway CA into the agent (replaces the
@@ -91,12 +91,18 @@ def launch_consolidated(
)
def teardown_consolidated(bottle_id: str, *, orchestrator_url: str) -> None:
def teardown_consolidated(
bottle_id: str, *, orchestrator_url: str, timeout: float | None = None,
) -> None:
"""Deregister the bottle and remove its git-gate state from the gateway
VM. Both steps are idempotent so this is safe from a cleanup trap. Does
NOT stop the infra VM — it's a persistent per-host singleton shared by
every bottle."""
OrchestratorClient(orchestrator_url).teardown_bottle(bottle_id)
from ...orchestrator.config_store import DEFAULT_TEARDOWN_TIMEOUT_SECONDS
OrchestratorClient(
orchestrator_url,
timeout=timeout if timeout is not None else DEFAULT_TEARDOWN_TIMEOUT_SECONDS,
).teardown_bottle(bottle_id)
deprovision_git_gate(infra_vm.gateway_transport(), bottle_id)
+2
View File
@@ -52,6 +52,7 @@ from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
from . import firecracker_vm, image_builder, isolation_probe, netpool, util
from .bottle import FirecrackerBottle
from .bottle_plan import FirecrackerBottlePlan
from ...orchestrator.config_store import resolve_teardown_timeout
from .consolidated_launch import (
launch_consolidated,
teardown_consolidated,
@@ -121,6 +122,7 @@ def launch(
stack.callback(
teardown_consolidated, ctx.bottle_id,
orchestrator_url=ctx.orchestrator_url,
timeout=resolve_teardown_timeout(),
)
# Step 5: install the SHARED gateway CA (replaces the per-bottle CA).
@@ -124,11 +124,17 @@ def register_agent(
)
def teardown_consolidated(bottle_id: str, *, orchestrator_url: str) -> None:
def teardown_consolidated(
bottle_id: str, *, orchestrator_url: str, timeout: float | None = None,
) -> None:
"""Deregister the bottle and remove its git-gate state from the gateway.
Both steps are idempotent so this is safe from a cleanup trap. Does NOT
stop the gateway — it's a persistent per-host singleton."""
OrchestratorClient(orchestrator_url).teardown_bottle(bottle_id)
from ...orchestrator.config_store import DEFAULT_TEARDOWN_TIMEOUT_SECONDS
OrchestratorClient(
orchestrator_url,
timeout=timeout if timeout is not None else DEFAULT_TEARDOWN_TIMEOUT_SECONDS,
).teardown_bottle(bottle_id)
deprovision_git_gate(AppleGatewayTransport(), bottle_id)
@@ -60,6 +60,7 @@ from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
from . import util as container_mod
from .bottle import MacosContainerBottle
from .bottle_plan import MacosContainerBottlePlan
from ...orchestrator.config_store import resolve_teardown_timeout
from .consolidated_launch import (
GatewayEndpoint,
ensure_gateway,
@@ -140,6 +141,7 @@ def launch(
stack.callback(
teardown_consolidated, ctx.bottle_id,
orchestrator_url=ctx.orchestrator_url,
timeout=resolve_teardown_timeout(),
)
info(
f"agent {plan.container_name} registered "