fix: require authoritative container snapshot before reconciling
test / stage-firecracker-inputs (pull_request) Successful in 5s
tracker-policy-pr / check-pr (pull_request) Successful in 12s
test / integration-docker (pull_request) Successful in 33s
test / unit (pull_request) Successful in 39s
lint / lint (push) Successful in 49s
test / build-infra (pull_request) Successful in 3m23s
test / integration-firecracker (pull_request) Successful in 1m42s
test / coverage (pull_request) Failing after 1m26s
test / publish-infra (pull_request) Has been skipped
test / stage-firecracker-inputs (pull_request) Successful in 5s
tracker-policy-pr / check-pr (pull_request) Successful in 12s
test / integration-docker (pull_request) Successful in 33s
test / unit (pull_request) Successful in 39s
lint / lint (push) Successful in 49s
test / build-infra (pull_request) Successful in 3m23s
test / integration-firecracker (pull_request) Successful in 1m42s
test / coverage (pull_request) Failing after 1m26s
test / publish-infra (pull_request) Has been skipped
A failed container list or a partial per-container inspect were previously indistinguishable from a legitimately empty/partial live set, so reconciliation could silently unregister healthy bottles. - enumerate_active() now raises EnumerationError instead of returning [] when `container list` fails - Add inspect_container_network_ip() to util, which returns None on inspect failure (vs "" for "no DHCP address yet"), so live_source_ips can tell the two apart - live_source_ips() raises EnumerationError on either failure mode; register_agent() catches it and skips reconciliation, same as OrchestratorClientError - Update tests: rename test_empty_when_the_cli_fails to test_raises_when_the_cli_fails, update patching to the new function, add coverage for list failure, per-container inspect failure, and the launch-not-blocked path
This commit is contained in:
@@ -41,7 +41,7 @@ from ...orchestrator.client import OrchestratorClient, OrchestratorClientError
|
||||
from ...orchestrator.registration import registration_inputs
|
||||
from ..docker.gateway_provision import deprovision_git_gate, provision_git_gate
|
||||
from . import util as container_mod
|
||||
from .enumerate import CONTAINER_NAME_PREFIX, enumerate_active
|
||||
from .enumerate import CONTAINER_NAME_PREFIX, EnumerationError, enumerate_active
|
||||
from .gateway import GATEWAY_NETWORK
|
||||
from .gateway_provision import AppleGatewayTransport
|
||||
from .infra import MacosInfraService, OrchestratorStartError
|
||||
@@ -98,14 +98,21 @@ def live_source_ips(network: str) -> list[str]:
|
||||
The reconciliation input: the orchestrator lives inside the infra
|
||||
container and cannot enumerate the host's containers, so the host has to
|
||||
tell it which bottles are actually up. Containers that have not been
|
||||
assigned an address yet contribute nothing (the read is non-fatal) — the
|
||||
reap's grace window, not this list, is what protects an in-flight
|
||||
launch."""
|
||||
assigned an address yet contribute nothing — the reap's grace window, not
|
||||
this list, is what protects an in-flight launch.
|
||||
|
||||
Raises `EnumerationError` when the live set cannot be determined
|
||||
authoritatively: either the container listing fails or any individual
|
||||
inspect fails. Callers must skip reconciliation in that case to avoid
|
||||
unregistering healthy bottles."""
|
||||
ips: list[str] = []
|
||||
for agent in enumerate_active():
|
||||
ip = container_mod.try_container_ipv4_on_network(
|
||||
f"{CONTAINER_NAME_PREFIX}{agent.slug}", network,
|
||||
)
|
||||
name = f"{CONTAINER_NAME_PREFIX}{agent.slug}"
|
||||
ip = container_mod.inspect_container_network_ip(name, network)
|
||||
if ip is None:
|
||||
raise EnumerationError(
|
||||
f"container inspect {name!r} failed; live set is not authoritative"
|
||||
)
|
||||
if ip:
|
||||
ips.append(ip)
|
||||
return ips
|
||||
@@ -133,7 +140,7 @@ def register_agent(
|
||||
# reconciliation failure must not block an otherwise-fine launch.
|
||||
try:
|
||||
client.reconcile(live_source_ips(endpoint.network))
|
||||
except OrchestratorClientError as e:
|
||||
except (OrchestratorClientError, EnumerationError) as e:
|
||||
info(f"registry reconciliation skipped: {e}")
|
||||
inputs = registration_inputs(egress_plan)
|
||||
reg = client.register_bottle(
|
||||
|
||||
@@ -19,6 +19,10 @@ CONTAINER_NAME_PREFIX = "bot-bottle-"
|
||||
_INFRA_NAMES = frozenset({INFRA_NAME})
|
||||
|
||||
|
||||
class EnumerationError(RuntimeError):
|
||||
"""container list failed; the resulting live set is not authoritative."""
|
||||
|
||||
|
||||
def enumerate_active() -> list[ActiveAgent]:
|
||||
result = subprocess.run(
|
||||
["container", "list", "--quiet"],
|
||||
@@ -27,7 +31,10 @@ def enumerate_active() -> list[ActiveAgent]:
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
return []
|
||||
raise EnumerationError(
|
||||
f"container list failed: "
|
||||
f"{(result.stderr or '').strip() or '<no stderr>'}"
|
||||
)
|
||||
out: list[ActiveAgent] = []
|
||||
for name in sorted(line.strip() for line in result.stdout.splitlines()):
|
||||
if not name.startswith(CONTAINER_NAME_PREFIX) or name in _INFRA_NAMES:
|
||||
|
||||
@@ -572,6 +572,41 @@ def try_container_ipv4_on_network(name: str, network: str) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
def inspect_container_network_ip(name: str, network: str) -> str | None:
|
||||
"""IP of `name` on `network`, distinguishing inspect failure from "not yet".
|
||||
|
||||
Returns:
|
||||
- the IP string when the container has one on `network`
|
||||
- "" when inspect succeeds but no address is assigned yet (in-flight DHCP)
|
||||
- None when the inspect command itself fails (authoritative list impossible)
|
||||
"""
|
||||
result = subprocess.run(
|
||||
[_CONTAINER, "inspect", name],
|
||||
capture_output=True, text=True, check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
return None
|
||||
try:
|
||||
data = json.loads(result.stdout or "[]")
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
if isinstance(data, list):
|
||||
data = data[0] if data else {}
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
status = data.get("status")
|
||||
networks = status.get("networks") if isinstance(status, dict) else None
|
||||
if not isinstance(networks, list):
|
||||
return ""
|
||||
for entry in networks:
|
||||
if not isinstance(entry, dict) or entry.get("network") != network:
|
||||
continue
|
||||
raw = entry.get("ipv4Address")
|
||||
if isinstance(raw, str) and raw:
|
||||
return raw.split("/", 1)[0]
|
||||
return ""
|
||||
|
||||
|
||||
def wait_container_ipv4_on_network(
|
||||
name: str, network: str, *, timeout: float = 15.0, poll: float = 0.25,
|
||||
) -> str:
|
||||
|
||||
Reference in New Issue
Block a user