fix(firecracker): serialize infra-VM create + agent builds (PR #354 review)

Codex flagged that parallel `start`s race on shared state: two cold
launches could both stop/build/boot the singleton on the same rootfs/PID,
concurrent builds share the infra VM's buildah store, the cleanup did
`buildah rm -a` (nuking a peer build's container), and the rootfs cache
was populated non-atomically.

- `infra_vm.ensure_running`: a host flock (`singleton.lock`) around the
  cold stop/build/boot path, with a double-checked health re-test under
  the lock so a second launcher adopts rather than re-boots. The healthy
  fast-path stays lock-free.
- `image_builder.build_agent_rootfs_dir`: a host flock (`.build.lock`)
  around cache-lookup + build + publish; build into `.building-<digest>`
  and publish by atomic `os.rename`, so a partial build never appears as
  `agent-<digest>`.
- scoped cleanup: per-build named working containers
  (`<tag>-smoke`/`-export`) removed by name instead of `buildah rm -a`;
  also cleared before the build to recover from a crashed prior run.

Verified: agent image builds via the locked/atomic path, infra VM stays
healthy, agent boots, `claude --version` = 2.1.172. Full unit suite +
pyright green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WBMWTEtQdJ4W5UrWuLHCck

bot_bottle/backend/firecracker/infra_vm.py

@@ -17,6 +17,7 @@ surface.
 
 from __future__ import annotations
 
+import fcntl
 import hashlib
 import os
 import shlex
@@ -26,8 +27,10 @@ import subprocess
 import time
 import urllib.error
 import urllib.request
+from contextlib import contextmanager
 from dataclasses import dataclass
 from pathlib import Path
+from typing import Generator
 
 from ...log import die, info
 from ..docker import util as docker_mod
@@ -134,7 +137,11 @@ def ensure_running() -> InfraVm:
     """Idempotent per-host singleton. Adopt the infra VM if its control plane
     is already healthy (a prior launcher booted it — it outlives short-lived
     `start` processes); otherwise clear any stale VM and boot a fresh one.
-    Returns a handle usable for CA fetch / git-gate provisioning."""
+    Returns a handle usable for CA fetch / git-gate provisioning.
+
+    Concurrency-safe: the cold stop/build/boot path is serialized by a host
+    flock, so two simultaneous first launches don't both boot on the same
+    rootfs/PID. The healthy fast-path takes no lock."""
     slot = netpool.orch_slot()
     url = f"http://{slot.guest_ip}:{CONTROL_PLANE_PORT}"
     key = _infra_dir() / "id_ed25519"
@@ -142,11 +149,31 @@ def ensure_running() -> InfraVm:
         info(f"adopting running infra VM at {url}")
         return InfraVm(guest_ip=slot.guest_ip, private_key=key)
 
-    stop()  # clear a stale/hung VM holding the link before booting fresh
-    ensure_built()
-    infra = boot()
-    wait_for_health(infra)
-    return infra
+    with _singleton_lock():
+        # Re-check under the lock: another launcher may have booted it while
+        # we waited for the lock (double-checked, so we adopt not re-boot).
+        if key.exists() and _health_ok(url):
+            info(f"adopting running infra VM at {url}")
+            return InfraVm(guest_ip=slot.guest_ip, private_key=key)
+        stop()  # clear a stale/hung VM holding the link before booting fresh
+        ensure_built()
+        infra = boot()
+        wait_for_health(infra)
+        return infra
+
+
+@contextmanager
+def _singleton_lock() -> Generator[None, None, None]:
+    """Host-level exclusive lock serializing the infra VM's cold create path
+    (`stop`/`ensure_built`/`boot`). flock auto-releases if the launcher
+    crashes, so the lock is never leaked."""
+    lock_path = _infra_dir() / "singleton.lock"
+    handle = open(lock_path, "w", encoding="utf-8")
+    try:
+        fcntl.flock(handle, fcntl.LOCK_EX)
+        yield
+    finally:
+        handle.close()
 
 
 def stop() -> None: