fix(gateway): persist git-gate state across gateway restarts

Per-bottle git-gate state (bare repos under /git/<id>, deploy creds under
/git-gate/creds/<id>) was provisioned once at bottle launch and lived only
in the gateway's ephemeral storage. A gateway rebuild/restart wiped it and
nothing re-provisioned already-running bottles, so their agents 404'd on
fetch/push. Same class of bug as the CA (#510); the orchestrator restores
only egress tokens, not git-gate declarations.

Persist the state on both backends, mirroring the CA-persistence approach:

- firecracker: attach a second persistent data drive (/dev/vdc) to the
  gateway VM and bind-mount its git/ + creds/ subdirs onto /git and
  /git-gate/creds in the gateway guest init, before the data plane starts.
  Generalize the VM config to a stable-ordered data_drives tuple (CA=vdb,
  git=vdc; orchestrator registry stays vdb).
- docker: bind-mount host dirs (host_gateway_git_dir / creds_dir, under the
  never-pruned app-data root) onto /git and /git-gate/creds, with
  BOT_BOTTLE_DOCKER_GIT_MOUNT / _CREDS_MOUNT env overrides so CI isolates
  them to per-run volumes it cleans up.

Teardown already rm -rf's /git/<id> + creds, so the persistent store
self-cleans over the normal lifecycle.

Closes #512

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

bot_bottle/backend/docker/gateway.py

@@ -9,6 +9,8 @@ from .gateway_transport import DockerGatewayTransport
 from ...paths import (
     ORCHESTRATOR_AUTH_JWT_ENV,
     host_gateway_ca_dir,
+    host_gateway_git_dir,
+    host_gateway_creds_dir,
 )
 from ... import resources
 from ...gateway import (
@@ -40,6 +42,8 @@ class DockerGateway(Gateway):
         dockerfile: str | None = GATEWAY_DOCKERFILE,
         host_port_bindings: tuple[int, ...] = (),
         ca_mount_source: str | Path | None = None,
+        git_mount_source: str | Path | None = None,
+        creds_mount_source: str | Path | None = None,
         subnet: str | None = None,
     ) -> None:
         self.image_ref = image_ref
@@ -74,6 +78,17 @@ class DockerGateway(Gateway):
         self._ca_mount_source = str(
             ca_mount_source or configured_ca or host_gateway_ca_dir()
         )
+        # The persistent git-gate mounts (/git bare repos, /git-gate/creds deploy
+        # creds) — same host-bind-mount rationale as the CA (issue #512). The env
+        # overrides let CI point them at per-run named volumes it cleans up.
+        configured_git = os.environ.get("BOT_BOTTLE_DOCKER_GIT_MOUNT", "").strip()
+        self._git_mount_source = str(
+            git_mount_source or configured_git or host_gateway_git_dir()
+        )
+        configured_creds = os.environ.get("BOT_BOTTLE_DOCKER_CREDS_MOUNT", "").strip()
+        self._creds_mount_source = str(
+            creds_mount_source or configured_creds or host_gateway_creds_dir()
+        )
 
     def image_exists(self) -> bool:
         return run_docker(["docker", "image", "inspect", self.image_ref]).returncode == 0
@@ -198,6 +213,12 @@ class DockerGateway(Gateway):
             # container recreation AND docker volume pruning (agents trust it)
             # — see host_gateway_ca_dir / issue #450.
             "--volume", f"{self._ca_mount_source}:{MITMPROXY_HOME}",
+            # Persist per-bottle git-gate state (bare repos + deploy creds) on
+            # the host so a gateway restart doesn't drop already-running bottles'
+            # repos — they would otherwise 404 on fetch/push (issue #512). Same
+            # host-bind-mount rationale as the CA.
+            "--volume", f"{self._git_mount_source}:/git",
+            "--volume", f"{self._creds_mount_source}:/git-gate/creds",
             # No DB mount: the data plane (egress / supervise / git-gate) reaches
             # the supervise queue over the control-plane RPC and never opens
             # bot-bottle.db, so the gateway container gets no file handle on it

bot_bottle/backend/firecracker/firecracker_vm.py

@@ -84,7 +84,7 @@ def _config(
     vcpus: int,
     mem_mib: int,
     guest_mac: str,
-    data_drive: Path | None = None,
+    data_drives: tuple[Path, ...] = (),
     extra_boot_args: str = "",
 ) -> dict[str, object]:
     drives: list[dict[str, object]] = [
@@ -95,12 +95,15 @@ def _config(
             "is_read_only": False,
         }
     ]
-    # A second virtio-block device (guest /dev/vdb) — the infra VM's
-    # persistent registry "volume", a host-side ext4 file that outlives the
-    # ephemeral rootfs across VM restarts.
-    if data_drive is not None:
+    # Extra virtio-block devices — the infra VMs' persistent "volumes",
+    # host-side ext4 files that outlive the ephemeral rootfs across VM restarts.
+    # They appear to the guest as /dev/vdb, /dev/vdc, ... in list order (after
+    # the root /dev/vda), so callers must keep the order stable: the orchestrator
+    # attaches its registry (vdb); the gateway attaches its CA (vdb) then its
+    # git-gate state (vdc).
+    for i, data_drive in enumerate(data_drives):
         drives.append({
-            "drive_id": "data",
+            "drive_id": f"data{i}",
             "path_on_host": str(data_drive),
             "is_root_device": False,
             "is_read_only": False,
@@ -138,7 +141,7 @@ def boot(
     mem_mib: int = 2048,
     guest_mac: str = "06:00:AC:10:00:02",
     detached: bool = False,
-    data_drive: Path | None = None,
+    data_drives: tuple[Path, ...] = (),
     extra_boot_args: str = "",
 ) -> VmHandle:
     """Write the config and launch the VMM. Returns once the process is
@@ -155,7 +158,7 @@ def boot(
         _config(
             rootfs=rootfs, tap=tap, guest_ip=guest_ip, host_ip=host_ip,
             pubkey=pubkey, vcpus=vcpus, mem_mib=mem_mib, guest_mac=guest_mac,
-            data_drive=data_drive, extra_boot_args=extra_boot_args,
+            data_drives=data_drives, extra_boot_args=extra_boot_args,
         ),
         indent=2,
     ))

bot_bottle/backend/firecracker/gateway.py

@@ -61,6 +61,16 @@ _GATEWAY_CA_PATH = "/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem"
 # which outlives the ephemeral rootfs. mitmproxy is tiny; 16M is ample.
 _CA_VOLUME_SIZE = "16M"
 
+# The gateway VM's persistent git-gate volume — a (sparse) ext4 file the gateway
+# init mounts, then bind-mounts onto /git + /git-gate/creds (see
+# `infra_vm._GATEWAY_GIT_MOUNT`), so per-bottle bare repos and deploy creds
+# SURVIVE a gateway-VM rebuild. Without it a restart drops every already-running
+# bottle's git-gate state and its agent 404s on fetch/push (same class as the CA
+# — issue #512). Bare repos hold upstream history, so this is sized far larger
+# than the CA volume; mke2fs leaves the file sparse, so the host only stores
+# blocks actually used.
+_GIT_VOLUME_SIZE = "8G"
+
 _CA_FETCH_TIMEOUT_SECONDS = 15.0
 
 
@@ -106,15 +116,18 @@ class FirecrackerGateway(Gateway):
                 f"cannot resolve orchestrator guest IP from {self._orchestrator_url!r}"
             )
         # Boot on the gateway link from the gateway rootfs, then push the token
-        # the init waits for before starting the data plane. The persistent CA
-        # volume (/dev/vdb, mounted at mitmproxy's confdir by the gateway init)
-        # keeps the CA STABLE across rebuilds so already-running bottles keep
-        # trusting the gateway's TLS interception (issue #450).
+        # the init waits for before starting the data plane. Two persistent
+        # volumes ride along, attached in a FIXED order the gateway init depends
+        # on: the CA volume as /dev/vdb (mounted at mitmproxy's confdir) and the
+        # git-gate volume as /dev/vdc (bind-mounted onto /git + /git-gate/creds).
+        # Both keep gateway-side state STABLE across rebuilds so already-running
+        # bottles keep working — TLS interception (issue #450) and git-gate fetch
+        # /push (issue #512) respectively.
         vm = infra_vm.boot_vm(
             name=GATEWAY_NAME, slot=netpool.gw_slot(), run_dir=infra_vm._gw_dir(),
             role="gateway", mem_mib=_GW_MEM_MIB,
             extra_boot_args=f"bb_orch={orchestrator_guest_ip}",
-            data_drive=self._ensure_ca_volume(),
+            data_drives=(self._ensure_ca_volume(), self._ensure_git_volume()),
         )
         infra_vm.push_secret(
             vm, self._gateway_token, _GUEST_GATEWAY_JWT_PATH,
@@ -152,6 +165,26 @@ class FirecrackerGateway(Gateway):
             die(f"creating gateway CA volume failed: {proc.stderr.strip()}")
         return vol
 
+    def _ensure_git_volume(self) -> Path:
+        """Create the empty ext4 git-gate volume on first use; reuse it after.
+
+        Empty on first boot (the gateway init lays out `git/` + `creds/` subdirs
+        and bind-mounts them); every later boot reuses whatever repos + creds the
+        volume already holds — which is what keeps git-gate state stable across a
+        gateway-VM rebuild (issue #512). Sibling of `_ensure_ca_volume`."""
+        vol = infra_vm._gw_dir() / "gateway-git.ext4"
+        if vol.exists():
+            return vol
+        info(f"creating gateway git-gate volume {vol} ({_GIT_VOLUME_SIZE})")
+        proc = subprocess.run(
+            ["mke2fs", "-q", "-t", "ext4", "-F", str(vol), _GIT_VOLUME_SIZE],
+            capture_output=True, text=True, check=False,
+        )
+        if proc.returncode != 0:
+            vol.unlink(missing_ok=True)
+            die(f"creating gateway git-gate volume failed: {proc.stderr.strip()}")
+        return vol
+
     def address(self) -> str:
         """The gateway VM's guest IP — the agent-facing target agent VMs'
         gateway-port traffic is DNAT'd to."""

bot_bottle/backend/firecracker/infra_vm.py

@@ -63,6 +63,15 @@ _GUEST_GATEWAY_JWT_PATH = "/var/lib/bot-bottle/gateway-jwt"
 # survives a gateway-VM rebuild; without it every rebuild mints a fresh CA that
 # already-running bottles distrust, breaking the TLS handshake (issue #450).
 _GATEWAY_CA_MOUNT = "/home/mitmproxy/.mitmproxy"
+# The gateway VM's persistent git-gate volume staging mount (/dev/vdc). The
+# gateway boots with this volume (see `FirecrackerGateway._ensure_git_volume`)
+# and the init bind-mounts its `git/` + `creds/` subdirs onto the load-bearing
+# `/git` and `/git-gate/creds` paths, so per-bottle bare repos + deploy creds
+# survive a gateway-VM rebuild; without it a restart drops every already-running
+# bottle's git-gate state and its agent 404s on fetch/push (issue #512).
+_GATEWAY_GIT_MOUNT = "/var/lib/bot-bottle-gitgate"
+_GATEWAY_GIT_REPO_ROOT = "/git"
+_GATEWAY_GIT_CREDS_DIR = "/git-gate/creds"
 
 # The two per-plane rootfs source images. The orchestrator VM boots a control
 # plane + buildah rootfs (Dockerfile.orchestrator.fc, FROM orchestrator); the
@@ -198,11 +207,12 @@ def boot_vm(
     run_dir: Path,
     role: str,
     mem_mib: int,
-    data_drive: Path | None = None,
+    data_drives: tuple[Path, ...] = (),
     extra_boot_args: str = "",
 ) -> InfraVm:
     """Boot the `role` infra VM from its per-plane rootfs on `slot`'s link.
-    Records the PID."""
+    Records the PID. `data_drives` are attached as /dev/vdb, /dev/vdc, ... in
+    order, so callers must keep the order stable (see `firecracker_vm._config`)."""
     if not netpool.tap_present(slot.iface):
         die(f"infra link {slot.iface} not present.\n"
             f"    ./cli.py backend setup --backend=firecracker")
@@ -224,7 +234,7 @@ def boot_vm(
         name=name, rootfs=rootfs, tap=slot.iface,
         guest_ip=slot.guest_ip, host_ip=slot.host_ip, pubkey=pubkey,
         run_dir=run_dir, mem_mib=mem_mib, detached=True,
-        data_drive=data_drive, extra_boot_args=boot_args,
+        data_drives=data_drives, extra_boot_args=boot_args,
     )
     _pid_file(run_dir).write_text(str(vm.process.pid))
     return InfraVm(guest_ip=slot.guest_ip, private_key=private_key, vm=vm)
@@ -461,6 +471,17 @@ def _gateway_init() -> str:
 # mitmproxy) starts.
 mkdir -p {_GATEWAY_CA_MOUNT}
 mount -t ext4 /dev/vdb {_GATEWAY_CA_MOUNT} 2>/dev/null || true
+# Persistent git-gate volume (/dev/vdc): its git/ + creds/ subdirs are
+# bind-mounted onto the load-bearing /git and /git-gate/creds so per-bottle bare
+# repos + deploy creds survive a gateway-VM rebuild (issue #512). On first boot
+# the volume is empty; the subdirs are created here. Must mount BEFORE the data
+# plane (hence git-http) starts, and before any per-bottle provisioning writes.
+mkdir -p {_GATEWAY_GIT_MOUNT}
+mount -t ext4 /dev/vdc {_GATEWAY_GIT_MOUNT} 2>/dev/null || true
+mkdir -p {_GATEWAY_GIT_MOUNT}/git {_GATEWAY_GIT_MOUNT}/creds
+mkdir -p {_GATEWAY_GIT_REPO_ROOT} {_GATEWAY_GIT_CREDS_DIR}
+mount --bind {_GATEWAY_GIT_MOUNT}/git {_GATEWAY_GIT_REPO_ROOT} 2>/dev/null || true
+mount --bind {_GATEWAY_GIT_MOUNT}/creds {_GATEWAY_GIT_CREDS_DIR} 2>/dev/null || true
 ORCH=$(sed -n 's/.*bb_orch=\\([^ ]*\\).*/\\1/p' /proc/cmdline)
 GW_JWT=""
 i=0

bot_bottle/backend/firecracker/orchestrator.py

@@ -104,7 +104,7 @@ class FirecrackerOrchestrator(Orchestrator):
         vm = infra_vm.boot_vm(
             name=ORCHESTRATOR_NAME, slot=netpool.orch_slot(),
             run_dir=infra_vm._orch_dir(), role="orchestrator", mem_mib=_ORCH_MEM_MIB,
-            data_drive=self._ensure_registry_volume(),
+            data_drives=(self._ensure_registry_volume(),),
         )
         # Push the host-canonical signing key (the init waits for it before
         # starting the control plane). It comes through the shared provisioning

bot_bottle/paths.py

@@ -53,6 +53,15 @@ ORCHESTRATOR_AUTH_JWT_ENV = "BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT"
 # the shared gateway's TLS interception, so it must not rotate on restart. See
 # host_gateway_ca_dir() for why this is a host bind-mount, not a named volume.
 GATEWAY_CA_DIRNAME = "gateway-ca"
+# The host directories holding the gateway's persistent git-gate state — the
+# per-bottle bare repos (`gateway-git`) and deploy creds (`gateway-creds`).
+# Bind-mounted into the gateway container at /git and /git-gate/creds so they
+# survive container recreation; without it a gateway restart drops every
+# already-running bottle's git-gate state and its agent 404s on fetch/push
+# (issue #512). Host bind-mounts (not named volumes) for the same reason as the
+# CA dir — see host_gateway_ca_dir().
+GATEWAY_GIT_DIRNAME = "gateway-git"
+GATEWAY_CREDS_DIRNAME = "gateway-creds"
 
 
 def bot_bottle_root() -> Path:
@@ -97,6 +106,27 @@ def host_gateway_ca_dir() -> Path:
     return ca_dir
 
 
+def host_gateway_git_dir() -> Path:
+    """The directory holding the gateway's persistent per-bottle bare repos,
+    created if missing. Bind-mounted into the gateway container at /git so the
+    repos survive container recreation (issue #512). A host bind-mount under the
+    app-data root, never pruned — same rationale as host_gateway_ca_dir()."""
+    git_dir = bot_bottle_root() / GATEWAY_GIT_DIRNAME
+    git_dir.mkdir(parents=True, exist_ok=True)
+    return git_dir
+
+
+def host_gateway_creds_dir() -> Path:
+    """The directory holding the gateway's persistent per-bottle git-gate deploy
+    creds, created if missing. Bind-mounted into the gateway container at
+    /git-gate/creds so the creds survive container recreation (issue #512). A
+    host bind-mount under the app-data root — same rationale as
+    host_gateway_ca_dir()."""
+    creds_dir = bot_bottle_root() / GATEWAY_CREDS_DIRNAME
+    creds_dir.mkdir(parents=True, exist_ok=True)
+    return creds_dir
+
+
 def host_signing_key(filename: str) -> str:
     """A per-host signing key at `<root>/<filename>`, minted (256-bit, url-safe)
     and persisted 0600 on first use, then reused.
@@ -143,10 +173,14 @@ __all__ = [
     "ORCHESTRATOR_TOKEN_ENV",
     "ORCHESTRATOR_AUTH_JWT_ENV",
     "GATEWAY_CA_DIRNAME",
+    "GATEWAY_GIT_DIRNAME",
+    "GATEWAY_CREDS_DIRNAME",
     "bot_bottle_root",
     "host_db_path",
     "host_db_dir",
     "host_gateway_ca_dir",
+    "host_gateway_git_dir",
+    "host_gateway_creds_dir",
     "host_signing_key",
     "host_orchestrator_token",
 ]