fix(gateway): persist git-gate state across gateway restarts

Per-bottle git-gate state (bare repos under /git/<id>, deploy creds under
/git-gate/creds/<id>) was provisioned once at bottle launch and lived only
in the gateway's ephemeral storage. A gateway rebuild/restart wiped it and
nothing re-provisioned already-running bottles, so their agents 404'd on
fetch/push. Same class of bug as the CA (#510); the orchestrator restores
only egress tokens, not git-gate declarations.

Persist the state on both backends, mirroring the CA-persistence approach:

- firecracker: attach a second persistent data drive (/dev/vdc) to the
  gateway VM and bind-mount its git/ + creds/ subdirs onto /git and
  /git-gate/creds in the gateway guest init, before the data plane starts.
  Generalize the VM config to a stable-ordered data_drives tuple (CA=vdb,
  git=vdc; orchestrator registry stays vdb).
- docker: bind-mount host dirs (host_gateway_git_dir / creds_dir, under the
  never-pruned app-data root) onto /git and /git-gate/creds, with
  BOT_BOTTLE_DOCKER_GIT_MOUNT / _CREDS_MOUNT env overrides so CI isolates
  them to per-run volumes it cleans up.

Teardown already rm -rf's /git/<id> + creds, so the persistent store
self-cleans over the normal lifecycle.

Closes #512

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

tests/unit/test_firecracker_backend.py

@@ -404,16 +404,19 @@ class TestBootArgs(unittest.TestCase):
         self.assertEqual("bbfc0", cfg["network-interfaces"][0]["host_dev_name"])
         self.assertEqual(1, len(cfg["drives"]))  # no data drive by default
 
-    def test_config_adds_data_drive(self):
+    def test_config_adds_data_drives_in_order(self):
         cfg = cast(Any, firecracker_vm._config(
             rootfs=Path("/run/rootfs.ext4"), tap="bbfc0",
             guest_ip="100.64.0.1", host_ip="100.64.0.0", pubkey="k",
             vcpus=2, mem_mib=2048, guest_mac="06:00:AC:10:00:02",
-            data_drive=Path("/run/registry.ext4"),
+            data_drives=(Path("/run/ca.ext4"), Path("/run/git.ext4")),
         ))
-        self.assertEqual(2, len(cfg["drives"]))
+        # rootfs (vda) + two data drives, attached in list order so the guest
+        # sees them as /dev/vdb, /dev/vdc — an order callers depend on.
+        self.assertEqual(3, len(cfg["drives"]))
         self.assertFalse(cfg["drives"][1]["is_root_device"])
-        self.assertEqual("/run/registry.ext4", cfg["drives"][1]["path_on_host"])
+        self.assertEqual("/run/ca.ext4", cfg["drives"][1]["path_on_host"])
+        self.assertEqual("/run/git.ext4", cfg["drives"][2]["path_on_host"])
 
 
 class TestBottleExecClose(unittest.TestCase):

tests/unit/test_firecracker_gateway.py

@@ -60,9 +60,12 @@ class TestFirecrackerGatewayConnect(unittest.TestCase):
         gw = FirecrackerGateway()
         booted = infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k"))
         ca_vol = Path("/gw/gateway-ca.ext4")
+        git_vol = Path("/gw/gateway-git.ext4")
         with patch.object(infra_vm, "boot_vm", return_value=booted) as boot, \
                 patch.object(FirecrackerGateway, "_ensure_ca_volume",
                              return_value=ca_vol), \
+                patch.object(FirecrackerGateway, "_ensure_git_volume",
+                             return_value=git_vol), \
                 patch.object(infra_vm, "push_secret") as push:
             gw.connect_to_orchestrator(_ORCH_URL, _TOKEN)
         # Booted on the gateway link with the gateway role; the orchestrator's
@@ -70,9 +73,9 @@ class TestFirecrackerGatewayConnect(unittest.TestCase):
         kw = boot.call_args.kwargs
         self.assertEqual("gateway", kw["role"])
         self.assertIn("bb_orch=10.243.255.1", kw["extra_boot_args"])
-        # The persistent CA volume rides as /dev/vdb so the mitmproxy CA
-        # survives a gateway-VM rebuild (issue #450).
-        self.assertEqual(ca_vol, kw.get("data_drive"))
+        # The persistent volumes ride in a FIXED order — CA as /dev/vdb, git-gate
+        # as /dev/vdc — so both survive a gateway-VM rebuild (issues #450, #512).
+        self.assertEqual((ca_vol, git_vol), kw.get("data_drives"))
         # The host-minted token (never the key — #469) is pushed to the guest.
         push.assert_called_once()
         self.assertEqual(_TOKEN, push.call_args.args[1])
@@ -107,6 +110,34 @@ class TestGatewayCaVolume(unittest.TestCase):
             self.assertIn(str(out), argv)
 
 
+class TestGatewayGitVolume(unittest.TestCase):
+    """The persistent git-gate volume that keeps per-bottle bare repos + creds
+    stable across a gateway-VM rebuild (issue #512)."""
+
+    def test_reuses_existing_volume(self) -> None:
+        gw = FirecrackerGateway()
+        with tempfile.TemporaryDirectory() as td:
+            vol = Path(td) / "gateway-git.ext4"
+            vol.write_bytes(b"")  # already present
+            with patch.object(infra_vm, "_gw_dir", return_value=Path(td)), \
+                    patch(f"{_GW}.subprocess.run") as run:
+                out = gw._ensure_git_volume()
+            run.assert_not_called()  # no mke2fs when it exists — the repos survive
+            self.assertEqual(vol, out)
+
+    def test_creates_volume_when_missing(self) -> None:
+        gw = FirecrackerGateway()
+        with tempfile.TemporaryDirectory() as td:
+            with patch.object(infra_vm, "_gw_dir", return_value=Path(td)), \
+                    patch(f"{_GW}.subprocess.run",
+                          return_value=CompletedProcess([], 0)) as run:
+                out = gw._ensure_git_volume()
+            argv = run.call_args.args[0]
+            self.assertIn("mke2fs", argv)
+            self.assertEqual(str(Path(td) / "gateway-git.ext4"), out.__fspath__())
+            self.assertIn(str(out), argv)
+
+
 class TestFirecrackerGatewaySurface(unittest.TestCase):
     def test_is_running_reads_the_gateway_pidfile(self) -> None:
         gw = FirecrackerGateway()

tests/unit/test_firecracker_infra_vm.py

@@ -66,6 +66,18 @@ class TestRoleInits(unittest.TestCase):
         self.assertIn(f"mount -t ext4 /dev/vdb {infra_vm._GATEWAY_CA_MOUNT}", init)
         self.assertLess(init.index("/dev/vdb"),
                         init.index("bot_bottle.gateway.bootstrap"))
+        # Persistent git-gate volume (/dev/vdc) bind-mounted onto /git and
+        # /git-gate/creds so per-bottle repos + creds survive a rebuild (#512),
+        # also before the data plane (and any provisioning writes).
+        self.assertIn(f"mount -t ext4 /dev/vdc {infra_vm._GATEWAY_GIT_MOUNT}", init)
+        self.assertIn(
+            f"mount --bind {infra_vm._GATEWAY_GIT_MOUNT}/git "
+            f"{infra_vm._GATEWAY_GIT_REPO_ROOT}", init)
+        self.assertIn(
+            f"mount --bind {infra_vm._GATEWAY_GIT_MOUNT}/creds "
+            f"{infra_vm._GATEWAY_GIT_CREDS_DIR}", init)
+        self.assertLess(init.index("/dev/vdc"),
+                        init.index("bot_bottle.gateway.bootstrap"))
         self.assertIn("export PATH=", init)  # shared preamble
         self.assertIn("BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise", init)
         self.assertIn(f"cat {infra_vm._GUEST_GATEWAY_JWT_PATH}", init)  # host-minted JWT

tests/unit/test_firecracker_orchestrator.py

@@ -52,7 +52,7 @@ class TestEnsureRunning(unittest.TestCase):
         kw = boot.call_args.kwargs
         self.assertEqual("orchestrator", kw["role"])
         self.assertEqual(4096, kw["mem_mib"])          # orchestrator keeps build headroom
-        self.assertEqual(Path("/reg"), kw["data_drive"])  # DB volume on the CP
+        self.assertEqual((Path("/reg"),), kw["data_drives"])  # DB volume on the CP
         # The host-canonical signing key is pushed to the guest signing-key path.
         self.assertEqual("host-key", push.call_args.args[1])
         self.assertEqual(infra_vm._GUEST_SIGNING_KEY_PATH, push.call_args.args[2])

tests/unit/test_orchestrator_gateway.py

@@ -161,6 +161,26 @@ class TestDockerGateway(unittest.TestCase):
             "ci-ca-volume:/home/mitmproxy/.mitmproxy", argv
         )
 
+    def test_persists_git_gate_state_across_recreation(self) -> None:
+        # Per-bottle bare repos + deploy creds are bind-mounted from the host so
+        # a gateway restart doesn't drop already-running bottles' git-gate state
+        # (issue #512). Named sources here stand in for CI's per-run volumes.
+        sc = DockerGateway(
+            "bot-bottle-gateway:latest",
+            git_mount_source="ci-git-volume",
+            creds_mount_source="ci-creds-volume",
+        )
+
+        def fake(argv: list[str], **_kw: object) -> Mock:
+            return _proc(stdout="") if argv[:2] == ["docker", "ps"] else _proc()
+
+        with patch(_RUN_DOCKER, side_effect=fake) as run:
+            sc.connect_to_orchestrator(_ORCH_URL, _TOKEN)
+        argv = next(c.args[0] for c in run.call_args_list
+                    if c.args[0][:2] == ["docker", "run"])
+        self.assertIn("ci-git-volume:/git", argv)
+        self.assertIn("ci-creds-volume:/git-gate/creds", argv)
+
     def test_connect_injects_the_pre_minted_gateway_token(self) -> None:
         # The gateway presents the token the orchestrator handed it — it never
         # mints (holds no signing key). The value rides the env (bare `--env