From dff811f19084c623307a10964cfe5e331fd1e504 Mon Sep 17 00:00:00 2001 From: didericis Date: Sun, 26 Jul 2026 17:34:22 -0400 Subject: [PATCH] fix(gateway): persist git-gate state across gateway restarts Per-bottle git-gate state (bare repos under /git/, deploy creds under /git-gate/creds/) was provisioned once at bottle launch and lived only in the gateway's ephemeral storage. A gateway rebuild/restart wiped it and nothing re-provisioned already-running bottles, so their agents 404'd on fetch/push. Same class of bug as the CA (#510); the orchestrator restores only egress tokens, not git-gate declarations. Persist the state on both backends, mirroring the CA-persistence approach: - firecracker: attach a second persistent data drive (/dev/vdc) to the gateway VM and bind-mount its git/ + creds/ subdirs onto /git and /git-gate/creds in the gateway guest init, before the data plane starts. Generalize the VM config to a stable-ordered data_drives tuple (CA=vdb, git=vdc; orchestrator registry stays vdb). - docker: bind-mount host dirs (host_gateway_git_dir / creds_dir, under the never-pruned app-data root) onto /git and /git-gate/creds, with BOT_BOTTLE_DOCKER_GIT_MOUNT / _CREDS_MOUNT env overrides so CI isolates them to per-run volumes it cleans up. Teardown already rm -rf's /git/ + creds, so the persistent store self-cleans over the normal lifecycle. Closes #512 Co-Authored-By: Claude Opus 4.8 --- .gitea/workflows/pre-release-test.yml | 6 ++- .gitea/workflows/test.yml | 6 ++- bot_bottle/backend/docker/gateway.py | 21 +++++++++ .../backend/firecracker/firecracker_vm.py | 19 ++++---- bot_bottle/backend/firecracker/gateway.py | 43 ++++++++++++++++--- bot_bottle/backend/firecracker/infra_vm.py | 27 ++++++++++-- .../backend/firecracker/orchestrator.py | 2 +- bot_bottle/paths.py | 34 +++++++++++++++ tests/unit/test_firecracker_backend.py | 11 +++-- tests/unit/test_firecracker_gateway.py | 37 ++++++++++++++-- tests/unit/test_firecracker_infra_vm.py | 12 ++++++ tests/unit/test_firecracker_orchestrator.py | 2 +- tests/unit/test_orchestrator_gateway.py | 20 +++++++++ 13 files changed, 213 insertions(+), 27 deletions(-) diff --git a/.gitea/workflows/pre-release-test.yml b/.gitea/workflows/pre-release-test.yml index 4263914f..f72f160c 100644 --- a/.gitea/workflows/pre-release-test.yml +++ b/.gitea/workflows/pre-release-test.yml @@ -100,6 +100,8 @@ jobs: export BOT_BOTTLE_DOCKER_CLIENT_NETWORK="$DOCKER_CLIENT_NETWORK" export BOT_BOTTLE_DOCKER_ROOT_MOUNT="bot-bottle-ci-root-$RUN_KEY" export BOT_BOTTLE_DOCKER_CA_MOUNT="bot-bottle-ci-ca-$RUN_KEY" + export BOT_BOTTLE_DOCKER_GIT_MOUNT="bot-bottle-ci-git-$RUN_KEY" + export BOT_BOTTLE_DOCKER_CREDS_MOUNT="bot-bottle-ci-creds-$RUN_KEY" python3 -m coverage run -m scripts.unittest_gate \ -t . -s tests/integration -v \ --minimum-executed 22 --fail-on-skip @@ -110,7 +112,9 @@ jobs: RUN_KEY="${GITHUB_RUN_ID:-${GITHUB_RUN_NUMBER:-0}}" docker volume rm --force \ "bot-bottle-ci-root-$RUN_KEY" \ - "bot-bottle-ci-ca-$RUN_KEY" 2>/dev/null || true + "bot-bottle-ci-ca-$RUN_KEY" \ + "bot-bottle-ci-git-$RUN_KEY" \ + "bot-bottle-ci-creds-$RUN_KEY" 2>/dev/null || true # Non-dot name so upload-artifact's dotfile-skipping glob picks it up. - name: Stage docker coverage for upload diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index 963e7d0f..5599d358 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -116,6 +116,8 @@ jobs: export BOT_BOTTLE_DOCKER_CLIENT_NETWORK="$DOCKER_CLIENT_NETWORK" export BOT_BOTTLE_DOCKER_ROOT_MOUNT="bot-bottle-ci-root-$RUN_KEY" export BOT_BOTTLE_DOCKER_CA_MOUNT="bot-bottle-ci-ca-$RUN_KEY" + export BOT_BOTTLE_DOCKER_GIT_MOUNT="bot-bottle-ci-git-$RUN_KEY" + export BOT_BOTTLE_DOCKER_CREDS_MOUNT="bot-bottle-ci-creds-$RUN_KEY" python3 -m coverage run -m scripts.unittest_gate \ -t . -s tests/integration -v \ --minimum-executed 22 --fail-on-skip @@ -126,7 +128,9 @@ jobs: RUN_KEY="${GITHUB_RUN_ID:-${GITHUB_RUN_NUMBER:-0}}" docker volume rm --force \ "bot-bottle-ci-root-$RUN_KEY" \ - "bot-bottle-ci-ca-$RUN_KEY" 2>/dev/null || true + "bot-bottle-ci-ca-$RUN_KEY" \ + "bot-bottle-ci-git-$RUN_KEY" \ + "bot-bottle-ci-creds-$RUN_KEY" 2>/dev/null || true - name: Stage docker coverage for upload run: cp .coverage.docker coverage-docker.dat diff --git a/bot_bottle/backend/docker/gateway.py b/bot_bottle/backend/docker/gateway.py index 689ba5f0..887307c1 100644 --- a/bot_bottle/backend/docker/gateway.py +++ b/bot_bottle/backend/docker/gateway.py @@ -9,6 +9,8 @@ from .gateway_transport import DockerGatewayTransport from ...paths import ( ORCHESTRATOR_AUTH_JWT_ENV, host_gateway_ca_dir, + host_gateway_git_dir, + host_gateway_creds_dir, ) from ... import resources from ...gateway import ( @@ -40,6 +42,8 @@ class DockerGateway(Gateway): dockerfile: str | None = GATEWAY_DOCKERFILE, host_port_bindings: tuple[int, ...] = (), ca_mount_source: str | Path | None = None, + git_mount_source: str | Path | None = None, + creds_mount_source: str | Path | None = None, subnet: str | None = None, ) -> None: self.image_ref = image_ref @@ -74,6 +78,17 @@ class DockerGateway(Gateway): self._ca_mount_source = str( ca_mount_source or configured_ca or host_gateway_ca_dir() ) + # The persistent git-gate mounts (/git bare repos, /git-gate/creds deploy + # creds) — same host-bind-mount rationale as the CA (issue #512). The env + # overrides let CI point them at per-run named volumes it cleans up. + configured_git = os.environ.get("BOT_BOTTLE_DOCKER_GIT_MOUNT", "").strip() + self._git_mount_source = str( + git_mount_source or configured_git or host_gateway_git_dir() + ) + configured_creds = os.environ.get("BOT_BOTTLE_DOCKER_CREDS_MOUNT", "").strip() + self._creds_mount_source = str( + creds_mount_source or configured_creds or host_gateway_creds_dir() + ) def image_exists(self) -> bool: return run_docker(["docker", "image", "inspect", self.image_ref]).returncode == 0 @@ -198,6 +213,12 @@ class DockerGateway(Gateway): # container recreation AND docker volume pruning (agents trust it) # — see host_gateway_ca_dir / issue #450. "--volume", f"{self._ca_mount_source}:{MITMPROXY_HOME}", + # Persist per-bottle git-gate state (bare repos + deploy creds) on + # the host so a gateway restart doesn't drop already-running bottles' + # repos — they would otherwise 404 on fetch/push (issue #512). Same + # host-bind-mount rationale as the CA. + "--volume", f"{self._git_mount_source}:/git", + "--volume", f"{self._creds_mount_source}:/git-gate/creds", # No DB mount: the data plane (egress / supervise / git-gate) reaches # the supervise queue over the control-plane RPC and never opens # bot-bottle.db, so the gateway container gets no file handle on it diff --git a/bot_bottle/backend/firecracker/firecracker_vm.py b/bot_bottle/backend/firecracker/firecracker_vm.py index 6ccd81ce..cbdf5243 100644 --- a/bot_bottle/backend/firecracker/firecracker_vm.py +++ b/bot_bottle/backend/firecracker/firecracker_vm.py @@ -84,7 +84,7 @@ def _config( vcpus: int, mem_mib: int, guest_mac: str, - data_drive: Path | None = None, + data_drives: tuple[Path, ...] = (), extra_boot_args: str = "", ) -> dict[str, object]: drives: list[dict[str, object]] = [ @@ -95,12 +95,15 @@ def _config( "is_read_only": False, } ] - # A second virtio-block device (guest /dev/vdb) — the infra VM's - # persistent registry "volume", a host-side ext4 file that outlives the - # ephemeral rootfs across VM restarts. - if data_drive is not None: + # Extra virtio-block devices — the infra VMs' persistent "volumes", + # host-side ext4 files that outlive the ephemeral rootfs across VM restarts. + # They appear to the guest as /dev/vdb, /dev/vdc, ... in list order (after + # the root /dev/vda), so callers must keep the order stable: the orchestrator + # attaches its registry (vdb); the gateway attaches its CA (vdb) then its + # git-gate state (vdc). + for i, data_drive in enumerate(data_drives): drives.append({ - "drive_id": "data", + "drive_id": f"data{i}", "path_on_host": str(data_drive), "is_root_device": False, "is_read_only": False, @@ -138,7 +141,7 @@ def boot( mem_mib: int = 2048, guest_mac: str = "06:00:AC:10:00:02", detached: bool = False, - data_drive: Path | None = None, + data_drives: tuple[Path, ...] = (), extra_boot_args: str = "", ) -> VmHandle: """Write the config and launch the VMM. Returns once the process is @@ -155,7 +158,7 @@ def boot( _config( rootfs=rootfs, tap=tap, guest_ip=guest_ip, host_ip=host_ip, pubkey=pubkey, vcpus=vcpus, mem_mib=mem_mib, guest_mac=guest_mac, - data_drive=data_drive, extra_boot_args=extra_boot_args, + data_drives=data_drives, extra_boot_args=extra_boot_args, ), indent=2, )) diff --git a/bot_bottle/backend/firecracker/gateway.py b/bot_bottle/backend/firecracker/gateway.py index b1ea6440..0a17c9d4 100644 --- a/bot_bottle/backend/firecracker/gateway.py +++ b/bot_bottle/backend/firecracker/gateway.py @@ -61,6 +61,16 @@ _GATEWAY_CA_PATH = "/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem" # which outlives the ephemeral rootfs. mitmproxy is tiny; 16M is ample. _CA_VOLUME_SIZE = "16M" +# The gateway VM's persistent git-gate volume — a (sparse) ext4 file the gateway +# init mounts, then bind-mounts onto /git + /git-gate/creds (see +# `infra_vm._GATEWAY_GIT_MOUNT`), so per-bottle bare repos and deploy creds +# SURVIVE a gateway-VM rebuild. Without it a restart drops every already-running +# bottle's git-gate state and its agent 404s on fetch/push (same class as the CA +# — issue #512). Bare repos hold upstream history, so this is sized far larger +# than the CA volume; mke2fs leaves the file sparse, so the host only stores +# blocks actually used. +_GIT_VOLUME_SIZE = "8G" + _CA_FETCH_TIMEOUT_SECONDS = 15.0 @@ -106,15 +116,18 @@ class FirecrackerGateway(Gateway): f"cannot resolve orchestrator guest IP from {self._orchestrator_url!r}" ) # Boot on the gateway link from the gateway rootfs, then push the token - # the init waits for before starting the data plane. The persistent CA - # volume (/dev/vdb, mounted at mitmproxy's confdir by the gateway init) - # keeps the CA STABLE across rebuilds so already-running bottles keep - # trusting the gateway's TLS interception (issue #450). + # the init waits for before starting the data plane. Two persistent + # volumes ride along, attached in a FIXED order the gateway init depends + # on: the CA volume as /dev/vdb (mounted at mitmproxy's confdir) and the + # git-gate volume as /dev/vdc (bind-mounted onto /git + /git-gate/creds). + # Both keep gateway-side state STABLE across rebuilds so already-running + # bottles keep working — TLS interception (issue #450) and git-gate fetch + # /push (issue #512) respectively. vm = infra_vm.boot_vm( name=GATEWAY_NAME, slot=netpool.gw_slot(), run_dir=infra_vm._gw_dir(), role="gateway", mem_mib=_GW_MEM_MIB, extra_boot_args=f"bb_orch={orchestrator_guest_ip}", - data_drive=self._ensure_ca_volume(), + data_drives=(self._ensure_ca_volume(), self._ensure_git_volume()), ) infra_vm.push_secret( vm, self._gateway_token, _GUEST_GATEWAY_JWT_PATH, @@ -152,6 +165,26 @@ class FirecrackerGateway(Gateway): die(f"creating gateway CA volume failed: {proc.stderr.strip()}") return vol + def _ensure_git_volume(self) -> Path: + """Create the empty ext4 git-gate volume on first use; reuse it after. + + Empty on first boot (the gateway init lays out `git/` + `creds/` subdirs + and bind-mounts them); every later boot reuses whatever repos + creds the + volume already holds — which is what keeps git-gate state stable across a + gateway-VM rebuild (issue #512). Sibling of `_ensure_ca_volume`.""" + vol = infra_vm._gw_dir() / "gateway-git.ext4" + if vol.exists(): + return vol + info(f"creating gateway git-gate volume {vol} ({_GIT_VOLUME_SIZE})") + proc = subprocess.run( + ["mke2fs", "-q", "-t", "ext4", "-F", str(vol), _GIT_VOLUME_SIZE], + capture_output=True, text=True, check=False, + ) + if proc.returncode != 0: + vol.unlink(missing_ok=True) + die(f"creating gateway git-gate volume failed: {proc.stderr.strip()}") + return vol + def address(self) -> str: """The gateway VM's guest IP — the agent-facing target agent VMs' gateway-port traffic is DNAT'd to.""" diff --git a/bot_bottle/backend/firecracker/infra_vm.py b/bot_bottle/backend/firecracker/infra_vm.py index 9fc978a9..6a1ce691 100644 --- a/bot_bottle/backend/firecracker/infra_vm.py +++ b/bot_bottle/backend/firecracker/infra_vm.py @@ -63,6 +63,15 @@ _GUEST_GATEWAY_JWT_PATH = "/var/lib/bot-bottle/gateway-jwt" # survives a gateway-VM rebuild; without it every rebuild mints a fresh CA that # already-running bottles distrust, breaking the TLS handshake (issue #450). _GATEWAY_CA_MOUNT = "/home/mitmproxy/.mitmproxy" +# The gateway VM's persistent git-gate volume staging mount (/dev/vdc). The +# gateway boots with this volume (see `FirecrackerGateway._ensure_git_volume`) +# and the init bind-mounts its `git/` + `creds/` subdirs onto the load-bearing +# `/git` and `/git-gate/creds` paths, so per-bottle bare repos + deploy creds +# survive a gateway-VM rebuild; without it a restart drops every already-running +# bottle's git-gate state and its agent 404s on fetch/push (issue #512). +_GATEWAY_GIT_MOUNT = "/var/lib/bot-bottle-gitgate" +_GATEWAY_GIT_REPO_ROOT = "/git" +_GATEWAY_GIT_CREDS_DIR = "/git-gate/creds" # The two per-plane rootfs source images. The orchestrator VM boots a control # plane + buildah rootfs (Dockerfile.orchestrator.fc, FROM orchestrator); the @@ -198,11 +207,12 @@ def boot_vm( run_dir: Path, role: str, mem_mib: int, - data_drive: Path | None = None, + data_drives: tuple[Path, ...] = (), extra_boot_args: str = "", ) -> InfraVm: """Boot the `role` infra VM from its per-plane rootfs on `slot`'s link. - Records the PID.""" + Records the PID. `data_drives` are attached as /dev/vdb, /dev/vdc, ... in + order, so callers must keep the order stable (see `firecracker_vm._config`).""" if not netpool.tap_present(slot.iface): die(f"infra link {slot.iface} not present.\n" f" ./cli.py backend setup --backend=firecracker") @@ -224,7 +234,7 @@ def boot_vm( name=name, rootfs=rootfs, tap=slot.iface, guest_ip=slot.guest_ip, host_ip=slot.host_ip, pubkey=pubkey, run_dir=run_dir, mem_mib=mem_mib, detached=True, - data_drive=data_drive, extra_boot_args=boot_args, + data_drives=data_drives, extra_boot_args=boot_args, ) _pid_file(run_dir).write_text(str(vm.process.pid)) return InfraVm(guest_ip=slot.guest_ip, private_key=private_key, vm=vm) @@ -461,6 +471,17 @@ def _gateway_init() -> str: # mitmproxy) starts. mkdir -p {_GATEWAY_CA_MOUNT} mount -t ext4 /dev/vdb {_GATEWAY_CA_MOUNT} 2>/dev/null || true +# Persistent git-gate volume (/dev/vdc): its git/ + creds/ subdirs are +# bind-mounted onto the load-bearing /git and /git-gate/creds so per-bottle bare +# repos + deploy creds survive a gateway-VM rebuild (issue #512). On first boot +# the volume is empty; the subdirs are created here. Must mount BEFORE the data +# plane (hence git-http) starts, and before any per-bottle provisioning writes. +mkdir -p {_GATEWAY_GIT_MOUNT} +mount -t ext4 /dev/vdc {_GATEWAY_GIT_MOUNT} 2>/dev/null || true +mkdir -p {_GATEWAY_GIT_MOUNT}/git {_GATEWAY_GIT_MOUNT}/creds +mkdir -p {_GATEWAY_GIT_REPO_ROOT} {_GATEWAY_GIT_CREDS_DIR} +mount --bind {_GATEWAY_GIT_MOUNT}/git {_GATEWAY_GIT_REPO_ROOT} 2>/dev/null || true +mount --bind {_GATEWAY_GIT_MOUNT}/creds {_GATEWAY_GIT_CREDS_DIR} 2>/dev/null || true ORCH=$(sed -n 's/.*bb_orch=\\([^ ]*\\).*/\\1/p' /proc/cmdline) GW_JWT="" i=0 diff --git a/bot_bottle/backend/firecracker/orchestrator.py b/bot_bottle/backend/firecracker/orchestrator.py index f2d17157..eb271ce0 100644 --- a/bot_bottle/backend/firecracker/orchestrator.py +++ b/bot_bottle/backend/firecracker/orchestrator.py @@ -104,7 +104,7 @@ class FirecrackerOrchestrator(Orchestrator): vm = infra_vm.boot_vm( name=ORCHESTRATOR_NAME, slot=netpool.orch_slot(), run_dir=infra_vm._orch_dir(), role="orchestrator", mem_mib=_ORCH_MEM_MIB, - data_drive=self._ensure_registry_volume(), + data_drives=(self._ensure_registry_volume(),), ) # Push the host-canonical signing key (the init waits for it before # starting the control plane). It comes through the shared provisioning diff --git a/bot_bottle/paths.py b/bot_bottle/paths.py index 8cd69e04..365cf374 100644 --- a/bot_bottle/paths.py +++ b/bot_bottle/paths.py @@ -53,6 +53,15 @@ ORCHESTRATOR_AUTH_JWT_ENV = "BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT" # the shared gateway's TLS interception, so it must not rotate on restart. See # host_gateway_ca_dir() for why this is a host bind-mount, not a named volume. GATEWAY_CA_DIRNAME = "gateway-ca" +# The host directories holding the gateway's persistent git-gate state — the +# per-bottle bare repos (`gateway-git`) and deploy creds (`gateway-creds`). +# Bind-mounted into the gateway container at /git and /git-gate/creds so they +# survive container recreation; without it a gateway restart drops every +# already-running bottle's git-gate state and its agent 404s on fetch/push +# (issue #512). Host bind-mounts (not named volumes) for the same reason as the +# CA dir — see host_gateway_ca_dir(). +GATEWAY_GIT_DIRNAME = "gateway-git" +GATEWAY_CREDS_DIRNAME = "gateway-creds" def bot_bottle_root() -> Path: @@ -97,6 +106,27 @@ def host_gateway_ca_dir() -> Path: return ca_dir +def host_gateway_git_dir() -> Path: + """The directory holding the gateway's persistent per-bottle bare repos, + created if missing. Bind-mounted into the gateway container at /git so the + repos survive container recreation (issue #512). A host bind-mount under the + app-data root, never pruned — same rationale as host_gateway_ca_dir().""" + git_dir = bot_bottle_root() / GATEWAY_GIT_DIRNAME + git_dir.mkdir(parents=True, exist_ok=True) + return git_dir + + +def host_gateway_creds_dir() -> Path: + """The directory holding the gateway's persistent per-bottle git-gate deploy + creds, created if missing. Bind-mounted into the gateway container at + /git-gate/creds so the creds survive container recreation (issue #512). A + host bind-mount under the app-data root — same rationale as + host_gateway_ca_dir().""" + creds_dir = bot_bottle_root() / GATEWAY_CREDS_DIRNAME + creds_dir.mkdir(parents=True, exist_ok=True) + return creds_dir + + def host_signing_key(filename: str) -> str: """A per-host signing key at `/`, minted (256-bit, url-safe) and persisted 0600 on first use, then reused. @@ -143,10 +173,14 @@ __all__ = [ "ORCHESTRATOR_TOKEN_ENV", "ORCHESTRATOR_AUTH_JWT_ENV", "GATEWAY_CA_DIRNAME", + "GATEWAY_GIT_DIRNAME", + "GATEWAY_CREDS_DIRNAME", "bot_bottle_root", "host_db_path", "host_db_dir", "host_gateway_ca_dir", + "host_gateway_git_dir", + "host_gateway_creds_dir", "host_signing_key", "host_orchestrator_token", ] diff --git a/tests/unit/test_firecracker_backend.py b/tests/unit/test_firecracker_backend.py index 6ec358e5..a8f1032f 100644 --- a/tests/unit/test_firecracker_backend.py +++ b/tests/unit/test_firecracker_backend.py @@ -404,16 +404,19 @@ class TestBootArgs(unittest.TestCase): self.assertEqual("bbfc0", cfg["network-interfaces"][0]["host_dev_name"]) self.assertEqual(1, len(cfg["drives"])) # no data drive by default - def test_config_adds_data_drive(self): + def test_config_adds_data_drives_in_order(self): cfg = cast(Any, firecracker_vm._config( rootfs=Path("/run/rootfs.ext4"), tap="bbfc0", guest_ip="100.64.0.1", host_ip="100.64.0.0", pubkey="k", vcpus=2, mem_mib=2048, guest_mac="06:00:AC:10:00:02", - data_drive=Path("/run/registry.ext4"), + data_drives=(Path("/run/ca.ext4"), Path("/run/git.ext4")), )) - self.assertEqual(2, len(cfg["drives"])) + # rootfs (vda) + two data drives, attached in list order so the guest + # sees them as /dev/vdb, /dev/vdc — an order callers depend on. + self.assertEqual(3, len(cfg["drives"])) self.assertFalse(cfg["drives"][1]["is_root_device"]) - self.assertEqual("/run/registry.ext4", cfg["drives"][1]["path_on_host"]) + self.assertEqual("/run/ca.ext4", cfg["drives"][1]["path_on_host"]) + self.assertEqual("/run/git.ext4", cfg["drives"][2]["path_on_host"]) class TestBottleExecClose(unittest.TestCase): diff --git a/tests/unit/test_firecracker_gateway.py b/tests/unit/test_firecracker_gateway.py index c593af97..a11355d1 100644 --- a/tests/unit/test_firecracker_gateway.py +++ b/tests/unit/test_firecracker_gateway.py @@ -60,9 +60,12 @@ class TestFirecrackerGatewayConnect(unittest.TestCase): gw = FirecrackerGateway() booted = infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k")) ca_vol = Path("/gw/gateway-ca.ext4") + git_vol = Path("/gw/gateway-git.ext4") with patch.object(infra_vm, "boot_vm", return_value=booted) as boot, \ patch.object(FirecrackerGateway, "_ensure_ca_volume", return_value=ca_vol), \ + patch.object(FirecrackerGateway, "_ensure_git_volume", + return_value=git_vol), \ patch.object(infra_vm, "push_secret") as push: gw.connect_to_orchestrator(_ORCH_URL, _TOKEN) # Booted on the gateway link with the gateway role; the orchestrator's @@ -70,9 +73,9 @@ class TestFirecrackerGatewayConnect(unittest.TestCase): kw = boot.call_args.kwargs self.assertEqual("gateway", kw["role"]) self.assertIn("bb_orch=10.243.255.1", kw["extra_boot_args"]) - # The persistent CA volume rides as /dev/vdb so the mitmproxy CA - # survives a gateway-VM rebuild (issue #450). - self.assertEqual(ca_vol, kw.get("data_drive")) + # The persistent volumes ride in a FIXED order — CA as /dev/vdb, git-gate + # as /dev/vdc — so both survive a gateway-VM rebuild (issues #450, #512). + self.assertEqual((ca_vol, git_vol), kw.get("data_drives")) # The host-minted token (never the key — #469) is pushed to the guest. push.assert_called_once() self.assertEqual(_TOKEN, push.call_args.args[1]) @@ -107,6 +110,34 @@ class TestGatewayCaVolume(unittest.TestCase): self.assertIn(str(out), argv) +class TestGatewayGitVolume(unittest.TestCase): + """The persistent git-gate volume that keeps per-bottle bare repos + creds + stable across a gateway-VM rebuild (issue #512).""" + + def test_reuses_existing_volume(self) -> None: + gw = FirecrackerGateway() + with tempfile.TemporaryDirectory() as td: + vol = Path(td) / "gateway-git.ext4" + vol.write_bytes(b"") # already present + with patch.object(infra_vm, "_gw_dir", return_value=Path(td)), \ + patch(f"{_GW}.subprocess.run") as run: + out = gw._ensure_git_volume() + run.assert_not_called() # no mke2fs when it exists — the repos survive + self.assertEqual(vol, out) + + def test_creates_volume_when_missing(self) -> None: + gw = FirecrackerGateway() + with tempfile.TemporaryDirectory() as td: + with patch.object(infra_vm, "_gw_dir", return_value=Path(td)), \ + patch(f"{_GW}.subprocess.run", + return_value=CompletedProcess([], 0)) as run: + out = gw._ensure_git_volume() + argv = run.call_args.args[0] + self.assertIn("mke2fs", argv) + self.assertEqual(str(Path(td) / "gateway-git.ext4"), out.__fspath__()) + self.assertIn(str(out), argv) + + class TestFirecrackerGatewaySurface(unittest.TestCase): def test_is_running_reads_the_gateway_pidfile(self) -> None: gw = FirecrackerGateway() diff --git a/tests/unit/test_firecracker_infra_vm.py b/tests/unit/test_firecracker_infra_vm.py index 44cc280d..a1b664e5 100644 --- a/tests/unit/test_firecracker_infra_vm.py +++ b/tests/unit/test_firecracker_infra_vm.py @@ -66,6 +66,18 @@ class TestRoleInits(unittest.TestCase): self.assertIn(f"mount -t ext4 /dev/vdb {infra_vm._GATEWAY_CA_MOUNT}", init) self.assertLess(init.index("/dev/vdb"), init.index("bot_bottle.gateway.bootstrap")) + # Persistent git-gate volume (/dev/vdc) bind-mounted onto /git and + # /git-gate/creds so per-bottle repos + creds survive a rebuild (#512), + # also before the data plane (and any provisioning writes). + self.assertIn(f"mount -t ext4 /dev/vdc {infra_vm._GATEWAY_GIT_MOUNT}", init) + self.assertIn( + f"mount --bind {infra_vm._GATEWAY_GIT_MOUNT}/git " + f"{infra_vm._GATEWAY_GIT_REPO_ROOT}", init) + self.assertIn( + f"mount --bind {infra_vm._GATEWAY_GIT_MOUNT}/creds " + f"{infra_vm._GATEWAY_GIT_CREDS_DIR}", init) + self.assertLess(init.index("/dev/vdc"), + init.index("bot_bottle.gateway.bootstrap")) self.assertIn("export PATH=", init) # shared preamble self.assertIn("BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise", init) self.assertIn(f"cat {infra_vm._GUEST_GATEWAY_JWT_PATH}", init) # host-minted JWT diff --git a/tests/unit/test_firecracker_orchestrator.py b/tests/unit/test_firecracker_orchestrator.py index 5e1524ba..32f19634 100644 --- a/tests/unit/test_firecracker_orchestrator.py +++ b/tests/unit/test_firecracker_orchestrator.py @@ -52,7 +52,7 @@ class TestEnsureRunning(unittest.TestCase): kw = boot.call_args.kwargs self.assertEqual("orchestrator", kw["role"]) self.assertEqual(4096, kw["mem_mib"]) # orchestrator keeps build headroom - self.assertEqual(Path("/reg"), kw["data_drive"]) # DB volume on the CP + self.assertEqual((Path("/reg"),), kw["data_drives"]) # DB volume on the CP # The host-canonical signing key is pushed to the guest signing-key path. self.assertEqual("host-key", push.call_args.args[1]) self.assertEqual(infra_vm._GUEST_SIGNING_KEY_PATH, push.call_args.args[2]) diff --git a/tests/unit/test_orchestrator_gateway.py b/tests/unit/test_orchestrator_gateway.py index 7c8cbd2e..ede18065 100644 --- a/tests/unit/test_orchestrator_gateway.py +++ b/tests/unit/test_orchestrator_gateway.py @@ -161,6 +161,26 @@ class TestDockerGateway(unittest.TestCase): "ci-ca-volume:/home/mitmproxy/.mitmproxy", argv ) + def test_persists_git_gate_state_across_recreation(self) -> None: + # Per-bottle bare repos + deploy creds are bind-mounted from the host so + # a gateway restart doesn't drop already-running bottles' git-gate state + # (issue #512). Named sources here stand in for CI's per-run volumes. + sc = DockerGateway( + "bot-bottle-gateway:latest", + git_mount_source="ci-git-volume", + creds_mount_source="ci-creds-volume", + ) + + def fake(argv: list[str], **_kw: object) -> Mock: + return _proc(stdout="") if argv[:2] == ["docker", "ps"] else _proc() + + with patch(_RUN_DOCKER, side_effect=fake) as run: + sc.connect_to_orchestrator(_ORCH_URL, _TOKEN) + argv = next(c.args[0] for c in run.call_args_list + if c.args[0][:2] == ["docker", "run"]) + self.assertIn("ci-git-volume:/git", argv) + self.assertIn("ci-creds-volume:/git-gate/creds", argv) + def test_connect_injects_the_pre_minted_gateway_token(self) -> None: # The gateway presents the token the orchestrator handed it — it never # mints (holds no signing key). The value rides the env (bare `--env