diff --git a/Dockerfile.infra.fc b/Dockerfile.infra.fc deleted file mode 100644 index c2e7dc4f..00000000 --- a/Dockerfile.infra.fc +++ /dev/null @@ -1,34 +0,0 @@ -# Firecracker infra VM image (PRD 0070 Stage B). -# -# The shared rootfs both firecracker infra VMs boot: the gateway data plane + -# the orchestrator control plane + the in-VM agent-image builder. A `bb_role=` -# kernel-cmdline arg selects which plane the guest init starts (see -# `infra_vm._infra_init`), so there is one artifact to build/publish/pull. -# -# `Dockerfile.orchestrator` is the single definition of the orchestrator -# content (the lean `bot_bottle` package on trixie `python:3.12-slim`); it's -# pulled in via `COPY --from` (multi-`FROM` can't union two bases). Both images -# share the trixie base, so the copy is clean. -# -# The Firecracker backend builds users' agent Dockerfiles *inside the -# orchestrator VM* with buildah (rootless, daemonless) instead of on the host — -# no host Docker daemon, no root-equivalent `docker` group. Requires the trixie -# base from bot-bottle-gateway (buildah 1.39: bookworm's 1.28 can't parse the -# Dockerfile heredocs agent images use). `crun` is the OCI runtime; `netavark` + -# `aardvark-dns` are the network backend for `FROM` pulls + `RUN` egress. `vfs` + -# `chroot`: buildah works as root in the bare microVM (no fuse-overlayfs / -# overlay module / subuid maps). Matches image_builder. -FROM bot-bottle-gateway:latest - -# The orchestrator content, from its single definition. The gateway image -# already has the flat daemon modules under /app; this adds the full -# `bot_bottle` package so `python3 -m bot_bottle.orchestrator` resolves in the -# orchestrator-role VM. -COPY --from=bot-bottle-orchestrator:latest /app/bot_bottle /app/bot_bottle - -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - buildah crun netavark aardvark-dns \ - && rm -rf /var/lib/apt/lists/* -ENV STORAGE_DRIVER=vfs \ - BUILDAH_ISOLATION=chroot diff --git a/Dockerfile.orchestrator.fc b/Dockerfile.orchestrator.fc new file mode 100644 index 00000000..2fa69acc --- /dev/null +++ b/Dockerfile.orchestrator.fc @@ -0,0 +1,23 @@ +# Firecracker orchestrator-VM image (PRD 0070). +# +# The control-plane rootfs the orchestrator microVM boots: the lean orchestrator +# image + the in-VM agent-image builder. The Firecracker backend builds users' +# agent Dockerfiles *inside this VM* with buildah (rootless, daemonless) instead +# of on the host — no host Docker daemon, no root-equivalent `docker` group. The +# gateway VM boots a *separate*, slimmer rootfs (bot-bottle-gateway:latest) that +# carries none of this build tooling — the exposed data plane stays minimal. +# +# `crun` is the OCI runtime; `netavark` + `aardvark-dns` are the network backend +# for `FROM` pulls + `RUN` egress. `vfs` + `chroot`: buildah works as root in the +# bare microVM (no fuse-overlayfs / overlay module / subuid maps). The trixie +# base (from Dockerfile.orchestrator's python:3.12-slim) carries buildah 1.39, +# which parses the Dockerfile heredocs agent images use (bookworm's 1.28 can't). +# Matches image_builder. +FROM bot-bottle-orchestrator:latest + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + buildah crun netavark aardvark-dns \ + && rm -rf /var/lib/apt/lists/* +ENV STORAGE_DRIVER=vfs \ + BUILDAH_ISOLATION=chroot diff --git a/bot_bottle/backend/firecracker/infra_artifact.py b/bot_bottle/backend/firecracker/infra_artifact.py index 2d847de5..b5b7ae60 100644 --- a/bot_bottle/backend/firecracker/infra_artifact.py +++ b/bot_bottle/backend/firecracker/infra_artifact.py @@ -1,22 +1,26 @@ -"""Prebuilt infra-VM rootfs, pulled as an artifact (PRD 0069 Stage 2). +"""Prebuilt infra-VM rootfs images, pulled as artifacts (PRD 0069 Stage 2). -The Firecracker infra VM boots a fixed rootfs (orchestrator control plane + -gateway + buildah, control-plane init as PID 1) that does not vary per launch — -the per-boot bits (authorized_keys, guest IP) ride the kernel cmdline, so one -rootfs boots on any host. Instead of building that rootfs on the launch host -with Docker, we build it **off-host** and publish it as a versioned, ready-to- -boot ext4 (gzip-compressed) to a Gitea **generic package**; the launch host -downloads + verifies + boots it. No Docker, no image tooling on the launch -host — just an HTTP fetch and gunzip. +The two Firecracker infra VMs each boot a fixed per-plane rootfs that does not +vary per launch — the per-boot bits (authorized_keys, guest IP) ride the kernel +cmdline, so one rootfs boots on any host: + + * `orchestrator` — the control plane + buildah (in-VM agent-image builds); + * `gateway` — the slim data plane (no build tooling on the exposed VM). + +Instead of building them on the launch host with Docker, we build them +**off-host** and publish each as a versioned, ready-to-boot ext4 (gzip- +compressed) to a per-role Gitea **generic package**; the launch host downloads + +verifies + boots them. No Docker, no image tooling on the launch host — just an +HTTP fetch and gunzip. publish (off-host, see publish_infra.py): docker build -> rootfs dir -> mke2fs -> gzip -> PUT generic package pull (this module, launch host): GET .../rootfs.ext4.gz (+ .sha256) -> verify -> gunzip -> boot -The artifact **version** is a content hash of everything baked into the rootfs -(the shipped bot_bottle package, the three Dockerfiles, and the init), so a -launch host always pulls the artifact matching its code and a content change +Each artifact **version** is a content hash of everything baked into that rootfs +(the shipped bot_bottle package, the role's Dockerfiles, and the role init), so +a launch host always pulls the artifact matching its code and a content change can't silently boot a stale rootfs. A checksum mismatch fails closed. Set `BOT_BOTTLE_INFRA_BUILD=local` to skip the pull and build the rootfs @@ -41,11 +45,23 @@ from . import util _ARTIFACT_FORMAT = "1" _REPO_ROOT = Path(__file__).resolve().parents[3] -_DOCKERFILES = ("Dockerfile.orchestrator", "Dockerfile.gateway", "Dockerfile.infra.fc") + +# The two per-plane infra VM roles. Each publishes/pulls its own rootfs artifact +# from its own generic package; the Dockerfiles baked into each differ (only the +# orchestrator rootfs carries buildah), so the versions are hashed separately. +ROLES = ("orchestrator", "gateway") +_DOCKERFILES = { + "orchestrator": ("Dockerfile.orchestrator", "Dockerfile.orchestrator.fc"), + "gateway": ("Dockerfile.gateway",), +} _DEFAULT_BASE = "https://gitea.dideric.is" _DEFAULT_OWNER = "didericis" -_PACKAGE = "bot-bottle-firecracker-infra" + + +def _package(role: str) -> str: + return f"bot-bottle-firecracker-{role}" + # Streaming copy chunk for the (hundreds-of-MB) download. _CHUNK = 1 << 20 @@ -57,21 +73,24 @@ def local_build_requested() -> bool: return os.environ.get("BOT_BOTTLE_INFRA_BUILD", "").strip().lower() == "local" -def infra_artifact_version(init_script: str, *, repo_root: Path = _REPO_ROOT) -> str: - """Content hash (16 hex) of everything baked into the infra rootfs: the - whole shipped `bot_bottle` package, the three fixed Dockerfiles, and the - guest init. Deterministic across the publish host and the launch host when - both run the same checkout, so the tag the launch host pulls is exactly the - tag publish produced. +def infra_artifact_version( + init_script: str, role: str, *, repo_root: Path = _REPO_ROOT, +) -> str: + """Content hash (16 hex) of everything baked into `role`'s infra rootfs: the + whole shipped `bot_bottle` package, that role's Dockerfiles, and its guest + init. Deterministic across the publish host and the launch host when both run + the same checkout, so the tag the launch host pulls is exactly the tag + publish produced. - The package is `COPY bot_bottle /app/bot_bottle`'d wholesale into the image, - so hash *every* regular file under it — not just `*.py`. Non-Python inputs - (e.g. `gateway/egress/entrypoint.sh`, `netpool.defaults.env`) are baked in too, and - a change to one must bump the version or a launch host could boot a stale - rootfs whose code differs from its checkout. `__pycache__`/`.pyc` are the - only exclusions — build artifacts, never copied.""" + The package is baked into both images wholesale (orchestrator `COPY`s it, + gateway `pip install`s it), so hash *every* regular file under it — not just + `*.py`. Non-Python inputs (e.g. `gateway/egress/entrypoint.sh`, + `netpool.defaults.env`) are baked in too, and a change to one must bump the + version or a launch host could boot a stale rootfs whose code differs from + its checkout. `__pycache__`/`.pyc` are the only exclusions — build artifacts, + never copied.""" h = hashlib.sha256() - h.update(f"format={_ARTIFACT_FORMAT}\n".encode()) + h.update(f"format={_ARTIFACT_FORMAT}\nrole={role}\n".encode()) pkg = repo_root / "bot_bottle" for path in sorted(pkg.rglob("*")): if not path.is_file(): @@ -81,7 +100,7 @@ def infra_artifact_version(init_script: str, *, repo_root: Path = _REPO_ROOT) -> h.update(str(path.relative_to(repo_root)).encode()) h.update(b"\0") h.update(path.read_bytes()) - for name in _DOCKERFILES: + for name in _DOCKERFILES[role]: h.update(name.encode()) h.update(b"\0") h.update((repo_root / name).read_bytes()) @@ -107,11 +126,11 @@ def _config() -> tuple[str, str, str]: return base, owner, token -def artifact_url(version: str, filename: str) -> str: - """The generic-package download URL for one file of this version's - artifact (`rootfs.ext4.gz` / `rootfs.ext4.gz.sha256`).""" +def artifact_url(version: str, filename: str, *, role: str) -> str: + """The generic-package download URL for one file of `role`'s artifact at + `version` (`rootfs.ext4.gz` / `rootfs.ext4.gz.sha256`).""" base, owner, _ = _config() - return f"{base}/api/packages/{owner}/generic/{_PACKAGE}/{version}/{filename}" + return f"{base}/api/packages/{owner}/generic/{_package(role)}/{version}/{filename}" _GZ_NAME = "rootfs.ext4.gz" @@ -119,8 +138,8 @@ _SHA_NAME = "rootfs.ext4.gz.sha256" _CANDIDATE_DIR_ENV = "BOT_BOTTLE_INFRA_ARTIFACT_DIR" -def _cache_root(version: str) -> Path: - return util.cache_dir() / "infra-artifact" / version +def _cache_root(version: str, role: str) -> Path: + return util.cache_dir() / "infra-artifact" / role / version def _open(url: str) -> urllib.request.Request: @@ -162,13 +181,17 @@ def _sha256_file(path: Path) -> str: return h.hexdigest() -def ensure_artifact_gz(version: str) -> Path: - """The verified, cached `rootfs.ext4.gz` for `version` — downloading it (and - its `.sha256`) once, then reusing it. Fail-closed on a checksum mismatch: - the partial is removed and we die rather than boot an unverified rootfs.""" +def ensure_artifact_gz(version: str, *, role: str) -> Path: + """The verified, cached `rootfs.ext4.gz` for `role` at `version` — + downloading it (and its `.sha256`) once, then reusing it. Fail-closed on a + checksum mismatch: the partial is removed and we die rather than boot an + unverified rootfs. + + A pre-staged candidate bundle (`BOT_BOTTLE_INFRA_ARTIFACT_DIR`) holds each + role under its own `//` subdir.""" candidate_dir = os.environ.get(_CANDIDATE_DIR_ENV, "").strip() if candidate_dir: - root = Path(candidate_dir) + root = Path(candidate_dir) / role version_file = root / "version.txt" # Guard the read so a missing version.txt is a clean error, not a raw # FileNotFoundError. @@ -177,7 +200,7 @@ def ensure_artifact_gz(version: str) -> Path: declared = version_file.read_text(encoding="utf-8").strip() if declared != version: die( - f"infra candidate version mismatch: expected {version}, " + f"infra candidate version mismatch ({role}): expected {version}, " f"bundle contains {declared or ''}" ) gz = root / _GZ_NAME @@ -188,22 +211,22 @@ def ensure_artifact_gz(version: str) -> Path: actual = _sha256_file(gz) if actual != expected: die( - f"infra candidate checksum mismatch for {version}:\n" + f"infra candidate checksum mismatch ({role}) for {version}:\n" f" expected {expected}\n actual {actual}" ) return gz - root = _cache_root(version) + root = _cache_root(version, role) root.mkdir(parents=True, exist_ok=True) gz = root / _GZ_NAME ok = root / ".verified" if gz.is_file() and ok.is_file(): return gz - info(f"pulling infra rootfs artifact {_PACKAGE}/{version}") - _download(artifact_url(version, _GZ_NAME), gz) + info(f"pulling infra rootfs artifact {_package(role)}/{version}") + _download(artifact_url(version, _GZ_NAME, role=role), gz) sha = root / _SHA_NAME - _download(artifact_url(version, _SHA_NAME), sha) + _download(artifact_url(version, _SHA_NAME, role=role), sha) expected = sha.read_text().split()[0].strip().lower() actual = _sha256_file(gz) @@ -211,7 +234,7 @@ def ensure_artifact_gz(version: str) -> Path: gz.unlink(missing_ok=True) sha.unlink(missing_ok=True) die( - f"infra artifact checksum mismatch for {version}:\n" + f"infra artifact checksum mismatch ({role}) for {version}:\n" f" expected {expected}\n" f" actual {actual}\n" f" refusing to boot an unverified rootfs." @@ -220,13 +243,13 @@ def ensure_artifact_gz(version: str) -> Path: return gz -def materialize_ext4(version: str, dest: Path) -> None: - """Ensure the verified artifact is cached, then gunzip it to `dest` — a - fresh, writable per-boot rootfs (the VM mutates it; the cached `.gz` stays +def materialize_ext4(version: str, dest: Path, *, role: str) -> None: + """Ensure the verified `role` artifact is cached, then gunzip it to `dest` — + a fresh, writable per-boot rootfs (the VM mutates it; the cached `.gz` stays pristine). Atomic via a `.part` sibling.""" - gz = ensure_artifact_gz(version) + gz = ensure_artifact_gz(version, role=role) tmp = dest.with_suffix(dest.suffix + ".part") - info(f"expanding infra rootfs -> {dest}") + info(f"expanding {role} infra rootfs -> {dest}") with gzip.open(gz, "rb") as src, open(tmp, "wb") as out: shutil.copyfileobj(src, out, _CHUNK) tmp.replace(dest) diff --git a/bot_bottle/backend/firecracker/infra_vm.py b/bot_bottle/backend/firecracker/infra_vm.py index 12e0770e..9eddfb65 100644 --- a/bot_bottle/backend/firecracker/infra_vm.py +++ b/bot_bottle/backend/firecracker/infra_vm.py @@ -17,11 +17,11 @@ Two persistent microVMs, split now that #469 got the DB off the data plane key); reaches the orchestrator's control plane at `orch_guest:8099` over the one nft forward rule that link allows. -Both VMs boot the **same** shared infra rootfs (gateway + orchestrator + -buildah); a `bb_role=` kernel-cmdline arg selects which plane a VM's PID-1 -init starts, so there is still a single published artifact to build/pull. The -gateway VM's slimmer memory ceiling (buildah is present on disk but unused -there) is set at boot. +Each VM boots its **own** per-plane rootfs — the orchestrator rootfs carries the +control plane + buildah (in-VM agent builds), the gateway rootfs is the slim +data plane with no build tooling on the exposed VM. Two artifacts, built/pulled +per role (`infra_artifact`); each rootfs bakes only its own role init as PID 1. +The gateway VM also runs a slimmer memory ceiling. SSH is left enabled for debugging + provisioning; the control plane is the load-bearing surface. @@ -61,15 +61,22 @@ _GUEST_SIGNING_KEY_PATH = "/var/lib/bot-bottle/orchestrator-token" # the gateway daemons present it to the orchestrator, and never see the key. _GUEST_GATEWAY_JWT_PATH = "/var/lib/bot-bottle/gateway-jwt" -# The single shared infra image: gateway data plane + COPY'd control-plane -# `bot_bottle` package + buildah (Dockerfile.infra.fc, FROM gateway). Built from -# source by default; a pull-from-registry mode lands later. Both VMs boot from -# it, the `bb_role` cmdline selecting the plane. -_INFRA_IMAGE = "bot-bottle-infra:latest" +# The two per-plane rootfs source images. The orchestrator VM boots a control +# plane + buildah rootfs (Dockerfile.orchestrator.fc, FROM orchestrator); the +# gateway VM boots the slim data-plane image directly (no build tooling on the +# exposed VM). Built from source by default; the launch host pulls prebuilt +# artifacts instead (`infra_artifact`). _GATEWAY_IMAGE = "bot-bottle-gateway:latest" _ORCHESTRATOR_IMAGE = "bot-bottle-orchestrator:latest" +_ORCHESTRATOR_FC_IMAGE = "bot-bottle-orchestrator-fc:latest" _REPO_ROOT = Path(__file__).resolve().parents[3] +# Per-role rootfs source image + the extra free space `mke2fs` leaves for the +# guest to grow into. The orchestrator keeps buildah's large build slack; the +# gateway carries no build tooling, so its rootfs is much smaller. +_ROOTFS_IMAGE = {"orchestrator": _ORCHESTRATOR_FC_IMAGE, "gateway": _GATEWAY_IMAGE} +_ROOTFS_SLACK_MIB = {"orchestrator": 8192, "gateway": 1024} + ORCHESTRATOR_PORT = 8099 # Gateway data-plane ports (agent-facing): egress proxy, supervise MCP, # git-http. Reached by agent VMs via the PREROUTING DNAT to the gateway VM. @@ -119,45 +126,54 @@ class InfraEndpoint: return self.gateway.ca_cert_pem(timeout=timeout) +def role_init(role: str) -> str: + """The guest PID-1 init for `role` (each per-plane rootfs bakes only its + own — no `bb_role` branch, since the rootfs *is* the role).""" + return _orchestrator_init() if role == "orchestrator" else _gateway_init() + + +def _role_version(role: str) -> str: + return infra_artifact.infra_artifact_version(role_init(role), role) + + def ensure_built() -> None: - """Ensure the infra rootfs is available before boot. + """Ensure both infra rootfs artifacts are available before boot. Default (docker-free, PRD 0069 Stage 2): download + verify the prebuilt - rootfs artifact matching this code version (see `infra_artifact`); the - launch host needs no Docker. `BOT_BOTTLE_INFRA_BUILD=local` instead builds - the fixed images from source with host Docker — the infra image is `FROM` - the gateway image and `COPY --from`s the orchestrator image, so both must - exist first — for iterating on the Dockerfiles.""" + orchestrator + gateway rootfs artifacts matching this code version (see + `infra_artifact`); the launch host needs no Docker. + `BOT_BOTTLE_INFRA_BUILD=local` instead builds the images from source with + host Docker (the orchestrator-fc image is `FROM` the orchestrator image, so + it must exist first) — for iterating on the Dockerfiles.""" if infra_artifact.local_build_requested(): build_infra_images_with_docker() return - infra_artifact.ensure_artifact_gz( - infra_artifact.infra_artifact_version(_infra_init())) + for role in infra_artifact.ROLES: + infra_artifact.ensure_artifact_gz(_role_version(role), role=role) def build_infra_images_with_docker() -> None: - """Build the three fixed images from source with host Docker: orchestrator, - gateway, then the Firecracker infra image (Dockerfile.infra.fc: FROM gateway - + COPY --from orchestrator + buildah). The launch host uses this only in - `BOT_BOTTLE_INFRA_BUILD=local` mode; `publish_infra` uses it off-host to - produce the published artifact.""" + """Build the fixed images from source with host Docker: orchestrator, + gateway, then the orchestrator-fc image (Dockerfile.orchestrator.fc: FROM + orchestrator + buildah). The gateway VM boots the gateway image directly. + The launch host uses this only in `BOT_BOTTLE_INFRA_BUILD=local` mode; + `publish_infra` uses it off-host to produce the published artifacts.""" docker_mod.build_image( _ORCHESTRATOR_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.orchestrator") docker_mod.build_image( _GATEWAY_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.gateway") docker_mod.build_image( - _INFRA_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.infra.fc") + _ORCHESTRATOR_FC_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.orchestrator.fc") -def build_infra_rootfs_dir() -> Path: - """The infra VMs' shared base rootfs: the infra image prepared with the - role-branched init as PID 1. The init's content is folded into the cache - key so an init change rebuilds the rootfs (the base image digest alone - wouldn't catch it).""" - init = _infra_init() +def build_rootfs_dir(role: str) -> Path: + """`role`'s base rootfs dir: its source image prepared with the role init as + PID 1. The init's content is folded into the cache key so an init change + rebuilds the rootfs (the base image digest alone wouldn't catch it).""" + init = role_init(role) tag = hashlib.sha256(init.encode()).hexdigest()[:8] return util.build_base_rootfs_dir( - _INFRA_IMAGE, variant=f"-infra-{tag}", init_script=init, + _ROOTFS_IMAGE[role], variant=f"-{role}-{tag}", init_script=init, ) @@ -259,8 +275,8 @@ def boot_vm( data_drive: Path | None = None, extra_boot_args: str = "", ) -> InfraVm: - """Boot one infra VM from the shared rootfs on `slot`'s link, tagged with - its `bb_role` so the guest init starts the right plane. Records the PID.""" + """Boot the `role` infra VM from its per-plane rootfs on `slot`'s link. + Records the PID.""" if not netpool.tap_present(slot.iface): die(f"infra link {slot.iface} not present.\n" f" ./cli.py backend setup --backend=firecracker") @@ -268,18 +284,16 @@ def boot_vm( run_dir.mkdir(parents=True, exist_ok=True) rootfs = run_dir / "rootfs.ext4" if infra_artifact.local_build_requested(): - util.build_rootfs_ext4(build_infra_rootfs_dir(), rootfs, slack_mib=8192) + util.build_rootfs_ext4( + build_rootfs_dir(role), rootfs, slack_mib=_ROOTFS_SLACK_MIB[role]) else: - # Prebuilt artifact already carries the buildah build slack; expand it - # to a fresh writable rootfs for this boot. - infra_artifact.materialize_ext4( - infra_artifact.infra_artifact_version(_infra_init()), rootfs) + # Prebuilt artifact already carries the role's build slack; expand it to + # a fresh writable rootfs for this boot. + infra_artifact.materialize_ext4(_role_version(role), rootfs, role=role) private_key, pubkey = _stable_keypair() info(f"booting {role} VM on {slot.iface} (guest {slot.guest_ip})") - boot_args = f"bb_role={role}" - if extra_boot_args: - boot_args = f"{boot_args} {extra_boot_args}" + boot_args = extra_boot_args vm = firecracker_vm.boot( name=name, rootfs=rootfs, tap=slot.iface, guest_ip=slot.guest_ip, host_ip=slot.host_ip, pubkey=pubkey, @@ -323,7 +337,9 @@ def _version_file() -> Path: def _expected_version() -> str: - return infra_artifact.infra_artifact_version(_infra_init()) + """The combined marker for the running pair: both per-plane artifact + versions, so a change to either rootfs dislodges the adopted pair.""" + return " ".join(f"{role}={_role_version(role)}" for role in infra_artifact.ROLES) def _adoptable(key: Path, url: str, want: str) -> bool: @@ -455,19 +471,12 @@ def _health_ok(url: str) -> bool: return False -def _infra_init() -> str: - """PID-1 init for the infra VMs. Shared setup (pseudo-filesystems, PATH, - resolver, debug SSH), then a `bb_role` cmdline branch selecting the plane: - - * orchestrator — mount the persistent registry volume, wait for the - host-seeded signing key, start ONLY the control plane; - * gateway — wait for the host-seeded `gateway` JWT, start ONLY the - data-plane daemons (multi-tenant against the orchestrator at the - `bb_orch` cmdline address). - - Both VMs boot this same init (one published artifact); the cmdline selects - the role, so no orchestrator IP is baked in (an IP_BASE override doesn't - change the artifact version).""" +def _init_head() -> str: + """The shared PID-1 preamble both role inits open with: mount the pseudo- + filesystems, export a real PATH (a bare-init shell's built-in exec path + isn't in the *environment*, so backgrounded `python3 ...` children would + find no PATH), set the direct upstream resolver, install the per-boot SSH + pubkey from the cmdline, and start dropbear for debug/provisioning.""" return f"""#!/bin/sh # bot-bottle Firecracker infra VM init (PID 1). mount -t proc proc /proc 2>/dev/null @@ -476,10 +485,6 @@ mount -t devtmpfs dev /dev 2>/dev/null mkdir -p /dev/pts && mount -t devpts devpts /dev/pts 2>/dev/null mount -o remount,rw / 2>/dev/null -# Export a real PATH: a bare-init shell resolves its own execs via a -# built-in default path, but that isn't in the *environment*, so -# gateway_init's subprocess daemons (spawned as `python3 ...`) would -# inherit no PATH and fail to find python3. Export it for all children. export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin # Direct upstream resolver (control-plane / gateway egress + buildah). @@ -497,62 +502,70 @@ mkdir -p /etc/dropbear /run /var/lib/bot-bottle /bb-dropbear -R -E -p 22 & -ROLE=$(sed -n 's/.*bb_role=\\([^ ]*\\).*/\\1/p' /proc/cmdline) cd /app +""" -if [ "$ROLE" = gateway ]; then - # Gateway data plane, multi-tenant: each request resolves source-IP -> - # policy against the orchestrator VM (its guest IP is on the cmdline as - # bb_orch). The VM backend reaches git over git-http (9420), so the git:// - # daemon (git-gate, needs a per-bottle entrypoint the consolidated model - # doesn't use) is left out. No SUPERVISE_DB_PATH: the data plane reaches the - # supervise queue over the control-plane RPC and never opens bot-bottle.db - # (PRD 0070 / #469). It presents the pre-minted `gateway` JWT the launcher - # pushed; if it never arrives, REFUSE to start rather than run without auth. - ORCH=$(sed -n 's/.*bb_orch=\\([^ ]*\\).*/\\1/p' /proc/cmdline) - GW_JWT="" - i=0 - while [ "$i" -lt 600 ]; do - GW_JWT=$(cat {_GUEST_GATEWAY_JWT_PATH} 2>/dev/null) - [ -n "$GW_JWT" ] && break - i=$((i + 1)) - sleep 0.1 - done - if [ -z "$GW_JWT" ]; then - echo "infra gateway: gateway JWT never arrived; refusing to start the data plane" >&2 - else - chmod 600 {_GUEST_GATEWAY_JWT_PATH} 2>/dev/null || true - BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise \\ - BOT_BOTTLE_ORCHESTRATOR_URL=http://$ORCH:{ORCHESTRATOR_PORT} \\ - BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT="$GW_JWT" \\ - python3 -m bot_bottle.gateway.bootstrap & - fi -else - # Control plane. Source is baked at /app; the package is stdlib-only. - # Persistent registry volume (second virtio-block device, /dev/vdb) mounted - # at the DB dir, so bot-bottle.db survives orchestrator-VM restarts. - mount -t ext4 /dev/vdb /var/lib/bot-bottle 2>/dev/null || true - # Wait for the launcher to push the host-canonical signing key over SSH, - # then hand it ONLY to the orchestrator (to verify tokens). If it never - # arrives, REFUSE to start rather than run OPEN — open mode would grant - # every unauthenticated caller the `cli` role (issue #469). - CP_KEY="" - i=0 - while [ "$i" -lt 600 ]; do - CP_KEY=$(cat {_GUEST_SIGNING_KEY_PATH} 2>/dev/null) - [ -n "$CP_KEY" ] && break - i=$((i + 1)) - sleep 0.1 - done - if [ -z "$CP_KEY" ]; then - echo "infra: control-plane signing key never arrived; refusing to start the control plane (would run OPEN)" >&2 - else - chmod 600 {_GUEST_SIGNING_KEY_PATH} 2>/dev/null || true - BOT_BOTTLE_ROOT=/var/lib/bot-bottle BOT_BOTTLE_ORCHESTRATOR_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator \\ - --host 0.0.0.0 --port {ORCHESTRATOR_PORT} --broker stub & - fi -fi +_INIT_TAIL = """ # Reap as PID 1; children are backgrounded, so `wait` blocks. while : ; do wait ; done """ + + +def _gateway_init() -> str: + """PID-1 init for the gateway (data-plane) VM. Waits for the host-seeded + `gateway` JWT, then starts ONLY the data-plane daemons, multi-tenant against + the orchestrator at the `bb_orch` cmdline address. The VM backend reaches git + over git-http (9420), so the git:// daemon (a per-bottle entrypoint the + consolidated model doesn't use) is left out. No SUPERVISE_DB_PATH: the data + plane reaches the supervise queue over the control-plane RPC and never opens + bot-bottle.db (PRD 0070 / #469). If the JWT never arrives, REFUSE to start + rather than run without auth.""" + return _init_head() + f""" +ORCH=$(sed -n 's/.*bb_orch=\\([^ ]*\\).*/\\1/p' /proc/cmdline) +GW_JWT="" +i=0 +while [ "$i" -lt 600 ]; do + GW_JWT=$(cat {_GUEST_GATEWAY_JWT_PATH} 2>/dev/null) + [ -n "$GW_JWT" ] && break + i=$((i + 1)) + sleep 0.1 +done +if [ -z "$GW_JWT" ]; then + echo "infra gateway: gateway JWT never arrived; refusing to start the data plane" >&2 +else + chmod 600 {_GUEST_GATEWAY_JWT_PATH} 2>/dev/null || true + BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise \\ + BOT_BOTTLE_ORCHESTRATOR_URL=http://$ORCH:{ORCHESTRATOR_PORT} \\ + BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT="$GW_JWT" \\ + python3 -m bot_bottle.gateway.bootstrap & +fi +""" + _INIT_TAIL + + +def _orchestrator_init() -> str: + """PID-1 init for the orchestrator (control-plane) VM. Mounts the persistent + registry volume (/dev/vdb — bot-bottle.db survives a VM restart), waits for + the host-seeded signing key, then starts ONLY the control plane. If the key + never arrives, REFUSE to start rather than run OPEN — open mode would grant + every unauthenticated caller the `cli` role (#469).""" + return _init_head() + f""" +# Persistent registry volume (second virtio-block device, /dev/vdb) mounted at +# the DB dir, so bot-bottle.db survives orchestrator-VM restarts. +mount -t ext4 /dev/vdb /var/lib/bot-bottle 2>/dev/null || true +CP_KEY="" +i=0 +while [ "$i" -lt 600 ]; do + CP_KEY=$(cat {_GUEST_SIGNING_KEY_PATH} 2>/dev/null) + [ -n "$CP_KEY" ] && break + i=$((i + 1)) + sleep 0.1 +done +if [ -z "$CP_KEY" ]; then + echo "infra: control-plane signing key never arrived; refusing to start the control plane (would run OPEN)" >&2 +else + chmod 600 {_GUEST_SIGNING_KEY_PATH} 2>/dev/null || true + BOT_BOTTLE_ROOT=/var/lib/bot-bottle BOT_BOTTLE_ORCHESTRATOR_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator \\ + --host 0.0.0.0 --port {ORCHESTRATOR_PORT} --broker stub & +fi +""" + _INIT_TAIL diff --git a/bot_bottle/backend/firecracker/publish_infra.py b/bot_bottle/backend/firecracker/publish_infra.py index f295e86b..6cfb651a 100644 --- a/bot_bottle/backend/firecracker/publish_infra.py +++ b/bot_bottle/backend/firecracker/publish_infra.py @@ -1,20 +1,22 @@ -"""Build the infra rootfs and publish it as a Gitea generic package. +"""Build the infra rootfs artifacts and publish them as Gitea generic packages. The off-host (build / CI) half of PRD 0069 Stage 2: this DOES use Docker, but never on the launch host. It runs the same pipeline the launch host used to run -locally — `docker build` the three fixed images, export to a rootfs dir, inject -the guest boot, `mke2fs` to an ext4 with the buildah build slack — then gzips -the ext4 and PUTs it (plus a `.sha256`) to -`…/api/packages//generic/bot-bottle-firecracker-infra//`. +locally — `docker build` the fixed images, export each per-plane rootfs, inject +the guest boot, `mke2fs` to an ext4 — then gzips each and PUTs it (plus a +`.sha256`) to `…/api/packages//generic/bot-bottle-firecracker-//`. -The `` is `infra_artifact.infra_artifact_version(...)`, the content -hash of the rootfs inputs, so a launch host at the same code checkout resolves -the exact artifact this produced. +There are two artifacts, one per plane (`orchestrator`, `gateway`); the +orchestrator rootfs carries buildah, the gateway rootfs is slim. Each +`` is `infra_artifact.infra_artifact_version(...)`, the content hash of +that rootfs's inputs, so a launch host at the same code checkout resolves the +exact artifacts this produced. python3 -m bot_bottle.backend.firecracker.publish_infra --output DIR python3 -m bot_bottle.backend.firecracker.publish_infra --publish-dir DIR -Auth: a token with `write:package` on the target owner, from +A candidate bundle holds each role under its own `DIR//` subdir. Auth: a +token with `write:package` on the target owner, from `BOT_BOTTLE_INFRA_ARTIFACT_TOKEN`. """ @@ -33,17 +35,27 @@ from . import infra_artifact, infra_vm, util _CHUNK = 1 << 20 -# A human-readable description shipped alongside the artifact — generic packages +_GZ_NAME = "rootfs.ext4.gz" +_SHA_NAME = "rootfs.ext4.gz.sha256" + +# A human-readable description shipped alongside each artifact — generic packages # have no description field, so this file *is* the description on the package # page. Uploaded on every publish so it never goes stale. _ABOUT_NAME = "about.txt" -_ABOUT_TEXT = ( - "bot-bottle infra rootfs for the Firecracker backend (PRD 0069 Stage 2, " - "#348): the per-host infra VM (orchestrator control plane + gateway + " - "buildah). Prebuilt off-host, gzip ext4; the launch host downloads + " - "sha256-verifies + boots it, no host Docker. The version tag is a content " - "hash of the rootfs inputs. Files: rootfs.ext4.gz + rootfs.ext4.gz.sha256.\n" -) + + +def _about_text(role: str) -> str: + return ( + f"bot-bottle firecracker {role} rootfs (PRD 0069 Stage 2 / PRD 0070): " + f"the per-host {role} infra VM. Prebuilt off-host, gzip ext4; the launch " + f"host downloads + sha256-verifies + boots it, no host Docker. The " + f"version tag is a content hash of the rootfs inputs. Files: " + f"{_GZ_NAME} + {_SHA_NAME}.\n" + ) + + +def _role_version(role: str) -> str: + return infra_artifact.infra_artifact_version(infra_vm.role_init(role), role) def _gzip(src: Path, dest: Path) -> None: @@ -109,34 +121,35 @@ def _delete(url: str, token: str) -> None: raise SystemExit(f"registry unreachable: {url} ({e.reason})") -def build_artifact(out_dir: Path) -> tuple[str, Path, Path]: - """Build the infra rootfs ext4, gzip it, and write the checksum. Returns - `(version, gz_path, sha_path)`. Uses host Docker (off-host / CI).""" - version = infra_artifact.infra_artifact_version(infra_vm._infra_init()) - print(f"building infra rootfs artifact {version} (docker)") - infra_vm.build_infra_images_with_docker() - base = infra_vm.build_infra_rootfs_dir() +def build_role_artifact(role: str, role_dir: Path) -> str: + """Build `role`'s rootfs ext4, gzip it, and write the checksum + version into + `role_dir`. Returns the version. Assumes the docker images are already + built (`infra_vm.build_infra_images_with_docker`). Uses host Docker.""" + version = _role_version(role) + print(f"building {role} rootfs artifact {version} (docker)") + base = infra_vm.build_rootfs_dir(role) - ext4 = out_dir / "rootfs.ext4" - util.build_rootfs_ext4(base, ext4, slack_mib=8192) - gz = out_dir / "rootfs.ext4.gz" - print("compressing rootfs") + ext4 = role_dir / "rootfs.ext4" + util.build_rootfs_ext4(base, ext4, slack_mib=infra_vm._ROOTFS_SLACK_MIB[role]) + gz = role_dir / _GZ_NAME + print(f"compressing {role} rootfs") _gzip(ext4, gz) ext4.unlink(missing_ok=True) - sha = out_dir / "rootfs.ext4.gz.sha256" + sha = role_dir / _SHA_NAME digest = _sha256(gz) - sha.write_text(f"{digest} rootfs.ext4.gz\n") - print(f" {gz.name}: {gz.stat().st_size / 1e6:.0f} MB sha256={digest}") - return version, gz, sha + sha.write_text(f"{digest} {_GZ_NAME}\n") + (role_dir / "version.txt").write_text(version + "\n", encoding="utf-8") + print(f" {role}/{gz.name}: {gz.stat().st_size / 1e6:.0f} MB sha256={digest}") + return version -def _try_download_published(out_dir: Path) -> tuple[str, Path, Path] | None: - """If this version's artifact is already in the registry, download the gz - and sha to out_dir and return (version, gz_path, sha_path). Returns None - when not yet published.""" - version = infra_artifact.infra_artifact_version(infra_vm._infra_init()) - sha_url = infra_artifact.artifact_url(version, "rootfs.ext4.gz.sha256") +def _try_download_published(role: str, role_dir: Path) -> str | None: + """If `role`'s artifact for this version is already in the registry, download + the gz + sha into `role_dir` and return the version. None when not yet + published.""" + version = _role_version(role) + sha_url = infra_artifact.artifact_url(version, _SHA_NAME, role=role) try: with urllib.request.urlopen(infra_artifact._open(sha_url)): pass @@ -146,70 +159,70 @@ def _try_download_published(out_dir: Path) -> tuple[str, Path, Path] | None: raise SystemExit(f"registry check failed (HTTP {e.code}): {sha_url}") except urllib.error.URLError as e: raise SystemExit(f"registry unreachable: {sha_url} ({e.reason})") - print(f"infra rootfs {version} already published — downloading instead of building") - gz = out_dir / "rootfs.ext4.gz" - sha = out_dir / "rootfs.ext4.gz.sha256" - infra_artifact._download(infra_artifact.artifact_url(version, "rootfs.ext4.gz"), gz) - infra_artifact._download(infra_artifact.artifact_url(version, "rootfs.ext4.gz.sha256"), sha) - return version, gz, sha + print(f"{role} rootfs {version} already published — downloading instead of building") + infra_artifact._download( + infra_artifact.artifact_url(version, _GZ_NAME, role=role), role_dir / _GZ_NAME) + infra_artifact._download(sha_url, role_dir / _SHA_NAME) + (role_dir / "version.txt").write_text(version + "\n", encoding="utf-8") + return version -def _publish_bundle(root: Path, token: str) -> str: - version_file = root / "version.txt" +def _publish_bundle(role: str, role_dir: Path, token: str) -> str: + version_file = role_dir / "version.txt" # Guard the read so a missing version.txt is a clean error, not a raw # FileNotFoundError. if not version_file.is_file(): - raise SystemExit(f"incomplete artifact bundle: {root}") + raise SystemExit(f"incomplete {role} artifact bundle: {role_dir}") version = version_file.read_text(encoding="utf-8").strip() - expected = infra_artifact.infra_artifact_version(infra_vm._infra_init()) + expected = _role_version(role) if version != expected: raise SystemExit( - f"artifact bundle version {version!r} does not match checkout {expected!r}" + f"{role} artifact bundle version {version!r} does not match checkout {expected!r}" ) - gz = root / "rootfs.ext4.gz" - sha = root / "rootfs.ext4.gz.sha256" + gz = role_dir / _GZ_NAME + sha = role_dir / _SHA_NAME if not gz.is_file() or not sha.is_file(): - raise SystemExit(f"incomplete artifact bundle: {root}") + raise SystemExit(f"incomplete {role} artifact bundle: {role_dir}") expected_sha = sha.read_text().split()[0].strip().lower() if _sha256(gz) != expected_sha: - raise SystemExit("artifact bundle checksum mismatch") + raise SystemExit(f"{role} artifact bundle checksum mismatch") - gz_url = infra_artifact.artifact_url(version, gz.name) - sha_url = infra_artifact.artifact_url(version, sha.name) - about_url = infra_artifact.artifact_url(version, _ABOUT_NAME) + gz_url = infra_artifact.artifact_url(version, _GZ_NAME, role=role) + sha_url = infra_artifact.artifact_url(version, _SHA_NAME, role=role) + about_url = infra_artifact.artifact_url(version, _ABOUT_NAME, role=role) # Publishing is idempotent. If this exact complete artifact is already - # present, a test-only main commit is a no-op. Otherwise clear any partial - # upload left by an interrupted prior attempt and upload the complete set. + # present, a re-publish is a no-op. Otherwise clear any partial upload left + # by an interrupted prior attempt and upload the complete set. try: with urllib.request.urlopen(infra_artifact._open(sha_url)) as resp: remote_sha = resp.read().decode("utf-8").split()[0].strip().lower() except urllib.error.HTTPError as e: if e.code != 404: - raise SystemExit(f"checking existing artifact failed (HTTP {e.code})") + raise SystemExit(f"checking existing {role} artifact failed (HTTP {e.code})") remote_sha = "" except urllib.error.URLError as e: raise SystemExit(f"registry unreachable: {sha_url} ({e.reason})") if remote_sha == expected_sha: - print(f"infra rootfs {version} already published") + print(f"{role} rootfs {version} already published") return version for url in (gz_url, sha_url, about_url): _delete(url, token) _put(gz_url, gz, token) _put(sha_url, sha.read_bytes(), token) - _put(about_url, _ABOUT_TEXT.encode(), token) + _put(about_url, _about_text(role).encode(), token) return version def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( - prog="publish_infra", description="Build + publish the infra rootfs artifact.") + prog="publish_infra", description="Build + publish the infra rootfs artifacts.") mode = parser.add_mutually_exclusive_group(required=True) mode.add_argument("--output", type=Path, - help="build a candidate bundle in DIR without publishing") + help="build candidate bundles in DIR// without publishing") mode.add_argument("--publish-dir", type=Path, - help="publish an already-built and tested candidate bundle") + help="publish already-built + tested candidate bundles under DIR") parser.add_argument("--reuse-published", action="store_true", help="with --output: download from registry if already published instead of building") args = parser.parse_args(argv) @@ -221,23 +234,28 @@ def main(argv: list[str] | None = None) -> int: "with write:package") if args.output is not None: - args.output.mkdir(parents=True, exist_ok=True) - reused = None - if args.reuse_published: - reused = _try_download_published(args.output) - if reused is not None: - version, _, _ = reused - (args.output / "version.txt").write_text(version + "\n", encoding="utf-8") - print(f"reused published infra rootfs candidate {version}") - return 0 - version, _gz, _sha = build_artifact(args.output) - (args.output / "version.txt").write_text(version + "\n", encoding="utf-8") - print(f"built infra rootfs candidate {version}") + # Build (or reuse) all roles. Images are built once, up front, only when + # something actually needs building. + pending = [] + for role in infra_artifact.ROLES: + role_dir = args.output / role + role_dir.mkdir(parents=True, exist_ok=True) + if args.reuse_published and _try_download_published(role, role_dir): + print(f"reused published {role} rootfs candidate") + continue + pending.append(role) + if pending: + print("building infra images (docker)") + infra_vm.build_infra_images_with_docker() + for role in pending: + build_role_artifact(role, args.output / role) + print(f"built {role} rootfs candidate") return 0 assert args.publish_dir is not None - version = _publish_bundle(args.publish_dir, token) - print(f"published infra rootfs {version}") + for role in infra_artifact.ROLES: + version = _publish_bundle(role, args.publish_dir / role, token) + print(f"published {role} rootfs {version}") return 0 diff --git a/tests/unit/test_firecracker_infra_vm.py b/tests/unit/test_firecracker_infra_vm.py index ba918070..20f78ed0 100644 --- a/tests/unit/test_firecracker_infra_vm.py +++ b/tests/unit/test_firecracker_infra_vm.py @@ -68,48 +68,55 @@ class TestInfraEndpoint(unittest.TestCase): ca.assert_called_once_with(timeout=1) -class TestBuildInfraRootfs(unittest.TestCase): - def test_uses_infra_variant_and_role_branched_init(self): - with patch.object(infra_vm.util, "build_base_rootfs_dir") as build: - build.return_value = Path("/cache/rootfs/x-infra") - infra_vm.build_infra_rootfs_dir() - build.assert_called_once() - self.assertEqual(infra_vm._INFRA_IMAGE, build.call_args.args[0]) - # variant is "-infra-" so an init change rebuilds the rootfs. - self.assertTrue(build.call_args.kwargs["variant"].startswith("-infra-")) - init = build.call_args.kwargs["init_script"] - # One shared init, role-branched off the kernel cmdline: it starts the - # control plane OR the gateway data plane, and exports PATH so the - # gateway daemons' subprocesses find python3. - self.assertIn("bb_role=", init) - self.assertIn('if [ "$ROLE" = gateway ]; then', init) - self.assertIn("bot_bottle.orchestrator", init) - self.assertIn("bot_bottle.gateway.bootstrap", init) - self.assertIn("export PATH=", init) +class TestRoleInits(unittest.TestCase): + def test_orchestrator_init_starts_only_the_control_plane(self): + init = infra_vm.role_init("orchestrator") + # No bb_role branch — the rootfs *is* the role. + self.assertNotIn("bb_role=", init) + self.assertNotIn("bot_bottle.gateway.bootstrap", init) + self.assertIn("export PATH=", init) # shared preamble # Persistent registry volume mounted at the DB dir on the orchestrator. self.assertIn("/dev/vdb", init) - # VM backend uses git-http (9420); the git:// daemon is left out. - self.assertIn("BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise", init) - # Role-scoped auth (#469): the orchestrator gets the host-seeded signing - # key; the gateway gets the host-minted `gateway` JWT (NOT the key — the - # JWT is minted on the host now, so the init never touches the key to - # mint it); each plane refuses to start without its secret. self.assertIn(f"cat {infra_vm._GUEST_SIGNING_KEY_PATH}", init) # host-seeded key - self.assertIn(f"cat {infra_vm._GUEST_GATEWAY_JWT_PATH}", init) # host-minted JWT - self.assertNotIn("ROLE_GATEWAY", init) # minting moved to the host self.assertIn("refusing to start the control plane", init) # no open mode - self.assertIn("refusing to start the data plane", init) # gateway fail-closed self.assertIn('BOT_BOTTLE_ORCHESTRATOR_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator', init) # key -> orchestrator only - self.assertIn('BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT="$GW_JWT"', init) # JWT -> gateway daemons + + def test_gateway_init_starts_only_the_data_plane(self): + init = infra_vm.role_init("gateway") + self.assertNotIn("bb_role=", init) + self.assertNotIn("bot_bottle.orchestrator", init) + self.assertNotIn("/dev/vdb", init) # no registry volume on the data plane + self.assertIn("export PATH=", init) # shared preamble + self.assertIn("BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise", init) + self.assertIn(f"cat {infra_vm._GUEST_GATEWAY_JWT_PATH}", init) # host-minted JWT + self.assertNotIn("ROLE_GATEWAY", init) # minting moved to the host + self.assertIn("refusing to start the data plane", init) # fail-closed + self.assertIn('BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT="$GW_JWT"', init) # JWT -> daemons # The gateway resolves the orchestrator's address off the cmdline # (bb_orch), so no IP is baked into the artifact. self.assertIn("bb_orch=", init) self.assertIn("BOT_BOTTLE_ORCHESTRATOR_URL=http://$ORCH:", init) +class TestBuildRootfsDir(unittest.TestCase): + def test_orchestrator_rootfs_uses_the_fc_image_and_role_variant(self): + with patch.object(infra_vm.util, "build_base_rootfs_dir") as build: + build.return_value = Path("/cache/rootfs/x") + infra_vm.build_rootfs_dir("orchestrator") + self.assertEqual(infra_vm._ORCHESTRATOR_FC_IMAGE, build.call_args.args[0]) + self.assertTrue(build.call_args.kwargs["variant"].startswith("-orchestrator-")) + + def test_gateway_rootfs_uses_the_gateway_image_directly(self): + with patch.object(infra_vm.util, "build_base_rootfs_dir") as build: + build.return_value = Path("/cache/rootfs/x") + infra_vm.build_rootfs_dir("gateway") + self.assertEqual(infra_vm._GATEWAY_IMAGE, build.call_args.args[0]) + self.assertTrue(build.call_args.kwargs["variant"].startswith("-gateway-")) + + class TestEnsureBuilt(unittest.TestCase): - def test_default_pulls_artifact_without_docker(self): + def test_default_pulls_both_artifacts_without_docker(self): # PRD 0069 Stage 2: the launch host pulls the prebuilt rootfs; no Docker. # Pin BOT_BOTTLE_INFRA_BUILD off: the coverage CI job exports it =local # for the integration suite, and that ambient value would otherwise send @@ -119,17 +126,21 @@ class TestEnsureBuilt(unittest.TestCase): patch.object(infra_vm.infra_artifact, "ensure_artifact_gz") as pull: infra_vm.ensure_built() build.assert_not_called() - pull.assert_called_once() + # One pull per plane. + roles = {c.kwargs["role"] for c in pull.call_args_list} + self.assertEqual({"orchestrator", "gateway"}, roles) - def test_local_mode_builds_deps_before_infra(self): + def test_local_mode_builds_orchestrator_before_its_fc_image(self): with patch.dict(os.environ, {"BOT_BOTTLE_INFRA_BUILD": "local"}), \ patch.object(infra_vm.docker_mod, "build_image") as build: infra_vm.ensure_built() tags = [c.args[0] for c in build.call_args_list] - # infra is FROM gateway and COPY --from orchestrator, so both first. - self.assertEqual(infra_vm._INFRA_IMAGE, tags[-1]) - self.assertIn(infra_vm._ORCHESTRATOR_IMAGE, tags[:-1]) - self.assertIn(infra_vm._GATEWAY_IMAGE, tags[:-1]) + # orchestrator-fc is FROM orchestrator, so the base is built first. + self.assertIn(infra_vm._ORCHESTRATOR_IMAGE, tags) + self.assertIn(infra_vm._GATEWAY_IMAGE, tags) + self.assertEqual(infra_vm._ORCHESTRATOR_FC_IMAGE, tags[-1]) + self.assertLess(tags.index(infra_vm._ORCHESTRATOR_IMAGE), + tags.index(infra_vm._ORCHESTRATOR_FC_IMAGE)) # The orchestrator + gateway services are imported lazily inside ensure_running diff --git a/tests/unit/test_infra_artifact.py b/tests/unit/test_infra_artifact.py index 7c5483d7..d9bf0aa6 100644 --- a/tests/unit/test_infra_artifact.py +++ b/tests/unit/test_infra_artifact.py @@ -3,6 +3,8 @@ The launch-host half — version hashing and download/verify/decompress — is what keeps a docker-free host from booting a stale or corrupted rootfs, so the checksum + fail-closed paths are locked here. Network is mocked; no Docker. +There are two per-plane artifacts now; these exercise one role (orchestrator) — +the pull/verify logic is role-agnostic, keyed only by the package name. """ from __future__ import annotations @@ -21,6 +23,8 @@ from unittest import mock from bot_bottle.backend.firecracker import infra_artifact as ia from bot_bottle.log import Die +_ROLE = "orchestrator" + def _gz(data: bytes) -> bytes: return gzip.compress(data) @@ -66,29 +70,36 @@ class _CacheMixin(unittest.TestCase): if sha_text is None: sha_text = f"{hashlib.sha256(gz_bytes).hexdigest()} rootfs.ext4.gz\n" net = _FakeNet({ - ia.artifact_url(version, "rootfs.ext4.gz"): gz_bytes, - ia.artifact_url(version, "rootfs.ext4.gz.sha256"): sha_text.encode(), + ia.artifact_url(version, "rootfs.ext4.gz", role=_ROLE): gz_bytes, + ia.artifact_url(version, "rootfs.ext4.gz.sha256", role=_ROLE): sha_text.encode(), }) return mock.patch.object(ia.urllib.request, "urlopen", net.urlopen), net class TestVersion(unittest.TestCase): def test_deterministic_16_hex(self) -> None: - v = ia.infra_artifact_version("#!/bin/sh\ntrue\n") - self.assertEqual(v, ia.infra_artifact_version("#!/bin/sh\ntrue\n")) + v = ia.infra_artifact_version("#!/bin/sh\ntrue\n", _ROLE) + self.assertEqual(v, ia.infra_artifact_version("#!/bin/sh\ntrue\n", _ROLE)) self.assertEqual(16, len(v)) int(v, 16) # hex def test_init_change_bumps_version(self) -> None: self.assertNotEqual( - ia.infra_artifact_version("a"), ia.infra_artifact_version("b")) + ia.infra_artifact_version("a", _ROLE), ia.infra_artifact_version("b", _ROLE)) + + def test_role_changes_version(self) -> None: + # The same init under a different role hashes differently (role is folded + # in, and each role hashes its own Dockerfiles). + self.assertNotEqual( + ia.infra_artifact_version("init", "orchestrator"), + ia.infra_artifact_version("init", "gateway")) class TestVersionInputs(unittest.TestCase): """The hash must cover *every* file baked into the rootfs, not just `*.py` - (`COPY bot_bottle` is wholesale) — else a non-Python change (e.g. the egress - entrypoint shell script) leaves the version unchanged and a launch host - boots a rootfs whose code differs from its checkout.""" + (the package is baked in wholesale) — else a non-Python change (e.g. the + egress entrypoint shell script) leaves the version unchanged and a launch + host boots a rootfs whose code differs from its checkout.""" def _fake_repo(self, root: Path) -> None: pkg = root / "bot_bottle" @@ -96,7 +107,8 @@ class TestVersionInputs(unittest.TestCase): (pkg / "app.py").write_text("print('hi')\n") (pkg / "egress_entrypoint.sh").write_text("#!/bin/sh\nexec mitmdump\n") (pkg / "netpool.defaults.env").write_text("FOO=1\n") - for name in ("Dockerfile.orchestrator", "Dockerfile.gateway", "Dockerfile.infra", "Dockerfile.infra.fc"): + for name in ("Dockerfile.orchestrator", "Dockerfile.orchestrator.fc", + "Dockerfile.gateway"): (root / name).write_text(f"FROM scratch # {name}\n") (root / "pyproject.toml").write_text("[project]\nname = 'bot-bottle'\n") @@ -104,10 +116,10 @@ class TestVersionInputs(unittest.TestCase): with tempfile.TemporaryDirectory() as d: root = Path(d) self._fake_repo(root) - before = ia.infra_artifact_version("init", repo_root=root) + before = ia.infra_artifact_version("init", _ROLE, repo_root=root) (root / "pyproject.toml").write_text( "[project]\nname = 'bot-bottle'\ndependencies = ['httpx']\n") - after = ia.infra_artifact_version("init", repo_root=root) + after = ia.infra_artifact_version("init", _ROLE, repo_root=root) self.assertNotEqual(before, after) def test_dropbear_change_bumps_version(self) -> None: @@ -119,85 +131,92 @@ class TestVersionInputs(unittest.TestCase): with mock.patch.dict(os.environ, { "BOT_BOTTLE_FC_DROPBEAR": str(dropbear), }): - before = ia.infra_artifact_version("init", repo_root=root) + before = ia.infra_artifact_version("init", _ROLE, repo_root=root) dropbear.write_bytes(b"dropbear-v2") - after = ia.infra_artifact_version("init", repo_root=root) + after = ia.infra_artifact_version("init", _ROLE, repo_root=root) self.assertNotEqual(before, after) def test_non_python_file_change_bumps_version(self) -> None: with tempfile.TemporaryDirectory() as d: root = Path(d) self._fake_repo(root) - before = ia.infra_artifact_version("init", repo_root=root) + before = ia.infra_artifact_version("init", _ROLE, repo_root=root) (root / "bot_bottle" / "egress_entrypoint.sh").write_text( "#!/bin/sh\nexec mitmdump --different\n") - after = ia.infra_artifact_version("init", repo_root=root) + after = ia.infra_artifact_version("init", _ROLE, repo_root=root) self.assertNotEqual(before, after) def test_pyc_and_pycache_ignored(self) -> None: with tempfile.TemporaryDirectory() as d: root = Path(d) self._fake_repo(root) - before = ia.infra_artifact_version("init", repo_root=root) + before = ia.infra_artifact_version("init", _ROLE, repo_root=root) cache = root / "bot_bottle" / "__pycache__" cache.mkdir() (cache / "app.cpython-312.pyc").write_bytes(b"\x00bytecode") (root / "bot_bottle" / "app.pyc").write_bytes(b"\x00bytecode") - after = ia.infra_artifact_version("init", repo_root=root) + after = ia.infra_artifact_version("init", _ROLE, repo_root=root) self.assertEqual(before, after) class TestEnsureArtifact(_CacheMixin): + def _candidate(self, root: Path, version: str, gz: bytes, sha_text: str | None = None) -> Path: + """Stage a candidate bundle for `_ROLE` under root//.""" + role_dir = root / _ROLE + role_dir.mkdir(parents=True, exist_ok=True) + (role_dir / "version.txt").write_text(version + "\n") + (role_dir / "rootfs.ext4.gz").write_bytes(gz) + if sha_text is None: + sha_text = f"{hashlib.sha256(gz).hexdigest()} rootfs.ext4.gz\n" + (role_dir / "rootfs.ext4.gz.sha256").write_text(sha_text) + return role_dir + def test_uses_verified_ci_candidate_without_network(self) -> None: version = "deadbeef00000000" gz = _gz(b"candidate ext4") with tempfile.TemporaryDirectory() as d: root = Path(d) - (root / "version.txt").write_text(version + "\n") - (root / "rootfs.ext4.gz").write_bytes(gz) - digest = hashlib.sha256(gz).hexdigest() - (root / "rootfs.ext4.gz.sha256").write_text( - f"{digest} rootfs.ext4.gz\n") + role_dir = self._candidate(root, version, gz) with mock.patch.dict(os.environ, { "BOT_BOTTLE_INFRA_ARTIFACT_DIR": str(root), }), mock.patch.object(ia.urllib.request, "urlopen") as net: - path = ia.ensure_artifact_gz(version) - self.assertEqual(root / "rootfs.ext4.gz", path) + path = ia.ensure_artifact_gz(version, role=_ROLE) + self.assertEqual(role_dir / "rootfs.ext4.gz", path) net.assert_not_called() def test_rejects_candidate_for_another_version(self) -> None: with tempfile.TemporaryDirectory() as d: root = Path(d) - (root / "version.txt").write_text("wrong\n") + (root / _ROLE).mkdir() + (root / _ROLE / "version.txt").write_text("wrong\n") with mock.patch.dict(os.environ, { "BOT_BOTTLE_INFRA_ARTIFACT_DIR": str(root), }): with self.assertRaises(Die) as ctx: - ia.ensure_artifact_gz("expected") + ia.ensure_artifact_gz("expected", role=_ROLE) self.assertIn("version mismatch", str(ctx.exception.message)) def test_rejects_incomplete_candidate(self) -> None: with tempfile.TemporaryDirectory() as d: root = Path(d) - (root / "version.txt").write_text("v1\n") + (root / _ROLE).mkdir() + (root / _ROLE / "version.txt").write_text("v1\n") with mock.patch.dict(os.environ, { "BOT_BOTTLE_INFRA_ARTIFACT_DIR": str(root), }): with self.assertRaises(Die) as ctx: - ia.ensure_artifact_gz("v1") + ia.ensure_artifact_gz("v1", role=_ROLE) self.assertIn("incomplete", str(ctx.exception.message)) def test_rejects_candidate_checksum_mismatch(self) -> None: with tempfile.TemporaryDirectory() as d: root = Path(d) - (root / "version.txt").write_text("v1\n") - (root / "rootfs.ext4.gz").write_bytes(b"bad") - (root / "rootfs.ext4.gz.sha256").write_text("0" * 64 + " rootfs.ext4.gz\n") + self._candidate(root, "v1", b"bad", sha_text="0" * 64 + " rootfs.ext4.gz\n") with mock.patch.dict(os.environ, { "BOT_BOTTLE_INFRA_ARTIFACT_DIR": str(root), }): with self.assertRaises(Die) as ctx: - ia.ensure_artifact_gz("v1") + ia.ensure_artifact_gz("v1", role=_ROLE) self.assertIn("checksum mismatch", str(ctx.exception.message)) def test_downloads_verifies_and_caches(self) -> None: @@ -205,12 +224,12 @@ class TestEnsureArtifact(_CacheMixin): gz = _gz(b"fake ext4 bytes") patcher, net = self._serve(version, gz) with patcher: - path = ia.ensure_artifact_gz(version) + path = ia.ensure_artifact_gz(version, role=_ROLE) self.assertTrue(path.is_file()) self.assertEqual(gz, path.read_bytes()) first_calls = len(net.calls) # Second call is a cache hit — no further network. - ia.ensure_artifact_gz(version) + ia.ensure_artifact_gz(version, role=_ROLE) self.assertEqual(first_calls, len(net.calls)) def test_checksum_mismatch_fails_closed(self) -> None: @@ -219,17 +238,17 @@ class TestEnsureArtifact(_CacheMixin): patcher, _ = self._serve(version, gz, sha_text="0" * 64 + " rootfs.ext4.gz\n") with patcher: with self.assertRaises(Die) as ctx: - ia.ensure_artifact_gz(version) + ia.ensure_artifact_gz(version, role=_ROLE) self.assertIn("checksum mismatch", str(ctx.exception.message)) # nothing left cached to accidentally boot - self.assertFalse((ia._cache_root(version) / "rootfs.ext4.gz").exists()) + self.assertFalse((ia._cache_root(version, _ROLE) / "rootfs.ext4.gz").exists()) def test_missing_artifact_points_at_publish(self) -> None: version = "0000000000000000" net = _FakeNet({}) # everything 404s with mock.patch.object(ia.urllib.request, "urlopen", net.urlopen): with self.assertRaises(Die) as ctx: - ia.ensure_artifact_gz(version) + ia.ensure_artifact_gz(version, role=_ROLE) self.assertIn("publish_infra", str(ctx.exception.message)) def test_materialize_gunzips_to_dest(self) -> None: @@ -238,7 +257,7 @@ class TestEnsureArtifact(_CacheMixin): patcher, _ = self._serve(version, _gz(raw)) with patcher, tempfile.TemporaryDirectory() as d: dest = Path(d) / "rootfs.ext4" - ia.materialize_ext4(version, dest) + ia.materialize_ext4(version, dest, role=_ROLE) self.assertEqual(raw, dest.read_bytes()) @@ -248,10 +267,10 @@ class TestConfig(unittest.TestCase): "BOT_BOTTLE_INFRA_ARTIFACT_BASE": "https://mirror.example/", "BOT_BOTTLE_INFRA_ARTIFACT_OWNER": "acme", }): - url = ia.artifact_url("v1", "rootfs.ext4.gz") + url = ia.artifact_url("v1", "rootfs.ext4.gz", role="gateway") self.assertEqual( "https://mirror.example/api/packages/acme/generic/" - "bot-bottle-firecracker-infra/v1/rootfs.ext4.gz", url) + "bot-bottle-firecracker-gateway/v1/rootfs.ext4.gz", url) def test_local_build_flag(self) -> None: with mock.patch.dict(os.environ, {"BOT_BOTTLE_INFRA_BUILD": "local"}): diff --git a/tests/unit/test_publish_infra.py b/tests/unit/test_publish_infra.py index 8bc06d26..265b17b7 100644 --- a/tests/unit/test_publish_infra.py +++ b/tests/unit/test_publish_infra.py @@ -61,155 +61,149 @@ class TestPut(unittest.TestCase): self.assertEqual(b"abc123 rootfs\n", captured[0].data) +_ROLE = "orchestrator" + + class TestPublishBundle(unittest.TestCase): - def _bundle(self, root: Path, version: str) -> None: + def _bundle(self, role_dir: Path, version: str) -> None: payload = b"candidate" - (root / "version.txt").write_text(version + "\n") - (root / "rootfs.ext4.gz").write_bytes(payload) + role_dir.mkdir(parents=True, exist_ok=True) + (role_dir / "version.txt").write_text(version + "\n") + (role_dir / "rootfs.ext4.gz").write_bytes(payload) digest = hashlib.sha256(payload).hexdigest() - (root / "rootfs.ext4.gz.sha256").write_text( + (role_dir / "rootfs.ext4.gz.sha256").write_text( f"{digest} rootfs.ext4.gz\n") def test_existing_identical_artifact_is_success(self) -> None: with tempfile.TemporaryDirectory() as d: - root = Path(d) - self._bundle(root, "v1") - sha = (root / "rootfs.ext4.gz.sha256").read_bytes() + role_dir = Path(d) / _ROLE + self._bundle(role_dir, "v1") + sha = (role_dir / "rootfs.ext4.gz.sha256").read_bytes() response = mock.MagicMock() response.__enter__.return_value.read.return_value = sha - with mock.patch.object( - pub.infra_artifact, "infra_artifact_version", return_value="v1" - ), mock.patch.object( - pub.urllib.request, "urlopen", return_value=response - ), mock.patch.object(pub, "_put") as put: - self.assertEqual("v1", pub._publish_bundle(root, "token")) + with mock.patch.object(pub, "_role_version", return_value="v1"), \ + mock.patch.object( + pub.urllib.request, "urlopen", return_value=response + ), mock.patch.object(pub, "_put") as put: + self.assertEqual("v1", pub._publish_bundle(_ROLE, role_dir, "token")) put.assert_not_called() def test_partial_artifact_is_replaced(self) -> None: with tempfile.TemporaryDirectory() as d: - root = Path(d) - self._bundle(root, "v1") + role_dir = Path(d) / _ROLE + self._bundle(role_dir, "v1") missing = urllib.error.HTTPError("u", 404, "missing", Message(), None) - with mock.patch.object( - pub.infra_artifact, "infra_artifact_version", return_value="v1" - ), mock.patch.object( - pub.urllib.request, "urlopen", side_effect=missing - ), mock.patch.object(pub, "_delete") as delete, \ + with mock.patch.object(pub, "_role_version", return_value="v1"), \ + mock.patch.object( + pub.urllib.request, "urlopen", side_effect=missing + ), mock.patch.object(pub, "_delete") as delete, \ mock.patch.object(pub, "_put") as put: - pub._publish_bundle(root, "token") + pub._publish_bundle(_ROLE, role_dir, "token") self.assertEqual(3, delete.call_count) self.assertEqual(3, put.call_count) def test_rejects_bundle_for_different_checkout(self) -> None: with tempfile.TemporaryDirectory() as d: - root = Path(d) - self._bundle(root, "old") - with mock.patch.object( - pub.infra_artifact, "infra_artifact_version", return_value="new" - ): + role_dir = Path(d) / _ROLE + self._bundle(role_dir, "old") + with mock.patch.object(pub, "_role_version", return_value="new"): with self.assertRaises(SystemExit) as ctx: - pub._publish_bundle(root, "token") + pub._publish_bundle(_ROLE, role_dir, "token") self.assertIn("does not match checkout", str(ctx.exception)) def test_rejects_bad_bundle_checksum(self) -> None: with tempfile.TemporaryDirectory() as d: - root = Path(d) - self._bundle(root, "v1") - (root / "rootfs.ext4.gz").write_bytes(b"tampered") - with mock.patch.object( - pub.infra_artifact, "infra_artifact_version", return_value="v1" - ): + role_dir = Path(d) / _ROLE + self._bundle(role_dir, "v1") + (role_dir / "rootfs.ext4.gz").write_bytes(b"tampered") + with mock.patch.object(pub, "_role_version", return_value="v1"): with self.assertRaises(SystemExit) as ctx: - pub._publish_bundle(root, "token") + pub._publish_bundle(_ROLE, role_dir, "token") self.assertIn("checksum mismatch", str(ctx.exception)) def test_registry_lookup_failure_is_reported(self) -> None: with tempfile.TemporaryDirectory() as d: - root = Path(d) - self._bundle(root, "v1") + role_dir = Path(d) / _ROLE + self._bundle(role_dir, "v1") failure = urllib.error.URLError("offline") - with mock.patch.object( - pub.infra_artifact, "infra_artifact_version", return_value="v1" - ), mock.patch.object(pub.urllib.request, "urlopen", side_effect=failure): + with mock.patch.object(pub, "_role_version", return_value="v1"), \ + mock.patch.object(pub.urllib.request, "urlopen", side_effect=failure): with self.assertRaises(SystemExit) as ctx: - pub._publish_bundle(root, "token") + pub._publish_bundle(_ROLE, role_dir, "token") self.assertIn("registry unreachable", str(ctx.exception)) class TestTryDownloadPublished(unittest.TestCase): def test_downloads_existing_artifact(self) -> None: with tempfile.TemporaryDirectory() as d, \ - mock.patch.object(pub.infra_vm, "_infra_init", return_value="init"), \ + mock.patch.object(pub, "_role_version", return_value="v1"), \ mock.patch.object( - pub.infra_artifact, "infra_artifact_version", return_value="v1" - ), mock.patch.object( pub.urllib.request, "urlopen", return_value=_Resp() ), mock.patch.object(pub.infra_artifact, "_download") as download: - root = Path(d) - result = pub._try_download_published(root) - - self.assertEqual( - ("v1", root / "rootfs.ext4.gz", root / "rootfs.ext4.gz.sha256"), - result, - ) - self.assertEqual(2, download.call_count) + role_dir = Path(d) / _ROLE + role_dir.mkdir() + result = pub._try_download_published(_ROLE, role_dir) + self.assertEqual("v1", result) + self.assertEqual("v1\n", (role_dir / "version.txt").read_text()) + self.assertEqual(2, download.call_count) def test_missing_artifact_returns_none(self) -> None: missing = urllib.error.HTTPError("u", 404, "missing", Message(), None) - with tempfile.TemporaryDirectory() as d, mock.patch.object( - pub.urllib.request, "urlopen", side_effect=missing - ): - self.assertIsNone(pub._try_download_published(Path(d))) + with tempfile.TemporaryDirectory() as d, \ + mock.patch.object(pub, "_role_version", return_value="v1"), \ + mock.patch.object(pub.urllib.request, "urlopen", side_effect=missing): + self.assertIsNone(pub._try_download_published(_ROLE, Path(d))) def test_registry_http_failure_is_reported(self) -> None: failure = urllib.error.HTTPError("u", 500, "failed", Message(), None) - with tempfile.TemporaryDirectory() as d, mock.patch.object( - pub.urllib.request, "urlopen", side_effect=failure - ): + with tempfile.TemporaryDirectory() as d, \ + mock.patch.object(pub, "_role_version", return_value="v1"), \ + mock.patch.object(pub.urllib.request, "urlopen", side_effect=failure): with self.assertRaises(SystemExit) as ctx: - pub._try_download_published(Path(d)) + pub._try_download_published(_ROLE, Path(d)) self.assertIn("registry check failed (HTTP 500)", str(ctx.exception)) def test_registry_connection_failure_is_reported(self) -> None: - with tempfile.TemporaryDirectory() as d, mock.patch.object( - pub.urllib.request, "urlopen", side_effect=urllib.error.URLError("offline") - ): + with tempfile.TemporaryDirectory() as d, \ + mock.patch.object(pub, "_role_version", return_value="v1"), \ + mock.patch.object( + pub.urllib.request, "urlopen", + side_effect=urllib.error.URLError("offline")): with self.assertRaises(SystemExit) as ctx: - pub._try_download_published(Path(d)) + pub._try_download_published(_ROLE, Path(d)) self.assertIn("registry unreachable", str(ctx.exception)) class TestMain(unittest.TestCase): - def test_output_builds_candidate_and_records_version(self) -> None: + def test_output_builds_a_candidate_per_role(self) -> None: with tempfile.TemporaryDirectory() as d: root = Path(d) / "candidate" - with mock.patch.object( - pub, "build_artifact", return_value=("v1", root / "g", root / "s") - ) as build: + with mock.patch.object(pub.infra_vm, "build_infra_images_with_docker") as images, \ + mock.patch.object(pub, "build_role_artifact", return_value="v1") as build: self.assertEqual(0, pub.main(["--output", str(root)])) - build.assert_called_once_with(root) - self.assertEqual("v1\n", (root / "version.txt").read_text()) + images.assert_called_once() + built_roles = {c.args[0] for c in build.call_args_list} + self.assertEqual({"orchestrator", "gateway"}, built_roles) - def test_output_reuses_published_candidate(self) -> None: + def test_output_reuses_published_candidates(self) -> None: with tempfile.TemporaryDirectory() as d: root = Path(d) / "candidate" - reused = ("v1", root / "rootfs.ext4.gz", root / "rootfs.ext4.gz.sha256") - with mock.patch.object( - pub, "_try_download_published", return_value=reused - ) as reuse, mock.patch.object(pub, "build_artifact") as build: + with mock.patch.object(pub, "_try_download_published", return_value="v1") as reuse, \ + mock.patch.object(pub.infra_vm, "build_infra_images_with_docker") as images, \ + mock.patch.object(pub, "build_role_artifact") as build: self.assertEqual( - 0, pub.main(["--output", str(root), "--reuse-published"]) - ) - reuse.assert_called_once_with(root) + 0, pub.main(["--output", str(root), "--reuse-published"])) + self.assertEqual(2, reuse.call_count) # once per role + images.assert_not_called() build.assert_not_called() - self.assertEqual("v1\n", (root / "version.txt").read_text()) - def test_publish_dir_publishes_existing_candidate(self) -> None: + def test_publish_dir_publishes_both_roles(self) -> None: with tempfile.TemporaryDirectory() as d, \ mock.patch.object(pub.infra_artifact, "_config", return_value=("", "", "t")), \ mock.patch.object(pub, "_publish_bundle", return_value="v1") as publish: self.assertEqual(0, pub.main(["--publish-dir", d])) - publish.assert_called_once_with(Path(d), "t") + published_roles = {c.args[0] for c in publish.call_args_list} + self.assertEqual({"orchestrator", "gateway"}, published_roles) if __name__ == "__main__":