Compare commits
11 Commits
a73aba935f
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| dfc693e0b6 | |||
| 39d47b8108 | |||
| d0a0ce8d60 | |||
| 5c08701983 | |||
| e3e195f866 | |||
| e3d24b7e41 | |||
| f2891a1634 | |||
| 8d8a88aeeb | |||
| 18f190b7e3 | |||
| bbb8913382 | |||
| 59be808ab1 |
@@ -1,9 +1,12 @@
|
|||||||
"""FirecrackerFreezer — snapshot a running microVM to a Docker image.
|
"""FirecrackerFreezer — snapshot a running microVM to a rootfs tar.
|
||||||
|
|
||||||
The VM is live and can't be block-copied safely, so — like the macOS
|
The VM is live and can't be block-copied safely, so — like the macOS
|
||||||
backend — we stream the guest root filesystem out over the control
|
backend — we stream the guest root filesystem out over the control
|
||||||
channel (SSH here) and rebuild an image from it. The bottle keeps
|
channel (SSH here). Unlike the other backends this needs no Docker: the
|
||||||
running after the snapshot.
|
tar *is* the resumable artifact. `resume` extracts it and rebuilds a
|
||||||
|
fresh per-bottle ext4 with `mke2fs -d` (see `util.build_committed_rootfs_dir`
|
||||||
|
and `launch._build_agent_base`). The bottle keeps running after the
|
||||||
|
snapshot.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -11,9 +14,9 @@ from __future__ import annotations
|
|||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
import tempfile
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ...bottle_state import committed_rootfs_path
|
||||||
from ...log import die, info
|
from ...log import die, info
|
||||||
from .. import ActiveAgent
|
from .. import ActiveAgent
|
||||||
from ..freeze import Freezer
|
from ..freeze import Freezer
|
||||||
@@ -30,14 +33,13 @@ class FirecrackerFreezer(Freezer):
|
|||||||
if not private_key.is_file() or not guest_ip:
|
if not private_key.is_file() or not guest_ip:
|
||||||
die(f"cannot freeze {agent.slug}: run dir {run_dir} is missing the "
|
die(f"cannot freeze {agent.slug}: run dir {run_dir} is missing the "
|
||||||
f"SSH key or VM config (is the bottle still running?)")
|
f"SSH key or VM config (is the bottle still running?)")
|
||||||
image_tag = f"bot-bottle-committed-{agent.slug}:latest"
|
tar_path = committed_rootfs_path(agent.slug)
|
||||||
_commit_via_ssh(private_key, guest_ip, image_tag)
|
_commit_rootfs_via_ssh(private_key, guest_ip, tar_path)
|
||||||
info(f"committed {agent.slug} -> {image_tag!r}")
|
info(f"committed {agent.slug} -> {tar_path}")
|
||||||
return image_tag
|
return str(tar_path)
|
||||||
|
|
||||||
def _export_hint(self, slug: str, image_ref: str) -> None:
|
def _export_hint(self, slug: str, image_ref: str) -> None:
|
||||||
info(f"to export for migration: docker image save {image_ref} "
|
info(f"to export for migration: cp {image_ref} {slug}.tar")
|
||||||
f"-o {slug}.tar")
|
|
||||||
|
|
||||||
|
|
||||||
def _guest_ip_from_config(config_path: Path) -> str:
|
def _guest_ip_from_config(config_path: Path) -> str:
|
||||||
@@ -53,24 +55,36 @@ def _guest_ip_from_config(config_path: Path) -> str:
|
|||||||
return ""
|
return ""
|
||||||
|
|
||||||
|
|
||||||
def _commit_via_ssh(private_key: Path, guest_ip: str, image_tag: str) -> None:
|
def _commit_rootfs_via_ssh(private_key: Path, guest_ip: str, tar_path: Path) -> None:
|
||||||
with tempfile.TemporaryDirectory(prefix="bot-bottle-fc-commit.") as tmp:
|
"""Stream the guest rootfs out over SSH into `tar_path`. Excludes the
|
||||||
rootfs_tar = os.path.join(tmp, "rootfs.tar")
|
virtual/live mounts (proc/sys/dev/run) — resume recreates those empty
|
||||||
ssh = util.ssh_base_argv(private_key, guest_ip)
|
mount points. Written to a `.partial` sibling and renamed on success so
|
||||||
with open(rootfs_tar, "wb") as tar_out:
|
a failed freeze never leaves a truncated artifact in its place."""
|
||||||
result = subprocess.run(
|
tar_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
[*ssh, "--", "tar", "--create", "--one-file-system",
|
partial = tar_path.with_name(tar_path.name + ".partial")
|
||||||
"--exclude=./proc", "--exclude=./sys", "--exclude=./dev",
|
ssh = util.ssh_base_argv(private_key, guest_ip)
|
||||||
"--exclude=./run", "--file=-", "--directory=/", "."],
|
# The snapshot can contain the bottle's private workspace, so keep it
|
||||||
stdout=tar_out, stderr=subprocess.PIPE, check=False,
|
# owner-only (0600) for the whole stream. The `os.open` mode only applies
|
||||||
)
|
# on *creation*, so unlink any leftover partial (a prior interrupted run
|
||||||
if result.returncode != 0:
|
# could have left it world-readable, or something could swap in a symlink
|
||||||
die(f"ssh tar for {guest_ip} failed: "
|
# at this predictable name) and exclusively recreate it — O_EXCL|O_NOFOLLOW
|
||||||
f"{(result.stderr or b'').decode().strip() or '<no stderr>'}")
|
# — then fchmod immediately so umask can't loosen it. Re-assert after the
|
||||||
with open(os.path.join(tmp, "Dockerfile"), "w", encoding="utf-8") as f:
|
# rename too (os.replace carries the source mode, but be explicit).
|
||||||
f.write("FROM scratch\nADD rootfs.tar /\nUSER node\nWORKDIR /home/node\n")
|
partial.unlink(missing_ok=True)
|
||||||
build = subprocess.run(
|
fd = os.open(
|
||||||
["docker", "build", "-t", image_tag, tmp], check=False,
|
partial, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600
|
||||||
|
)
|
||||||
|
os.fchmod(fd, 0o600)
|
||||||
|
with os.fdopen(fd, "wb") as tar_out:
|
||||||
|
result = subprocess.run(
|
||||||
|
[*ssh, "--", "tar", "--create", "--one-file-system",
|
||||||
|
"--exclude=./proc", "--exclude=./sys", "--exclude=./dev",
|
||||||
|
"--exclude=./run", "--file=-", "--directory=/", "."],
|
||||||
|
stdout=tar_out, stderr=subprocess.PIPE, check=False,
|
||||||
)
|
)
|
||||||
if build.returncode != 0:
|
if result.returncode != 0:
|
||||||
die(f"docker build for {image_tag!r} failed")
|
partial.unlink(missing_ok=True)
|
||||||
|
die(f"ssh tar for {guest_ip} failed: "
|
||||||
|
f"{(result.stderr or b'').decode().strip() or '<no stderr>'}")
|
||||||
|
os.replace(partial, tar_path)
|
||||||
|
os.chmod(tar_path, 0o600)
|
||||||
|
|||||||
@@ -0,0 +1,199 @@
|
|||||||
|
"""Prebuilt infra-VM rootfs, pulled as an artifact (PRD 0069 Stage 2).
|
||||||
|
|
||||||
|
The Firecracker infra VM boots a fixed rootfs (orchestrator control plane +
|
||||||
|
gateway + buildah, control-plane init as PID 1) that does not vary per launch —
|
||||||
|
the per-boot bits (authorized_keys, guest IP) ride the kernel cmdline, so one
|
||||||
|
rootfs boots on any host. Instead of building that rootfs on the launch host
|
||||||
|
with Docker, we build it **off-host** and publish it as a versioned, ready-to-
|
||||||
|
boot ext4 (gzip-compressed) to a Gitea **generic package**; the launch host
|
||||||
|
downloads + verifies + boots it. No Docker, no image tooling on the launch
|
||||||
|
host — just an HTTP fetch and gunzip.
|
||||||
|
|
||||||
|
publish (off-host, see publish_infra.py):
|
||||||
|
docker build -> rootfs dir -> mke2fs -> gzip -> PUT generic package
|
||||||
|
pull (this module, launch host):
|
||||||
|
GET .../rootfs.ext4.gz (+ .sha256) -> verify -> gunzip -> boot
|
||||||
|
|
||||||
|
The artifact **version** is a content hash of everything baked into the rootfs
|
||||||
|
(the shipped bot_bottle package, the three Dockerfiles, and the init), so a
|
||||||
|
launch host always pulls the artifact matching its code and a content change
|
||||||
|
can't silently boot a stale rootfs. A checksum mismatch fails closed.
|
||||||
|
|
||||||
|
Set `BOT_BOTTLE_INFRA_BUILD=local` to skip the pull and build the rootfs
|
||||||
|
locally with Docker (dev iteration on the Dockerfiles) — see `infra_vm`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ...log import die, info
|
||||||
|
from . import util
|
||||||
|
|
||||||
|
# Bump if the on-disk artifact *format* changes (compression, layout) so a new
|
||||||
|
# scheme can't collide with a cached/published artifact of the old one.
|
||||||
|
_ARTIFACT_FORMAT = "1"
|
||||||
|
|
||||||
|
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||||
|
_DOCKERFILES = ("Dockerfile.orchestrator", "Dockerfile.gateway", "Dockerfile.infra")
|
||||||
|
|
||||||
|
_DEFAULT_BASE = "https://gitea.dideric.is"
|
||||||
|
_DEFAULT_OWNER = "didericis"
|
||||||
|
_PACKAGE = "bot-bottle-firecracker-infra"
|
||||||
|
|
||||||
|
# Streaming copy chunk for the (hundreds-of-MB) download.
|
||||||
|
_CHUNK = 1 << 20
|
||||||
|
|
||||||
|
|
||||||
|
def local_build_requested() -> bool:
|
||||||
|
"""True when the operator opted into the dev Docker-build path instead of
|
||||||
|
pulling the published artifact (`BOT_BOTTLE_INFRA_BUILD=local`)."""
|
||||||
|
return os.environ.get("BOT_BOTTLE_INFRA_BUILD", "").strip().lower() == "local"
|
||||||
|
|
||||||
|
|
||||||
|
def infra_artifact_version(init_script: str, *, repo_root: Path = _REPO_ROOT) -> str:
|
||||||
|
"""Content hash (16 hex) of everything baked into the infra rootfs: the
|
||||||
|
whole shipped `bot_bottle` package, the three fixed Dockerfiles, and the
|
||||||
|
guest init. Deterministic across the publish host and the launch host when
|
||||||
|
both run the same checkout, so the tag the launch host pulls is exactly the
|
||||||
|
tag publish produced.
|
||||||
|
|
||||||
|
The package is `COPY bot_bottle /app/bot_bottle`'d wholesale into the image,
|
||||||
|
so hash *every* regular file under it — not just `*.py`. Non-Python inputs
|
||||||
|
(e.g. `egress_entrypoint.sh`, `netpool.defaults.env`) are baked in too, and
|
||||||
|
a change to one must bump the version or a launch host could boot a stale
|
||||||
|
rootfs whose code differs from its checkout. `__pycache__`/`.pyc` are the
|
||||||
|
only exclusions — build artifacts, never copied."""
|
||||||
|
h = hashlib.sha256()
|
||||||
|
h.update(f"format={_ARTIFACT_FORMAT}\n".encode())
|
||||||
|
pkg = repo_root / "bot_bottle"
|
||||||
|
for path in sorted(pkg.rglob("*")):
|
||||||
|
if not path.is_file():
|
||||||
|
continue
|
||||||
|
if "__pycache__" in path.parts or path.suffix == ".pyc":
|
||||||
|
continue
|
||||||
|
h.update(str(path.relative_to(repo_root)).encode())
|
||||||
|
h.update(b"\0")
|
||||||
|
h.update(path.read_bytes())
|
||||||
|
for name in _DOCKERFILES:
|
||||||
|
h.update(name.encode())
|
||||||
|
h.update(b"\0")
|
||||||
|
h.update((repo_root / name).read_bytes())
|
||||||
|
h.update(b"init\0")
|
||||||
|
h.update(init_script.encode())
|
||||||
|
return h.hexdigest()[:16]
|
||||||
|
|
||||||
|
|
||||||
|
def _config() -> tuple[str, str, str]:
|
||||||
|
"""(base_url, owner, token) for the generic-package endpoint. Base + owner
|
||||||
|
are overridable for other deployments / mirrors; the token comes solely from
|
||||||
|
`BOT_BOTTLE_INFRA_ARTIFACT_TOKEN` (a dedicated package-scoped token, kept
|
||||||
|
separate from the general-purpose Gitea token) and is optional — a public
|
||||||
|
package needs none to pull."""
|
||||||
|
base = os.environ.get("BOT_BOTTLE_INFRA_ARTIFACT_BASE", _DEFAULT_BASE).rstrip("/")
|
||||||
|
owner = os.environ.get("BOT_BOTTLE_INFRA_ARTIFACT_OWNER", _DEFAULT_OWNER)
|
||||||
|
token = os.environ.get("BOT_BOTTLE_INFRA_ARTIFACT_TOKEN", "")
|
||||||
|
return base, owner, token
|
||||||
|
|
||||||
|
|
||||||
|
def artifact_url(version: str, filename: str) -> str:
|
||||||
|
"""The generic-package download URL for one file of this version's
|
||||||
|
artifact (`rootfs.ext4.gz` / `rootfs.ext4.gz.sha256`)."""
|
||||||
|
base, owner, _ = _config()
|
||||||
|
return f"{base}/api/packages/{owner}/generic/{_PACKAGE}/{version}/{filename}"
|
||||||
|
|
||||||
|
|
||||||
|
_GZ_NAME = "rootfs.ext4.gz"
|
||||||
|
_SHA_NAME = "rootfs.ext4.gz.sha256"
|
||||||
|
|
||||||
|
|
||||||
|
def _cache_root(version: str) -> Path:
|
||||||
|
return util.cache_dir() / "infra-artifact" / version
|
||||||
|
|
||||||
|
|
||||||
|
def _open(url: str) -> urllib.request.Request:
|
||||||
|
_, _, token = _config()
|
||||||
|
req = urllib.request.Request(url)
|
||||||
|
if token:
|
||||||
|
req.add_header("Authorization", f"token {token}")
|
||||||
|
return req
|
||||||
|
|
||||||
|
|
||||||
|
def _download(url: str, dest: Path) -> None:
|
||||||
|
"""Stream `url` to `dest` (atomic via a `.part` sibling)."""
|
||||||
|
tmp = dest.with_suffix(dest.suffix + ".part")
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(_open(url)) as resp, open(tmp, "wb") as out:
|
||||||
|
shutil.copyfileobj(resp, out, _CHUNK)
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
tmp.unlink(missing_ok=True)
|
||||||
|
if e.code == 404:
|
||||||
|
die(
|
||||||
|
f"infra artifact not published for this code version.\n"
|
||||||
|
f" missing: {url}\n"
|
||||||
|
f" publish it from a build host (Docker):\n"
|
||||||
|
f" python3 -m bot_bottle.backend.firecracker.publish_infra\n"
|
||||||
|
f" or build the rootfs locally: BOT_BOTTLE_INFRA_BUILD=local"
|
||||||
|
)
|
||||||
|
die(f"downloading infra artifact failed (HTTP {e.code}): {url}")
|
||||||
|
except urllib.error.URLError as e:
|
||||||
|
tmp.unlink(missing_ok=True)
|
||||||
|
die(f"infra artifact registry unreachable: {url} ({e.reason})")
|
||||||
|
tmp.replace(dest)
|
||||||
|
|
||||||
|
|
||||||
|
def _sha256_file(path: Path) -> str:
|
||||||
|
h = hashlib.sha256()
|
||||||
|
with open(path, "rb") as fh:
|
||||||
|
for chunk in iter(lambda: fh.read(_CHUNK), b""):
|
||||||
|
h.update(chunk)
|
||||||
|
return h.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_artifact_gz(version: str) -> Path:
|
||||||
|
"""The verified, cached `rootfs.ext4.gz` for `version` — downloading it (and
|
||||||
|
its `.sha256`) once, then reusing it. Fail-closed on a checksum mismatch:
|
||||||
|
the partial is removed and we die rather than boot an unverified rootfs."""
|
||||||
|
root = _cache_root(version)
|
||||||
|
root.mkdir(parents=True, exist_ok=True)
|
||||||
|
gz = root / _GZ_NAME
|
||||||
|
ok = root / ".verified"
|
||||||
|
if gz.is_file() and ok.is_file():
|
||||||
|
return gz
|
||||||
|
|
||||||
|
info(f"pulling infra rootfs artifact {_PACKAGE}/{version}")
|
||||||
|
_download(artifact_url(version, _GZ_NAME), gz)
|
||||||
|
sha = root / _SHA_NAME
|
||||||
|
_download(artifact_url(version, _SHA_NAME), sha)
|
||||||
|
|
||||||
|
expected = sha.read_text().split()[0].strip().lower()
|
||||||
|
actual = _sha256_file(gz)
|
||||||
|
if actual != expected:
|
||||||
|
gz.unlink(missing_ok=True)
|
||||||
|
sha.unlink(missing_ok=True)
|
||||||
|
die(
|
||||||
|
f"infra artifact checksum mismatch for {version}:\n"
|
||||||
|
f" expected {expected}\n"
|
||||||
|
f" actual {actual}\n"
|
||||||
|
f" refusing to boot an unverified rootfs."
|
||||||
|
)
|
||||||
|
ok.write_text("ok\n")
|
||||||
|
return gz
|
||||||
|
|
||||||
|
|
||||||
|
def materialize_ext4(version: str, dest: Path) -> None:
|
||||||
|
"""Ensure the verified artifact is cached, then gunzip it to `dest` — a
|
||||||
|
fresh, writable per-boot rootfs (the VM mutates it; the cached `.gz` stays
|
||||||
|
pristine). Atomic via a `.part` sibling."""
|
||||||
|
gz = ensure_artifact_gz(version)
|
||||||
|
tmp = dest.with_suffix(dest.suffix + ".part")
|
||||||
|
info(f"expanding infra rootfs -> {dest}")
|
||||||
|
with gzip.open(gz, "rb") as src, open(tmp, "wb") as out:
|
||||||
|
shutil.copyfileobj(src, out, _CHUNK)
|
||||||
|
tmp.replace(dest)
|
||||||
@@ -35,7 +35,7 @@ from typing import Generator
|
|||||||
from ...log import die, info
|
from ...log import die, info
|
||||||
from ..docker import util as docker_mod
|
from ..docker import util as docker_mod
|
||||||
from ..docker.gateway_provision import GatewayProvisionError
|
from ..docker.gateway_provision import GatewayProvisionError
|
||||||
from . import firecracker_vm, netpool, util
|
from . import firecracker_vm, infra_artifact, netpool, util
|
||||||
|
|
||||||
# The single infra-VM image: gateway data plane + baked control-plane source
|
# The single infra-VM image: gateway data plane + baked control-plane source
|
||||||
# (Dockerfile.infra FROM the gateway image). Built from source by default;
|
# (Dockerfile.infra FROM the gateway image). Built from source by default;
|
||||||
@@ -109,10 +109,26 @@ class InfraVm:
|
|||||||
|
|
||||||
|
|
||||||
def ensure_built() -> None:
|
def ensure_built() -> None:
|
||||||
"""Build the infra image from source (bootstrap via host docker). The
|
"""Ensure the infra rootfs is available before boot.
|
||||||
infra image `COPY --from`s the orchestrator image and is `FROM` the
|
|
||||||
gateway image, so both must exist first. A pull-from-registry mode
|
Default (docker-free, PRD 0069 Stage 2): download + verify the prebuilt
|
||||||
replaces this later."""
|
rootfs artifact matching this code version (see `infra_artifact`); the
|
||||||
|
launch host needs no Docker. `BOT_BOTTLE_INFRA_BUILD=local` instead builds
|
||||||
|
the three fixed images from source with host Docker — the infra image
|
||||||
|
`COPY --from`s the orchestrator image and is `FROM` the gateway image, so
|
||||||
|
both must exist first — for iterating on the Dockerfiles."""
|
||||||
|
if infra_artifact.local_build_requested():
|
||||||
|
build_infra_images_with_docker()
|
||||||
|
return
|
||||||
|
infra_artifact.ensure_artifact_gz(
|
||||||
|
infra_artifact.infra_artifact_version(_infra_init()))
|
||||||
|
|
||||||
|
|
||||||
|
def build_infra_images_with_docker() -> None:
|
||||||
|
"""Build the three fixed images from source with host Docker: orchestrator,
|
||||||
|
gateway, then the combined infra image (`COPY --from` orchestrator, `FROM`
|
||||||
|
gateway). The launch host uses this only in `BOT_BOTTLE_INFRA_BUILD=local`
|
||||||
|
mode; `publish_infra` uses it off-host to produce the published artifact."""
|
||||||
docker_mod.build_image(
|
docker_mod.build_image(
|
||||||
_ORCHESTRATOR_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.orchestrator")
|
_ORCHESTRATOR_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.orchestrator")
|
||||||
docker_mod.build_image(
|
docker_mod.build_image(
|
||||||
@@ -190,10 +206,15 @@ def boot() -> InfraVm:
|
|||||||
die(f"orchestrator link {slot.iface} not present.\n"
|
die(f"orchestrator link {slot.iface} not present.\n"
|
||||||
f" ./cli.py backend setup --backend=firecracker")
|
f" ./cli.py backend setup --backend=firecracker")
|
||||||
|
|
||||||
base = build_infra_rootfs_dir()
|
|
||||||
run_dir = _infra_dir()
|
run_dir = _infra_dir()
|
||||||
rootfs = run_dir / "rootfs.ext4"
|
rootfs = run_dir / "rootfs.ext4"
|
||||||
util.build_rootfs_ext4(base, rootfs, slack_mib=8192)
|
if infra_artifact.local_build_requested():
|
||||||
|
util.build_rootfs_ext4(build_infra_rootfs_dir(), rootfs, slack_mib=8192)
|
||||||
|
else:
|
||||||
|
# Prebuilt artifact already carries the buildah build slack; expand it
|
||||||
|
# to a fresh writable rootfs for this boot.
|
||||||
|
infra_artifact.materialize_ext4(
|
||||||
|
infra_artifact.infra_artifact_version(_infra_init()), rootfs)
|
||||||
private_key, pubkey = _stable_keypair()
|
private_key, pubkey = _stable_keypair()
|
||||||
|
|
||||||
info(f"booting infra VM on {slot.iface} (guest {slot.guest_ip})")
|
info(f"booting infra VM on {slot.iface} (guest {slot.guest_ip})")
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
"""Launch flow for the Firecracker backend (PRD 0070, consolidated).
|
"""Launch flow for the Firecracker backend (PRD 0070, consolidated).
|
||||||
|
|
||||||
Per bottle:
|
Per bottle:
|
||||||
1. build the agent image (docker), export it to a cached ext4 rootfs;
|
1. build the agent rootfs in a builder VM (buildah, no host docker), or
|
||||||
|
resume a frozen bottle from its committed rootfs tar; cache the ext4;
|
||||||
2. ensure the per-host orchestrator + shared gateway are up;
|
2. ensure the per-host orchestrator + shared gateway are up;
|
||||||
3. claim a free TAP pool slot (rootless flock);
|
3. claim a free TAP pool slot (rootless flock);
|
||||||
4. register the bottle on the orchestrator by the VM's guest IP (the
|
4. register the bottle on the orchestrator by the VM's guest IP (the
|
||||||
@@ -31,6 +32,7 @@ from typing import Callable, Generator
|
|||||||
|
|
||||||
from ...agent_provider import runtime_for
|
from ...agent_provider import runtime_for
|
||||||
from ...bottle_state import (
|
from ...bottle_state import (
|
||||||
|
committed_rootfs_path,
|
||||||
egress_state_dir,
|
egress_state_dir,
|
||||||
git_gate_state_dir,
|
git_gate_state_dir,
|
||||||
read_committed_image,
|
read_committed_image,
|
||||||
@@ -45,7 +47,6 @@ from ...git_gate import (
|
|||||||
)
|
)
|
||||||
from ...log import info, warn
|
from ...log import info, warn
|
||||||
from ...supervise import SUPERVISE_PORT
|
from ...supervise import SUPERVISE_PORT
|
||||||
from ..docker import util as docker_mod
|
|
||||||
from ..docker.egress import EGRESS_PORT
|
from ..docker.egress import EGRESS_PORT
|
||||||
from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
|
from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
|
||||||
from . import firecracker_vm, image_builder, isolation_probe, netpool, util
|
from . import firecracker_vm, image_builder, isolation_probe, netpool, util
|
||||||
@@ -210,16 +211,13 @@ def _build_agent_base(
|
|||||||
) -> tuple[FirecrackerBottlePlan, Path]:
|
) -> tuple[FirecrackerBottlePlan, Path]:
|
||||||
"""Produce the agent's base rootfs dir. Primary path: build the Dockerfile
|
"""Produce the agent's base rootfs dir. Primary path: build the Dockerfile
|
||||||
inside a Firecracker builder VM (buildah, no host docker), smoke-testing
|
inside a Firecracker builder VM (buildah, no host docker), smoke-testing
|
||||||
the image before export. A committed snapshot (freeze/migrate) is still
|
the image before export. A committed snapshot (freeze/migrate) is resumed
|
||||||
exported via the host docker path until that is ported too."""
|
directly from the rootfs tar the freezer wrote — no host docker either."""
|
||||||
committed = read_committed_image(plan.slug)
|
committed = read_committed_image(plan.slug)
|
||||||
if committed and docker_mod.image_exists(committed):
|
committed_tar = committed_rootfs_path(plan.slug)
|
||||||
info(f"using committed image {committed!r}")
|
if committed and committed_tar.is_file():
|
||||||
plan = dataclasses.replace(
|
info(f"resuming from committed rootfs {committed_tar}")
|
||||||
plan,
|
return plan, util.build_committed_rootfs_dir(committed_tar)
|
||||||
agent_provision=dataclasses.replace(plan.agent_provision, image=committed),
|
|
||||||
)
|
|
||||||
return plan, util.build_base_rootfs_dir(committed)
|
|
||||||
base = image_builder.build_agent_rootfs_dir(
|
base = image_builder.build_agent_rootfs_dir(
|
||||||
Path(plan.dockerfile_path),
|
Path(plan.dockerfile_path),
|
||||||
image_tag=plan.image,
|
image_tag=plan.image,
|
||||||
|
|||||||
@@ -0,0 +1,169 @@
|
|||||||
|
"""Build the infra rootfs and publish it as a Gitea generic package.
|
||||||
|
|
||||||
|
The off-host (build / CI) half of PRD 0069 Stage 2: this DOES use Docker, but
|
||||||
|
never on the launch host. It runs the same pipeline the launch host used to run
|
||||||
|
locally — `docker build` the three fixed images, export to a rootfs dir, inject
|
||||||
|
the guest boot, `mke2fs` to an ext4 with the buildah build slack — then gzips
|
||||||
|
the ext4 and PUTs it (plus a `.sha256`) to
|
||||||
|
`…/api/packages/<owner>/generic/bot-bottle-firecracker-infra/<version>/`.
|
||||||
|
|
||||||
|
The `<version>` is `infra_artifact.infra_artifact_version(...)`, the content
|
||||||
|
hash of the rootfs inputs, so a launch host at the same code checkout resolves
|
||||||
|
the exact artifact this produced.
|
||||||
|
|
||||||
|
python3 -m bot_bottle.backend.firecracker.publish_infra [--dry-run] [--force]
|
||||||
|
|
||||||
|
Auth: a token with `write:package` on the target owner, from
|
||||||
|
`BOT_BOTTLE_INFRA_ARTIFACT_TOKEN`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import shutil
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from . import infra_artifact, infra_vm, util
|
||||||
|
|
||||||
|
_CHUNK = 1 << 20
|
||||||
|
|
||||||
|
# A human-readable description shipped alongside the artifact — generic packages
|
||||||
|
# have no description field, so this file *is* the description on the package
|
||||||
|
# page. Uploaded on every publish so it never goes stale.
|
||||||
|
_ABOUT_NAME = "about.txt"
|
||||||
|
_ABOUT_TEXT = (
|
||||||
|
"bot-bottle infra rootfs for the Firecracker backend (PRD 0069 Stage 2, "
|
||||||
|
"#348): the per-host infra VM (orchestrator control plane + gateway + "
|
||||||
|
"buildah). Prebuilt off-host, gzip ext4; the launch host downloads + "
|
||||||
|
"sha256-verifies + boots it, no host Docker. The version tag is a content "
|
||||||
|
"hash of the rootfs inputs. Files: rootfs.ext4.gz + rootfs.ext4.gz.sha256.\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _gzip(src: Path, dest: Path) -> None:
|
||||||
|
with open(src, "rb") as fh, gzip.open(dest, "wb") as out:
|
||||||
|
shutil.copyfileobj(fh, out, _CHUNK)
|
||||||
|
|
||||||
|
|
||||||
|
def _sha256(path: Path) -> str:
|
||||||
|
h = hashlib.sha256()
|
||||||
|
with open(path, "rb") as fh:
|
||||||
|
for chunk in iter(lambda: fh.read(_CHUNK), b""):
|
||||||
|
h.update(chunk)
|
||||||
|
return h.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _put(url: str, body: "bytes | Path", token: str) -> None:
|
||||||
|
"""PUT `body` (raw bytes, or a Path streamed from disk) to `url`. The rootfs
|
||||||
|
is hundreds of MB, so it is passed as a Path and streamed — `urlopen` reads
|
||||||
|
the open file in blocks rather than materializing it in memory (with an
|
||||||
|
explicit Content-Length, which Gitea requires and which also stops urllib
|
||||||
|
from `len()`-ing a non-bytes body)."""
|
||||||
|
handle = None
|
||||||
|
if isinstance(body, Path):
|
||||||
|
length = body.stat().st_size
|
||||||
|
handle = open(body, "rb")
|
||||||
|
data: object = handle
|
||||||
|
else:
|
||||||
|
length = len(body)
|
||||||
|
data = body
|
||||||
|
req = urllib.request.Request(url, data=data, method="PUT") # type: ignore[arg-type]
|
||||||
|
req.add_header("Content-Length", str(length))
|
||||||
|
if token:
|
||||||
|
req.add_header("Authorization", f"token {token}")
|
||||||
|
req.add_header("Content-Type", "application/octet-stream")
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req) as resp:
|
||||||
|
print(f" uploaded {url} (HTTP {resp.status})")
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
if e.code == 409:
|
||||||
|
raise SystemExit(
|
||||||
|
f"artifact already published at {url} (HTTP 409); "
|
||||||
|
f"bump the code version or pass --force to overwrite"
|
||||||
|
)
|
||||||
|
raise SystemExit(f"upload failed (HTTP {e.code}): {url}\n{e.read().decode(errors='replace')}")
|
||||||
|
except urllib.error.URLError as e:
|
||||||
|
raise SystemExit(f"registry unreachable: {url} ({e.reason})")
|
||||||
|
finally:
|
||||||
|
if handle is not None:
|
||||||
|
handle.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _delete(url: str, token: str) -> None:
|
||||||
|
req = urllib.request.Request(url, method="DELETE")
|
||||||
|
if token:
|
||||||
|
req.add_header("Authorization", f"token {token}")
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req):
|
||||||
|
pass
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
if e.code != 404:
|
||||||
|
raise SystemExit(f"could not overwrite existing artifact (HTTP {e.code}): {url}")
|
||||||
|
except urllib.error.URLError as e:
|
||||||
|
raise SystemExit(f"registry unreachable: {url} ({e.reason})")
|
||||||
|
|
||||||
|
|
||||||
|
def build_artifact(out_dir: Path) -> tuple[str, Path, Path]:
|
||||||
|
"""Build the infra rootfs ext4, gzip it, and write the checksum. Returns
|
||||||
|
`(version, gz_path, sha_path)`. Uses host Docker (off-host / CI)."""
|
||||||
|
version = infra_artifact.infra_artifact_version(infra_vm._infra_init())
|
||||||
|
print(f"building infra rootfs artifact {version} (docker)")
|
||||||
|
infra_vm.build_infra_images_with_docker()
|
||||||
|
base = infra_vm.build_infra_rootfs_dir()
|
||||||
|
|
||||||
|
ext4 = out_dir / "rootfs.ext4"
|
||||||
|
util.build_rootfs_ext4(base, ext4, slack_mib=8192)
|
||||||
|
gz = out_dir / "rootfs.ext4.gz"
|
||||||
|
print("compressing rootfs")
|
||||||
|
_gzip(ext4, gz)
|
||||||
|
ext4.unlink(missing_ok=True)
|
||||||
|
|
||||||
|
sha = out_dir / "rootfs.ext4.gz.sha256"
|
||||||
|
digest = _sha256(gz)
|
||||||
|
sha.write_text(f"{digest} rootfs.ext4.gz\n")
|
||||||
|
print(f" {gz.name}: {gz.stat().st_size / 1e6:.0f} MB sha256={digest}")
|
||||||
|
return version, gz, sha
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
prog="publish_infra", description="Build + publish the infra rootfs artifact.")
|
||||||
|
parser.add_argument("--dry-run", action="store_true",
|
||||||
|
help="build the artifact but do not upload")
|
||||||
|
parser.add_argument("--force", action="store_true",
|
||||||
|
help="overwrite an already-published artifact of this version")
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
|
||||||
|
_, _, token = infra_artifact._config()
|
||||||
|
if not args.dry_run and not token:
|
||||||
|
raise SystemExit(
|
||||||
|
"no publish token: set BOT_BOTTLE_INFRA_ARTIFACT_TOKEN to a token "
|
||||||
|
"with write:package")
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory(prefix="bb-publish-infra.") as tmp:
|
||||||
|
version, gz, sha = build_artifact(Path(tmp))
|
||||||
|
gz_url = infra_artifact.artifact_url(version, gz.name)
|
||||||
|
sha_url = infra_artifact.artifact_url(version, sha.name)
|
||||||
|
about_url = infra_artifact.artifact_url(version, _ABOUT_NAME)
|
||||||
|
if args.dry_run:
|
||||||
|
print(f"dry-run: would upload -> {gz_url}")
|
||||||
|
return 0
|
||||||
|
if args.force:
|
||||||
|
_delete(gz_url, token)
|
||||||
|
_delete(sha_url, token)
|
||||||
|
_delete(about_url, token)
|
||||||
|
_put(gz_url, gz, token) # streamed from disk (hundreds of MB)
|
||||||
|
_put(sha_url, sha.read_bytes(), token) # tiny, in-memory is fine
|
||||||
|
_put(about_url, _ABOUT_TEXT.encode(), token) # package description
|
||||||
|
print(f"published infra rootfs {version}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -14,6 +14,7 @@ and `./cli.py backend setup --backend=firecracker`.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
import os
|
import os
|
||||||
import platform
|
import platform
|
||||||
import shutil
|
import shutil
|
||||||
@@ -212,15 +213,80 @@ def build_base_rootfs_dir(
|
|||||||
return base
|
return base
|
||||||
|
|
||||||
|
|
||||||
|
def build_committed_rootfs_dir(tar_path: Path) -> Path:
|
||||||
|
"""Prepare a base rootfs dir from a frozen-bottle snapshot tar (the
|
||||||
|
freeze/resume path — no Docker). Extracts the snapshot, recreates the
|
||||||
|
virtual mount points the freezer excluded, and injects the guest init +
|
||||||
|
static dropbear, mirroring `build_base_rootfs_dir` but sourced from a tar
|
||||||
|
we control rather than a Docker image.
|
||||||
|
|
||||||
|
Cached under the rootfs cache, keyed by the tar's size+mtime so a
|
||||||
|
re-freeze re-extracts but repeated resumes of the same snapshot don't.
|
||||||
|
Returns the prepared directory (read as the `mke2fs -d` source)."""
|
||||||
|
st = tar_path.stat()
|
||||||
|
fingerprint = hashlib.sha256(
|
||||||
|
f"{tar_path}:{st.st_size}:{st.st_mtime_ns}".encode()
|
||||||
|
).hexdigest()[:16]
|
||||||
|
base = cache_dir() / "rootfs" / f"committed-{fingerprint}"
|
||||||
|
ready = base / ".bb-ready"
|
||||||
|
if ready.is_file():
|
||||||
|
return base
|
||||||
|
|
||||||
|
if base.exists():
|
||||||
|
shutil.rmtree(base, ignore_errors=True)
|
||||||
|
base.mkdir(parents=True)
|
||||||
|
|
||||||
|
info(f"extracting committed rootfs {tar_path} -> {base}")
|
||||||
|
result = subprocess.run(
|
||||||
|
["tar", "-x", "-f", str(tar_path), "-C", str(base)],
|
||||||
|
capture_output=True, text=True, check=False,
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
die(f"extracting committed rootfs {tar_path} failed: "
|
||||||
|
f"{result.stderr.strip() or '<no stderr>'}")
|
||||||
|
|
||||||
|
# The freezer excludes the live/virtual filesystems from the snapshot;
|
||||||
|
# recreate them as empty mount points so the guest init can mount
|
||||||
|
# proc/sys/dev and dropbear has a writable /run.
|
||||||
|
for mount_point in ("proc", "sys", "dev", "run"):
|
||||||
|
(base / mount_point).mkdir(mode=0o755, exist_ok=True)
|
||||||
|
|
||||||
|
inject_guest_boot(base)
|
||||||
|
ready.write_text("ok\n")
|
||||||
|
return base
|
||||||
|
|
||||||
|
|
||||||
def inject_guest_boot(rootfs: Path, init_script: str | None = None) -> None:
|
def inject_guest_boot(rootfs: Path, init_script: str | None = None) -> None:
|
||||||
"""Drop the static dropbear and the PID-1 init into the rootfs.
|
"""Drop the static dropbear and the PID-1 init into the rootfs.
|
||||||
`init_script` defaults to the SSH-only agent init; the infra VM
|
`init_script` defaults to the SSH-only agent init; the infra VM
|
||||||
passes its own (control plane + gateway) init."""
|
passes its own (control plane + gateway) init.
|
||||||
shutil.copy2(dropbear_path(), rootfs / "bb-dropbear")
|
|
||||||
os.chmod(rootfs / "bb-dropbear", 0o755)
|
A committed snapshot is guest-controlled, so `bb-dropbear`/`bb-init`
|
||||||
init = rootfs / "bb-init"
|
may already exist as symlinks aimed at a host file (e.g. bb-init ->
|
||||||
init.write_text(init_script or _GUEST_INIT)
|
~/.bashrc). Replace whatever is there and create the files with
|
||||||
os.chmod(init, 0o755)
|
O_EXCL|O_NOFOLLOW so the write always lands a fresh regular file in
|
||||||
|
the staging tree and never follows a planted symlink out of it."""
|
||||||
|
_write_staged_file(rootfs / "bb-dropbear", dropbear_path().read_bytes())
|
||||||
|
_write_staged_file(rootfs / "bb-init", (init_script or _GUEST_INIT).encode())
|
||||||
|
|
||||||
|
|
||||||
|
def _write_staged_file(path: Path, data: bytes) -> None:
|
||||||
|
"""Write `data` to `path` (mode 0755) as a fresh regular file inside a
|
||||||
|
staging rootfs, replacing any pre-existing entry without following a
|
||||||
|
symlink at `path`. Fails closed on anything unexpected there."""
|
||||||
|
if path.is_symlink() or path.exists():
|
||||||
|
if path.is_dir() and not path.is_symlink():
|
||||||
|
shutil.rmtree(path)
|
||||||
|
else:
|
||||||
|
path.unlink()
|
||||||
|
fd = os.open(
|
||||||
|
path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o755
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
os.write(fd, data)
|
||||||
|
finally:
|
||||||
|
os.close(fd)
|
||||||
|
os.chmod(path, 0o755)
|
||||||
|
|
||||||
|
|
||||||
def build_rootfs_ext4(base_dir: Path, out_path: Path, *, slack_mib: int = 1024) -> None:
|
def build_rootfs_ext4(base_dir: Path, out_path: Path, *, slack_mib: int = 1024) -> None:
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ from __future__ import annotations
|
|||||||
import dataclasses
|
import dataclasses
|
||||||
import json
|
import json
|
||||||
import secrets
|
import secrets
|
||||||
|
import socket
|
||||||
import string
|
import string
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -43,6 +44,7 @@ from .paths import bot_bottle_root
|
|||||||
_STATE_SUBDIR = "state"
|
_STATE_SUBDIR = "state"
|
||||||
_PER_BOTTLE_DOCKERFILE_NAME = "Dockerfile"
|
_PER_BOTTLE_DOCKERFILE_NAME = "Dockerfile"
|
||||||
_COMMITTED_IMAGE_NAME = "committed-image"
|
_COMMITTED_IMAGE_NAME = "committed-image"
|
||||||
|
_COMMITTED_ROOTFS_NAME = "committed-rootfs.tar"
|
||||||
_TRANSCRIPT_SUBDIR = "transcript"
|
_TRANSCRIPT_SUBDIR = "transcript"
|
||||||
# Per-daemon scratch subdirs. PRD 0018 chunk 2: bind-mount sources
|
# Per-daemon scratch subdirs. PRD 0018 chunk 2: bind-mount sources
|
||||||
# live here so chunk 3's `docker compose up` can find them at stable
|
# live here so chunk 3's `docker compose up` can find them at stable
|
||||||
@@ -87,6 +89,14 @@ def bottle_identity(agent_name: str) -> str:
|
|||||||
return f"{slug}-{suffix}"
|
return f"{slug}-{suffix}"
|
||||||
|
|
||||||
|
|
||||||
|
def globalize_slug(slug: str) -> str:
|
||||||
|
"""Return a globally-unique slug qualified with the current hostname.
|
||||||
|
|
||||||
|
Assumes slug is a value returned from mint_slug. Use wherever a slug
|
||||||
|
must be unique across hosts (e.g. deploy-key titles)."""
|
||||||
|
return f"{socket.gethostname()}-{slug}"
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class BottleMetadata:
|
class BottleMetadata:
|
||||||
"""Persistent record of how a bottle was launched, written at
|
"""Persistent record of how a bottle was launched, written at
|
||||||
@@ -191,6 +201,15 @@ def committed_image_path(identity: str) -> Path:
|
|||||||
return bottle_state_dir(identity) / _COMMITTED_IMAGE_NAME
|
return bottle_state_dir(identity) / _COMMITTED_IMAGE_NAME
|
||||||
|
|
||||||
|
|
||||||
|
def committed_rootfs_path(identity: str) -> Path:
|
||||||
|
"""Where the Firecracker freezer stores a snapshot of the bottle's
|
||||||
|
guest rootfs (a plain tar). This is the resumable/migratable artifact
|
||||||
|
the Firecracker backend boots from — no Docker image involved. The
|
||||||
|
matching `committed-image` state file records that a snapshot exists
|
||||||
|
(and its path); `resume` boots from this tar when both are present."""
|
||||||
|
return bottle_state_dir(identity) / _COMMITTED_ROOTFS_NAME
|
||||||
|
|
||||||
|
|
||||||
def write_committed_image(identity: str, image_tag: str) -> Path:
|
def write_committed_image(identity: str, image_tag: str) -> Path:
|
||||||
"""Persist the committed image tag for `identity`. The next
|
"""Persist the committed image tag for `identity`. The next
|
||||||
`cli.py resume <identity>` will boot from this image instead of
|
`cli.py resume <identity>` will boot from this image instead of
|
||||||
@@ -340,10 +359,12 @@ __all__ = [
|
|||||||
"BottleMetadata",
|
"BottleMetadata",
|
||||||
"agent_state_dir",
|
"agent_state_dir",
|
||||||
"bottle_identity",
|
"bottle_identity",
|
||||||
|
"globalize_slug",
|
||||||
"bottle_state_dir",
|
"bottle_state_dir",
|
||||||
"cleanup_state",
|
"cleanup_state",
|
||||||
"clear_preserve_marker",
|
"clear_preserve_marker",
|
||||||
"committed_image_path",
|
"committed_image_path",
|
||||||
|
"committed_rootfs_path",
|
||||||
"egress_state_dir",
|
"egress_state_dir",
|
||||||
"git_gate_state_dir",
|
"git_gate_state_dir",
|
||||||
"is_preserved",
|
"is_preserved",
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import dataclasses
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
from .bottle_state import globalize_slug
|
||||||
from .errors import MissingEnvVarError
|
from .errors import MissingEnvVarError
|
||||||
from .log import info
|
from .log import info
|
||||||
from .manifest import ManifestBottle, ManifestGitEntry
|
from .manifest import ManifestBottle, ManifestGitEntry
|
||||||
@@ -46,7 +47,7 @@ def _provision_dynamic_key(
|
|||||||
owner_repo = entry.UpstreamPath
|
owner_repo = entry.UpstreamPath
|
||||||
if owner_repo.endswith(".git"):
|
if owner_repo.endswith(".git"):
|
||||||
owner_repo = owner_repo[:-4]
|
owner_repo = owner_repo[:-4]
|
||||||
title = f"bot-bottle:{slug}:{entry.Name}"
|
title = f"bot-bottle:{globalize_slug(slug)}:{entry.Name}"
|
||||||
|
|
||||||
info(f"provisioning deploy key for git-gate.repos[{entry.Name!r}]")
|
info(f"provisioning deploy key for git-gate.repos[{entry.Name!r}]")
|
||||||
key_id, private_key_bytes = provisioner.create(owner_repo, title)
|
key_id, private_key_bytes = provisioner.create(owner_repo, title)
|
||||||
|
|||||||
@@ -8,16 +8,20 @@
|
|||||||
> **Superseded in part by [PRD 0070](0070-per-host-orchestrator.md) (#351):**
|
> **Superseded in part by [PRD 0070](0070-per-host-orchestrator.md) (#351):**
|
||||||
> the sidecar-consolidation framing here (Stage 1, per-host sidecar; Stage 4,
|
> the sidecar-consolidation framing here (Stage 1, per-host sidecar; Stage 4,
|
||||||
> sidecar-as-VM) is taken over by 0070's per-host orchestrator. This PRD still
|
> sidecar-as-VM) is taken over by 0070's per-host orchestrator. This PRD still
|
||||||
> owns the docker-free **image-building** work — Stage 2 (nix-built fixed
|
> owns the docker-free **image-provisioning** work — Stage 2 (pull the fixed
|
||||||
> images, a dependency of 0070) and Stage 3 (in-VM Dockerfile builder).
|
> images from an OCI registry instead of building them with host Docker, a
|
||||||
|
> dependency of 0070) and Stage 3 (in-VM Dockerfile builder).
|
||||||
|
|
||||||
## Summary
|
## Summary
|
||||||
|
|
||||||
Make the Firecracker backend depend on **firecracker + KVM only**, removing
|
Make the Firecracker backend depend on **firecracker + KVM only**, removing
|
||||||
Docker from the host. Two moves get us there: run the **sidecar bundle as a
|
Docker from the host. Two moves get us there: run the **sidecar bundle as a
|
||||||
persistent, per-host service** (eventually a Firecracker VM) instead of a
|
persistent, per-host service** (eventually a Firecracker VM) instead of a
|
||||||
per-bottle container, and **build agent rootfs images without a host Docker
|
per-bottle container, and **provision rootfs images without a host Docker
|
||||||
daemon** (nix for the fixed images; an in-VM builder for user Dockerfiles).
|
daemon** — pull the fixed images (orchestrator/gateway/infra) from an OCI
|
||||||
|
registry and unpack them daemonlessly, and build user Dockerfiles in an in-VM
|
||||||
|
builder. The images are still *built* with Docker, but off the launch host
|
||||||
|
(CI / a publish step) and pushed to the registry; the launch host only pulls.
|
||||||
|
|
||||||
## Motivation
|
## Motivation
|
||||||
|
|
||||||
@@ -93,13 +97,51 @@ torn down at exit."
|
|||||||
Can ship as a container first (quick resource/ops win) and become a VM in
|
Can ship as a container first (quick resource/ops win) and become a VM in
|
||||||
Stage 4.
|
Stage 4.
|
||||||
|
|
||||||
### Stage 2 — Fixed images built with nix (no Docker)
|
### Stage 2 — Fixed rootfs prebuilt + pulled as an artifact (no host Docker)
|
||||||
|
|
||||||
The images bot-bottle *ships* — the sidecar, the agent base, and the builder
|
The one fixed image the Firecracker backend needs at launch — the combined
|
||||||
(Stage 3) — are built declaratively with nix (`nixos-generators` /
|
**infra** rootfs the infra VM boots (orchestrator control plane + gateway +
|
||||||
`make-ext4-fs` / `pkgs.dockerTools` for the rootfs), producing an ext4 or
|
buildah, with the control-plane init as PID 1) — is **prebuilt end-to-end off
|
||||||
tar with correct ownership. Removes Docker for everything we own and gives
|
the launch host and published as a versioned, ready-to-boot ext4 artifact**.
|
||||||
the rootless-rootfs correctness (#347) for free on these images.
|
The launch host **downloads the `.ext4` and boots it directly** — no
|
||||||
|
`docker build`, no `docker export`, no `mke2fs`, no image tooling at all.
|
||||||
|
|
||||||
|
This is possible because the infra rootfs is already **host- and
|
||||||
|
bottle-agnostic**: the per-boot bits (authorized_keys, guest IP) arrive on the
|
||||||
|
**kernel cmdline**, not in the rootfs (see `build_base_rootfs_dir`). So one
|
||||||
|
published ext4 boots on any launch host.
|
||||||
|
|
||||||
|
- **Artifact.** `rootfs.ext4` + a `rootfs.ext4.sha256`, published as a Gitea
|
||||||
|
**generic package** (`bot-bottle-firecracker-infra/<tag>`) — generic packages take
|
||||||
|
arbitrary large binaries (no attachment size cap / file-type allowlist that
|
||||||
|
release attachments impose). The matching `vmlinux` kernel can ship the same
|
||||||
|
way, so the whole VM is fetchable.
|
||||||
|
- **Pull.** The launch host `GET`s
|
||||||
|
`…/api/packages/<owner>/generic/bot-bottle-firecracker-infra/<tag>/rootfs.ext4` (+
|
||||||
|
`.sha256`) for its pinned tag, verifies the checksum, caches it under the
|
||||||
|
tag, and attaches it as the infra VM's root disk. Host prerequisite is an
|
||||||
|
HTTP client — nothing else. Public packages need no auth to pull; a token
|
||||||
|
with `read:package` covers a private instance.
|
||||||
|
- **Registry.** The artifact base URL + owner are configurable, defaulting to
|
||||||
|
this deployment's Gitea (`https://gitea.dideric.is` / `didericis`);
|
||||||
|
overridable via env for other deployments / air-gapped mirrors.
|
||||||
|
- **Versioning.** A pinned tag bumped when the infra rootfs contents change
|
||||||
|
(bot_bottle's shipped files, the base deps, or the init), so a launch host
|
||||||
|
pulls the artifact matching its code and a content change can't silently
|
||||||
|
boot a stale rootfs. A checksum mismatch fails closed.
|
||||||
|
- **Publish.** A `publish` step (CLI subcommand / CI job) runs the full
|
||||||
|
pipeline **on a build/CI host** — `docker build` the three Dockerfiles →
|
||||||
|
export → inject guest boot → `mke2fs` → upload the `.ext4` + `.sha256`.
|
||||||
|
Building still uses Docker, but never on the launch/runner host, which is
|
||||||
|
the one #348 needs unprivileged.
|
||||||
|
- **Dev escape hatch.** An explicit opt-in still builds the rootfs locally
|
||||||
|
with Docker (for iterating on the Dockerfiles without a publish
|
||||||
|
round-trip); it is never the default path.
|
||||||
|
|
||||||
|
Removes Docker from the launch host entirely for the fixed image, and the
|
||||||
|
launch host needs no OCI/rootfs tooling — just fetch + boot. The build-time
|
||||||
|
cache / build-time-egress open problems a from-scratch build would face don't
|
||||||
|
arise: the launch host never builds, it downloads a finished disk.
|
||||||
|
|
||||||
### Stage 3 — User Dockerfiles built in a builder VM (the unlock)
|
### Stage 3 — User Dockerfiles built in a builder VM (the unlock)
|
||||||
|
|
||||||
|
|||||||
@@ -205,25 +205,29 @@ class TestFirecrackerFreezer(_FakeHomeMixin, unittest.TestCase):
|
|||||||
)
|
)
|
||||||
|
|
||||||
def test_snapshots_running_vm_without_stopping(self):
|
def test_snapshots_running_vm_without_stopping(self):
|
||||||
"""Commit should tar the running guest rootfs over SSH, not stop it."""
|
"""Commit should tar the running guest rootfs over SSH into the
|
||||||
|
committed-rootfs artifact (no Docker), not stop the VM."""
|
||||||
slug = "dev-abc12"
|
slug = "dev-abc12"
|
||||||
self._write_meta(slug)
|
self._write_meta(slug)
|
||||||
self._stage_run_dir(slug)
|
self._stage_run_dir(slug)
|
||||||
freezer = FirecrackerFreezer()
|
freezer = FirecrackerFreezer()
|
||||||
agent = _make_agent(slug, "firecracker")
|
agent = _make_agent(slug, "firecracker")
|
||||||
|
|
||||||
with patch("bot_bottle.backend.firecracker.freezer._commit_via_ssh") as mock_commit, \
|
commit_fn = "bot_bottle.backend.firecracker.freezer._commit_rootfs_via_ssh"
|
||||||
|
with patch(commit_fn) as mock_commit, \
|
||||||
patch("bot_bottle.backend.freeze.info"), \
|
patch("bot_bottle.backend.freeze.info"), \
|
||||||
patch("bot_bottle.backend.firecracker.freezer.info"):
|
patch("bot_bottle.backend.firecracker.freezer.info"):
|
||||||
freezer.commit(agent)
|
freezer.commit(agent)
|
||||||
|
|
||||||
image_tag = f"bot-bottle-committed-{slug}:latest"
|
tar_path = bottle_state.committed_rootfs_path(slug)
|
||||||
self.assertEqual(1, mock_commit.call_count)
|
self.assertEqual(1, mock_commit.call_count)
|
||||||
# (private_key, guest_ip, image_tag) — guest_ip parsed from config.
|
# (private_key, guest_ip, tar_path) — guest_ip parsed from config.
|
||||||
args = mock_commit.call_args.args
|
args = mock_commit.call_args.args
|
||||||
self.assertEqual("100.64.0.1", args[1])
|
self.assertEqual("100.64.0.1", args[1])
|
||||||
self.assertEqual(image_tag, args[2])
|
self.assertEqual(tar_path, args[2])
|
||||||
self.assertEqual(image_tag, bottle_state.read_committed_image(slug))
|
# The committed-image state records the artifact path; resume boots
|
||||||
|
# from the tar rather than a Docker image.
|
||||||
|
self.assertEqual(str(tar_path), bottle_state.read_committed_image(slug))
|
||||||
self.assertTrue(bottle_state.is_preserved(slug))
|
self.assertTrue(bottle_state.is_preserved(slug))
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -6,10 +6,13 @@ branches. Mock subprocess/os so nothing needs KVM or a live VM.
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
|
import stat
|
||||||
import subprocess
|
import subprocess
|
||||||
import tempfile
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
from bot_bottle.backend.firecracker import firecracker_vm, freezer, netpool, util
|
from bot_bottle.backend.firecracker import firecracker_vm, freezer, netpool, util
|
||||||
@@ -128,5 +131,172 @@ class TestRequireFirecracker(unittest.TestCase):
|
|||||||
util.require_firecracker()
|
util.require_firecracker()
|
||||||
|
|
||||||
|
|
||||||
|
class TestBuildCommittedRootfsDir(unittest.TestCase):
|
||||||
|
"""Resume prepares the base rootfs dir from the freezer's snapshot tar
|
||||||
|
with no Docker: extract, recreate the excluded mount points, inject the
|
||||||
|
guest boot bits."""
|
||||||
|
|
||||||
|
def _make_tar(self, tmp: Path) -> Path:
|
||||||
|
import tarfile
|
||||||
|
|
||||||
|
src = tmp / "src"
|
||||||
|
(src / "home" / "node").mkdir(parents=True)
|
||||||
|
(src / "home" / "node" / "hello").write_text("hi")
|
||||||
|
tar_path = tmp / "rootfs.tar"
|
||||||
|
with tarfile.open(tar_path, "w") as tar:
|
||||||
|
tar.add(src, arcname=".")
|
||||||
|
return tar_path
|
||||||
|
|
||||||
|
def test_extracts_recreates_mountpoints_and_injects_boot(self):
|
||||||
|
with tempfile.TemporaryDirectory(prefix="fc-committed.") as d:
|
||||||
|
tmp = Path(d)
|
||||||
|
tar_path = self._make_tar(tmp)
|
||||||
|
# Stand in for the static dropbear that inject_guest_boot copies.
|
||||||
|
dropbear = tmp / "dropbear"
|
||||||
|
dropbear.write_text("#!/bin/true\n")
|
||||||
|
cache = tmp / "cache"
|
||||||
|
cache.mkdir()
|
||||||
|
|
||||||
|
with patch.object(util, "cache_dir", return_value=cache), \
|
||||||
|
patch.object(util, "dropbear_path", return_value=dropbear), \
|
||||||
|
patch.object(util, "info"):
|
||||||
|
base = util.build_committed_rootfs_dir(tar_path)
|
||||||
|
|
||||||
|
self.assertEqual("hi", (base / "home" / "node" / "hello").read_text())
|
||||||
|
for mount_point in ("proc", "sys", "dev", "run"):
|
||||||
|
self.assertTrue((base / mount_point).is_dir(),
|
||||||
|
f"missing recreated mount point /{mount_point}")
|
||||||
|
self.assertTrue((base / "bb-dropbear").is_file())
|
||||||
|
self.assertTrue((base / "bb-init").is_file())
|
||||||
|
self.assertTrue((base / ".bb-ready").is_file())
|
||||||
|
|
||||||
|
def test_caches_on_repeat_and_reextracts_after_refreeze(self):
|
||||||
|
with tempfile.TemporaryDirectory(prefix="fc-committed.") as d:
|
||||||
|
tmp = Path(d)
|
||||||
|
tar_path = self._make_tar(tmp)
|
||||||
|
dropbear = tmp / "dropbear"
|
||||||
|
dropbear.write_text("#!/bin/true\n")
|
||||||
|
cache = tmp / "cache"
|
||||||
|
cache.mkdir()
|
||||||
|
|
||||||
|
ctx = [
|
||||||
|
patch.object(util, "cache_dir", return_value=cache),
|
||||||
|
patch.object(util, "dropbear_path", return_value=dropbear),
|
||||||
|
patch.object(util, "info"),
|
||||||
|
]
|
||||||
|
for c in ctx:
|
||||||
|
c.start()
|
||||||
|
self.addCleanup(lambda: [c.stop() for c in ctx])
|
||||||
|
|
||||||
|
real_run = subprocess.run
|
||||||
|
calls = {"n": 0}
|
||||||
|
|
||||||
|
def counting_run(argv: list[str], *a: Any, **k: Any) -> Any:
|
||||||
|
if argv and argv[0] == "tar":
|
||||||
|
calls["n"] += 1
|
||||||
|
return real_run(argv, *a, **k)
|
||||||
|
|
||||||
|
with patch.object(util.subprocess, "run", side_effect=counting_run):
|
||||||
|
first = util.build_committed_rootfs_dir(tar_path)
|
||||||
|
second = util.build_committed_rootfs_dir(tar_path)
|
||||||
|
self.assertEqual(first, second)
|
||||||
|
self.assertEqual(1, calls["n"]) # cached — no re-extract
|
||||||
|
|
||||||
|
# A re-freeze rewrites the tar; a new size/mtime -> new cache
|
||||||
|
# key -> re-extract. Force a distinct mtime so the test isn't
|
||||||
|
# at the mercy of filesystem timestamp granularity.
|
||||||
|
import tarfile
|
||||||
|
extra = tmp / "extra"
|
||||||
|
extra.mkdir()
|
||||||
|
(extra / "note").write_text("v2")
|
||||||
|
with tarfile.open(tar_path, "w") as tar:
|
||||||
|
tar.add(extra, arcname=".")
|
||||||
|
st = tar_path.stat()
|
||||||
|
os.utime(tar_path, ns=(st.st_atime_ns, st.st_mtime_ns + 1_000_000_000))
|
||||||
|
|
||||||
|
third = util.build_committed_rootfs_dir(tar_path)
|
||||||
|
self.assertNotEqual(first, third)
|
||||||
|
self.assertEqual(2, calls["n"])
|
||||||
|
|
||||||
|
|
||||||
|
class TestInjectGuestBootSymlinkSafe(unittest.TestCase):
|
||||||
|
"""A committed snapshot is guest-controlled: inject_guest_boot must not
|
||||||
|
follow a planted symlink and overwrite a host file during resume."""
|
||||||
|
|
||||||
|
def test_planted_symlink_does_not_escape_staging_tree(self):
|
||||||
|
with tempfile.TemporaryDirectory(prefix="fc-inject.") as d:
|
||||||
|
tmp = Path(d)
|
||||||
|
dropbear = tmp / "dropbear"
|
||||||
|
dropbear.write_bytes(b"DROPBEAR")
|
||||||
|
# A host file the malicious snapshot tries to clobber.
|
||||||
|
victim = tmp / "victim"
|
||||||
|
victim.write_text("original")
|
||||||
|
|
||||||
|
rootfs = tmp / "rootfs"
|
||||||
|
rootfs.mkdir()
|
||||||
|
# The snapshot planted bb-init/bb-dropbear as symlinks to it.
|
||||||
|
(rootfs / "bb-init").symlink_to(victim)
|
||||||
|
(rootfs / "bb-dropbear").symlink_to(victim)
|
||||||
|
|
||||||
|
with patch.object(util, "dropbear_path", return_value=dropbear):
|
||||||
|
util.inject_guest_boot(rootfs, init_script="#!/bin/sh\nreal\n")
|
||||||
|
|
||||||
|
# Host file untouched; the staged paths are fresh regular files.
|
||||||
|
self.assertEqual("original", victim.read_text())
|
||||||
|
self.assertFalse((rootfs / "bb-init").is_symlink())
|
||||||
|
self.assertFalse((rootfs / "bb-dropbear").is_symlink())
|
||||||
|
self.assertEqual("#!/bin/sh\nreal\n", (rootfs / "bb-init").read_text())
|
||||||
|
self.assertEqual(b"DROPBEAR", (rootfs / "bb-dropbear").read_bytes())
|
||||||
|
|
||||||
|
|
||||||
|
class TestCommitRootfsPermissions(unittest.TestCase):
|
||||||
|
"""The snapshot tar can hold the bottle's private workspace, so the
|
||||||
|
freezer must write it owner-only (0600)."""
|
||||||
|
|
||||||
|
def _commit(self, tar_path: Path) -> int:
|
||||||
|
"""Run _commit_rootfs_via_ssh with a stubbed ssh|tar pipe; return the
|
||||||
|
mode of the open partial observed mid-stream (from subprocess.run)."""
|
||||||
|
key = tar_path.parent.parent / "key"
|
||||||
|
key.write_text("K")
|
||||||
|
seen: dict[str, int] = {}
|
||||||
|
|
||||||
|
def fake_run(argv: list[str], *a: Any, **k: Any) -> Any:
|
||||||
|
out = k["stdout"]
|
||||||
|
seen["mode"] = stat.S_IMODE(os.fstat(out.fileno()).st_mode)
|
||||||
|
out.write(b"TARDATA")
|
||||||
|
return subprocess.CompletedProcess(argv, 0, b"", b"")
|
||||||
|
|
||||||
|
with patch.object(freezer.util, "ssh_base_argv", return_value=["ssh"]), \
|
||||||
|
patch.object(freezer.subprocess, "run", side_effect=fake_run):
|
||||||
|
freezer._commit_rootfs_via_ssh(key, "10.0.0.1", tar_path)
|
||||||
|
return seen["mode"]
|
||||||
|
|
||||||
|
def test_snapshot_created_owner_only(self):
|
||||||
|
with tempfile.TemporaryDirectory(prefix="fc-freeze.") as d:
|
||||||
|
tar_path = Path(d) / "state" / "committed-rootfs.tar"
|
||||||
|
tar_path.parent.mkdir()
|
||||||
|
stream_mode = self._commit(tar_path)
|
||||||
|
|
||||||
|
self.assertEqual(0o600, stream_mode) # private during the stream
|
||||||
|
self.assertEqual(b"TARDATA", tar_path.read_bytes())
|
||||||
|
self.assertEqual(0o600, stat.S_IMODE(tar_path.stat().st_mode))
|
||||||
|
|
||||||
|
def test_leftover_world_readable_partial_is_recreated_private(self):
|
||||||
|
"""A partial left 0644 by an interrupted prior run must not keep the
|
||||||
|
new snapshot world-readable while it streams."""
|
||||||
|
with tempfile.TemporaryDirectory(prefix="fc-freeze.") as d:
|
||||||
|
tar_path = Path(d) / "state" / "committed-rootfs.tar"
|
||||||
|
tar_path.parent.mkdir()
|
||||||
|
partial = tar_path.with_name(tar_path.name + ".partial")
|
||||||
|
partial.write_bytes(b"stale")
|
||||||
|
os.chmod(partial, 0o644)
|
||||||
|
|
||||||
|
stream_mode = self._commit(tar_path)
|
||||||
|
|
||||||
|
self.assertEqual(0o600, stream_mode)
|
||||||
|
self.assertEqual(b"TARDATA", tar_path.read_bytes())
|
||||||
|
self.assertEqual(0o600, stat.S_IMODE(tar_path.stat().st_mode))
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ decisions that must hold without a VM.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest.mock import MagicMock, patch
|
from unittest.mock import MagicMock, patch
|
||||||
@@ -95,8 +96,17 @@ class TestRegistryVolume(unittest.TestCase):
|
|||||||
|
|
||||||
|
|
||||||
class TestEnsureBuilt(unittest.TestCase):
|
class TestEnsureBuilt(unittest.TestCase):
|
||||||
def test_builds_deps_before_infra(self):
|
def test_default_pulls_artifact_without_docker(self):
|
||||||
with patch.object(infra_vm.docker_mod, "build_image") as build:
|
# PRD 0069 Stage 2: the launch host pulls the prebuilt rootfs; no Docker.
|
||||||
|
with patch.object(infra_vm.docker_mod, "build_image") as build, \
|
||||||
|
patch.object(infra_vm.infra_artifact, "ensure_artifact_gz") as pull:
|
||||||
|
infra_vm.ensure_built()
|
||||||
|
build.assert_not_called()
|
||||||
|
pull.assert_called_once()
|
||||||
|
|
||||||
|
def test_local_mode_builds_deps_before_infra(self):
|
||||||
|
with patch.dict(os.environ, {"BOT_BOTTLE_INFRA_BUILD": "local"}), \
|
||||||
|
patch.object(infra_vm.docker_mod, "build_image") as build:
|
||||||
infra_vm.ensure_built()
|
infra_vm.ensure_built()
|
||||||
tags = [c.args[0] for c in build.call_args_list]
|
tags = [c.args[0] for c in build.call_args_list]
|
||||||
# infra is FROM gateway and COPY --from orchestrator, so both first.
|
# infra is FROM gateway and COPY --from orchestrator, so both first.
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ Covers the pure `git_gate_render_gitconfig` renderer and the dynamic
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import socket
|
||||||
import tempfile
|
import tempfile
|
||||||
import types
|
import types
|
||||||
import unittest
|
import unittest
|
||||||
@@ -126,8 +127,9 @@ class TestProvisionDynamicKey(unittest.TestCase):
|
|||||||
self.assertEqual(b"PRIVATE-KEY-BYTES", key_file.read_bytes())
|
self.assertEqual(b"PRIVATE-KEY-BYTES", key_file.read_bytes())
|
||||||
id_file = Path(d) / "repo-deploy-key-id"
|
id_file = Path(d) / "repo-deploy-key-id"
|
||||||
self.assertEqual("kid123", id_file.read_text())
|
self.assertEqual("kid123", id_file.read_text())
|
||||||
# owner_repo had .git stripped; title carries slug + name
|
# owner_repo had .git stripped; title carries globalize_slug(slug) + name
|
||||||
self.assertEqual([("o/r", "bot-bottle:myslug:repo")], fake.created)
|
hostname = socket.gethostname()
|
||||||
|
self.assertEqual([("o/r", f"bot-bottle:{hostname}-myslug:repo")], fake.created)
|
||||||
|
|
||||||
def test_missing_token_raises(self) -> None:
|
def test_missing_token_raises(self) -> None:
|
||||||
with tempfile.TemporaryDirectory() as d, \
|
with tempfile.TemporaryDirectory() as d, \
|
||||||
|
|||||||
@@ -0,0 +1,182 @@
|
|||||||
|
"""Unit: the prebuilt infra-rootfs artifact pull (PRD 0069 Stage 2).
|
||||||
|
|
||||||
|
The launch-host half — version hashing and download/verify/decompress — is
|
||||||
|
what keeps a docker-free host from booting a stale or corrupted rootfs, so the
|
||||||
|
checksum + fail-closed paths are locked here. Network is mocked; no Docker.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import io
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
from bot_bottle.backend.firecracker import infra_artifact as ia
|
||||||
|
from bot_bottle.log import Die
|
||||||
|
|
||||||
|
|
||||||
|
def _gz(data: bytes) -> bytes:
|
||||||
|
return gzip.compress(data)
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeNet:
|
||||||
|
"""Map artifact URLs to bytes (or an HTTPError) for urlopen."""
|
||||||
|
|
||||||
|
def __init__(self, responses: "dict[str, bytes | Exception]") -> None:
|
||||||
|
self._responses = responses
|
||||||
|
self.calls: list[str] = []
|
||||||
|
|
||||||
|
def urlopen(self, req: urllib.request.Request, *a: object, **k: object) -> io.BytesIO:
|
||||||
|
url = req.full_url
|
||||||
|
self.calls.append(url)
|
||||||
|
val = self._responses.get(url)
|
||||||
|
if isinstance(val, Exception):
|
||||||
|
raise val
|
||||||
|
if val is None:
|
||||||
|
raise urllib.error.HTTPError(url, 404, "not found", {}, None) # type: ignore[arg-type]
|
||||||
|
return io.BytesIO(val)
|
||||||
|
|
||||||
|
|
||||||
|
class _CacheMixin(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self._tmp = tempfile.TemporaryDirectory()
|
||||||
|
self._env = mock.patch.dict(
|
||||||
|
os.environ,
|
||||||
|
{"BOT_BOTTLE_FC_CACHE": self._tmp.name,
|
||||||
|
"BOT_BOTTLE_INFRA_ARTIFACT_TOKEN": ""},
|
||||||
|
clear=False,
|
||||||
|
)
|
||||||
|
self._env.start()
|
||||||
|
self.addCleanup(self._env.stop)
|
||||||
|
self.addCleanup(self._tmp.cleanup)
|
||||||
|
|
||||||
|
def _serve(self, version: str, gz_bytes: bytes, sha_text: str | None = None):
|
||||||
|
if sha_text is None:
|
||||||
|
sha_text = f"{hashlib.sha256(gz_bytes).hexdigest()} rootfs.ext4.gz\n"
|
||||||
|
net = _FakeNet({
|
||||||
|
ia.artifact_url(version, "rootfs.ext4.gz"): gz_bytes,
|
||||||
|
ia.artifact_url(version, "rootfs.ext4.gz.sha256"): sha_text.encode(),
|
||||||
|
})
|
||||||
|
return mock.patch.object(ia.urllib.request, "urlopen", net.urlopen), net
|
||||||
|
|
||||||
|
|
||||||
|
class TestVersion(unittest.TestCase):
|
||||||
|
def test_deterministic_16_hex(self) -> None:
|
||||||
|
v = ia.infra_artifact_version("#!/bin/sh\ntrue\n")
|
||||||
|
self.assertEqual(v, ia.infra_artifact_version("#!/bin/sh\ntrue\n"))
|
||||||
|
self.assertEqual(16, len(v))
|
||||||
|
int(v, 16) # hex
|
||||||
|
|
||||||
|
def test_init_change_bumps_version(self) -> None:
|
||||||
|
self.assertNotEqual(
|
||||||
|
ia.infra_artifact_version("a"), ia.infra_artifact_version("b"))
|
||||||
|
|
||||||
|
|
||||||
|
class TestVersionInputs(unittest.TestCase):
|
||||||
|
"""The hash must cover *every* file baked into the rootfs, not just `*.py`
|
||||||
|
(`COPY bot_bottle` is wholesale) — else a non-Python change (e.g. the egress
|
||||||
|
entrypoint shell script) leaves the version unchanged and a launch host
|
||||||
|
boots a rootfs whose code differs from its checkout."""
|
||||||
|
|
||||||
|
def _fake_repo(self, root: Path) -> None:
|
||||||
|
pkg = root / "bot_bottle"
|
||||||
|
pkg.mkdir()
|
||||||
|
(pkg / "app.py").write_text("print('hi')\n")
|
||||||
|
(pkg / "egress_entrypoint.sh").write_text("#!/bin/sh\nexec mitmdump\n")
|
||||||
|
(pkg / "netpool.defaults.env").write_text("FOO=1\n")
|
||||||
|
for name in ("Dockerfile.orchestrator", "Dockerfile.gateway", "Dockerfile.infra"):
|
||||||
|
(root / name).write_text(f"FROM scratch # {name}\n")
|
||||||
|
|
||||||
|
def test_non_python_file_change_bumps_version(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
root = Path(d)
|
||||||
|
self._fake_repo(root)
|
||||||
|
before = ia.infra_artifact_version("init", repo_root=root)
|
||||||
|
(root / "bot_bottle" / "egress_entrypoint.sh").write_text(
|
||||||
|
"#!/bin/sh\nexec mitmdump --different\n")
|
||||||
|
after = ia.infra_artifact_version("init", repo_root=root)
|
||||||
|
self.assertNotEqual(before, after)
|
||||||
|
|
||||||
|
def test_pyc_and_pycache_ignored(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
root = Path(d)
|
||||||
|
self._fake_repo(root)
|
||||||
|
before = ia.infra_artifact_version("init", repo_root=root)
|
||||||
|
cache = root / "bot_bottle" / "__pycache__"
|
||||||
|
cache.mkdir()
|
||||||
|
(cache / "app.cpython-312.pyc").write_bytes(b"\x00bytecode")
|
||||||
|
(root / "bot_bottle" / "app.pyc").write_bytes(b"\x00bytecode")
|
||||||
|
after = ia.infra_artifact_version("init", repo_root=root)
|
||||||
|
self.assertEqual(before, after)
|
||||||
|
|
||||||
|
|
||||||
|
class TestEnsureArtifact(_CacheMixin):
|
||||||
|
def test_downloads_verifies_and_caches(self) -> None:
|
||||||
|
version = "deadbeef00000000"
|
||||||
|
gz = _gz(b"fake ext4 bytes")
|
||||||
|
patcher, net = self._serve(version, gz)
|
||||||
|
with patcher:
|
||||||
|
path = ia.ensure_artifact_gz(version)
|
||||||
|
self.assertTrue(path.is_file())
|
||||||
|
self.assertEqual(gz, path.read_bytes())
|
||||||
|
first_calls = len(net.calls)
|
||||||
|
# Second call is a cache hit — no further network.
|
||||||
|
ia.ensure_artifact_gz(version)
|
||||||
|
self.assertEqual(first_calls, len(net.calls))
|
||||||
|
|
||||||
|
def test_checksum_mismatch_fails_closed(self) -> None:
|
||||||
|
version = "beefbeefbeefbeef"
|
||||||
|
gz = _gz(b"payload")
|
||||||
|
patcher, _ = self._serve(version, gz, sha_text="0" * 64 + " rootfs.ext4.gz\n")
|
||||||
|
with patcher:
|
||||||
|
with self.assertRaises(Die) as ctx:
|
||||||
|
ia.ensure_artifact_gz(version)
|
||||||
|
self.assertIn("checksum mismatch", str(ctx.exception.message))
|
||||||
|
# nothing left cached to accidentally boot
|
||||||
|
self.assertFalse((ia._cache_root(version) / "rootfs.ext4.gz").exists())
|
||||||
|
|
||||||
|
def test_missing_artifact_points_at_publish(self) -> None:
|
||||||
|
version = "0000000000000000"
|
||||||
|
net = _FakeNet({}) # everything 404s
|
||||||
|
with mock.patch.object(ia.urllib.request, "urlopen", net.urlopen):
|
||||||
|
with self.assertRaises(Die) as ctx:
|
||||||
|
ia.ensure_artifact_gz(version)
|
||||||
|
self.assertIn("publish_infra", str(ctx.exception.message))
|
||||||
|
|
||||||
|
def test_materialize_gunzips_to_dest(self) -> None:
|
||||||
|
version = "1234123412341234"
|
||||||
|
raw = b"the real rootfs contents" * 100
|
||||||
|
patcher, _ = self._serve(version, _gz(raw))
|
||||||
|
with patcher, tempfile.TemporaryDirectory() as d:
|
||||||
|
dest = Path(d) / "rootfs.ext4"
|
||||||
|
ia.materialize_ext4(version, dest)
|
||||||
|
self.assertEqual(raw, dest.read_bytes())
|
||||||
|
|
||||||
|
|
||||||
|
class TestConfig(unittest.TestCase):
|
||||||
|
def test_base_and_owner_overridable(self) -> None:
|
||||||
|
with mock.patch.dict(os.environ, {
|
||||||
|
"BOT_BOTTLE_INFRA_ARTIFACT_BASE": "https://mirror.example/",
|
||||||
|
"BOT_BOTTLE_INFRA_ARTIFACT_OWNER": "acme",
|
||||||
|
}):
|
||||||
|
url = ia.artifact_url("v1", "rootfs.ext4.gz")
|
||||||
|
self.assertEqual(
|
||||||
|
"https://mirror.example/api/packages/acme/generic/"
|
||||||
|
"bot-bottle-firecracker-infra/v1/rootfs.ext4.gz", url)
|
||||||
|
|
||||||
|
def test_local_build_flag(self) -> None:
|
||||||
|
with mock.patch.dict(os.environ, {"BOT_BOTTLE_INFRA_BUILD": "local"}):
|
||||||
|
self.assertTrue(ia.local_build_requested())
|
||||||
|
with mock.patch.dict(os.environ, {"BOT_BOTTLE_INFRA_BUILD": ""}):
|
||||||
|
self.assertFalse(ia.local_build_requested())
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
"""Unit: the infra-artifact publisher's upload path (PRD 0069 Stage 2).
|
||||||
|
|
||||||
|
The rootfs is hundreds of MB, so `_put` must stream it from disk rather than
|
||||||
|
read it into memory. Network is mocked; no Docker, no real build.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
import urllib.request
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
from bot_bottle.backend.firecracker import publish_infra as pub
|
||||||
|
|
||||||
|
|
||||||
|
class _Resp:
|
||||||
|
status = 201
|
||||||
|
|
||||||
|
def __enter__(self) -> "_Resp":
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *a: object) -> bool:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
class TestPut(unittest.TestCase):
|
||||||
|
def test_streams_file_body_with_content_length(self) -> None:
|
||||||
|
captured: list[urllib.request.Request] = []
|
||||||
|
|
||||||
|
def fake_urlopen(req: urllib.request.Request, *a: object, **k: object) -> _Resp:
|
||||||
|
captured.append(req)
|
||||||
|
return _Resp()
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
f = Path(d) / "rootfs.ext4.gz"
|
||||||
|
payload = b"x" * 4096
|
||||||
|
f.write_bytes(payload)
|
||||||
|
with mock.patch.object(pub.urllib.request, "urlopen", fake_urlopen):
|
||||||
|
pub._put("https://reg/pkg", f, token="t")
|
||||||
|
|
||||||
|
req = captured[0]
|
||||||
|
# Body is the open file object (streamed), never the bytes in memory.
|
||||||
|
self.assertTrue(hasattr(req.data, "read"))
|
||||||
|
self.assertNotIsInstance(req.data, (bytes, bytearray))
|
||||||
|
self.assertEqual(str(len(payload)), req.get_header("Content-length"))
|
||||||
|
|
||||||
|
def test_small_bytes_body_still_works(self) -> None:
|
||||||
|
captured: list[urllib.request.Request] = []
|
||||||
|
|
||||||
|
def fake_urlopen(req: urllib.request.Request, *a: object, **k: object) -> _Resp:
|
||||||
|
captured.append(req)
|
||||||
|
return _Resp()
|
||||||
|
|
||||||
|
with mock.patch.object(pub.urllib.request, "urlopen", fake_urlopen):
|
||||||
|
pub._put("https://reg/sha", b"abc123 rootfs\n", token="")
|
||||||
|
self.assertEqual(b"abc123 rootfs\n", captured[0].data)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user