Compare commits
20 Commits
ef9f81ba83
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 6fea44067f | |||
| bf8ff91b31 | |||
| 315ed04979 | |||
| 9a04ab262b | |||
| cc094765fd | |||
| 0ba25352b9 | |||
| dba48706de | |||
| 854a8956ad | |||
| 7a9628fc03 | |||
| ca8b2a9f2c | |||
| 96f5be48a6 | |||
| 82cf9bab5a | |||
| 3c92e79775 | |||
| 220620bfcc | |||
| b0f012b8e6 | |||
| fa9fed4194 | |||
| d0b595828f | |||
| 182a28d724 | |||
| cfb2284b99 | |||
| 2cd06814e6 |
@@ -1,8 +1,23 @@
|
|||||||
"""Cleanup for the Firecracker backend.
|
"""Cleanup for the Firecracker backend.
|
||||||
|
|
||||||
Orphans are: firecracker VMM processes whose config lives under our run
|
Reaps *orphans* only — resources with no live VM behind them:
|
||||||
dir, and the per-bottle run dirs. TAP slots free themselves (the flock
|
|
||||||
drops when the launcher exits), so there is nothing to reclaim there.
|
* orphan run dirs: a per-bottle run dir (holding the ~1G rootfs.ext4)
|
||||||
|
whose firecracker process has exited. These leak when a launch is
|
||||||
|
hard-killed before its teardown runs (host OOM/crash, a cancelled CI
|
||||||
|
job, `kill -9`); the clean-exit path already removes its own dir in
|
||||||
|
launch.py.
|
||||||
|
* orphan VM pids: a firecracker process whose run dir is already gone
|
||||||
|
— a VMM left lingering after its dir was removed.
|
||||||
|
|
||||||
|
A run dir with a *live* firecracker process is a running bottle and is
|
||||||
|
left strictly alone: it is neither killed nor removed. (The backend's
|
||||||
|
`enumerate_active` registry is still a stub — #354 — so a live process
|
||||||
|
is the only reliable "this bottle is in use" signal we have. Once the
|
||||||
|
registry lands, registry-orphaned-but-running VMs can be reaped too.)
|
||||||
|
|
||||||
|
TAP slots free themselves (the flock drops when the launcher exits), so
|
||||||
|
there is nothing to reclaim there.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -22,38 +37,73 @@ def _run_root() -> Path:
|
|||||||
return util.cache_dir() / "run"
|
return util.cache_dir() / "run"
|
||||||
|
|
||||||
|
|
||||||
def _orphan_vm_pids() -> list[int]:
|
def _run_dir_of(cmd: str, run_root: Path) -> Path | None:
|
||||||
"""firecracker processes whose --config-file is under our run dir."""
|
"""The bottle run dir a firecracker cmdline belongs to, or None.
|
||||||
run_root = str(_run_root())
|
|
||||||
|
A bottle VM is launched with `--config-file <run_root>/<slug>/config.json`,
|
||||||
|
so the run dir is the config file's parent when it sits directly under
|
||||||
|
the run root. Anything else (a builder VM, the infra VM elsewhere) is
|
||||||
|
not ours to reap here.
|
||||||
|
"""
|
||||||
|
toks = cmd.split()
|
||||||
|
for i, tok in enumerate(toks):
|
||||||
|
if tok == "--config-file" and i + 1 < len(toks):
|
||||||
|
parent = Path(toks[i + 1]).parent
|
||||||
|
if parent.parent == run_root:
|
||||||
|
return parent
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _scan_processes(run_root: Path) -> tuple[set[str], list[int]]:
|
||||||
|
"""Inspect running firecracker VMs under ``run_root``.
|
||||||
|
|
||||||
|
Returns ``(live_run_dirs, orphan_pids)``:
|
||||||
|
* ``live_run_dirs`` — run dirs backed by a running VM (never reaped);
|
||||||
|
* ``orphan_pids`` — firecracker pids whose run dir no longer exists
|
||||||
|
(a lingering VMM to kill).
|
||||||
|
"""
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
["pgrep", "-a", "firecracker"],
|
["pgrep", "-a", "firecracker"],
|
||||||
capture_output=True, text=True, check=False,
|
capture_output=True, text=True, check=False,
|
||||||
)
|
)
|
||||||
if result.returncode != 0:
|
if result.returncode != 0:
|
||||||
return []
|
return set(), []
|
||||||
pids: list[int] = []
|
live: set[str] = set()
|
||||||
|
orphan_pids: list[int] = []
|
||||||
for line in result.stdout.splitlines():
|
for line in result.stdout.splitlines():
|
||||||
parts = line.split(None, 1)
|
parts = line.split(None, 1)
|
||||||
if len(parts) != 2 or run_root not in parts[1]:
|
if len(parts) != 2:
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
pids.append(int(parts[0]))
|
pid = int(parts[0])
|
||||||
except ValueError:
|
except ValueError:
|
||||||
continue
|
continue
|
||||||
return pids
|
run_dir = _run_dir_of(parts[1], run_root)
|
||||||
|
if run_dir is None:
|
||||||
|
continue
|
||||||
|
if run_dir.is_dir():
|
||||||
|
live.add(str(run_dir))
|
||||||
|
else:
|
||||||
|
orphan_pids.append(pid)
|
||||||
|
return live, orphan_pids
|
||||||
|
|
||||||
|
|
||||||
def _run_dirs() -> list[str]:
|
def _orphan_run_dirs(run_root: Path, live: set[str]) -> list[str]:
|
||||||
run_root = _run_root()
|
"""Run dirs with no live VM behind them — the leaked ones to remove."""
|
||||||
if not run_root.is_dir():
|
if not run_root.is_dir():
|
||||||
return []
|
return []
|
||||||
return sorted(str(p) for p in run_root.iterdir() if p.is_dir())
|
return sorted(
|
||||||
|
str(p) for p in run_root.iterdir()
|
||||||
|
if p.is_dir() and str(p) not in live
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def prepare_cleanup() -> FirecrackerBottleCleanupPlan:
|
def prepare_cleanup() -> FirecrackerBottleCleanupPlan:
|
||||||
|
run_root = _run_root()
|
||||||
|
live, orphan_pids = _scan_processes(run_root)
|
||||||
return FirecrackerBottleCleanupPlan(
|
return FirecrackerBottleCleanupPlan(
|
||||||
vm_pids=tuple(_orphan_vm_pids()),
|
vm_pids=tuple(orphan_pids),
|
||||||
run_dirs=tuple(_run_dirs()),
|
run_dirs=tuple(_orphan_run_dirs(run_root, live)),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import dataclasses
|
import dataclasses
|
||||||
import os
|
import os
|
||||||
|
import shutil
|
||||||
from contextlib import ExitStack, contextmanager
|
from contextlib import ExitStack, contextmanager
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Callable, Generator
|
from typing import Callable, Generator
|
||||||
@@ -167,6 +168,10 @@ def launch(
|
|||||||
# Step 6: build the per-bottle rootfs + SSH key, then boot.
|
# Step 6: build the per-bottle rootfs + SSH key, then boot.
|
||||||
run_dir = util.cache_dir() / "run" / plan.slug
|
run_dir = util.cache_dir() / "run" / plan.slug
|
||||||
run_dir.mkdir(parents=True, exist_ok=True)
|
run_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
# Remove the run dir on teardown so the per-bottle rootfs.ext4 (~1G)
|
||||||
|
# doesn't leak. Registered before vm.terminate below so it runs *after*
|
||||||
|
# it (ExitStack is LIFO): the VM is gone before we rm its rootfs.
|
||||||
|
stack.callback(lambda: shutil.rmtree(run_dir, ignore_errors=True))
|
||||||
rootfs = run_dir / "rootfs.ext4"
|
rootfs = run_dir / "rootfs.ext4"
|
||||||
util.build_rootfs_ext4(agent_base, rootfs)
|
util.build_rootfs_ext4(agent_base, rootfs)
|
||||||
private_key, pubkey = util.generate_keypair(run_dir)
|
private_key, pubkey = util.generate_keypair(run_dir)
|
||||||
|
|||||||
@@ -19,11 +19,6 @@ what would send podman down the `newuidmap` path that cannot work here.
|
|||||||
The agent still talks to `docker` and `docker compose`; those speak to
|
The agent still talks to `docker` and `docker compose`; those speak to
|
||||||
podman's Docker-compatible API socket, so nothing in the agent's habits
|
podman's Docker-compatible API socket, so nothing in the agent's habits
|
||||||
changes.
|
changes.
|
||||||
|
|
||||||
Nested containers run *within* the bottle boundary, not inside a new one: the
|
|
||||||
single-UID mapping means `root` in a nested container is the agent user
|
|
||||||
outside it. This is for build and test workloads, not for sandboxing
|
|
||||||
untrusted code.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -65,9 +60,11 @@ def build_image(
|
|||||||
) -> str:
|
) -> str:
|
||||||
"""Layer the nested-container tooling onto an already-built agent image.
|
"""Layer the nested-container tooling onto an already-built agent image.
|
||||||
|
|
||||||
Only what the flag is meant to gate lands here. Podman itself is already
|
Podman and its networking stack live here rather than in the base agent
|
||||||
in every built-in agent image (issue #451); the storage/network helpers,
|
images so that bottles without the flag pay no image-size cost.
|
||||||
the Docker CLI, and the compose plugin are the ~100MB this flag buys.
|
|
||||||
|
# TODO(#394): replace this hand-rolled Dockerfile with a docker-layer
|
||||||
|
# abstraction once that infrastructure exists.
|
||||||
"""
|
"""
|
||||||
image = f"{base_image}{IMAGE_SUFFIX}"
|
image = f"{base_image}{IMAGE_SUFFIX}"
|
||||||
init_script = Path(__file__).with_name("nested-containers-init.sh")
|
init_script = Path(__file__).with_name("nested-containers-init.sh")
|
||||||
@@ -82,9 +79,11 @@ def build_image(
|
|||||||
"COPY --from=docker_cli /usr/local/libexec/docker/cli-plugins/"
|
"COPY --from=docker_cli /usr/local/libexec/docker/cli-plugins/"
|
||||||
"docker-compose /usr/local/libexec/docker/cli-plugins/docker-compose\n"
|
"docker-compose /usr/local/libexec/docker/cli-plugins/docker-compose\n"
|
||||||
"RUN apt-get update \\\n"
|
"RUN apt-get update \\\n"
|
||||||
# podman 5's networking stack, installed explicitly because the
|
# podman 5's networking stack, installed explicitly because
|
||||||
# base image's `--no-install-recommends` podman does not pull it
|
# --no-install-recommends omits it and each missing piece fails
|
||||||
# in and each piece fails at a different, misleading layer:
|
# at a different, misleading layer:
|
||||||
|
# podman -> moved here from the base agent images so that
|
||||||
|
# bottles without nested_containers pay no cost
|
||||||
# passt -> `pasta`, the default rootless netns helper
|
# passt -> `pasta`, the default rootless netns helper
|
||||||
# (podman 4 used slirp4netns); without it
|
# (podman 4 used slirp4netns); without it
|
||||||
# nothing starts: "could not find pasta"
|
# nothing starts: "could not find pasta"
|
||||||
@@ -95,7 +94,7 @@ def build_image(
|
|||||||
# looks healthy
|
# looks healthy
|
||||||
# slirp4netns stays as the documented fallback for pasta.
|
# slirp4netns stays as the documented fallback for pasta.
|
||||||
" && apt-get install -y --no-install-recommends "
|
" && apt-get install -y --no-install-recommends "
|
||||||
"aardvark-dns fuse-overlayfs netavark nftables passt "
|
"aardvark-dns fuse-overlayfs netavark nftables passt podman "
|
||||||
"slirp4netns uidmap \\\n"
|
"slirp4netns uidmap \\\n"
|
||||||
" && rm -rf /var/lib/apt/lists/* \\\n"
|
" && rm -rf /var/lib/apt/lists/* \\\n"
|
||||||
# Deliberate: an empty subordinate range keeps podman on the
|
# Deliberate: an empty subordinate range keeps podman on the
|
||||||
|
|||||||
@@ -26,7 +26,6 @@ RUN apt-get update \
|
|||||||
ca-certificates \
|
ca-certificates \
|
||||||
curl \
|
curl \
|
||||||
openssh-client \
|
openssh-client \
|
||||||
podman \
|
|
||||||
ripgrep \
|
ripgrep \
|
||||||
iproute2 \
|
iproute2 \
|
||||||
dnsutils \
|
dnsutils \
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ RUN apt-get update \
|
|||||||
ca-certificates \
|
ca-certificates \
|
||||||
curl \
|
curl \
|
||||||
openssh-client \
|
openssh-client \
|
||||||
podman \
|
|
||||||
procps \
|
procps \
|
||||||
ripgrep \
|
ripgrep \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ RUN apt-get update \
|
|||||||
curl \
|
curl \
|
||||||
fd-find \
|
fd-find \
|
||||||
openssh-client \
|
openssh-client \
|
||||||
podman \
|
|
||||||
ripgrep \
|
ripgrep \
|
||||||
&& ln -s /usr/bin/fdfind /usr/local/bin/fd \
|
&& ln -s /usr/bin/fdfind /usr/local/bin/fd \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|||||||
@@ -49,6 +49,10 @@ class ManifestBottle:
|
|||||||
# costs image weight, a resident service, and relaxed guest device modes
|
# costs image weight, a resident service, and relaxed guest device modes
|
||||||
# that the majority of bottles never need.
|
# that the majority of bottles never need.
|
||||||
nested_containers: bool = False
|
nested_containers: bool = False
|
||||||
|
# Source fields retained across extends/runtime composition. Boolean
|
||||||
|
# defaults otherwise erase the distinction between "omitted" and an
|
||||||
|
# explicitly declared value (especially False).
|
||||||
|
declared_fields: frozenset[str] = frozenset()
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_dict(cls, name: str, raw: object) -> "ManifestBottle":
|
def from_dict(cls, name: str, raw: object) -> "ManifestBottle":
|
||||||
@@ -139,4 +143,5 @@ class ManifestBottle:
|
|||||||
env=env, agent_provider=agent_provider, git=git,
|
env=env, agent_provider=agent_provider, git=git,
|
||||||
git_user=git_user, egress=egress, supervise=supervise_raw,
|
git_user=git_user, egress=egress, supervise=supervise_raw,
|
||||||
nested_containers=nested_raw,
|
nested_containers=nested_raw,
|
||||||
|
declared_fields=frozenset(d),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -8,6 +8,17 @@ from .manifest_git import ManifestGitUser, parse_git_gate_config
|
|||||||
from .manifest_util import ManifestError, as_json_object
|
from .manifest_util import ManifestError, as_json_object
|
||||||
|
|
||||||
|
|
||||||
|
def _overlay_declared_bool(
|
||||||
|
base: ManifestBottle,
|
||||||
|
override: ManifestBottle,
|
||||||
|
field: str,
|
||||||
|
) -> bool:
|
||||||
|
"""Overlay a defaulted boolean only when override declared it."""
|
||||||
|
value = getattr(override if field in override.declared_fields else base, field)
|
||||||
|
assert isinstance(value, bool)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
def merge_bottles_runtime(bottles: "list[ManifestBottle]") -> "ManifestBottle":
|
def merge_bottles_runtime(bottles: "list[ManifestBottle]") -> "ManifestBottle":
|
||||||
"""Merge an ordered list of pre-resolved ManifestBottle objects.
|
"""Merge an ordered list of pre-resolved ManifestBottle objects.
|
||||||
|
|
||||||
@@ -15,16 +26,13 @@ def merge_bottles_runtime(bottles: "list[ManifestBottle]") -> "ManifestBottle":
|
|||||||
the same field-merge rules as the file-based extends machinery:
|
the same field-merge rules as the file-based extends machinery:
|
||||||
env: dict merge, later wins; git_user: per-field overlay, later
|
env: dict merge, later wins; git_user: per-field overlay, later
|
||||||
wins on non-empty; git (repos): union by name, later wins; egress
|
wins on non-empty; git (repos): union by name, later wins; egress
|
||||||
routes: concatenate; agent_provider, supervise: later
|
routes: concatenate; agent_provider, supervise, nested_containers:
|
||||||
replaces; nested_containers: OR (see below).
|
later replaces (presence-aware).
|
||||||
|
|
||||||
nested_containers is OR'd rather than replaced because these objects
|
Defaulted booleans use presence-aware replacement: if the later bottle
|
||||||
are already resolved: a bottle that never mentions the key is
|
was loaded from a source that explicitly declared the key, its value
|
||||||
indistinguishable from one that sets it false, so "later replaces"
|
wins (so an explicit `false` can override an earlier `true`). If the
|
||||||
would let any bottle composed after a container-enabled one silently
|
later bottle never mentioned the key, the earlier value is preserved.
|
||||||
drop the capability. The file-based `extends:` path still sees the
|
|
||||||
raw keys, so there an explicit `nested_containers: false` in a child
|
|
||||||
turns it back off.
|
|
||||||
"""
|
"""
|
||||||
if not bottles:
|
if not bottles:
|
||||||
raise ValueError("merge_bottles_runtime requires at least one bottle")
|
raise ValueError("merge_bottles_runtime requires at least one bottle")
|
||||||
@@ -62,8 +70,11 @@ def _merge_two_bottles_runtime(base: "ManifestBottle", override: "ManifestBottle
|
|||||||
git=merged_git,
|
git=merged_git,
|
||||||
git_user=merged_git_user,
|
git_user=merged_git_user,
|
||||||
egress=merged_egress,
|
egress=merged_egress,
|
||||||
supervise=override.supervise,
|
supervise=_overlay_declared_bool(base, override, "supervise"),
|
||||||
nested_containers=base.nested_containers or override.nested_containers,
|
nested_containers=_overlay_declared_bool(
|
||||||
|
base, override, "nested_containers"
|
||||||
|
),
|
||||||
|
declared_fields=base.declared_fields | override.declared_fields,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -215,8 +226,11 @@ def _fold_two_bottles(
|
|||||||
git=merged_git,
|
git=merged_git,
|
||||||
git_user=merged_git_user,
|
git_user=merged_git_user,
|
||||||
egress=merged_egress,
|
egress=merged_egress,
|
||||||
supervise=later.supervise,
|
supervise=_overlay_declared_bool(earlier, later, "supervise"),
|
||||||
nested_containers=earlier.nested_containers or later.nested_containers,
|
nested_containers=_overlay_declared_bool(
|
||||||
|
earlier, later, "nested_containers"
|
||||||
|
),
|
||||||
|
declared_fields=earlier.declared_fields | later.declared_fields,
|
||||||
), merged_repos_raw
|
), merged_repos_raw
|
||||||
|
|
||||||
|
|
||||||
@@ -274,13 +288,9 @@ def _merge_bottles(
|
|||||||
if "agent_provider" in child_raw
|
if "agent_provider" in child_raw
|
||||||
else parent.agent_provider
|
else parent.agent_provider
|
||||||
)
|
)
|
||||||
merged_supervise = (
|
merged_supervise = _overlay_declared_bool(parent, child, "supervise")
|
||||||
child.supervise if "supervise" in child_raw else parent.supervise
|
merged_nested_containers = _overlay_declared_bool(
|
||||||
)
|
parent, child, "nested_containers"
|
||||||
merged_nested_containers = (
|
|
||||||
child.nested_containers
|
|
||||||
if "nested_containers" in child_raw
|
|
||||||
else parent.nested_containers
|
|
||||||
)
|
)
|
||||||
validate_egress_routes(name, merged_egress.routes)
|
validate_egress_routes(name, merged_egress.routes)
|
||||||
|
|
||||||
@@ -292,6 +302,7 @@ def _merge_bottles(
|
|||||||
egress=merged_egress,
|
egress=merged_egress,
|
||||||
supervise=merged_supervise,
|
supervise=merged_supervise,
|
||||||
nested_containers=merged_nested_containers,
|
nested_containers=merged_nested_containers,
|
||||||
|
declared_fields=parent.declared_fields | child.declared_fields,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,178 @@
|
|||||||
|
# Firecracker Image Remote Store
|
||||||
|
|
||||||
|
**Date:** 2026-07-22
|
||||||
|
**Context:** PR #459 (run-dir leak fix) surfaced that committed Firecracker snapshots
|
||||||
|
currently live only on the host machine. Once a host is wiped or a run-dir is
|
||||||
|
evicted, a user's preserved bottle is gone. This note investigates a secure remote
|
||||||
|
store so committed images survive host turnover and can be restored without the
|
||||||
|
user having to pre-flag which sessions to keep.
|
||||||
|
|
||||||
|
## Verdict
|
||||||
|
|
||||||
|
Backblaze B2 + Cloudflare CDN is the cost-optimal choice for most deployments.
|
||||||
|
Cloudflare R2 is the simpler zero-config option at slightly higher storage cost.
|
||||||
|
Self-hosted MinIO is the right call for air-gapped or on-premises installs.
|
||||||
|
|
||||||
|
On the image-size front, zstd-compressing the committed tar before upload
|
||||||
|
produces roughly a 60–70% reduction with negligible impact on restore latency.
|
||||||
|
OverlayFS (for in-flight working rootfs, not for the committed artifact) cuts
|
||||||
|
per-instance disk use to ~10–50 MB per extra bottle sharing the same base.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## What Gets Stored
|
||||||
|
|
||||||
|
The Firecracker backend produces two artifact types:
|
||||||
|
|
||||||
|
| Artifact | Created by | Size | Lifetime |
|
||||||
|
|----------|-----------|------|---------|
|
||||||
|
| `rootfs/agent-<digest>/` (dir) | `image_builder.py` → `mke2fs -d` | ~1 GB as ext4 | Cached per Dockerfile hash; evictable |
|
||||||
|
| `committed/<slug>/rootfs.tar` | `FirecrackerFreezer._freeze` via SSH tar | 500 MB–1 GB | User-preserved; must survive host wipe |
|
||||||
|
|
||||||
|
The committed artifact is a tar of the guest's live filesystem streamed out over
|
||||||
|
SSH (`freezer.py:58–90`). At resume time `launch.py` calls `mke2fs -d` to
|
||||||
|
rebuild a fresh ext4 from this tar. The tar — not the ext4 — is what needs to be
|
||||||
|
pushed to remote storage and pulled back at restore time.
|
||||||
|
|
||||||
|
The base image cache (`rootfs/agent-<digest>/`) is derivable from the Dockerfile
|
||||||
|
and can be rebuilt on demand; it is lower priority for remote storage.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Storage Candidates
|
||||||
|
|
||||||
|
### Object storage
|
||||||
|
|
||||||
|
| Provider | Storage | Egress | Notes |
|
||||||
|
|----------|---------|--------|-------|
|
||||||
|
| **Backblaze B2** | $0.006/GB | Free (via Cloudflare Bandwidth Alliance) | Cheapest storage; pairs with Cloudflare CDN to eliminate egress |
|
||||||
|
| **Cloudflare R2** | $0.015/GB | $0 always | Zero-config egress; no lifecycle transitions (limitation) |
|
||||||
|
| **Wasabi** | $0.0069/GB | Free (1:1 ratio) | 90-day minimum retention; good for archival; lifecycle evaluated daily |
|
||||||
|
| **AWS S3** | $0.023/GB | $0.09/GB | Richest lifecycle support; expensive at scale; avoid unless already in AWS |
|
||||||
|
| **MinIO** (self-hosted) | Host cost only | None | S3-compatible; best for private/on-prem deployments |
|
||||||
|
|
||||||
|
**B2 + Cloudflare CDN** is effectively $0.006/GB with zero egress — about 18×
|
||||||
|
cheaper than S3 for restore-heavy workloads. **R2** is the zero-config choice
|
||||||
|
($0 egress by default, no Bandwidth Alliance pairing needed) at a slightly
|
||||||
|
higher storage rate.
|
||||||
|
|
||||||
|
**Cloudflare R2's missing lifecycle support** is the main caveat: auto-eviction
|
||||||
|
rules (evict images older than N days) cannot currently be expressed natively in
|
||||||
|
R2. Wasabi and S3 both support declarative lifecycle policies.
|
||||||
|
|
||||||
|
### Retention policy recommendation
|
||||||
|
|
||||||
|
The comment proposes:
|
||||||
|
- Retain images for ~1 week by default
|
||||||
|
- Warn when approaching a capacity threshold
|
||||||
|
- Auto-evict oldest images once threshold is exceeded
|
||||||
|
|
||||||
|
This maps cleanly to an application-level policy (not a provider lifecycle rule),
|
||||||
|
which avoids the R2 limitation and works consistently across providers:
|
||||||
|
|
||||||
|
1. On `commit`: upload tar, record `(slug, size_bytes, uploaded_at)` in a local
|
||||||
|
or remote manifest file.
|
||||||
|
2. On startup / on `list`: scan the manifest, warn if total stored size exceeds
|
||||||
|
e.g. 80% of the configured threshold.
|
||||||
|
3. On eviction run (CLI or cron): delete objects older than `retention_days`
|
||||||
|
(default 7) that push total over `max_capacity`; oldest-first.
|
||||||
|
|
||||||
|
This keeps the policy logic in bot-bottle and the storage provider as a dumb
|
||||||
|
object store — no vendor-specific lifecycle API required.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Image Size Reduction
|
||||||
|
|
||||||
|
### Current artifact sizes
|
||||||
|
|
||||||
|
A typical committed tar for a Claude Code agent image is 500 MB–1 GB uncompressed.
|
||||||
|
The per-run ext4 (copy of the base, written at `start`) adds another ~1 GB of
|
||||||
|
local disk. The leak fix in this PR addresses the ext4 copies; the remote store
|
||||||
|
addresses the committed tars.
|
||||||
|
|
||||||
|
### Compression
|
||||||
|
|
||||||
|
zstd compression of the committed tar before upload is the highest-leverage
|
||||||
|
single change:
|
||||||
|
|
||||||
|
| Codec | Typical size (1 GB rootfs) | Compress speed | Decompress speed |
|
||||||
|
|-------|---------------------------|---------------|-----------------|
|
||||||
|
| gzip | 350–430 MB | ~100 MB/s | ~500 MB/s |
|
||||||
|
| **zstd (default)** | **330–360 MB** | **~400 MB/s** | **~2 GB/s** |
|
||||||
|
| xz | 290–320 MB | ~20 MB/s | ~200 MB/s |
|
||||||
|
|
||||||
|
**zstd is the best trade-off**: 65–67% size reduction, near-instantaneous
|
||||||
|
decompression. The `tar` call in `freezer.py` could pipe through `zstd` before
|
||||||
|
writing to disk and to the remote; `resume` decompresses on the way back. A
|
||||||
|
`.tar.zst` suffix marks compressed artifacts so old tars remain restorable
|
||||||
|
without the codec.
|
||||||
|
|
||||||
|
### SquashFS for the base image cache
|
||||||
|
|
||||||
|
The `rootfs/agent-<digest>/` directory (the buildah-exported tree) is rebuilt by
|
||||||
|
`image_builder.py` and turned into per-run ext4 by `mke2fs -d`. Storing the
|
||||||
|
base as a SquashFS image instead of a flat directory tree would reduce it from
|
||||||
|
~1 GB to ~330–360 MB and make the cache remote-friendly. Firecracker does not
|
||||||
|
directly boot SquashFS, but the existing `mke2fs -d` path reads a directory tree
|
||||||
|
— a SquashFS mount could serve as the source. This is a larger change and lower
|
||||||
|
priority than tar compression.
|
||||||
|
|
||||||
|
### OverlayFS for per-run rootfs
|
||||||
|
|
||||||
|
Multiple simultaneous bottles sharing the same agent image today each get a full
|
||||||
|
`mke2fs -d` copy (~1 GB). OverlayFS (read-only base + writable sparse overlay)
|
||||||
|
would reduce this to ~10–50 MB per instance beyond the first:
|
||||||
|
|
||||||
|
- Mount the base image directory as read-only lower layer
|
||||||
|
- Attach a sparse ext4 or tmpfs writable layer per bottle
|
||||||
|
- Pass the merged overlay to Firecracker as the block device
|
||||||
|
|
||||||
|
E2B's public write-up on Firecracker + OverlayFS confirms this approach works
|
||||||
|
at scale. The `launch.py` changes would be non-trivial (device mapper or
|
||||||
|
`fuse-overlayfs` plumbing), so this is a follow-up rather than a prerequisite
|
||||||
|
for the remote store.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Recommended Approach
|
||||||
|
|
||||||
|
**Phase 1 — remote store with zstd (tight scope, actionable now)**
|
||||||
|
|
||||||
|
1. Add `--zstd` to the `tar` call in `FirecrackerFreezer._freeze`; name the
|
||||||
|
artifact `rootfs.tar.zst`. Keep uncompressed restore path for legacy tars.
|
||||||
|
2. Add a `bb firecracker upload <slug>` / `bb firecracker pull <slug>` pair that
|
||||||
|
pushes/fetches the compressed tar to the configured object store (S3-compatible
|
||||||
|
API, so B2, R2, MinIO, and Wasabi all work with the same client).
|
||||||
|
3. Store a `manifest.json` in the bucket (or a local mirror) tracking slug →
|
||||||
|
`{size, uploaded_at}`. Use it for threshold warnings and eviction.
|
||||||
|
4. Default retention: 7 days, configurable via `firecracker.image_retention_days`
|
||||||
|
in `~/.config/bot-bottle/config.toml` (or equivalent).
|
||||||
|
5. Warn at 80% of `max_capacity` (default e.g. 50 GB); evict oldest on commit
|
||||||
|
once at 100%.
|
||||||
|
|
||||||
|
**Storage recommendation:** Cloudflare R2 for hosted deployments (zero egress,
|
||||||
|
zero config), MinIO for private/on-premises.
|
||||||
|
|
||||||
|
**Phase 2 — base image cache compression**
|
||||||
|
|
||||||
|
Compress the `agent-<digest>` cache dir as a `.tar.zst` to save ~65% on repeated
|
||||||
|
image uploads. Low urgency since the base image is rebuildable.
|
||||||
|
|
||||||
|
**Phase 3 — OverlayFS per-run disk**
|
||||||
|
|
||||||
|
Replace the full per-run ext4 copy with an OverlayFS sparse layer. Largest disk
|
||||||
|
impact (~90–95% savings per concurrent bottle) but highest implementation
|
||||||
|
complexity. Track as a separate PRD.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Open Questions
|
||||||
|
|
||||||
|
- Does the host have a configured object-store credential path, or should the
|
||||||
|
remote store be an opt-in with an explicit `bb config set image-store.url ...`?
|
||||||
|
- Should `commit` automatically upload, or should upload be an explicit step to
|
||||||
|
avoid surprise egress?
|
||||||
|
- What is the acceptable cold-start latency for a restore from remote? A 330 MB
|
||||||
|
zstd tar at 100 Mbit/s takes ~26 s; at 1 Gbit/s, ~2.6 s. This bounds the
|
||||||
|
retention strategy (evict from local after successful upload vs keep local copy).
|
||||||
@@ -21,10 +21,12 @@ class TestBuiltinAgentImages(unittest.TestCase):
|
|||||||
r"(?m)^FROM node:22-trixie-slim\s*$",
|
r"(?m)^FROM node:22-trixie-slim\s*$",
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_all_install_podman(self):
|
def test_none_install_podman(self):
|
||||||
|
# podman lives in the nested-containers derived layer (nested_containers.py),
|
||||||
|
# not in the base agent images, so bottles without the flag pay no cost.
|
||||||
for dockerfile in _AGENT_DOCKERFILES:
|
for dockerfile in _AGENT_DOCKERFILES:
|
||||||
with self.subTest(provider=dockerfile.parent.name):
|
with self.subTest(provider=dockerfile.parent.name):
|
||||||
self.assertRegex(
|
self.assertNotRegex(
|
||||||
dockerfile.read_text(),
|
dockerfile.read_text(),
|
||||||
re.compile(r"(?m)^\s*podman(?:\s|\\|$)"),
|
re.compile(r"(?m)^\s*podman(?:\s|\\|$)"),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -10,7 +10,9 @@ classmethods forward to their module.
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
from bot_bottle.backend.firecracker import cleanup as fc_cleanup
|
from bot_bottle.backend.firecracker import cleanup as fc_cleanup
|
||||||
@@ -23,32 +25,69 @@ def _proc(stdout: str = "", returncode: int = 0) -> "subprocess.CompletedProcess
|
|||||||
return subprocess.CompletedProcess([], returncode, stdout=stdout, stderr="")
|
return subprocess.CompletedProcess([], returncode, stdout=stdout, stderr="")
|
||||||
|
|
||||||
|
|
||||||
class TestOrphanEnumeration(unittest.TestCase):
|
class TestProcessScan(unittest.TestCase):
|
||||||
def test_orphan_vm_pids_filters_by_run_dir(self):
|
def test_run_dir_of_matches_only_direct_children(self):
|
||||||
run_root = str(fc_cleanup._run_root())
|
run_root = Path("/cache/run")
|
||||||
out = (
|
self.assertEqual(
|
||||||
f"111 firecracker --config-file {run_root}/dev-a/config.json\n"
|
Path("/cache/run/dev-a"),
|
||||||
"222 firecracker --config-file /somewhere/else/config.json\n"
|
fc_cleanup._run_dir_of(
|
||||||
"notanint firecracker --config-file " + run_root + "/x\n"
|
f"firecracker --config-file {run_root}/dev-a/config.json", run_root
|
||||||
|
),
|
||||||
|
)
|
||||||
|
# infra/builder VMs elsewhere, or nested paths, are not ours.
|
||||||
|
self.assertIsNone(
|
||||||
|
fc_cleanup._run_dir_of("firecracker --config-file /elsewhere/config.json", run_root)
|
||||||
|
)
|
||||||
|
self.assertIsNone(
|
||||||
|
fc_cleanup._run_dir_of("firecracker --no-config", run_root)
|
||||||
)
|
)
|
||||||
with patch.object(fc_cleanup.subprocess, "run", return_value=_proc(out)):
|
|
||||||
self.assertEqual([111], fc_cleanup._orphan_vm_pids())
|
|
||||||
|
|
||||||
def test_orphan_vm_pids_empty_when_pgrep_fails(self):
|
def test_scan_splits_live_dirs_from_orphan_pids(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
run_root = Path(tmp)
|
||||||
|
(run_root / "live-a").mkdir() # dir present -> live VM, protected
|
||||||
|
# "gone-b" dir intentionally absent -> lingering VMM, orphan pid
|
||||||
|
out = (
|
||||||
|
f"111 firecracker --config-file {run_root}/live-a/config.json\n"
|
||||||
|
f"222 firecracker --config-file {run_root}/gone-b/config.json\n"
|
||||||
|
"333 firecracker --config-file /elsewhere/config.json\n"
|
||||||
|
"notanint firecracker --config-file x\n"
|
||||||
|
)
|
||||||
|
with patch.object(fc_cleanup.subprocess, "run", return_value=_proc(out)):
|
||||||
|
live, orphan_pids = fc_cleanup._scan_processes(run_root)
|
||||||
|
self.assertEqual({str(run_root / "live-a")}, live)
|
||||||
|
self.assertEqual([222], orphan_pids)
|
||||||
|
|
||||||
|
def test_scan_empty_when_pgrep_fails(self):
|
||||||
with patch.object(fc_cleanup.subprocess, "run", return_value=_proc(returncode=1)):
|
with patch.object(fc_cleanup.subprocess, "run", return_value=_proc(returncode=1)):
|
||||||
self.assertEqual([], fc_cleanup._orphan_vm_pids())
|
self.assertEqual((set(), []), fc_cleanup._scan_processes(Path("/x")))
|
||||||
|
|
||||||
def test_run_dirs_empty_when_absent(self):
|
def test_orphan_run_dirs_excludes_live_and_missing_root(self):
|
||||||
with patch.object(fc_cleanup.util, "cache_dir") as cache:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
cache.return_value.__truediv__.return_value.is_dir.return_value = False
|
run_root = Path(tmp)
|
||||||
self.assertEqual([], fc_cleanup._run_dirs())
|
(run_root / "live-a").mkdir()
|
||||||
|
(run_root / "dead-b").mkdir()
|
||||||
|
live = {str(run_root / "live-a")}
|
||||||
|
self.assertEqual(
|
||||||
|
[str(run_root / "dead-b")],
|
||||||
|
fc_cleanup._orphan_run_dirs(run_root, live),
|
||||||
|
)
|
||||||
|
# absent run root -> nothing to reap
|
||||||
|
self.assertEqual([], fc_cleanup._orphan_run_dirs(Path("/nope/run"), set()))
|
||||||
|
|
||||||
def test_prepare_cleanup_assembles_plan(self):
|
def test_prepare_cleanup_reaps_orphans_only(self):
|
||||||
with patch.object(fc_cleanup, "_orphan_vm_pids", return_value=[7]), \
|
"""The live VM's dir is never in the plan; the dead one is."""
|
||||||
patch.object(fc_cleanup, "_run_dirs", return_value=["/run/x"]):
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
plan = fc_cleanup.prepare_cleanup()
|
run_root = Path(tmp)
|
||||||
self.assertEqual((7,), plan.vm_pids)
|
(run_root / "live-a").mkdir()
|
||||||
self.assertEqual(("/run/x",), plan.run_dirs)
|
(run_root / "dead-b").mkdir()
|
||||||
|
out = f"111 firecracker --config-file {run_root}/live-a/config.json\n"
|
||||||
|
with patch.object(fc_cleanup, "_run_root", return_value=run_root), \
|
||||||
|
patch.object(fc_cleanup.subprocess, "run", return_value=_proc(out)):
|
||||||
|
plan = fc_cleanup.prepare_cleanup()
|
||||||
|
self.assertEqual((), plan.vm_pids)
|
||||||
|
self.assertEqual((str(run_root / "dead-b"),), plan.run_dirs)
|
||||||
|
self.assertNotIn(str(run_root / "live-a"), plan.run_dirs)
|
||||||
|
|
||||||
|
|
||||||
class TestCleanupRemoval(unittest.TestCase):
|
class TestCleanupRemoval(unittest.TestCase):
|
||||||
|
|||||||
@@ -108,7 +108,7 @@ class TestNestedContainersImage(unittest.TestCase):
|
|||||||
calls.append((image, context, dockerfile))
|
calls.append((image, context, dockerfile))
|
||||||
text = Path(dockerfile).read_text(encoding="utf-8")
|
text = Path(dockerfile).read_text(encoding="utf-8")
|
||||||
self.assertIn("FROM agent:base", text)
|
self.assertIn("FROM agent:base", text)
|
||||||
self.assertIn("aardvark-dns fuse-overlayfs netavark nftables passt", text)
|
self.assertIn("aardvark-dns fuse-overlayfs netavark nftables passt podman", text)
|
||||||
self.assertIn("USER node", text)
|
self.assertIn("USER node", text)
|
||||||
self.assertTrue((Path(context) / "nested-containers-init.sh").is_file())
|
self.assertTrue((Path(context) / "nested-containers-init.sh").is_file())
|
||||||
|
|
||||||
@@ -128,7 +128,7 @@ class TestNestedContainersImage(unittest.TestCase):
|
|||||||
seen.append(Path(dockerfile).read_text(encoding="utf-8"))
|
seen.append(Path(dockerfile).read_text(encoding="utf-8"))
|
||||||
|
|
||||||
nested_containers.build_image("agent:base", build)
|
nested_containers.build_image("agent:base", build)
|
||||||
for package in ("passt", "nftables", "aardvark-dns"):
|
for package in ("podman", "passt", "nftables", "aardvark-dns"):
|
||||||
self.assertIn(package, seen[0])
|
self.assertIn(package, seen[0])
|
||||||
|
|
||||||
def test_strips_subordinate_ranges_so_podman_avoids_newuidmap(self) -> None:
|
def test_strips_subordinate_ranges_so_podman_avoids_newuidmap(self) -> None:
|
||||||
|
|||||||
@@ -56,16 +56,56 @@ class TestMergeBottlesRuntime(unittest.TestCase):
|
|||||||
result = merge_bottles_runtime([base, override])
|
result = merge_bottles_runtime([base, override])
|
||||||
self.assertFalse(result.supervise)
|
self.assertFalse(result.supervise)
|
||||||
|
|
||||||
def test_nested_containers_survives_a_later_bottle(self):
|
def test_supervise_survives_a_later_bottle_that_omits_the_key(self):
|
||||||
"""OR, not replace: a resolved bottle that never mentioned the key is
|
disabled = _bottle(supervise=False)
|
||||||
indistinguishable from one that set it false, so `--bottle with-docker
|
quiet = _bottle(env={"X": "1"})
|
||||||
--bottle claude-dev` must not silently drop the capability."""
|
self.assertFalse(merge_bottles_runtime([disabled, quiet]).supervise)
|
||||||
|
|
||||||
|
def test_supervise_explicit_true_overrides_earlier_false(self):
|
||||||
|
disabled = _bottle(supervise=False)
|
||||||
|
enabled = _bottle(supervise=True)
|
||||||
|
self.assertTrue(merge_bottles_runtime([disabled, enabled]).supervise)
|
||||||
|
|
||||||
|
def test_nested_containers_survives_a_later_bottle_that_omits_the_key(self):
|
||||||
|
"""A bottle that never mentions nested_containers must not silently
|
||||||
|
drop the capability: `--bottle with-docker --bottle claude-dev`."""
|
||||||
enabled = _bottle(nested_containers=True)
|
enabled = _bottle(nested_containers=True)
|
||||||
quiet = _bottle(env={"X": "1"})
|
quiet = _bottle(env={"X": "1"})
|
||||||
self.assertTrue(merge_bottles_runtime([enabled, quiet]).nested_containers)
|
self.assertTrue(merge_bottles_runtime([enabled, quiet]).nested_containers)
|
||||||
self.assertTrue(merge_bottles_runtime([quiet, enabled]).nested_containers)
|
self.assertTrue(merge_bottles_runtime([quiet, enabled]).nested_containers)
|
||||||
self.assertFalse(merge_bottles_runtime([quiet, quiet]).nested_containers)
|
self.assertFalse(merge_bottles_runtime([quiet, quiet]).nested_containers)
|
||||||
|
|
||||||
|
def test_nested_containers_explicit_false_overrides_earlier_true(self):
|
||||||
|
"""An explicit nested_containers: false in a later bottle must win
|
||||||
|
over an earlier true (last-wins, presence-aware)."""
|
||||||
|
enabled = _bottle(nested_containers=True)
|
||||||
|
disabled = _bottle(nested_containers=False)
|
||||||
|
self.assertFalse(merge_bottles_runtime([enabled, disabled]).nested_containers)
|
||||||
|
|
||||||
|
def test_nested_containers_explicit_true_overrides_earlier_false(self):
|
||||||
|
enabled = _bottle(nested_containers=True)
|
||||||
|
disabled = _bottle(nested_containers=False)
|
||||||
|
self.assertTrue(merge_bottles_runtime([disabled, enabled]).nested_containers)
|
||||||
|
|
||||||
|
def test_extended_boolean_values_remain_explicit_at_runtime(self):
|
||||||
|
idx = _index(
|
||||||
|
bottles={
|
||||||
|
"enabled": {"nested_containers": True, "supervise": True},
|
||||||
|
"parent": {},
|
||||||
|
"disabled_child": {
|
||||||
|
"extends": "parent",
|
||||||
|
"nested_containers": False,
|
||||||
|
"supervise": False,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
agents={"impl": {"bottle": "enabled", "skills": [], "prompt": ""}},
|
||||||
|
)
|
||||||
|
result = idx.load_for_agent(
|
||||||
|
"impl", ("enabled", "disabled_child")
|
||||||
|
).bottle
|
||||||
|
self.assertFalse(result.nested_containers)
|
||||||
|
self.assertFalse(result.supervise)
|
||||||
|
|
||||||
def test_three_bottles_merged_left_to_right(self):
|
def test_three_bottles_merged_left_to_right(self):
|
||||||
b1 = _bottle(env={"A": "1", "B": "1", "C": "1"})
|
b1 = _bottle(env={"A": "1", "B": "1", "C": "1"})
|
||||||
b2 = _bottle(env={"B": "2", "C": "2"})
|
b2 = _bottle(env={"B": "2", "C": "2"})
|
||||||
|
|||||||
@@ -68,6 +68,23 @@ class TestExtendsBasic(unittest.TestCase):
|
|||||||
self.assertTrue(m.bottles["base"].supervise)
|
self.assertTrue(m.bottles["base"].supervise)
|
||||||
self.assertFalse(m.bottles["off"].supervise)
|
self.assertFalse(m.bottles["off"].supervise)
|
||||||
|
|
||||||
|
def test_child_overrides_nested_containers_scalar(self):
|
||||||
|
m = _build(
|
||||||
|
base={"nested_containers": True},
|
||||||
|
off={"extends": "base", "nested_containers": False},
|
||||||
|
)
|
||||||
|
self.assertTrue(m.bottles["base"].nested_containers)
|
||||||
|
self.assertFalse(m.bottles["off"].nested_containers)
|
||||||
|
|
||||||
|
def test_inherited_boolean_declaration_is_preserved(self):
|
||||||
|
m = _build(
|
||||||
|
base={"nested_containers": True, "supervise": False},
|
||||||
|
child={"extends": "base"},
|
||||||
|
)
|
||||||
|
child = m.bottles["child"]
|
||||||
|
self.assertIn("nested_containers", child.declared_fields)
|
||||||
|
self.assertIn("supervise", child.declared_fields)
|
||||||
|
|
||||||
def test_parent_resolved_once_for_multiple_children(self):
|
def test_parent_resolved_once_for_multiple_children(self):
|
||||||
# Two children sharing one parent: both inherit; the parent
|
# Two children sharing one parent: both inherit; the parent
|
||||||
# is resolved once + cached. (Cache behavior is internal; we
|
# is resolved once + cached. (Cache behavior is internal; we
|
||||||
@@ -477,6 +494,26 @@ class TestExtendsMultiParent(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
self.assertTrue(m.bottles["child"].supervise)
|
self.assertTrue(m.bottles["child"].supervise)
|
||||||
|
|
||||||
|
def test_later_parent_omitting_boole_preserves_earlier_values(self):
|
||||||
|
m = _build(
|
||||||
|
p1={"nested_containers": True, "supervise": False},
|
||||||
|
p2={"env": {"FROM_P2": "1"}},
|
||||||
|
child={"extends": ["p1", "p2"]},
|
||||||
|
)
|
||||||
|
child = m.bottles["child"]
|
||||||
|
self.assertTrue(child.nested_containers)
|
||||||
|
self.assertFalse(child.supervise)
|
||||||
|
|
||||||
|
def test_later_parent_explicit_boole_override_earlier_values(self):
|
||||||
|
m = _build(
|
||||||
|
p1={"nested_containers": True, "supervise": False},
|
||||||
|
p2={"nested_containers": False, "supervise": True},
|
||||||
|
child={"extends": ["p1", "p2"]},
|
||||||
|
)
|
||||||
|
child = m.bottles["child"]
|
||||||
|
self.assertFalse(child.nested_containers)
|
||||||
|
self.assertTrue(child.supervise)
|
||||||
|
|
||||||
def test_child_supervise_overrides_all_parents(self):
|
def test_child_supervise_overrides_all_parents(self):
|
||||||
m = _build(
|
m = _build(
|
||||||
p1={"supervise": True},
|
p1={"supervise": True},
|
||||||
|
|||||||
Reference in New Issue
Block a user