spike(macos): rootless containers in bottles — Docker impossible, podman works #449

Closed
didericis-codex wants to merge 5 commits from spike/rootless-docker-macos into main
14 changed files with 408 additions and 7 deletions
Showing only changes of commit 72e35a1343 - Show all commits
+8
View File
@@ -75,6 +75,14 @@ On compatible macOS hosts, the default backend requires Apple's `container` CLI
Use `BOT_BOTTLE_BACKEND=docker ./cli.py start <agent>` on hosts where neither Apple Container nor KVM is available and Docker is the desired backend.
> **Experimental Docker-in-bottle spike (#392):** a bottle may set
> `docker_access: true`. On the macOS backend this starts a guest-local,
> rootless Docker daemon after the bottle is registered. It does not mount
> Docker Desktop's socket or add outer VM capabilities. The spike currently
> requires live-macOS validation before it is considered supported; startup
> fails closed when the Apple guest kernel cannot satisfy rootless Docker's
> user-namespace requirements.
### Firecracker on Linux
On Linux, a KVM-capable host defaults to the Firecracker backend. It needs:
@@ -20,6 +20,7 @@ class MacosContainerBottlePlan(BottlePlan):
# bottle is registered. See launch.py's stamp for why it lives here and not
# only in the exec-time proxy env.
identity_token: str = ""
docker_access: bool = False
@property
def container_name(self) -> str:
+20 -5
View File
@@ -64,6 +64,7 @@ from .gateway_hosts import (
refresh_gateway_host,
set_gateway_host,
)
from . import rootless_docker
from .bottle_plan import MacosContainerBottlePlan
from ...orchestrator.config_store import resolve_teardown_timeout
from .consolidated_launch import (
@@ -171,6 +172,10 @@ def launch(
# token above, so — unlike the run-time env — the plan CAN carry it.
plan = dataclasses.replace(plan, identity_token=ctx.identity_token)
exec_env = {
**_identity_proxy_env(endpoint, ctx.identity_token),
**rootless_docker.guest_env(plan.docker_access),
}
bottle = MacosContainerBottle(
plan.container_name,
teardown,
@@ -184,10 +189,13 @@ def launch(
),
terminal_color=plan.spec.color,
agent_workdir=plan.workspace_plan.workdir,
exec_env=_identity_proxy_env(endpoint, ctx.identity_token),
exec_env=exec_env,
)
bottle.prompt_path = provision(plan, bottle)
if plan.docker_access:
rootless_docker.start(bottle)
yield bottle
finally:
teardown()
@@ -199,15 +207,22 @@ def _build_images(plan: MacosContainerBottlePlan) -> MacosContainerBottlePlan:
committed = read_committed_image(plan.slug)
if committed and container_mod.image_exists(committed):
info(f"using committed image {committed!r}")
return dataclasses.replace(
plan = dataclasses.replace(
plan,
agent_provision=dataclasses.replace(
plan.agent_provision, image=committed,
),
)
container_mod.build_image(
plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path,
)
else:
container_mod.build_image(
plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path,
)
if plan.docker_access:
image = rootless_docker.build_image(plan.image, container_mod.build_image)
plan = dataclasses.replace(
plan,
agent_provision=dataclasses.replace(plan.agent_provision, image=image),
)
return plan
@@ -44,4 +44,5 @@ def resolve_plan(
egress_plan=egress_plan,
supervise_plan=supervise_plan,
agent_provision=agent_provision_plan,
docker_access=manifest.bottle.docker_access,
)
@@ -0,0 +1,57 @@
#!/bin/sh
set -eu
uid="$(id -u)"
if [ "$uid" -eq 0 ]; then
echo "refusing to run rootless Docker as root" >&2
exit 1
fi
for command in dockerd-rootless.sh rootlesskit slirp4netns newuidmap newgidmap docker; do
command -v "$command" >/dev/null 2>&1 || {
echo "missing rootless Docker prerequisite: $command" >&2
exit 1
}
done
grep -q "^$(id -un):.*:65536$" /etc/subuid || {
echo "missing 65536-entry subordinate UID range for $(id -un)" >&2
exit 1
}
grep -q "^$(id -gn):.*:65536$" /etc/subgid || {
echo "missing 65536-entry subordinate GID range for $(id -gn)" >&2
exit 1
}
export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/tmp/bot-bottle-docker-run}"
mkdir -p "$XDG_RUNTIME_DIR" "$HOME/.docker"
chmod 700 "$XDG_RUNTIME_DIR"
# Docker uses this config for build and child-container proxy injection. The
# token-bearing proxy URL is already available to the agent; persisting it
# inside this disposable VM does not broaden its authority.
python3 - <<'PY'
import json
import os
from pathlib import Path
proxy = os.environ.get("HTTPS_PROXY") or os.environ.get("https_proxy", "")
no_proxy = os.environ.get("NO_PROXY") or os.environ.get("no_proxy", "")
config = {"proxies": {"default": {
"httpProxy": proxy,
"httpsProxy": proxy,
"noProxy": no_proxy,
}}}
path = Path.home() / ".docker" / "config.json"
path.write_text(json.dumps(config), encoding="utf-8")
path.chmod(0o600)
PY
if docker info >/dev/null 2>&1; then
exit 0
fi
log=/tmp/bot-bottle-rootless-docker.log
nohup dockerd-rootless.sh \
--storage-driver=fuse-overlayfs \
>"$log" 2>&1 </dev/null &
Outdated
Review

@didericis-claude This was only needed for the rootless docker test, correct? podman doesn't require this? Further clarify whether podman required anything in this PR and whether this issue is worded correctly/podman would enable docker compose (minus the token issue)

@didericis-claude This was only needed for the rootless *docker* test, correct? podman doesn't require this? Further clarify whether podman required anything in this PR and whether [this issue](https://gitea.dideric.is/didericis/bot-bottle/issues/451) is worded correctly/podman would enable docker compose (minus the token issue)
@@ -0,0 +1,93 @@
"""Experimental rootless Docker bootstrap for Apple-container bottles.
The daemon and every child remain inside the existing per-bottle VM. This
module deliberately refuses to compensate for missing prerequisites with
outer capabilities, a privileged container, or a host Docker socket.
"""
from __future__ import annotations
import shlex
import shutil
import tempfile
import time
from pathlib import Path
from typing import Callable
from ...log import die, info
_INIT = "/usr/local/libexec/bot-bottle/rootless-docker-init"
_RUNTIME_DIR = "/tmp/bot-bottle-docker-run"
_SOCKET = f"{_RUNTIME_DIR}/docker.sock"
READY_RETRIES = 30
def build_image(
base_image: str,
build: Callable[..., None],
) -> str:
"""Layer spike-only tooling on an already-built provider image."""
image = f"{base_image}-rootless-docker"
init_script = Path(__file__).with_name("rootless-docker-init.sh")
with tempfile.TemporaryDirectory(prefix="bot-bottle-rootless-docker.") as tmp:
context = Path(tmp)
shutil.copy2(init_script, context / "rootless-docker-init.sh")
(context / "Dockerfile").write_text(
"FROM docker:28-cli AS docker_cli\n"
f"FROM {base_image}\n"
"USER root\n"
"COPY --from=docker_cli /usr/local/libexec/docker/cli-plugins/"
"docker-compose /usr/local/libexec/docker/cli-plugins/docker-compose\n"
"RUN apt-get update \\\n"
" && apt-get install -y --no-install-recommends docker.io uidmap "
"rootlesskit slirp4netns fuse-overlayfs \\\n"
" && rm -rf /var/lib/apt/lists/* \\\n"
" && echo 'node:100000:65536' >> /etc/subuid \\\n"
" && echo 'node:100000:65536' >> /etc/subgid \\\n"
" && ln -s /usr/share/docker.io/contrib/dockerd-rootless.sh "
"/usr/local/bin/dockerd-rootless.sh\n"
"COPY rootless-docker-init.sh "
"/usr/local/libexec/bot-bottle/rootless-docker-init\n"
"RUN chmod 0755 /usr/local/libexec/bot-bottle/rootless-docker-init\n"
"USER node\n",
encoding="utf-8",
)
build(image, str(context), dockerfile=str(context / "Dockerfile"))
return image
def guest_env(enabled: bool) -> dict[str, str]:
"""Environment consumed by the Docker CLI inside an enabled bottle."""
if not enabled:
return {}
return {
"DOCKER_HOST": f"unix://{_SOCKET}",
"XDG_RUNTIME_DIR": _RUNTIME_DIR,
}
def start(bottle: object) -> None:
"""Start and verify the unprivileged daemon through the bottle exec API."""
info("starting experimental rootless Docker daemon")
result = bottle.exec(shlex.quote(_INIT)) # type: ignore[attr-defined]
if result.returncode != 0:
detail = (result.stderr or result.stdout or "").strip()
die(f"rootless Docker bootstrap failed: {detail or '<no output>'}")
for _ in range(READY_RETRIES):
result = bottle.exec("docker info >/dev/null 2>&1") # type: ignore[attr-defined]
if result.returncode == 0:
info("rootless Docker daemon is ready")
return
time.sleep(0.2)
logs = bottle.exec( # type: ignore[attr-defined]
"tail -n 80 /tmp/bot-bottle-rootless-docker.log 2>/dev/null || true"
)
die(
"rootless Docker did not become ready without additional outer "
f"privileges:\n{(logs.stdout or logs.stderr or '<no log>').strip()}"
)
__all__ = ["build_image", "guest_env", "start"]
+11
View File
@@ -44,6 +44,9 @@ class ManifestBottle:
# daemon that exposes egress MCP tools to the agent. Set
# `supervise: false` to skip the gateway.
supervise: bool = True
# Experimental guest-local container engine (issue #392). Backends must
# implement this without granting access to a host/shared daemon.
docker_access: bool = False
@classmethod
def from_dict(cls, name: str, raw: object) -> "ManifestBottle":
@@ -123,7 +126,15 @@ class ManifestBottle:
f"(was {type(supervise_raw).__name__})"
)
docker_access_raw = d.get("docker_access", False)
if not isinstance(docker_access_raw, bool):
raise ManifestError(
f"bottle '{name}' docker_access must be a boolean "
f"(was {type(docker_access_raw).__name__})"
)
return cls(
env=env, agent_provider=agent_provider, git=git,
git_user=git_user, egress=egress, supervise=supervise_raw,
docker_access=docker_access_raw,
)
+8
View File
@@ -54,6 +54,7 @@ def _merge_two_bottles_runtime(base: "ManifestBottle", override: "ManifestBottle
git_user=merged_git_user,
egress=merged_egress,
supervise=override.supervise,
docker_access=override.docker_access,
)
@@ -206,6 +207,7 @@ def _fold_two_bottles(
git_user=merged_git_user,
egress=merged_egress,
supervise=later.supervise,
docker_access=later.docker_access,
), merged_repos_raw
@@ -266,6 +268,11 @@ def _merge_bottles(
merged_supervise = (
child.supervise if "supervise" in child_raw else parent.supervise
)
merged_docker_access = (
child.docker_access
if "docker_access" in child_raw
else parent.docker_access
)
validate_egress_routes(name, merged_egress.routes)
return ManifestBottle(
@@ -275,6 +282,7 @@ def _merge_bottles(
git_user=merged_git_user,
egress=merged_egress,
supervise=merged_supervise,
docker_access=merged_docker_access,
)
+4 -1
View File
@@ -16,7 +16,10 @@ _FILENAME_RX = re.compile(r"^[a-z][a-z0-9-]*$")
# sets dies with a "did you mean" pointer: typos should not silently
# ghost into an empty config.
BOTTLE_KEYS = frozenset(
{"env", "extends", "agent_provider", "git-gate", "egress", "supervise"}
{
"env", "extends", "agent_provider", "git-gate", "egress", "supervise",
"docker_access",
}
)
AGENT_KEYS_REQUIRED: frozenset[str] = frozenset()
AGENT_KEYS_OPTIONAL = frozenset({"bottle", "skills", "git-gate"})
@@ -0,0 +1,103 @@
"""Live-Mac acceptance spike for guest-local rootless Docker (issue #392).
Run explicitly on an Apple Silicon/macOS 26 host:
BOT_BOTTLE_ROOTLESS_DOCKER_SPIKE=1 \
python3 -m unittest tests.integration.test_macos_rootless_docker_spike -v
The opt-in is deliberate: ordinary Linux CI cannot execute Apple Container.
"""
from __future__ import annotations
import os
import platform
import shutil
import tempfile
import unittest
from pathlib import Path
from bot_bottle.backend import BottleSpec, get_bottle_backend
from bot_bottle.manifest import ManifestIndex
@unittest.skipUnless(
platform.system() == "Darwin"
and os.environ.get("BOT_BOTTLE_ROOTLESS_DOCKER_SPIKE") == "1",
"requires an explicit live-Mac rootless-Docker spike run",
)
class TestMacosRootlessDockerSpike(unittest.TestCase):
def test_compose_stays_inside_registered_bottle(self) -> None:
workspace = Path(tempfile.mkdtemp(prefix="rootless-docker-spike."))
stage = Path(tempfile.mkdtemp(prefix="rootless-docker-stage."))
try:
(workspace / "index.html").write_text("bottle-compose-ok\n")
(workspace / "compose.yaml").write_text(
"services:\n"
" web:\n"
" image: python:3.12-alpine\n"
" working_dir: /workspace\n"
" command: python -m http.server 8000\n"
" volumes: ['.:/workspace']\n"
" ports: ['18080:8000']\n",
encoding="utf-8",
)
manifest = ManifestIndex.from_json_obj({
"bottles": {"dev": {
"docker_access": True,
"egress": {"routes": [
{"host": "auth.docker.io"},
{"host": "registry-1.docker.io"},
{"host": "production.cloudflare.docker.com"},
]},
}},
"agents": {"spike": {
"bottle": "dev", "skills": [], "prompt": "",
}},
})
spec = BottleSpec(
manifest=manifest,
agent_name="spike",
copy_cwd=True,
user_cwd=str(workspace),
)
backend = get_bottle_backend("macos-container")
plan = backend.prepare(spec, stage_dir=stage)
with backend.launch(plan) as bottle:
workdir = plan.workspace_plan.workdir
checks = (
"docker info >/dev/null && docker compose version && "
f"cd {workdir} && docker compose up -d --wait && "
"curl --fail --silent http://127.0.0.1:18080/ | "
"grep -q bottle-compose-ok"
)
result = bottle.exec(checks)
self.assertEqual(
0, result.returncode,
f"stdout={result.stdout!r}\nstderr={result.stderr!r}",
)
inspect = bottle.exec(
"docker info --format '{{json .SecurityOptions}}'"
)
self.assertIn("rootless", inspect.stdout.lower())
self.assertNotEqual(
0,
bottle.exec("test -S /var/run/docker.sock").returncode,
"spike must never expose a host/rootful Docker socket",
)
direct = bottle.exec(
"docker run --rm --env HTTP_PROXY= --env HTTPS_PROXY= "
"--env http_proxy= --env https_proxy= python:3.12-alpine "
"wget -T 4 -qO- https://evil.example.com/"
)
self.assertNotEqual(
0, direct.returncode,
"an inner container obtained direct, unproxied egress",
)
finally:
shutil.rmtree(workspace, ignore_errors=True)
shutil.rmtree(stage, ignore_errors=True)
if __name__ == "__main__":
unittest.main()
@@ -22,6 +22,7 @@ from bot_bottle.backend.macos_container.launch import (
_agent_run_argv,
_identity_proxy_env,
)
from bot_bottle.backend.macos_container.rootless_docker import guest_env
from bot_bottle.manifest import ManifestIndex
_BOTTLE = "bot_bottle.backend.macos_container.bottle"
@@ -76,6 +77,7 @@ def _plan(
),
agent_git_gate_url=agent_git_gate_url,
agent_supervise_url=agent_supervise_url,
docker_access=False,
))
@@ -178,6 +180,18 @@ class TestIdentityTokenDelivery(unittest.TestCase):
self.assertNotIn("--env", argv)
class TestRootlessDockerEnvironment(unittest.TestCase):
def test_disabled_bottle_gets_no_docker_environment(self) -> None:
self.assertEqual({}, guest_env(False))
def test_enabled_bottle_uses_only_guest_local_socket(self) -> None:
env = guest_env(True)
self.assertEqual(
"unix:///tmp/bot-bottle-docker-run/docker.sock", env["DOCKER_HOST"],
)
self.assertNotIn("/var/run/docker.sock", " ".join(env.values()))
class TestPlanIdentityToken(unittest.TestCase):
"""git-gate's gitconfig extraHeader and the supervise MCP --header read
`getattr(plan, "identity_token", "")` at provision time and both bypass the
+74
View File
@@ -0,0 +1,74 @@
"""Unit coverage for the fail-closed macOS rootless-Docker spike."""
from __future__ import annotations
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
from bot_bottle.backend.macos_container import rootless_docker
class _Bottle:
def __init__(self, results: list[SimpleNamespace]) -> None:
self.results = results
self.commands: list[str] = []
def exec(self, command: str) -> SimpleNamespace:
self.commands.append(command)
return self.results.pop(0)
def _result(returncode: int, *, stdout: str = "", stderr: str = "") -> SimpleNamespace:
return SimpleNamespace(returncode=returncode, stdout=stdout, stderr=stderr)
class TestRootlessDockerStart(unittest.TestCase):
def test_bootstraps_then_waits_for_guest_local_daemon(self) -> None:
bottle = _Bottle([_result(0), _result(1), _result(0)])
with patch.object(rootless_docker.time, "sleep"):
rootless_docker.start(bottle)
self.assertIn("rootless-docker-init", bottle.commands[0])
self.assertEqual(2, bottle.commands.count("docker info >/dev/null 2>&1"))
def test_bootstrap_failure_is_fatal_without_privilege_fallback(self) -> None:
bottle = _Bottle([_result(1, stderr="newuidmap missing")])
with patch.object(rootless_docker, "die", side_effect=RuntimeError) as die:
with self.assertRaises(RuntimeError):
rootless_docker.start(bottle)
self.assertIn("newuidmap missing", die.call_args.args[0])
self.assertEqual(1, len(bottle.commands))
def test_timeout_reports_guest_log(self) -> None:
bottle = _Bottle(
[_result(0)]
+ [_result(1) for _ in range(rootless_docker.READY_RETRIES)]
+ [_result(0, stdout="operation not permitted")]
)
with patch.object(rootless_docker.time, "sleep"), \
patch.object(rootless_docker, "die", side_effect=RuntimeError) as die:
with self.assertRaises(RuntimeError):
rootless_docker.start(bottle)
self.assertIn("operation not permitted", die.call_args.args[0])
class TestRootlessDockerImage(unittest.TestCase):
def test_layers_tooling_without_changing_base_image(self) -> None:
calls: list[tuple[str, str, str]] = []
def build(image: str, context: str, *, dockerfile: str) -> None:
calls.append((image, context, dockerfile))
text = Path(dockerfile).read_text(encoding="utf-8")
self.assertIn("FROM agent:base", text)
self.assertIn("docker.io uidmap rootlesskit slirp4netns", text)
self.assertIn("USER node", text)
self.assertTrue((Path(context) / "rootless-docker-init.sh").is_file())
image = rootless_docker.build_image("agent:base", build)
self.assertEqual("agent:base-rootless-docker", image)
self.assertEqual("agent:base-rootless-docker", calls[0][0])
if __name__ == "__main__":
unittest.main()
+6
View File
@@ -56,6 +56,12 @@ class TestMergeBottlesRuntime(unittest.TestCase):
result = merge_bottles_runtime([base, override])
self.assertFalse(result.supervise)
def test_docker_access_later_wins(self):
result = merge_bottles_runtime([
_bottle(docker_access=False), _bottle(docker_access=True),
])
self.assertTrue(result.docker_access)
def test_three_bottles_merged_left_to_right(self):
b1 = _bottle(env={"A": "1", "B": "1", "C": "1"})
b2 = _bottle(env={"B": "2", "C": "2"})
+8 -1
View File
@@ -44,13 +44,20 @@ class TestBottleValidation(unittest.TestCase):
with self.assertRaises(ManifestError):
ManifestBottle.from_dict("b", {"supervise": "yes"})
def test_docker_access_not_bool(self) -> None:
with self.assertRaises(ManifestError):
ManifestBottle.from_dict("b", {"docker_access": "yes"})
def test_removed_runtime_field(self) -> None:
with self.assertRaises(ManifestError):
ManifestBottle.from_dict("b", {"runtime": "runsc"})
def test_valid_minimal(self) -> None:
b = ManifestBottle.from_dict("b", {"supervise": False, "env": {"X": "1"}})
b = ManifestBottle.from_dict(
"b", {"supervise": False, "docker_access": True, "env": {"X": "1"}},
)
self.assertFalse(b.supervise)
self.assertTrue(b.docker_access)
self.assertEqual({"X": "1"}, dict(b.env))