Add cached image quickstart #336

Open
didericis-codex wants to merge 9 commits from issue-330-cached-images into main
15 changed files with 397 additions and 5 deletions
Showing only changes of commit 1b4f64767a - Show all commits
+3
View File
@@ -78,6 +78,9 @@ class BottleSpec:
# True when launched via --headless (no TTY, no interactive prompts).
# The git-gate host-key preflight uses this to error rather than prompt.
headless: bool = False
# Image startup policy. "fresh" preserves the normal build path;
# "cached" reuses the current local image/artifact without rebuilding.
image_policy: str = "fresh"
@dataclass(frozen=True)
2
+5 -4
View File
@@ -180,10 +180,6 @@ def _sidecar_bundle_service(plan: DockerBottlePlan) -> dict[str, Any]:
service: dict[str, Any] = {
"image": SIDECAR_BUNDLE_IMAGE,
"build": {
"context": _REPO_DIR,
"dockerfile": SIDECAR_BUNDLE_DOCKERFILE,
},
"container_name": sidecar_bundle_container_name(plan.slug),
"networks": {
"internal": {"aliases": internal_aliases},
@@ -192,6 +188,11 @@ def _sidecar_bundle_service(plan: DockerBottlePlan) -> dict[str, Any]:
"environment": env,
"volumes": volumes,
}
if plan.spec.image_policy != "cached":
service["build"] = {
"context": _REPO_DIR,
"dockerfile": SIDECAR_BUNDLE_DOCKERFILE,
}
return service
+30 -1
View File
@@ -41,7 +41,8 @@ from ...git_gate import (
provision_git_gate_dynamic_keys,
revoke_git_gate_provisioned_keys,
)
from ...log import info, warn
from ...image_cache import warn_if_stale
from ...log import die, info, warn
from . import network as network_mod
from . import util as docker_mod
from .bottle import DockerBottle
@@ -63,6 +64,7 @@ from .compose import (
write_compose_file,
)
from .egress import egress_tls_init
from .sidecar_bundle import SIDECAR_BUNDLE_IMAGE
# Where the repo root lives, for `docker build` context. Computed once.
@@ -100,12 +102,39 @@ def launch(
# Dockerfile. Sidecar images get built lazily by `docker compose
# up` via the renderer's `build:` directives.
committed = read_committed_image(plan.slug)
cached_policy = plan.spec.image_policy == "cached"
if committed and docker_mod.image_exists(committed):
info(f"using committed image {committed!r}")
if cached_policy:
warn_if_stale(
f"agent image {committed!r}",
docker_mod.image_created_at(committed),
)
plan = dataclasses.replace(
plan,
agent_provision=dataclasses.replace(plan.agent_provision, image=committed),
)
elif cached_policy:
if not docker_mod.image_exists(plan.image):
die(
f"cached agent image {plan.image!r} not found; "
"run without --cached-images to build it"
)
if not docker_mod.image_exists(SIDECAR_BUNDLE_IMAGE):
die(
f"cached sidecar image {SIDECAR_BUNDLE_IMAGE!r} not found; "
"run without --cached-images to build it"
)
info(f"using cached agent image {plan.image!r}")
info(f"using cached sidecar image {SIDECAR_BUNDLE_IMAGE!r}")
warn_if_stale(
f"agent image {plan.image!r}",
docker_mod.image_created_at(plan.image),
)
warn_if_stale(
f"sidecar image {SIDECAR_BUNDLE_IMAGE!r}",
docker_mod.image_created_at(SIDECAR_BUNDLE_IMAGE),
)
else:
docker_mod.build_image(
plan.image, _REPO_DIR,
+40
View File
@@ -4,6 +4,7 @@ existence, and building images."""
from __future__ import annotations
from datetime import datetime, timezone
import re
import shutil
import subprocess
@@ -186,6 +187,45 @@ def image_id(ref: str) -> str:
return r.stdout.strip()
def image_created_at(ref: str) -> datetime:
"""Return Docker's image Created timestamp as an aware UTC datetime."""
r = subprocess.run(
["docker", "image", "inspect", "--format", "{{.Created}}", ref],
capture_output=True,
text=True,
check=False,
)
if r.returncode != 0:
die(
f"docker image inspect for {ref!r} failed: "
f"{(r.stderr or '').strip() or '<no stderr>'}"
)
raw = r.stdout.strip()
try:
return _parse_docker_timestamp(raw)
except ValueError:
die(f"docker image inspect for {ref!r} returned invalid Created timestamp: {raw!r}")
def _parse_docker_timestamp(raw: str) -> datetime:
text = raw.strip()
if text.endswith("Z"):
text = text[:-1] + "+00:00"
dot = text.find(".")
if dot != -1:
tz_plus = text.find("+", dot)
tz_minus = text.find("-", dot)
tz_candidates = [pos for pos in (tz_plus, tz_minus) if pos != -1]
if tz_candidates:
tz_pos = min(tz_candidates)
frac = text[dot + 1:tz_pos]
text = text[:dot + 1] + frac[:6].ljust(6, "0") + text[tz_pos:]
dt = datetime.fromisoformat(text)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
return dt.astimezone(timezone.utc)
def save(ref: str, output: str) -> None:
"""`docker save REF -o OUTPUT`. Writes a tarball of the image
layers + manifest to the host path. Used by smolmachines
+23
View File
@@ -63,6 +63,14 @@ def cmd_start(argv: list[str]) -> int:
"skip all prompts. For orchestrators, CI, and webhooks."
),
)
parser.add_argument(
"--cached-images",
action="store_true",
help=(
"quickstart with existing local agent and sidecar images; "
"only valid with --headless"
),
)
parser.add_argument(
"--bottle",
action="append",
@@ -95,6 +103,8 @@ def cmd_start(argv: list[str]) -> int:
help="agent name defined in bot-bottle.json (omit to pick interactively)",
)
args = parser.parse_args(argv)
if args.cached_images and not args.headless:
die("--cached-images is only supported with --headless")
dry_run = args.dry_run or os.environ.get("BOT_BOTTLE_DRY_RUN") == "1"
@@ -142,6 +152,10 @@ def cmd_start(argv: list[str]) -> int:
label, color = tui.name_color_modal(default_label=agent_name)
label, color = _resolve_unique_label(label, color)
image_policy = _select_image_policy()
if image_policy is None:
return 0
spec = BottleSpec(
manifest=manifest,
agent_name=agent_name,
@@ -150,6 +164,7 @@ def cmd_start(argv: list[str]) -> int:
label=label,
color=color,
bottle_names=bottle_names,
image_policy=image_policy,
)
return _launch_bottle(
spec,
@@ -210,6 +225,7 @@ def _start_headless(
color=args.color or "",
bottle_names=bottle_names,
headless=True,
image_policy="cached" if args.cached_images else "fresh",
)
return _launch_bottle(
spec,
@@ -390,6 +406,13 @@ def _text_prompt_yes() -> bool:
return reply in ("y", "Y", "yes", "YES")
def _select_image_policy() -> str | None:
return tui.filter_select(
["fresh", "cached"],
title="Select image startup mode",
)
def _text_render_preflight():
def _render(plan: DockerBottlePlan, backend_name: str) -> None:
print(file=sys.stderr)
+79
View File
@@ -0,0 +1,79 @@
"""SQLite-backed bot-bottle configuration store."""
from __future__ import annotations
from pathlib import Path
try:
from .db_store import DbStore
from .migrations import TableMigrations
from .supervise_types import host_db_path
except ImportError:
from db_store import DbStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
from migrations import TableMigrations # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
from supervise_types import host_db_path # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
CACHED_IMAGE_STALE_WARNING_DAYS = "cached_image_stale_warning_days"
DEFAULT_CACHED_IMAGE_STALE_WARNING_DAYS = 1
class ConfigStore(DbStore):
"""SQLite key/value configuration for host-side bot-bottle settings."""
def __init__(self, db_path: Path | None = None) -> None:
migrations = TableMigrations("config_store", [
# v1 — generic string key/value settings
"""
CREATE TABLE IF NOT EXISTS bot_bottle_config (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
)
didericis-codex marked this conversation as resolved Outdated
Outdated
Review

Will be easier to surface migration issues or problems with config changes if we make the specific config keys explicit. Think we should just put them in the schema here rather than having a generic key.

Will be easier to surface migration issues or problems with config changes if we make the specific config keys explicit. Think we should just put them in the schema here rather than having a generic `key`.
Outdated
Review

Done in 0c70de0e07. The config store schema now uses explicit settings columns with cached_image_stale_warning_days directly on the table, and the generic key/value accessors are gone. I also added a schema assertion test so this stays explicit.

Done in 0c70de0e07efdd7dca22bf85860c6b2d00198b52. The config store schema now uses explicit settings columns with `cached_image_stale_warning_days` directly on the table, and the generic key/value accessors are gone. I also added a schema assertion test so this stays explicit.
""",
])
super().__init__(db_path or host_db_path(), migrations)
def get(self, key: str) -> str | None:
if not self.db_path.is_file():
return None
with self._connect() as conn:
row = conn.execute(
"SELECT value FROM bot_bottle_config WHERE key = ?",
(key,),
).fetchone()
return str(row["value"]) if row is not None else None
def set(self, key: str, value: str) -> Path:
with self._connect() as conn:
conn.execute(
"""
INSERT INTO bot_bottle_config (key, value)
VALUES (?, ?)
ON CONFLICT(key) DO UPDATE SET value = excluded.value
""",
(key, value),
)
self._chmod()
return self.db_path
def get_int(self, key: str, default: int) -> int:
raw = self.get(key)
if raw is None:
return default
try:
return int(raw)
except ValueError:
return default
def cached_image_stale_warning_days(self) -> int:
return self.get_int(
CACHED_IMAGE_STALE_WARNING_DAYS,
DEFAULT_CACHED_IMAGE_STALE_WARNING_DAYS,
)
__all__ = [
"CACHED_IMAGE_STALE_WARNING_DAYS",
"DEFAULT_CACHED_IMAGE_STALE_WARNING_DAYS",
"ConfigStore",
]
+38
View File
@@ -0,0 +1,38 @@
"""Shared helpers for cached-image quickstart warnings."""
from __future__ import annotations
from datetime import datetime, timezone
from pathlib import Path
try:
from .config_store import ConfigStore
from .log import warn
except ImportError:
from config_store import ConfigStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
from log import warn # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
def warn_if_stale(label: str, created_at: datetime) -> None:
threshold_days = ConfigStore().cached_image_stale_warning_days()
if threshold_days < 0:
return
now = datetime.now(timezone.utc)
created = created_at.astimezone(timezone.utc)
age = now - created
if age.total_seconds() <= threshold_days * 86400:
return
warn(
f"cached {label} is {age.days} day(s) old; "
"quickstart does not verify it matches the current Dockerfile/context"
)
def warn_if_stale_path(label: str, path: Path) -> None:
warn_if_stale(
label,
datetime.fromtimestamp(path.stat().st_mtime, tz=timezone.utc),
)
__all__ = ["warn_if_stale", "warn_if_stale_path"]
+4
View File
@@ -6,9 +6,11 @@ from pathlib import Path
try:
from .audit_store import AuditStore
from .config_store import ConfigStore
from .queue_store import QueueStore
except ImportError:
from audit_store import AuditStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
from config_store import ConfigStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
from queue_store import QueueStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
_instance: StoreManager | None = None
@@ -50,11 +52,13 @@ class StoreManager:
return (
QueueStore("", self.db_path).is_migrated()
and AuditStore(self.db_path).is_migrated()
and ConfigStore(self.db_path).is_migrated()
)
def migrate(self) -> None:
QueueStore("", self.db_path).migrate()
AuditStore(self.db_path).migrate()
ConfigStore(self.db_path).migrate()
__all__ = ["StoreManager"]
+12
View File
@@ -184,6 +184,18 @@ class TestCmdStartHeadless(unittest.TestCase):
)
self.assertEqual("docker", self._launch_mock.call_args[1]["backend_name"])
def test_cached_images_sets_cached_policy(self):
start_mod.cmd_start(
["--headless", "--cached-images", "researcher", "--bottle", "claude",
"--prompt", "Do it"]
)
self.assertEqual("cached", self._spec().image_policy)
def test_cached_images_requires_headless(self):
with self.assertRaises(Die):
start_mod.cmd_start(["--cached-images", "researcher"])
self._launch_mock.assert_not_called()
class TestPrepareWithPreflight(unittest.TestCase):
"""prepare_with_preflight calls render_preflight with the plan and backend name."""
+21
View File
@@ -57,6 +57,12 @@ class TestCmdStartSelector(unittest.TestCase):
self._bottle_picker_mock = self._bottle_picker_patch.start()
self._bottle_picker_mock.return_value = ["claude"] # default: one bottle selected
self._image_policy_patch = patch(
"bot_bottle.cli.start._select_image_policy",
return_value="fresh",
)
self._image_policy_patch.start()
self._env_patch = patch.dict(os.environ, {}, clear=False)
self._env_patch.start()
os.environ.pop("BOT_BOTTLE_BACKEND", None)
@@ -66,6 +72,7 @@ class TestCmdStartSelector(unittest.TestCase):
self._launch_patch.stop()
self._agent_picker_patch.stop()
self._bottle_picker_patch.stop()
self._image_policy_patch.stop()
self._env_patch.stop()
# ------------------------------------------------------------------
@@ -124,6 +131,19 @@ class TestCmdStartSelector(unittest.TestCase):
spec = self._launch_mock.call_args[0][0]
self.assertEqual(("claude", "dev"), spec.bottle_names)
def test_image_policy_forwarded_to_spec(self):
with patch("bot_bottle.cli.start._select_image_policy", return_value="cached"):
start_mod.cmd_start(["researcher"])
self._launch_mock.assert_called_once()
spec = self._launch_mock.call_args[0][0]
self.assertEqual("cached", spec.image_policy)
def test_image_policy_cancel_returns_0(self):
with patch("bot_bottle.cli.start._select_image_policy", return_value=None):
rc = start_mod.cmd_start(["researcher"])
self.assertEqual(0, rc)
self._launch_mock.assert_not_called()
def test_empty_bottle_selection_forwarded(self):
self._bottle_picker_mock.return_value = []
start_mod.cmd_start(["researcher"])
@@ -215,6 +235,7 @@ class TestCmdStartLabelCollision(unittest.TestCase):
).start()
# Stub the bottle picker to always return a selection.
patch.object(tui_mod, "filter_multiselect", return_value=["claude"]).start()
patch("bot_bottle.cli.start._select_image_policy", return_value="fresh").start()
self.addCleanup(patch.stopall)
def test_no_collision_proceeds_without_reprompt(self):
+7
View File
@@ -118,6 +118,7 @@ def _plan(
with_egress: bool = False,
supervise: bool = False,
canary: bool = False,
image_policy: str = "fresh",
) -> DockerBottlePlan:
"""Build a fully-resolved DockerBottlePlan. Toggles cover the
matrix the renderer's conditional-service logic branches on."""
@@ -148,6 +149,7 @@ def _plan(
agent_name="demo",
copy_cwd=False,
user_cwd="/tmp/x",
image_policy=image_policy,
)
return DockerBottlePlan(
spec=spec,
@@ -300,6 +302,11 @@ class TestSidecarBundleShape(unittest.TestCase):
self.assertEqual("bot-bottle-sidecars:latest", sc["image"])
self.assertEqual("Dockerfile.sidecars", sc["build"]["dockerfile"])
def test_cached_policy_omits_bundle_build(self):
sc = self._render(image_policy="cached")["services"]["sidecars"]
self.assertEqual("bot-bottle-sidecars:latest", sc["image"])
self.assertNotIn("build", sc)
def test_bundle_container_name_uses_sidecars_prefix(self):
sc = self._render()["services"]["sidecars"]
self.assertEqual(f"bot-bottle-sidecars-{SLUG}", sc["container_name"])
+44
View File
@@ -0,0 +1,44 @@
"""Unit tests for the host-side configuration store."""
from __future__ import annotations
import tempfile
import unittest
from pathlib import Path
from bot_bottle.config_store import (
CACHED_IMAGE_STALE_WARNING_DAYS,
DEFAULT_CACHED_IMAGE_STALE_WARNING_DAYS,
ConfigStore,
)
from bot_bottle.store_manager import StoreManager
class TestConfigStore(unittest.TestCase):
def test_cached_image_warning_days_defaults_to_one(self) -> None:
with tempfile.TemporaryDirectory(prefix="config-store.") as tmp:
store = ConfigStore(Path(tmp) / "bot-bottle.db")
store.migrate()
self.assertEqual(
DEFAULT_CACHED_IMAGE_STALE_WARNING_DAYS,
store.cached_image_stale_warning_days(),
)
def test_cached_image_warning_days_reads_value(self) -> None:
with tempfile.TemporaryDirectory(prefix="config-store.") as tmp:
store = ConfigStore(Path(tmp) / "bot-bottle.db")
store.migrate()
store.set(CACHED_IMAGE_STALE_WARNING_DAYS, "7")
self.assertEqual(7, store.cached_image_stale_warning_days())
def test_store_manager_includes_config_store(self) -> None:
with tempfile.TemporaryDirectory(prefix="config-store.") as tmp:
db = Path(tmp) / "bot-bottle.db"
manager = StoreManager(db)
self.assertFalse(manager.is_migrated())
manager.migrate()
self.assertTrue(manager.is_migrated())
if __name__ == "__main__":
unittest.main()
@@ -3,6 +3,7 @@
from __future__ import annotations
import contextlib
import dataclasses
import io
import tempfile
import unittest
@@ -16,6 +17,7 @@ from bot_bottle.backend.docker import launch as launch_mod
from bot_bottle.backend.docker.bottle_plan import DockerBottlePlan
from bot_bottle.egress import EgressPlan
from bot_bottle.git_gate import GitGatePlan
from bot_bottle.log import Die
from bot_bottle.manifest import ManifestIndex
@@ -103,6 +105,10 @@ class TestLaunchCommittedImage(unittest.TestCase):
launch_mod.docker_mod, "image_exists", return_value=image_present,
), mock.patch.object(
launch_mod.docker_mod, "build_image", side_effect=fake_build,
), mock.patch.object(
launch_mod.docker_mod, "image_created_at",
), mock.patch.object(
launch_mod, "warn_if_stale",
), mock.patch.object(
launch_mod, "egress_tls_init",
return_value=(Path("/egress_ca"), Path("/egress_cert")),
@@ -180,6 +186,24 @@ class TestLaunchCommittedImage(unittest.TestCase):
built = self._run_launch(plan, committed_tag=None)
self.assertEqual([_DEFAULT_IMAGE], built)
def test_cached_images_skip_build_when_present(self) -> None:
base = _plan(self._tmp)
plan = dataclasses.replace(
base,
spec=dataclasses.replace(base.spec, image_policy="cached"),
)
built = self._run_launch(plan, committed_tag=None, image_present=True)
self.assertEqual([], built)
def test_cached_images_die_when_agent_missing(self) -> None:
base = _plan(self._tmp)
plan = dataclasses.replace(
base,
spec=dataclasses.replace(base.spec, image_policy="cached"),
)
with self.assertRaises(Die):
self._run_launch(plan, committed_tag=None, image_present=False)
def test_falls_back_to_build_when_committed_image_missing_from_daemon(self) -> None:
plan = _plan(self._tmp)
built = self._run_launch(
+17
View File
@@ -9,6 +9,7 @@ from __future__ import annotations
import subprocess
import unittest
from datetime import timezone
from unittest.mock import patch
from bot_bottle.backend.docker import util as docker_mod
@@ -54,6 +55,22 @@ class TestImageId(unittest.TestCase):
self.assertIn("missing:tag", die.call_args.args[0])
class TestImageCreatedAt(unittest.TestCase):
def test_parses_docker_timestamp_with_nanoseconds(self):
with patch.object(
docker_mod.subprocess, "run",
return_value=_ok(stdout="2026-07-06T15:33:47.123456789Z\n"),
) as run:
created = docker_mod.image_created_at("bot-bottle-claude:latest")
self.assertEqual(2026, created.year)
self.assertEqual(123456, created.microsecond)
self.assertEqual(timezone.utc, created.tzinfo)
self.assertEqual(
["docker", "image", "inspect", "--format", "{{.Created}}", "bot-bottle-claude:latest"],
run.call_args.args[0],
)
class TestSave(unittest.TestCase):
def test_save_runs_docker_save(self):
with patch.object(
@@ -187,6 +187,56 @@ class TestAgentFromPath(unittest.TestCase):
dockerfile="/repo/Dockerfile",
)
def test_cached_policy_uses_existing_artifact_without_build(self):
with tempfile.TemporaryDirectory(prefix="cached-smolmachine.") as tmp:
digest = "abcdef0123456789"
artifact = Path(tmp) / f"{digest}.smolmachine.smolmachine"
artifact.write_text("")
plan = SimpleNamespace(
slug="dev-abc12",
agent_image="bot-bottle-claude:latest",
spec=SimpleNamespace(image_policy="cached"),
)
with patch.object(
_launch_mod, "_SMOLMACHINE_CACHE_DIR", Path(tmp),
), patch.object(
_launch_mod, "read_committed_image", return_value="",
), patch.object(
_launch_mod.docker_mod, "image_exists", return_value=True,
), patch.object(
_launch_mod.docker_mod, "image_id",
return_value=f"sha256:{digest}fffffffffffffffff",
), patch.object(
_launch_mod, "warn_if_stale_path",
), patch.object(
_launch_mod, "_ensure_smolmachine",
) as ensure:
result = _launch_mod._agent_from_path(cast(Any, plan))
self.assertEqual(artifact, result)
ensure.assert_not_called()
def test_cached_policy_dies_when_artifact_missing(self):
with tempfile.TemporaryDirectory(prefix="cached-smolmachine.") as tmp:
plan = SimpleNamespace(
slug="dev-abc12",
agent_image="bot-bottle-claude:latest",
spec=SimpleNamespace(image_policy="cached"),
)
with patch.object(
_launch_mod, "_SMOLMACHINE_CACHE_DIR", Path(tmp),
), patch.object(
_launch_mod, "read_committed_image", return_value="",
), patch.object(
_launch_mod.docker_mod, "image_exists", return_value=True,
), patch.object(
_launch_mod.docker_mod, "image_id",
return_value="sha256:abcdef0123456789fffffffffffffffff",
):
from bot_bottle.log import Die
with self.assertRaises(Die):
_launch_mod._agent_from_path(cast(Any, plan))
if __name__ == "__main__":
unittest.main()