From 5a31c6f9b2ef75d15774bab6e61d6e8320511811 Mon Sep 17 00:00:00 2001 From: claude Date: Sun, 26 Jul 2026 00:04:35 +0000 Subject: [PATCH] fix: make installed wheel self-contained + harden install.sh prereqs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the review on PR #481. Self-contained wheel (review point 1): the gateway/infra/orchestrator images build from a context that must hold bot_bottle/, pyproject.toml, and the root-level Dockerfiles. Modules previously located these by walking __file__ to the repo root, so an installed wheel (package in site-packages, no repo root) passed `doctor` but failed `start`. - Add bot_bottle/resources.py: build_root() returns the repo root in a checkout (unchanged) or a staged copy from the wheel's bundled _resources/ otherwise; dockerfile()/nix_netpool_module()/ netpool_script() derive from it. - setup.py bundles the root Dockerfiles, nix module, netpool script, and pyproject.toml into bot_bottle/_resources/ at build; MANIFEST.in ships them in the sdist. - Route every _REPO_ROOT/_REPO_DIR call site (docker/macos launch, macos infra, firecracker infra_vm/infra_artifact/setup, orchestrator lifecycle/gateway) through resources. Checkout behavior is unchanged. install.sh prerequisites (review point 2): check for git when installing a git+ spec, and — before the pip fallback — that pip is usable and the interpreter isn't externally managed (PEP 668), pointing at pipx. Tests: test_resources covers checkout + staged-wheel layouts; test_wheel_install builds the wheel, installs it into an isolated venv, and asserts `doctor` runs and build_root() yields a valid context. Running `start` end-to-end still needs a Docker/KVM host (CI). Co-Authored-By: Claude Opus 4.8 --- .coveragerc | 6 + .gitignore | 3 + MANIFEST.in | 8 ++ bot_bottle/backend/docker/gateway.py | 12 +- bot_bottle/backend/docker/infra.py | 9 +- bot_bottle/backend/docker/launch.py | 8 +- bot_bottle/backend/docker/orchestrator.py | 9 +- .../backend/firecracker/infra_artifact.py | 7 +- bot_bottle/backend/firecracker/infra_vm.py | 9 +- bot_bottle/backend/firecracker/setup.py | 9 +- bot_bottle/backend/macos_container/gateway.py | 9 +- bot_bottle/backend/macos_container/infra.py | 9 +- bot_bottle/backend/macos_container/launch.py | 5 +- .../backend/macos_container/orchestrator.py | 8 +- bot_bottle/gateway/__init__.py | 1 - bot_bottle/resources.py | 131 ++++++++++++++++++ docs/prds/prd-new-install-script.md | 50 ++++++- install.sh | 35 +++++ pyproject.toml | 2 +- setup.py | 46 ++++++ tests/unit/test_install_script.py | 15 ++ tests/unit/test_macos_nested_containers.py | 2 +- tests/unit/test_resources.py | 126 +++++++++++++++++ tests/unit/test_wheel_install.py | 116 ++++++++++++++++ 24 files changed, 583 insertions(+), 52 deletions(-) create mode 100644 MANIFEST.in create mode 100644 bot_bottle/resources.py create mode 100644 setup.py create mode 100644 tests/unit/test_resources.py create mode 100644 tests/unit/test_wheel_install.py diff --git a/.coveragerc b/.coveragerc index 161fde3f..7bf4ae03 100644 --- a/.coveragerc +++ b/.coveragerc @@ -20,3 +20,9 @@ omit = bot_bottle/cli/tui.py bot_bottle/cli/init.py tests/* + # Build-time only: setuptools invokes it out-of-process to build the + # wheel/sdist (it's never imported by the running app), so in-process + # coverage can't reach it. Its one job — bundling the root resources into + # bot_bottle/_resources/ — is exercised end-to-end by test_wheel_install, + # which builds and installs a real wheel and checks the result. + setup.py diff --git a/.gitignore b/.gitignore index 03d76f4e..8c0aa223 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,9 @@ __pycache__/ *.py[cod] *$py.class *.egg-info/ +# setuptools/build_meta output (wheels, sdists, build tree) +/build/ +/dist/ .venv/ venv/ .pytest_cache/ diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..a39c977c --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,8 @@ +# Root-level build resources copied into bot_bottle/_resources/ at build time +# (see setup.py). Included in the sdist so `pip install` from an sdist can +# still bundle them into the wheel. +include Dockerfile.gateway +include Dockerfile.orchestrator +include Dockerfile.orchestrator.fc +include nix/firecracker-netpool.nix +include scripts/firecracker-netpool.sh diff --git a/bot_bottle/backend/docker/gateway.py b/bot_bottle/backend/docker/gateway.py index 1cac5d32..612b3d77 100644 --- a/bot_bottle/backend/docker/gateway.py +++ b/bot_bottle/backend/docker/gateway.py @@ -10,9 +10,10 @@ from ...paths import ( ORCHESTRATOR_AUTH_JWT_ENV, host_gateway_ca_dir, ) +from ... import resources from ...gateway import ( Gateway, GatewayTransport, GATEWAY_IMAGE, GATEWAY_NAME, GATEWAY_NETWORK, - GATEWAY_DOCKERFILE, REPO_ROOT, GATEWAY_LABEL, MITMPROXY_HOME, + GATEWAY_DOCKERFILE, GATEWAY_LABEL, MITMPROXY_HOME, DEFAULT_CA_TIMEOUT_SECONDS, CA_POLL_SECONDS, GATEWAY_CA_CERT, GatewayError ) @@ -50,7 +51,9 @@ class DockerGateway(Gateway): # `address` / `stop` work on an already-running gateway without it. self._orchestrator_url = "" self._gateway_token = "" - self._build_context = build_context or REPO_ROOT + # Resolved lazily in ensure_built() so merely constructing a gateway to + # read its CA never stages a build root from an installed wheel. + self._build_context = build_context self._dockerfile = dockerfile # Ports published on the host (0.0.0.0). Used by the Firecracker # backend's dev-harness gateway so VMs can reach it via their TAP link; @@ -72,9 +75,10 @@ class DockerGateway(Gateway): forces a full rebuild (parity with `start --no-cache`).""" if self._dockerfile is None: return + context = self._build_context or resources.build_root() argv = ["docker", "build", "-t", self.image_ref, - "-f", str(self._build_context / self._dockerfile), - str(self._build_context)] + "-f", str(context / self._dockerfile), + str(context)] if os.environ.get("BOT_BOTTLE_NO_CACHE"): argv.insert(2, "--no-cache") proc = run_docker(argv) diff --git a/bot_bottle/backend/docker/infra.py b/bot_bottle/backend/docker/infra.py index 1320146a..66e26837 100644 --- a/bot_bottle/backend/docker/infra.py +++ b/bot_bottle/backend/docker/infra.py @@ -34,6 +34,7 @@ from .orchestrator import ( ORCHESTRATOR_NETWORK, ) from ...paths import bot_bottle_root +from ... import resources from ...gateway import ( GATEWAY_IMAGE, GATEWAY_NAME, @@ -50,8 +51,6 @@ from ...orchestrator.lifecycle import ( # the pair's public identity. INFRA_NAME = GATEWAY_NAME # the container agents attribute against is the gateway -_REPO_ROOT = Path(__file__).resolve().parents[3] - class DockerInfraService(InfraService): """Composes the per-host control plane + gateway as two containers. @@ -68,7 +67,7 @@ class DockerInfraService(InfraService): control_network: str = ORCHESTRATOR_NETWORK, orchestrator_image: str = ORCHESTRATOR_IMAGE, gateway_image: str = GATEWAY_IMAGE, - repo_root: Path = _REPO_ROOT, + repo_root: Path | None = None, host_root: Path | None = None, orchestrator_name: str = ORCHESTRATOR_NAME, orchestrator_label: str = ORCHESTRATOR_LABEL, @@ -79,7 +78,9 @@ class DockerInfraService(InfraService): self.control_network = control_network self.orchestrator_image = orchestrator_image self.gateway_image = gateway_image - self._repo_root = repo_root + # Build context / bind-mount source: the repo root in a checkout, a + # staged copy from the installed wheel otherwise (bot_bottle.resources). + self._repo_root = repo_root if repo_root is not None else resources.build_root() self._host_root = host_root or bot_bottle_root() self._orchestrator_name = orchestrator_name self._orchestrator_label = orchestrator_label diff --git a/bot_bottle/backend/docker/launch.py b/bot_bottle/backend/docker/launch.py index 3cec0b6a..1b96f01d 100644 --- a/bot_bottle/backend/docker/launch.py +++ b/bot_bottle/backend/docker/launch.py @@ -33,7 +33,6 @@ from __future__ import annotations import dataclasses import os from contextlib import ExitStack, contextmanager -from pathlib import Path from typing import Callable, Generator from ...agent_provider import runtime_for @@ -65,10 +64,7 @@ from ...orchestrator.store.config_store import resolve_teardown_timeout from .consolidated_launch import launch_consolidated, deprovision_consolidated from .infra import INFRA_NAME from .gateway import DockerGateway - - -# Where the repo root lives, for `docker build` context. Computed once. -_REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent) +from ... import resources def build_or_load_images(plan: DockerBottlePlan) -> BottleImages: @@ -88,7 +84,7 @@ def build_or_load_images(plan: DockerBottlePlan) -> BottleImages: ) info(f"using cached agent image {plan.image!r}") return BottleImages(agent=plan.image) - docker_mod.build_image(plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path) + docker_mod.build_image(plan.image, str(resources.build_root()), dockerfile=plan.dockerfile_path) docker_mod.verify_agent_image( plan.image, runtime_for(plan.agent_provider_template).smoke_test, ) diff --git a/bot_bottle/backend/docker/orchestrator.py b/bot_bottle/backend/docker/orchestrator.py index 550d99ea..70846459 100644 --- a/bot_bottle/backend/docker/orchestrator.py +++ b/bot_bottle/backend/docker/orchestrator.py @@ -15,6 +15,7 @@ import time from pathlib import Path from ... import log +from ... import resources from .util import run_docker from ...paths import ( ORCHESTRATOR_TOKEN_ENV, @@ -56,8 +57,6 @@ _ROOT_IN_CONTAINER = "/bot-bottle-root" _HEALTH_POLL_SECONDS = 0.25 -_REPO_ROOT = Path(__file__).resolve().parents[3] - class DockerOrchestrator(Orchestrator): """The control plane as a single fixed-name container. `ensure_built` builds @@ -72,7 +71,7 @@ class DockerOrchestrator(Orchestrator): label: str = ORCHESTRATOR_LABEL, port: int = DEFAULT_PORT, control_network: str = ORCHESTRATOR_NETWORK, - repo_root: Path = _REPO_ROOT, + repo_root: Path | None = None, host_root: Path | None = None, dockerfile: str | None = ORCHESTRATOR_DOCKERFILE, ) -> None: @@ -81,7 +80,9 @@ class DockerOrchestrator(Orchestrator): self.label = label self.port = port self.control_network = control_network - self._repo_root = repo_root + # Build context / bind-mount source: the repo root in a checkout, a + # staged copy from the installed wheel otherwise (bot_bottle.resources). + self._repo_root = repo_root if repo_root is not None else resources.build_root() self._host_root = host_root or bot_bottle_root() self._dockerfile = dockerfile diff --git a/bot_bottle/backend/firecracker/infra_artifact.py b/bot_bottle/backend/firecracker/infra_artifact.py index b5b7ae60..ba0ca075 100644 --- a/bot_bottle/backend/firecracker/infra_artifact.py +++ b/bot_bottle/backend/firecracker/infra_artifact.py @@ -37,6 +37,7 @@ import urllib.error import urllib.request from pathlib import Path +from ... import resources from ...log import die, info from . import util @@ -44,8 +45,6 @@ from . import util # scheme can't collide with a cached/published artifact of the old one. _ARTIFACT_FORMAT = "1" -_REPO_ROOT = Path(__file__).resolve().parents[3] - # The two per-plane infra VM roles. Each publishes/pulls its own rootfs artifact # from its own generic package; the Dockerfiles baked into each differ (only the # orchestrator rootfs carries buildah), so the versions are hashed separately. @@ -74,7 +73,7 @@ def local_build_requested() -> bool: def infra_artifact_version( - init_script: str, role: str, *, repo_root: Path = _REPO_ROOT, + init_script: str, role: str, *, repo_root: Path | None = None, ) -> str: """Content hash (16 hex) of everything baked into `role`'s infra rootfs: the whole shipped `bot_bottle` package, that role's Dockerfiles, and its guest @@ -89,6 +88,8 @@ def infra_artifact_version( version or a launch host could boot a stale rootfs whose code differs from its checkout. `__pycache__`/`.pyc` are the only exclusions — build artifacts, never copied.""" + if repo_root is None: + repo_root = resources.build_root() h = hashlib.sha256() h.update(f"format={_ARTIFACT_FORMAT}\nrole={role}\n".encode()) pkg = repo_root / "bot_bottle" diff --git a/bot_bottle/backend/firecracker/infra_vm.py b/bot_bottle/backend/firecracker/infra_vm.py index 9183136c..0fa03f0f 100644 --- a/bot_bottle/backend/firecracker/infra_vm.py +++ b/bot_bottle/backend/firecracker/infra_vm.py @@ -42,6 +42,7 @@ from dataclasses import dataclass from pathlib import Path from typing import Generator +from ... import resources from ...log import die, info from ..docker import util as docker_mod from . import firecracker_vm, infra_artifact, netpool, util @@ -65,7 +66,6 @@ _GUEST_GATEWAY_JWT_PATH = "/var/lib/bot-bottle/gateway-jwt" _GATEWAY_IMAGE = "bot-bottle-gateway:latest" _ORCHESTRATOR_IMAGE = "bot-bottle-orchestrator:latest" _ORCHESTRATOR_FC_IMAGE = "bot-bottle-orchestrator-fc:latest" -_REPO_ROOT = Path(__file__).resolve().parents[3] # Per-role rootfs source image + the extra free space `mke2fs` leaves for the # guest to grow into. The orchestrator keeps buildah's large build slack; the @@ -130,12 +130,13 @@ def build_infra_images_with_docker() -> None: orchestrator + buildah). The gateway VM boots the gateway image directly. The launch host uses this only in `BOT_BOTTLE_INFRA_BUILD=local` mode; `publish_infra` uses it off-host to produce the published artifacts.""" + root = str(resources.build_root()) docker_mod.build_image( - _ORCHESTRATOR_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.orchestrator") + _ORCHESTRATOR_IMAGE, root, dockerfile="Dockerfile.orchestrator") docker_mod.build_image( - _GATEWAY_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.gateway") + _GATEWAY_IMAGE, root, dockerfile="Dockerfile.gateway") docker_mod.build_image( - _ORCHESTRATOR_FC_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.orchestrator.fc") + _ORCHESTRATOR_FC_IMAGE, root, dockerfile="Dockerfile.orchestrator.fc") def build_rootfs_dir(role: str) -> Path: diff --git a/bot_bottle/backend/firecracker/setup.py b/bot_bottle/backend/firecracker/setup.py index d63dbb50..b7c93ed2 100644 --- a/bot_bottle/backend/firecracker/setup.py +++ b/bot_bottle/backend/firecracker/setup.py @@ -20,6 +20,7 @@ import subprocess import sys from pathlib import Path +from ... import resources from . import netpool from . import util @@ -42,13 +43,13 @@ def _has_systemd() -> bool: def _module_path() -> str: - """Absolute path to the importable NixOS module in this checkout.""" - return str(Path(__file__).resolve().parents[3] / "nix" / "firecracker-netpool.nix") + """Absolute path to the importable NixOS module (checkout or wheel).""" + return str(resources.nix_netpool_module()) def _script_path() -> str: - """Absolute path to the bundled bring-up script in this checkout.""" - return str(Path(__file__).resolve().parents[3] / "scripts" / "firecracker-netpool.sh") + """Absolute path to the bundled bring-up script (checkout or wheel).""" + return str(resources.netpool_script()) def _print_prereqs() -> None: diff --git a/bot_bottle/backend/macos_container/gateway.py b/bot_bottle/backend/macos_container/gateway.py index e81b8e9c..4b84193e 100644 --- a/bot_bottle/backend/macos_container/gateway.py +++ b/bot_bottle/backend/macos_container/gateway.py @@ -28,6 +28,7 @@ from ...paths import ( ORCHESTRATOR_AUTH_JWT_ENV, host_gateway_ca_dir, ) +from ... import resources from .. import util as backend_util from . import util as container_mod @@ -52,8 +53,6 @@ GATEWAY_DAEMONS = "egress,git-http,supervise" GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest") -_REPO_ROOT = Path(__file__).resolve().parents[3] - def ensure_networks( network: str = GATEWAY_NETWORK, @@ -84,14 +83,16 @@ class MacosGateway(Gateway): network: str = GATEWAY_NETWORK, egress_network: str = GATEWAY_EGRESS_NETWORK, control_network: str = CONTROL_NETWORK, - repo_root: Path = _REPO_ROOT, + repo_root: Path | None = None, ) -> None: self.image_ref = image_ref self.name = name self.network = network self.egress_network = egress_network self.control_network = control_network - self._repo_root = repo_root + # Build context: the repo root in a checkout, a staged copy from the + # installed wheel otherwise (bot_bottle.resources). + self._repo_root = repo_root if repo_root is not None else resources.build_root() # Set by `connect_to_orchestrator`: the URL the daemons resolve policy # against + the pre-minted `gateway` token they present. The gateway # never mints, so it never holds the signing key (#469). diff --git a/bot_bottle/backend/macos_container/infra.py b/bot_bottle/backend/macos_container/infra.py index 2327480d..7d250cd5 100644 --- a/bot_bottle/backend/macos_container/infra.py +++ b/bot_bottle/backend/macos_container/infra.py @@ -24,6 +24,7 @@ from __future__ import annotations from pathlib import Path +from ... import resources from ...orchestrator.lifecycle import ( DEFAULT_PORT, DEFAULT_STARTUP_TIMEOUT_SECONDS, @@ -53,8 +54,6 @@ from .orchestrator import ( # still import it (probe / reprovision attribute against the gateway). INFRA_NAME = GATEWAY_NAME -_REPO_ROOT = Path(__file__).resolve().parents[3] - class MacosInfraService(InfraService): """Composes the per-host orchestrator + gateway containers. Callers use @@ -70,7 +69,7 @@ class MacosInfraService(InfraService): control_network: str = CONTROL_NETWORK, gateway_image: str = GATEWAY_IMAGE, orchestrator_image: str = ORCHESTRATOR_IMAGE, - repo_root: Path = _REPO_ROOT, + repo_root: Path | None = None, orchestrator_name: str = ORCHESTRATOR_NAME, gateway_name: str = INFRA_NAME, db_volume: str = ORCHESTRATOR_DB_VOLUME, @@ -81,7 +80,9 @@ class MacosInfraService(InfraService): self.control_network = control_network self.gateway_image = gateway_image self.orchestrator_image = orchestrator_image - self._repo_root = repo_root + # Build context / bind-mount source: the repo root in a checkout, a + # staged copy from the installed wheel otherwise (bot_bottle.resources). + self._repo_root = repo_root if repo_root is not None else resources.build_root() self._orchestrator_name = orchestrator_name self._gateway_name = gateway_name self._db_volume = db_volume diff --git a/bot_bottle/backend/macos_container/launch.py b/bot_bottle/backend/macos_container/launch.py index 34793fba..e3b87fa4 100644 --- a/bot_bottle/backend/macos_container/launch.py +++ b/bot_bottle/backend/macos_container/launch.py @@ -36,7 +36,6 @@ import dataclasses import os import subprocess from contextlib import ExitStack, contextmanager -from pathlib import Path from typing import Callable, Generator from ...bottle_state import ( @@ -49,6 +48,7 @@ from ...git_gate import GitGate from ...gateway.git_gate.http_backend import DEFAULT_PORT as _GIT_HTTP_PORT from ...image_cache import check_stale from ...log import die, info, warn +from ... import resources from .. import BottleImages from ...supervisor.types import SUPERVISE_PORT from ..docker.egress import EGRESS_PORT @@ -71,7 +71,6 @@ from .consolidated_launch import ( deprovision_consolidated, ) -_REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent) _AGENT_SLEEP_SECONDS = "2147483647" @@ -94,7 +93,7 @@ def _agent_image(plan: MacosContainerBottlePlan) -> str: ) info(f"using cached agent image {plan.image!r}") return plan.image - container_mod.build_image(plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path) + container_mod.build_image(plan.image, str(resources.build_root()), dockerfile=plan.dockerfile_path) return plan.image diff --git a/bot_bottle/backend/macos_container/orchestrator.py b/bot_bottle/backend/macos_container/orchestrator.py index 32802760..9939de29 100644 --- a/bot_bottle/backend/macos_container/orchestrator.py +++ b/bot_bottle/backend/macos_container/orchestrator.py @@ -18,6 +18,7 @@ import urllib.request from pathlib import Path from ... import log +from ... import resources from ...paths import ( ORCHESTRATOR_TOKEN_ENV, host_orchestrator_token, @@ -48,7 +49,6 @@ _DB_ROOT_IN_CONTAINER = "/var/lib/bot-bottle" _SRC_IN_CONTAINER = "/bot-bottle-src" _HEALTH_POLL_SECONDS = 0.25 -_REPO_ROOT = Path(__file__).resolve().parents[3] class MacosOrchestrator(Orchestrator): @@ -64,7 +64,7 @@ class MacosOrchestrator(Orchestrator): label: str = ORCHESTRATOR_LABEL, port: int = DEFAULT_PORT, control_network: str = CONTROL_NETWORK, - repo_root: Path = _REPO_ROOT, + repo_root: Path | None = None, db_volume: str = ORCHESTRATOR_DB_VOLUME, ) -> None: self.image_ref = image_ref @@ -72,7 +72,9 @@ class MacosOrchestrator(Orchestrator): self.label = label self.port = port self.control_network = control_network - self._repo_root = repo_root + # Build context / bind-mount source: the repo root in a checkout, a + # staged copy from the installed wheel otherwise (bot_bottle.resources). + self._repo_root = repo_root if repo_root is not None else resources.build_root() self._db_volume = db_volume def url(self) -> str: diff --git a/bot_bottle/gateway/__init__.py b/bot_bottle/gateway/__init__.py index 568ba7c5..27925748 100644 --- a/bot_bottle/gateway/__init__.py +++ b/bot_bottle/gateway/__init__.py @@ -62,7 +62,6 @@ GATEWAY_CA_GLOB = "mitmproxy-ca*" # that lands. Env override matches the backend's BOT_BOTTLE_GATEWAY_IMAGE. GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest") GATEWAY_DOCKERFILE = "Dockerfile.gateway" -REPO_ROOT = Path(__file__).resolve().parents[2] def rotate_gateway_ca(ca_dir: Path | None = None) -> list[Path]: diff --git a/bot_bottle/resources.py b/bot_bottle/resources.py new file mode 100644 index 00000000..fe6837a1 --- /dev/null +++ b/bot_bottle/resources.py @@ -0,0 +1,131 @@ +"""Locate build-time resources whether bot-bottle runs from a source +checkout or an installed wheel. + +The gateway / infra / orchestrator images are built from a Docker (or Apple +`container`) build context that must contain the `bot_bottle` package, +`pyproject.toml`, and the root-level Dockerfiles as siblings. In a source +checkout that context is simply the repo root, one level above the package. +An installed wheel has no repo root: the same root-level files are shipped +inside the package under ``bot_bottle/_resources/`` (see ``setup.py``), and a +repo-root-shaped build context is staged on demand into the app-data dir. + +``build_root()`` is the single source of truth — it returns a directory laid +out like a repo root (has ``bot_bottle/``, ``pyproject.toml``, the +Dockerfiles, ``nix/``, ``scripts/``). Every caller that needs a build +context, a Dockerfile path, the nix netpool module, or the netpool script +derives from it, so checkout and wheel installs share one downstream path. +""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +from .paths import bot_bottle_root + +_PKG = Path(__file__).resolve().parent # …/bot_bottle +_CHECKOUT_ROOT = _PKG.parent # repo root in a checkout +_BUNDLED = _PKG / "_resources" # wheel-shipped copies + +# Root-level files bundled into the wheel under ``_resources/`` (paths are +# relative to the checkout root, and preserved verbatim under ``_resources/`` +# and in the staged build root). ``setup.py`` copies exactly this set; keep +# the two lists in sync (``test_resources`` guards that every entry exists). +BUNDLED_RESOURCES: tuple[str, ...] = ( + "pyproject.toml", + "Dockerfile.gateway", + "Dockerfile.orchestrator", + "Dockerfile.orchestrator.fc", + "nix/firecracker-netpool.nix", + "scripts/firecracker-netpool.sh", +) + +# Present at a checkout root, never in a bare installed package — the cheap +# tell for which layout we're in. +_CHECKOUT_MARKER = "Dockerfile.gateway" + + +class ResourceError(RuntimeError): + """Build resources are missing from the install (corrupt/partial wheel).""" + + +def is_source_checkout() -> bool: + """True when running from a source tree (the root Dockerfiles sit beside + the package); False from an installed wheel.""" + return (_CHECKOUT_ROOT / _CHECKOUT_MARKER).is_file() + + +def build_root() -> Path: + """A directory shaped like a repo root: ``bot_bottle/``, ``pyproject.toml``, + the root Dockerfiles, ``nix/``, and ``scripts/``. + + A checkout returns the repo root itself (no copying). An installed wheel + returns a staged copy under the app-data dir, materialized once and reused + (keyed by distribution version).""" + if is_source_checkout(): + return _CHECKOUT_ROOT + return _stage_build_root() + + +def dockerfile(name: str) -> Path: + """Absolute path to a root-level Dockerfile, e.g. ``Dockerfile.gateway``.""" + return build_root() / name + + +def nix_netpool_module() -> Path: + """Absolute path to the firecracker netpool NixOS module.""" + return build_root() / "nix" / "firecracker-netpool.nix" + + +def netpool_script() -> Path: + """Absolute path to the firecracker netpool bring-up script.""" + return build_root() / "scripts" / "firecracker-netpool.sh" + + +def _dist_version() -> str: + """Installed distribution version, for keying the staged build root. Falls + back to ``dev`` when metadata is unavailable (e.g. running from a tree).""" + try: + from importlib.metadata import PackageNotFoundError, version + try: + return version("bot-bottle") + except PackageNotFoundError: + return "dev" + except Exception: # pragma: no cover - importlib.metadata is always present + return "dev" + + +def _stage_build_root() -> Path: + """Materialize a repo-root-shaped build context from the installed wheel's + bundled resources. Idempotent and cached on disk by version.""" + if not _BUNDLED.is_dir(): + raise ResourceError( + "bot-bottle build resources are missing from this install " + f"(expected {_BUNDLED}). Reinstall the package." + ) + dest = bot_bottle_root() / "build-root" / _dist_version() + if (dest / ".complete").is_file(): + return dest + + staging = dest.with_name(dest.name + ".staging") + shutil.rmtree(staging, ignore_errors=True) + staging.mkdir(parents=True, exist_ok=True) + + # The package itself, minus caches and the bundled-resource copies, so the + # staged ``bot_bottle/`` matches a checkout's (keeps the firecracker + # infra-artifact content hash stable across checkout and wheel installs). + shutil.copytree( + _PKG, + staging / "bot_bottle", + ignore=shutil.ignore_patterns("__pycache__", "*.pyc", "_resources"), + ) + # The bundled root files, restored to their checkout-relative layout. + for rel in BUNDLED_RESOURCES: + dst = staging / rel + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(_BUNDLED / rel, dst) + + shutil.rmtree(dest, ignore_errors=True) + staging.rename(dest) + (dest / ".complete").write_text("") + return dest diff --git a/docs/prds/prd-new-install-script.md b/docs/prds/prd-new-install-script.md index 54e302b9..5cbe5eb2 100644 --- a/docs/prds/prd-new-install-script.md +++ b/docs/prds/prd-new-install-script.md @@ -67,10 +67,45 @@ bot-bottle = "bot_bottle.cli:main" ``` `bot_bottle.cli:main` already exists (the `cli.py` shim calls it), so no -refactor of the entry point is needed. `package-data` ships the container -Dockerfiles, `egress_entrypoint.sh`, the firecracker netpool defaults, and -the macos-container init script so an installed wheel can still build its -sidecar/agent images. +refactor of the entry point is needed. `package-data` ships the non-Python +assets that live *inside* the package (`egress_entrypoint.sh`, the contrib +Dockerfiles, the firecracker netpool defaults, the macos-container init +script). + +### Self-contained wheel (build resources) + +The gateway / infra / orchestrator images are built from a Docker (or Apple +`container`) build context that must contain the `bot_bottle` package, +`pyproject.toml`, and the **root-level** Dockerfiles as siblings. Several +modules used to locate that context by walking `__file__`'s parents to the +repo root (`_REPO_ROOT = Path(__file__)…parents[N]`) and reading +`Dockerfile.gateway`, `nix/firecracker-netpool.nix`, and +`scripts/firecracker-netpool.sh` from it. In an installed wheel the package +lives in `site-packages` with no repo root above it, so those reads fail — +`doctor` passes but `start` / backend setup breaks. + +Fix: a single resolver, `bot_bottle/resources.py`. + +- `build_root()` returns a directory shaped like a repo root (has + `bot_bottle/`, `pyproject.toml`, the Dockerfiles, `nix/`, `scripts/`). + In a **checkout** it's the repo root itself — unchanged behavior. From an + **installed wheel** it stages a copy under the app-data dir, once, keyed + by version. +- The root-level resources are shipped inside the wheel under + `bot_bottle/_resources/` by a `setup.py` `build_py` step (kept in sync + with `resources.BUNDLED_RESOURCES`); `MANIFEST.in` includes them in the + sdist. +- Every former `_REPO_ROOT` / `_REPO_DIR` call site now derives from + `resources`: the docker/macos agent-image launch, each backend's + `orchestrator` / `gateway` / `infra` service, firecracker `infra_vm` / + `infra_artifact` / `setup`, and the shared `gateway` build context. So + checkout and wheel installs share one downstream path. + +Verification: `test_resources` exercises both layouts (including the staged +wheel context); `test_wheel_install` builds the wheel, installs it into an +isolated venv, and asserts `bot-bottle doctor` runs and `build_root()` +produces a valid context. Running `start` end-to-end still needs a +Docker/KVM host (CI), not a source checkout. ### `install.sh` @@ -78,8 +113,11 @@ A POSIX `sh` bootstrapper that: 1. Checks `python3` is present and ≥ 3.11; exits with a clear message otherwise. -2. Creates `~/.bot-bottle/{agents,bottles,contrib}`. -3. Installs via `pipx` if available, else `python3 -m pip install --user`. +2. Checks `git` when installing a `git+` spec, and — when falling back to + pip — that pip is usable and the interpreter isn't externally managed + (PEP 668), pointing at pipx otherwise. +3. Creates `~/.bot-bottle/{agents,bottles,contrib}`. +4. Installs via `pipx` if available, else `python3 -m pip install --user`. The spec defaults to the git URL and is overridable via `BOT_BOTTLE_INSTALL_SPEC` (used by tests / local installs). 4. Locates the `bot-bottle` entry point (PATH or `~/.local/bin`). diff --git a/install.sh b/install.sh index 667b4b8c..0af73145 100755 --- a/install.sh +++ b/install.sh @@ -40,6 +40,41 @@ want = (int(sys.argv[1]), int(sys.argv[2])) raise SystemExit(0 if sys.version_info[:2] >= want else 1) PY +# Installing a `git+` spec (the default) shells out to git under the hood, +# whether via pipx or pip. Fail early with a clear message rather than deep +# inside the installer's output. +case "${PACKAGE_SPEC}" in + git+*|*.git) + command -v git >/dev/null 2>&1 || die \ + "git is required to install from '${PACKAGE_SPEC}'. Install git, or set "\ +"BOT_BOTTLE_INSTALL_SPEC to a non-git spec (e.g. a wheel path or a package index name)." + ;; +esac + +# The pip fallback needs a usable pip. Externally-managed interpreters +# (PEP 668, common on Debian/Ubuntu/Homebrew) reject `pip install --user`; +# pipx sidesteps that, so recommend it when pip can't be used. +if ! command -v pipx >/dev/null 2>&1; then + python3 -m pip --version >/dev/null 2>&1 || die \ + "neither pipx nor a usable 'python3 -m pip' was found. Install pipx "\ +"(recommended): 'python3 -m pip install --user pipx' or your OS package manager." + if python3 - <<'PY' +import os +import sys +import sysconfig + +# PEP 668: an EXTERNALLY-MANAGED marker in the stdlib dir means pip refuses +# to install into this interpreter without --break-system-packages. +marker = os.path.join(sysconfig.get_path("stdlib"), "EXTERNALLY-MANAGED") +raise SystemExit(0 if os.path.exists(marker) else 1) +PY + then + die "this Python is externally managed (PEP 668), so 'pip install --user' is "\ +"blocked. Install pipx and re-run: 'python3 -m pip install --user --break-system-packages pipx', "\ +"then 'pipx ensurepath'." + fi +fi + # --- config directories ------------------------------------------------------ mkdir -p \ diff --git a/pyproject.toml b/pyproject.toml index 6d600f2c..615afb68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ include = ["bot_bottle*"] # files shipped under bot_bottle/; test_pyproject.py asserts they exist. [tool.setuptools.package-data] bot_bottle = [ - "egress_entrypoint.sh", + "gateway/egress/entrypoint.sh", "contrib/claude/Dockerfile", "contrib/codex/Dockerfile", "contrib/pi/Dockerfile", diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..331c9e49 --- /dev/null +++ b/setup.py @@ -0,0 +1,46 @@ +"""Build shim. Project metadata lives in ``pyproject.toml``; this only adds a +build step that copies the root-level build resources (the Dockerfiles, the +nix netpool module, the netpool script, and ``pyproject.toml``) into +``bot_bottle/_resources/`` so an installed wheel is self-contained and can +build its gateway/infra/orchestrator images without a source checkout. + +Kept in sync with ``bot_bottle.resources.BUNDLED_RESOURCES`` — the +``test_resources`` suite guards against drift between the two lists. +""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +from setuptools import setup +from setuptools.command.build_py import build_py + +_ROOT = Path(__file__).resolve().parent + +# Must match bot_bottle.resources.BUNDLED_RESOURCES (paths relative to root). +_BUNDLED_RESOURCES = ( + "pyproject.toml", + "Dockerfile.gateway", + "Dockerfile.orchestrator", + "Dockerfile.orchestrator.fc", + "nix/firecracker-netpool.nix", + "scripts/firecracker-netpool.sh", +) + + +class _BundleResources(build_py): + """Copy the root-level build resources into the built package tree so they + ship inside the wheel under ``bot_bottle/_resources/``.""" + + def run(self) -> None: + super().run() + pkg_resources = Path(self.build_lib) / "bot_bottle" / "_resources" + for rel in _BUNDLED_RESOURCES: + src = _ROOT / rel + dst = pkg_resources / rel + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) + + +setup(cmdclass={"build_py": _BundleResources}) diff --git a/tests/unit/test_install_script.py b/tests/unit/test_install_script.py index 48475ed8..3186a5cf 100644 --- a/tests/unit/test_install_script.py +++ b/tests/unit/test_install_script.py @@ -53,6 +53,21 @@ class TestInstallScript(unittest.TestCase): # Tests / local installs point BOT_BOTTLE_INSTALL_SPEC at a checkout. self.assertIn("BOT_BOTTLE_INSTALL_SPEC", self.text) + def test_requires_git_for_git_specs(self): + # A git+ / .git spec (the default) shells out to git; the script must + # gate on it rather than failing opaquely inside pipx/pip. + self.assertIn("command -v git", self.text) + self.assertIn("git+*|*.git", self.text) + + def test_checks_pip_usable_before_fallback(self): + self.assertIn("python3 -m pip --version", self.text) + + def test_detects_externally_managed_python(self): + # PEP 668: 'pip install --user' is blocked on externally-managed + # interpreters; the script must detect this and point at pipx. + self.assertIn("EXTERNALLY-MANAGED", self.text) + self.assertIn("pipx", self.text) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_macos_nested_containers.py b/tests/unit/test_macos_nested_containers.py index c6e5fdb0..731dc320 100644 --- a/tests/unit/test_macos_nested_containers.py +++ b/tests/unit/test_macos_nested_containers.py @@ -283,7 +283,7 @@ class TestBuildOrLoadImages(unittest.TestCase): images = launch_mod.build_or_load_images(plan) build.assert_called_once_with( - "agent:base", launch_mod._REPO_DIR, # pylint: disable=protected-access + "agent:base", str(launch_mod.resources.build_root()), dockerfile="/repo/Dockerfile", ) derived.assert_called_once_with("agent:base", build) diff --git a/tests/unit/test_resources.py b/tests/unit/test_resources.py new file mode 100644 index 00000000..b517cf6f --- /dev/null +++ b/tests/unit/test_resources.py @@ -0,0 +1,126 @@ +"""Unit: bot_bottle.resources — build-resource resolution for both a source +checkout and an installed wheel. + +The checkout path is what the whole test suite already runs under; the wheel +path is exercised here by faking an installed layout (a package dir with a +bundled ``_resources/`` and no sibling Dockerfiles) and asserting that +``build_root()`` stages a repo-root-shaped context. See +``test_wheel_install.py`` for the end-to-end build+install check. +""" + +from __future__ import annotations + +import unittest +from pathlib import Path +from unittest.mock import patch + +from bot_bottle import resources + +from tests.unit import use_bottle_root + + +class TestCheckoutMode(unittest.TestCase): + """The environment the suite runs in: a real source checkout.""" + + def test_is_source_checkout(self): + self.assertTrue(resources.is_source_checkout()) + + def test_build_root_is_repo_root(self): + root = resources.build_root() + self.assertTrue((root / "bot_bottle").is_dir()) + self.assertTrue((root / "pyproject.toml").is_file()) + self.assertTrue((root / "Dockerfile.gateway").is_file()) + + def test_resource_helpers_resolve(self): + self.assertTrue(resources.dockerfile("Dockerfile.gateway").is_file()) + self.assertTrue(resources.nix_netpool_module().is_file()) + self.assertTrue(resources.netpool_script().is_file()) + + def test_bundled_resources_all_exist_at_root(self): + # Drift guard: every path setup.py bundles must exist in the checkout. + root = resources.build_root() + for rel in resources.BUNDLED_RESOURCES: + self.assertTrue((root / rel).is_file(), f"missing bundled resource: {rel}") + + +class TestDistVersion(unittest.TestCase): + """The version key for the staged build root, from importlib.metadata.""" + + def test_returns_installed_version(self): + import importlib.metadata as md + + with patch.object(md, "version", return_value="9.9.9"): + self.assertEqual("9.9.9", resources._dist_version()) # pylint: disable=protected-access + + def test_falls_back_when_not_installed(self): + import importlib.metadata as md + + with patch.object(md, "version", side_effect=md.PackageNotFoundError()): + self.assertEqual("dev", resources._dist_version()) # pylint: disable=protected-access + + +class TestWheelMode(unittest.TestCase): + """Fake an installed wheel: a package dir with _resources/ and no + checkout Dockerfiles beside it.""" + + def _fake_install(self, tmp: Path) -> Path: + pkg = tmp / "site-packages" / "bot_bottle" + (pkg / "cli").mkdir(parents=True) + (pkg / "__init__.py").write_text("") + (pkg / "cli" / "__init__.py").write_text("# module\n") + bundled = pkg / "_resources" + for rel in resources.BUNDLED_RESOURCES: + dst = bundled / rel + dst.parent.mkdir(parents=True, exist_ok=True) + dst.write_text(f"# fake {rel}\n") + return pkg + + def test_stage_and_resolve(self): + import tempfile + + with tempfile.TemporaryDirectory() as tmpname: + tmp = Path(tmpname) + pkg = self._fake_install(tmp) + appdata = tmp / "appdata" + restore = use_bottle_root(appdata) + self.addCleanup(restore) + with patch.object(resources, "_PKG", pkg), \ + patch.object(resources, "_BUNDLED", pkg / "_resources"), \ + patch.object(resources, "_CHECKOUT_ROOT", tmp / "no-checkout"): + self.assertFalse(resources.is_source_checkout()) + + root = resources.build_root() + # Staged context looks like a repo root. + self.assertTrue((root / "bot_bottle" / "__init__.py").is_file()) + self.assertTrue((root / "bot_bottle" / "cli" / "__init__.py").is_file()) + self.assertTrue((root / "pyproject.toml").is_file()) + self.assertTrue((root / "Dockerfile.gateway").is_file()) + self.assertTrue((root / "nix" / "firecracker-netpool.nix").is_file()) + self.assertTrue((root / "scripts" / "firecracker-netpool.sh").is_file()) + # The bundled-resource copies are NOT re-nested under the staged + # package (keeps it byte-identical to a checkout package). + self.assertFalse((root / "bot_bottle" / "_resources").exists()) + # Helpers resolve off the staged root. + self.assertEqual(root / "Dockerfile.gateway", + resources.dockerfile("Dockerfile.gateway")) + # Idempotent: second call returns the same completed dir. + self.assertEqual(root, resources.build_root()) + + def test_missing_bundle_raises(self): + import tempfile + + with tempfile.TemporaryDirectory() as tmpname: + tmp = Path(tmpname) + pkg = tmp / "bot_bottle" + pkg.mkdir() + restore = use_bottle_root(tmp / "appdata") + self.addCleanup(restore) + with patch.object(resources, "_PKG", pkg), \ + patch.object(resources, "_BUNDLED", pkg / "_resources"), \ + patch.object(resources, "_CHECKOUT_ROOT", tmp / "no-checkout"): + with self.assertRaises(resources.ResourceError): + resources.build_root() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_wheel_install.py b/tests/unit/test_wheel_install.py new file mode 100644 index 00000000..a96add9d --- /dev/null +++ b/tests/unit/test_wheel_install.py @@ -0,0 +1,116 @@ +"""Integration: build the wheel, install it into an isolated venv, and prove +the installed distribution is self-contained. + +This is the boundary a source-tree existence test can't reach (issue #197 +review): under an installed wheel the package lives in ``site-packages`` with +no repo root above it, so anything resolving Dockerfiles / nix / scripts from +``__file__``'s parents would break. Here we install for real and assert that +``bot-bottle doctor`` runs from the console script and that +``bot_bottle.resources`` stages a valid, repo-root-shaped build context. + +It does NOT run `start` — building images needs a Docker/KVM host (CI). It +skips cleanly when the build/venv toolchain isn't available. +""" + +from __future__ import annotations + +import importlib.util +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def _tooling_available() -> bool: + return importlib.util.find_spec("build") is not None + + +@unittest.skipUnless(_tooling_available(), "python 'build' module not installed") +class TestWheelInstall(unittest.TestCase): + _tmp: "tempfile.TemporaryDirectory[str]" + venv_py: Path + app_root: Path + + @classmethod + def setUpClass(cls): + cls._tmp = tempfile.TemporaryDirectory( # pylint: disable=consider-using-with + prefix="bb-wheel-") + tmp = Path(cls._tmp.name) + dist = tmp / "dist" + + build = subprocess.run( + [sys.executable, "-m", "build", "--wheel", "--outdir", str(dist), str(REPO_ROOT)], + capture_output=True, text=True, check=False, + ) + if build.returncode != 0: + raise unittest.SkipTest(f"wheel build unavailable:\n{build.stderr[-1500:]}") + wheels = list(dist.glob("*.whl")) + if not wheels: + raise unittest.SkipTest("no wheel produced") + + venv = tmp / "venv" + made = subprocess.run([sys.executable, "-m", "venv", str(venv)], + capture_output=True, text=True, check=False) + if made.returncode != 0: + raise unittest.SkipTest(f"venv unavailable:\n{made.stderr[-1500:]}") + cls.venv_py = venv / "bin" / "python" + + install = subprocess.run( + [str(cls.venv_py), "-m", "pip", "install", "--quiet", str(wheels[0])], + capture_output=True, text=True, check=False, + ) + if install.returncode != 0: + raise unittest.SkipTest(f"pip install failed:\n{install.stderr[-1500:]}") + + # Isolate the staged build root the wheel writes under the app-data dir. + cls.app_root = tmp / "appdata" + + @classmethod + def tearDownClass(cls): + cls._tmp.cleanup() + + def _run(self, tail: "list[str]") -> "subprocess.CompletedProcess[str]": + """Run the installed venv's python with `tail` appended, from a neutral + cwd so the source checkout isn't on sys.path — we must import the + *installed* package, not the repo we built from.""" + env = {"BOT_BOTTLE_ROOT": str(self.app_root), "PATH": "/usr/bin:/bin"} + return subprocess.run( + [str(self.venv_py), *tail], + capture_output=True, text=True, env=env, cwd=self._tmp.name, check=False, + ) + + def test_console_entry_point_installed(self): + # The `bot-bottle` script the wheel declares must exist in the venv. + script = self.venv_py.parent / "bot-bottle" + self.assertTrue(script.is_file(), "bot-bottle console script not installed") + + def test_doctor_runs_from_installed_package(self): + proc = self._run(["-m", "bot_bottle.cli", "doctor"]) + # doctor exits non-zero here (no backend), but it must RUN and report. + self.assertIn("python", proc.stdout) + self.assertIn(proc.returncode, (0, 1)) + + def test_installed_wheel_is_self_contained(self): + # From the installed layout (not a checkout), resources must resolve + # Dockerfiles and stage a repo-root-shaped build context. + script = ( + "import bot_bottle.resources as r\n" + "assert not r.is_source_checkout(), 'should not look like a checkout'\n" + "assert r.dockerfile('Dockerfile.gateway').is_file()\n" + "assert r.nix_netpool_module().is_file()\n" + "assert r.netpool_script().is_file()\n" + "root = r.build_root()\n" + "assert (root / 'bot_bottle' / '__init__.py').is_file(), 'no package in context'\n" + "assert (root / 'pyproject.toml').is_file(), 'no pyproject in context'\n" + "assert (root / 'Dockerfile.gateway').is_file(), 'no Dockerfile in context'\n" + "print('SELF_CONTAINED_OK')\n" + ) + proc = self._run(["-c", script]) + self.assertIn("SELF_CONTAINED_OK", proc.stdout, msg=proc.stderr) + + +if __name__ == "__main__": + unittest.main()