fix: make installed wheel self-contained + harden install.sh prereqs
tracker-policy-pr / check-pr (pull_request) Successful in 15s
test / integration-docker (pull_request) Successful in 41s
test / unit (pull_request) Successful in 49s
lint / lint (push) Successful in 1m7s
test / integration-firecracker (pull_request) Successful in 3m55s
test / coverage (pull_request) Failing after 25s
test / publish-infra (pull_request) Has been skipped
tracker-policy-pr / check-pr (pull_request) Successful in 15s
test / integration-docker (pull_request) Successful in 41s
test / unit (pull_request) Successful in 49s
lint / lint (push) Successful in 1m7s
test / integration-firecracker (pull_request) Successful in 3m55s
test / coverage (pull_request) Failing after 25s
test / publish-infra (pull_request) Has been skipped
Addresses the review on PR #481. Self-contained wheel (review point 1): the gateway/infra/orchestrator images build from a context that must hold bot_bottle/, pyproject.toml, and the root-level Dockerfiles. Modules previously located these by walking __file__ to the repo root, so an installed wheel (package in site-packages, no repo root) passed `doctor` but failed `start`. - Add bot_bottle/resources.py: build_root() returns the repo root in a checkout (unchanged) or a staged copy from the wheel's bundled _resources/ otherwise; dockerfile()/nix_netpool_module()/ netpool_script() derive from it. - setup.py bundles the root Dockerfiles, nix module, netpool script, and pyproject.toml into bot_bottle/_resources/ at build; MANIFEST.in ships them in the sdist. - Route every _REPO_ROOT/_REPO_DIR call site (docker/macos launch, macos infra, firecracker infra_vm/infra_artifact/setup, orchestrator lifecycle/gateway) through resources. Checkout behavior is unchanged. install.sh prerequisites (review point 2): check for git when installing a git+ spec, and — before the pip fallback — that pip is usable and the interpreter isn't externally managed (PEP 668), pointing at pipx. Tests: test_resources covers checkout + staged-wheel layouts; test_wheel_install builds the wheel, installs it into an isolated venv, and asserts `doctor` runs and build_root() yields a valid context. Running `start` end-to-end still needs a Docker/KVM host (CI). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -17,6 +17,9 @@ __pycache__/
|
|||||||
*.py[cod]
|
*.py[cod]
|
||||||
*$py.class
|
*$py.class
|
||||||
*.egg-info/
|
*.egg-info/
|
||||||
|
# setuptools/build_meta output (wheels, sdists, build tree)
|
||||||
|
/build/
|
||||||
|
/dist/
|
||||||
.venv/
|
.venv/
|
||||||
venv/
|
venv/
|
||||||
.pytest_cache/
|
.pytest_cache/
|
||||||
|
|||||||
@@ -0,0 +1,8 @@
|
|||||||
|
# Root-level build resources copied into bot_bottle/_resources/ at build time
|
||||||
|
# (see setup.py). Included in the sdist so `pip install` from an sdist can
|
||||||
|
# still bundle them into the wheel.
|
||||||
|
include Dockerfile.gateway
|
||||||
|
include Dockerfile.orchestrator
|
||||||
|
include Dockerfile.orchestrator.fc
|
||||||
|
include nix/firecracker-netpool.nix
|
||||||
|
include scripts/firecracker-netpool.sh
|
||||||
@@ -10,9 +10,10 @@ from ...paths import (
|
|||||||
ORCHESTRATOR_AUTH_JWT_ENV,
|
ORCHESTRATOR_AUTH_JWT_ENV,
|
||||||
host_gateway_ca_dir,
|
host_gateway_ca_dir,
|
||||||
)
|
)
|
||||||
|
from ... import resources
|
||||||
from ...gateway import (
|
from ...gateway import (
|
||||||
Gateway, GatewayTransport, GATEWAY_IMAGE, GATEWAY_NAME, GATEWAY_NETWORK,
|
Gateway, GatewayTransport, GATEWAY_IMAGE, GATEWAY_NAME, GATEWAY_NETWORK,
|
||||||
GATEWAY_DOCKERFILE, REPO_ROOT, GATEWAY_LABEL, MITMPROXY_HOME,
|
GATEWAY_DOCKERFILE, GATEWAY_LABEL, MITMPROXY_HOME,
|
||||||
DEFAULT_CA_TIMEOUT_SECONDS, CA_POLL_SECONDS, GATEWAY_CA_CERT, GatewayError
|
DEFAULT_CA_TIMEOUT_SECONDS, CA_POLL_SECONDS, GATEWAY_CA_CERT, GatewayError
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -50,7 +51,9 @@ class DockerGateway(Gateway):
|
|||||||
# `address` / `stop` work on an already-running gateway without it.
|
# `address` / `stop` work on an already-running gateway without it.
|
||||||
self._orchestrator_url = ""
|
self._orchestrator_url = ""
|
||||||
self._gateway_token = ""
|
self._gateway_token = ""
|
||||||
self._build_context = build_context or REPO_ROOT
|
# Resolved lazily in ensure_built() so merely constructing a gateway to
|
||||||
|
# read its CA never stages a build root from an installed wheel.
|
||||||
|
self._build_context = build_context
|
||||||
self._dockerfile = dockerfile
|
self._dockerfile = dockerfile
|
||||||
# Ports published on the host (0.0.0.0). Used by the Firecracker
|
# Ports published on the host (0.0.0.0). Used by the Firecracker
|
||||||
# backend's dev-harness gateway so VMs can reach it via their TAP link;
|
# backend's dev-harness gateway so VMs can reach it via their TAP link;
|
||||||
@@ -72,9 +75,10 @@ class DockerGateway(Gateway):
|
|||||||
forces a full rebuild (parity with `start --no-cache`)."""
|
forces a full rebuild (parity with `start --no-cache`)."""
|
||||||
if self._dockerfile is None:
|
if self._dockerfile is None:
|
||||||
return
|
return
|
||||||
|
context = self._build_context or resources.build_root()
|
||||||
argv = ["docker", "build", "-t", self.image_ref,
|
argv = ["docker", "build", "-t", self.image_ref,
|
||||||
"-f", str(self._build_context / self._dockerfile),
|
"-f", str(context / self._dockerfile),
|
||||||
str(self._build_context)]
|
str(context)]
|
||||||
if os.environ.get("BOT_BOTTLE_NO_CACHE"):
|
if os.environ.get("BOT_BOTTLE_NO_CACHE"):
|
||||||
argv.insert(2, "--no-cache")
|
argv.insert(2, "--no-cache")
|
||||||
proc = run_docker(argv)
|
proc = run_docker(argv)
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ from .orchestrator import (
|
|||||||
ORCHESTRATOR_NETWORK,
|
ORCHESTRATOR_NETWORK,
|
||||||
)
|
)
|
||||||
from ...paths import bot_bottle_root
|
from ...paths import bot_bottle_root
|
||||||
|
from ... import resources
|
||||||
from ...gateway import (
|
from ...gateway import (
|
||||||
GATEWAY_IMAGE,
|
GATEWAY_IMAGE,
|
||||||
GATEWAY_NAME,
|
GATEWAY_NAME,
|
||||||
@@ -50,8 +51,6 @@ from ...orchestrator.lifecycle import (
|
|||||||
# the pair's public identity.
|
# the pair's public identity.
|
||||||
INFRA_NAME = GATEWAY_NAME # the container agents attribute against is the gateway
|
INFRA_NAME = GATEWAY_NAME # the container agents attribute against is the gateway
|
||||||
|
|
||||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
|
||||||
|
|
||||||
|
|
||||||
class DockerInfraService(InfraService):
|
class DockerInfraService(InfraService):
|
||||||
"""Composes the per-host control plane + gateway as two containers.
|
"""Composes the per-host control plane + gateway as two containers.
|
||||||
@@ -68,7 +67,7 @@ class DockerInfraService(InfraService):
|
|||||||
control_network: str = ORCHESTRATOR_NETWORK,
|
control_network: str = ORCHESTRATOR_NETWORK,
|
||||||
orchestrator_image: str = ORCHESTRATOR_IMAGE,
|
orchestrator_image: str = ORCHESTRATOR_IMAGE,
|
||||||
gateway_image: str = GATEWAY_IMAGE,
|
gateway_image: str = GATEWAY_IMAGE,
|
||||||
repo_root: Path = _REPO_ROOT,
|
repo_root: Path | None = None,
|
||||||
host_root: Path | None = None,
|
host_root: Path | None = None,
|
||||||
orchestrator_name: str = ORCHESTRATOR_NAME,
|
orchestrator_name: str = ORCHESTRATOR_NAME,
|
||||||
orchestrator_label: str = ORCHESTRATOR_LABEL,
|
orchestrator_label: str = ORCHESTRATOR_LABEL,
|
||||||
@@ -79,7 +78,9 @@ class DockerInfraService(InfraService):
|
|||||||
self.control_network = control_network
|
self.control_network = control_network
|
||||||
self.orchestrator_image = orchestrator_image
|
self.orchestrator_image = orchestrator_image
|
||||||
self.gateway_image = gateway_image
|
self.gateway_image = gateway_image
|
||||||
self._repo_root = repo_root
|
# Build context / bind-mount source: the repo root in a checkout, a
|
||||||
|
# staged copy from the installed wheel otherwise (bot_bottle.resources).
|
||||||
|
self._repo_root = repo_root if repo_root is not None else resources.build_root()
|
||||||
self._host_root = host_root or bot_bottle_root()
|
self._host_root = host_root or bot_bottle_root()
|
||||||
self._orchestrator_name = orchestrator_name
|
self._orchestrator_name = orchestrator_name
|
||||||
self._orchestrator_label = orchestrator_label
|
self._orchestrator_label = orchestrator_label
|
||||||
|
|||||||
@@ -33,7 +33,6 @@ from __future__ import annotations
|
|||||||
import dataclasses
|
import dataclasses
|
||||||
import os
|
import os
|
||||||
from contextlib import ExitStack, contextmanager
|
from contextlib import ExitStack, contextmanager
|
||||||
from pathlib import Path
|
|
||||||
from typing import Callable, Generator
|
from typing import Callable, Generator
|
||||||
|
|
||||||
from ...agent_provider import runtime_for
|
from ...agent_provider import runtime_for
|
||||||
@@ -65,10 +64,7 @@ from ...orchestrator.store.config_store import resolve_teardown_timeout
|
|||||||
from .consolidated_launch import launch_consolidated, deprovision_consolidated
|
from .consolidated_launch import launch_consolidated, deprovision_consolidated
|
||||||
from .infra import INFRA_NAME
|
from .infra import INFRA_NAME
|
||||||
from .gateway import DockerGateway
|
from .gateway import DockerGateway
|
||||||
|
from ... import resources
|
||||||
|
|
||||||
# Where the repo root lives, for `docker build` context. Computed once.
|
|
||||||
_REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent)
|
|
||||||
|
|
||||||
|
|
||||||
def build_or_load_images(plan: DockerBottlePlan) -> BottleImages:
|
def build_or_load_images(plan: DockerBottlePlan) -> BottleImages:
|
||||||
@@ -88,7 +84,7 @@ def build_or_load_images(plan: DockerBottlePlan) -> BottleImages:
|
|||||||
)
|
)
|
||||||
info(f"using cached agent image {plan.image!r}")
|
info(f"using cached agent image {plan.image!r}")
|
||||||
return BottleImages(agent=plan.image)
|
return BottleImages(agent=plan.image)
|
||||||
docker_mod.build_image(plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path)
|
docker_mod.build_image(plan.image, str(resources.build_root()), dockerfile=plan.dockerfile_path)
|
||||||
docker_mod.verify_agent_image(
|
docker_mod.verify_agent_image(
|
||||||
plan.image, runtime_for(plan.agent_provider_template).smoke_test,
|
plan.image, runtime_for(plan.agent_provider_template).smoke_test,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import time
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from ... import log
|
from ... import log
|
||||||
|
from ... import resources
|
||||||
from .util import run_docker
|
from .util import run_docker
|
||||||
from ...paths import (
|
from ...paths import (
|
||||||
ORCHESTRATOR_TOKEN_ENV,
|
ORCHESTRATOR_TOKEN_ENV,
|
||||||
@@ -56,8 +57,6 @@ _ROOT_IN_CONTAINER = "/bot-bottle-root"
|
|||||||
|
|
||||||
_HEALTH_POLL_SECONDS = 0.25
|
_HEALTH_POLL_SECONDS = 0.25
|
||||||
|
|
||||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
|
||||||
|
|
||||||
|
|
||||||
class DockerOrchestrator(Orchestrator):
|
class DockerOrchestrator(Orchestrator):
|
||||||
"""The control plane as a single fixed-name container. `ensure_built` builds
|
"""The control plane as a single fixed-name container. `ensure_built` builds
|
||||||
@@ -72,7 +71,7 @@ class DockerOrchestrator(Orchestrator):
|
|||||||
label: str = ORCHESTRATOR_LABEL,
|
label: str = ORCHESTRATOR_LABEL,
|
||||||
port: int = DEFAULT_PORT,
|
port: int = DEFAULT_PORT,
|
||||||
control_network: str = ORCHESTRATOR_NETWORK,
|
control_network: str = ORCHESTRATOR_NETWORK,
|
||||||
repo_root: Path = _REPO_ROOT,
|
repo_root: Path | None = None,
|
||||||
host_root: Path | None = None,
|
host_root: Path | None = None,
|
||||||
dockerfile: str | None = ORCHESTRATOR_DOCKERFILE,
|
dockerfile: str | None = ORCHESTRATOR_DOCKERFILE,
|
||||||
) -> None:
|
) -> None:
|
||||||
@@ -81,7 +80,9 @@ class DockerOrchestrator(Orchestrator):
|
|||||||
self.label = label
|
self.label = label
|
||||||
self.port = port
|
self.port = port
|
||||||
self.control_network = control_network
|
self.control_network = control_network
|
||||||
self._repo_root = repo_root
|
# Build context / bind-mount source: the repo root in a checkout, a
|
||||||
|
# staged copy from the installed wheel otherwise (bot_bottle.resources).
|
||||||
|
self._repo_root = repo_root if repo_root is not None else resources.build_root()
|
||||||
self._host_root = host_root or bot_bottle_root()
|
self._host_root = host_root or bot_bottle_root()
|
||||||
self._dockerfile = dockerfile
|
self._dockerfile = dockerfile
|
||||||
|
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ import urllib.error
|
|||||||
import urllib.request
|
import urllib.request
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ... import resources
|
||||||
from ...log import die, info
|
from ...log import die, info
|
||||||
from . import util
|
from . import util
|
||||||
|
|
||||||
@@ -44,8 +45,6 @@ from . import util
|
|||||||
# scheme can't collide with a cached/published artifact of the old one.
|
# scheme can't collide with a cached/published artifact of the old one.
|
||||||
_ARTIFACT_FORMAT = "1"
|
_ARTIFACT_FORMAT = "1"
|
||||||
|
|
||||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
|
||||||
|
|
||||||
# The two per-plane infra VM roles. Each publishes/pulls its own rootfs artifact
|
# The two per-plane infra VM roles. Each publishes/pulls its own rootfs artifact
|
||||||
# from its own generic package; the Dockerfiles baked into each differ (only the
|
# from its own generic package; the Dockerfiles baked into each differ (only the
|
||||||
# orchestrator rootfs carries buildah), so the versions are hashed separately.
|
# orchestrator rootfs carries buildah), so the versions are hashed separately.
|
||||||
@@ -74,7 +73,7 @@ def local_build_requested() -> bool:
|
|||||||
|
|
||||||
|
|
||||||
def infra_artifact_version(
|
def infra_artifact_version(
|
||||||
init_script: str, role: str, *, repo_root: Path = _REPO_ROOT,
|
init_script: str, role: str, *, repo_root: Path | None = None,
|
||||||
) -> str:
|
) -> str:
|
||||||
"""Content hash (16 hex) of everything baked into `role`'s infra rootfs: the
|
"""Content hash (16 hex) of everything baked into `role`'s infra rootfs: the
|
||||||
whole shipped `bot_bottle` package, that role's Dockerfiles, and its guest
|
whole shipped `bot_bottle` package, that role's Dockerfiles, and its guest
|
||||||
@@ -89,6 +88,8 @@ def infra_artifact_version(
|
|||||||
version or a launch host could boot a stale rootfs whose code differs from
|
version or a launch host could boot a stale rootfs whose code differs from
|
||||||
its checkout. `__pycache__`/`.pyc` are the only exclusions — build artifacts,
|
its checkout. `__pycache__`/`.pyc` are the only exclusions — build artifacts,
|
||||||
never copied."""
|
never copied."""
|
||||||
|
if repo_root is None:
|
||||||
|
repo_root = resources.build_root()
|
||||||
h = hashlib.sha256()
|
h = hashlib.sha256()
|
||||||
h.update(f"format={_ARTIFACT_FORMAT}\nrole={role}\n".encode())
|
h.update(f"format={_ARTIFACT_FORMAT}\nrole={role}\n".encode())
|
||||||
pkg = repo_root / "bot_bottle"
|
pkg = repo_root / "bot_bottle"
|
||||||
|
|||||||
@@ -42,6 +42,7 @@ from dataclasses import dataclass
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Generator
|
from typing import Generator
|
||||||
|
|
||||||
|
from ... import resources
|
||||||
from ...log import die, info
|
from ...log import die, info
|
||||||
from ..docker import util as docker_mod
|
from ..docker import util as docker_mod
|
||||||
from . import firecracker_vm, infra_artifact, netpool, util
|
from . import firecracker_vm, infra_artifact, netpool, util
|
||||||
@@ -65,7 +66,6 @@ _GUEST_GATEWAY_JWT_PATH = "/var/lib/bot-bottle/gateway-jwt"
|
|||||||
_GATEWAY_IMAGE = "bot-bottle-gateway:latest"
|
_GATEWAY_IMAGE = "bot-bottle-gateway:latest"
|
||||||
_ORCHESTRATOR_IMAGE = "bot-bottle-orchestrator:latest"
|
_ORCHESTRATOR_IMAGE = "bot-bottle-orchestrator:latest"
|
||||||
_ORCHESTRATOR_FC_IMAGE = "bot-bottle-orchestrator-fc:latest"
|
_ORCHESTRATOR_FC_IMAGE = "bot-bottle-orchestrator-fc:latest"
|
||||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
|
||||||
|
|
||||||
# Per-role rootfs source image + the extra free space `mke2fs` leaves for the
|
# Per-role rootfs source image + the extra free space `mke2fs` leaves for the
|
||||||
# guest to grow into. The orchestrator keeps buildah's large build slack; the
|
# guest to grow into. The orchestrator keeps buildah's large build slack; the
|
||||||
@@ -130,12 +130,13 @@ def build_infra_images_with_docker() -> None:
|
|||||||
orchestrator + buildah). The gateway VM boots the gateway image directly.
|
orchestrator + buildah). The gateway VM boots the gateway image directly.
|
||||||
The launch host uses this only in `BOT_BOTTLE_INFRA_BUILD=local` mode;
|
The launch host uses this only in `BOT_BOTTLE_INFRA_BUILD=local` mode;
|
||||||
`publish_infra` uses it off-host to produce the published artifacts."""
|
`publish_infra` uses it off-host to produce the published artifacts."""
|
||||||
|
root = str(resources.build_root())
|
||||||
docker_mod.build_image(
|
docker_mod.build_image(
|
||||||
_ORCHESTRATOR_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.orchestrator")
|
_ORCHESTRATOR_IMAGE, root, dockerfile="Dockerfile.orchestrator")
|
||||||
docker_mod.build_image(
|
docker_mod.build_image(
|
||||||
_GATEWAY_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.gateway")
|
_GATEWAY_IMAGE, root, dockerfile="Dockerfile.gateway")
|
||||||
docker_mod.build_image(
|
docker_mod.build_image(
|
||||||
_ORCHESTRATOR_FC_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.orchestrator.fc")
|
_ORCHESTRATOR_FC_IMAGE, root, dockerfile="Dockerfile.orchestrator.fc")
|
||||||
|
|
||||||
|
|
||||||
def build_rootfs_dir(role: str) -> Path:
|
def build_rootfs_dir(role: str) -> Path:
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ import subprocess
|
|||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ... import resources
|
||||||
from . import netpool
|
from . import netpool
|
||||||
from . import util
|
from . import util
|
||||||
|
|
||||||
@@ -42,13 +43,13 @@ def _has_systemd() -> bool:
|
|||||||
|
|
||||||
|
|
||||||
def _module_path() -> str:
|
def _module_path() -> str:
|
||||||
"""Absolute path to the importable NixOS module in this checkout."""
|
"""Absolute path to the importable NixOS module (checkout or wheel)."""
|
||||||
return str(Path(__file__).resolve().parents[3] / "nix" / "firecracker-netpool.nix")
|
return str(resources.nix_netpool_module())
|
||||||
|
|
||||||
|
|
||||||
def _script_path() -> str:
|
def _script_path() -> str:
|
||||||
"""Absolute path to the bundled bring-up script in this checkout."""
|
"""Absolute path to the bundled bring-up script (checkout or wheel)."""
|
||||||
return str(Path(__file__).resolve().parents[3] / "scripts" / "firecracker-netpool.sh")
|
return str(resources.netpool_script())
|
||||||
|
|
||||||
|
|
||||||
def _print_prereqs() -> None:
|
def _print_prereqs() -> None:
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ from ...paths import (
|
|||||||
ORCHESTRATOR_AUTH_JWT_ENV,
|
ORCHESTRATOR_AUTH_JWT_ENV,
|
||||||
host_gateway_ca_dir,
|
host_gateway_ca_dir,
|
||||||
)
|
)
|
||||||
|
from ... import resources
|
||||||
from .. import util as backend_util
|
from .. import util as backend_util
|
||||||
from . import util as container_mod
|
from . import util as container_mod
|
||||||
|
|
||||||
@@ -52,8 +53,6 @@ GATEWAY_DAEMONS = "egress,git-http,supervise"
|
|||||||
|
|
||||||
GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest")
|
GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest")
|
||||||
|
|
||||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
|
||||||
|
|
||||||
|
|
||||||
def ensure_networks(
|
def ensure_networks(
|
||||||
network: str = GATEWAY_NETWORK,
|
network: str = GATEWAY_NETWORK,
|
||||||
@@ -84,14 +83,16 @@ class MacosGateway(Gateway):
|
|||||||
network: str = GATEWAY_NETWORK,
|
network: str = GATEWAY_NETWORK,
|
||||||
egress_network: str = GATEWAY_EGRESS_NETWORK,
|
egress_network: str = GATEWAY_EGRESS_NETWORK,
|
||||||
control_network: str = CONTROL_NETWORK,
|
control_network: str = CONTROL_NETWORK,
|
||||||
repo_root: Path = _REPO_ROOT,
|
repo_root: Path | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.image_ref = image_ref
|
self.image_ref = image_ref
|
||||||
self.name = name
|
self.name = name
|
||||||
self.network = network
|
self.network = network
|
||||||
self.egress_network = egress_network
|
self.egress_network = egress_network
|
||||||
self.control_network = control_network
|
self.control_network = control_network
|
||||||
self._repo_root = repo_root
|
# Build context: the repo root in a checkout, a staged copy from the
|
||||||
|
# installed wheel otherwise (bot_bottle.resources).
|
||||||
|
self._repo_root = repo_root if repo_root is not None else resources.build_root()
|
||||||
# Set by `connect_to_orchestrator`: the URL the daemons resolve policy
|
# Set by `connect_to_orchestrator`: the URL the daemons resolve policy
|
||||||
# against + the pre-minted `gateway` token they present. The gateway
|
# against + the pre-minted `gateway` token they present. The gateway
|
||||||
# never mints, so it never holds the signing key (#469).
|
# never mints, so it never holds the signing key (#469).
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ... import resources
|
||||||
from ...orchestrator.lifecycle import (
|
from ...orchestrator.lifecycle import (
|
||||||
DEFAULT_PORT,
|
DEFAULT_PORT,
|
||||||
DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||||
@@ -53,8 +54,6 @@ from .orchestrator import (
|
|||||||
# still import it (probe / reprovision attribute against the gateway).
|
# still import it (probe / reprovision attribute against the gateway).
|
||||||
INFRA_NAME = GATEWAY_NAME
|
INFRA_NAME = GATEWAY_NAME
|
||||||
|
|
||||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
|
||||||
|
|
||||||
|
|
||||||
class MacosInfraService(InfraService):
|
class MacosInfraService(InfraService):
|
||||||
"""Composes the per-host orchestrator + gateway containers. Callers use
|
"""Composes the per-host orchestrator + gateway containers. Callers use
|
||||||
@@ -70,7 +69,7 @@ class MacosInfraService(InfraService):
|
|||||||
control_network: str = CONTROL_NETWORK,
|
control_network: str = CONTROL_NETWORK,
|
||||||
gateway_image: str = GATEWAY_IMAGE,
|
gateway_image: str = GATEWAY_IMAGE,
|
||||||
orchestrator_image: str = ORCHESTRATOR_IMAGE,
|
orchestrator_image: str = ORCHESTRATOR_IMAGE,
|
||||||
repo_root: Path = _REPO_ROOT,
|
repo_root: Path | None = None,
|
||||||
orchestrator_name: str = ORCHESTRATOR_NAME,
|
orchestrator_name: str = ORCHESTRATOR_NAME,
|
||||||
gateway_name: str = INFRA_NAME,
|
gateway_name: str = INFRA_NAME,
|
||||||
db_volume: str = ORCHESTRATOR_DB_VOLUME,
|
db_volume: str = ORCHESTRATOR_DB_VOLUME,
|
||||||
@@ -81,7 +80,9 @@ class MacosInfraService(InfraService):
|
|||||||
self.control_network = control_network
|
self.control_network = control_network
|
||||||
self.gateway_image = gateway_image
|
self.gateway_image = gateway_image
|
||||||
self.orchestrator_image = orchestrator_image
|
self.orchestrator_image = orchestrator_image
|
||||||
self._repo_root = repo_root
|
# Build context / bind-mount source: the repo root in a checkout, a
|
||||||
|
# staged copy from the installed wheel otherwise (bot_bottle.resources).
|
||||||
|
self._repo_root = repo_root if repo_root is not None else resources.build_root()
|
||||||
self._orchestrator_name = orchestrator_name
|
self._orchestrator_name = orchestrator_name
|
||||||
self._gateway_name = gateway_name
|
self._gateway_name = gateway_name
|
||||||
self._db_volume = db_volume
|
self._db_volume = db_volume
|
||||||
|
|||||||
@@ -36,7 +36,6 @@ import dataclasses
|
|||||||
import os
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
from contextlib import ExitStack, contextmanager
|
from contextlib import ExitStack, contextmanager
|
||||||
from pathlib import Path
|
|
||||||
from typing import Callable, Generator
|
from typing import Callable, Generator
|
||||||
|
|
||||||
from ...bottle_state import (
|
from ...bottle_state import (
|
||||||
@@ -49,6 +48,7 @@ from ...git_gate import GitGate
|
|||||||
from ...gateway.git_gate.http_backend import DEFAULT_PORT as _GIT_HTTP_PORT
|
from ...gateway.git_gate.http_backend import DEFAULT_PORT as _GIT_HTTP_PORT
|
||||||
from ...image_cache import check_stale
|
from ...image_cache import check_stale
|
||||||
from ...log import die, info, warn
|
from ...log import die, info, warn
|
||||||
|
from ... import resources
|
||||||
from .. import BottleImages
|
from .. import BottleImages
|
||||||
from ...supervisor.types import SUPERVISE_PORT
|
from ...supervisor.types import SUPERVISE_PORT
|
||||||
from ..docker.egress import EGRESS_PORT
|
from ..docker.egress import EGRESS_PORT
|
||||||
@@ -71,7 +71,6 @@ from .consolidated_launch import (
|
|||||||
deprovision_consolidated,
|
deprovision_consolidated,
|
||||||
)
|
)
|
||||||
|
|
||||||
_REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent)
|
|
||||||
_AGENT_SLEEP_SECONDS = "2147483647"
|
_AGENT_SLEEP_SECONDS = "2147483647"
|
||||||
|
|
||||||
|
|
||||||
@@ -94,7 +93,7 @@ def _agent_image(plan: MacosContainerBottlePlan) -> str:
|
|||||||
)
|
)
|
||||||
info(f"using cached agent image {plan.image!r}")
|
info(f"using cached agent image {plan.image!r}")
|
||||||
return plan.image
|
return plan.image
|
||||||
container_mod.build_image(plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path)
|
container_mod.build_image(plan.image, str(resources.build_root()), dockerfile=plan.dockerfile_path)
|
||||||
return plan.image
|
return plan.image
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import urllib.request
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from ... import log
|
from ... import log
|
||||||
|
from ... import resources
|
||||||
from ...paths import (
|
from ...paths import (
|
||||||
ORCHESTRATOR_TOKEN_ENV,
|
ORCHESTRATOR_TOKEN_ENV,
|
||||||
host_orchestrator_token,
|
host_orchestrator_token,
|
||||||
@@ -48,7 +49,6 @@ _DB_ROOT_IN_CONTAINER = "/var/lib/bot-bottle"
|
|||||||
_SRC_IN_CONTAINER = "/bot-bottle-src"
|
_SRC_IN_CONTAINER = "/bot-bottle-src"
|
||||||
|
|
||||||
_HEALTH_POLL_SECONDS = 0.25
|
_HEALTH_POLL_SECONDS = 0.25
|
||||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
|
||||||
|
|
||||||
|
|
||||||
class MacosOrchestrator(Orchestrator):
|
class MacosOrchestrator(Orchestrator):
|
||||||
@@ -64,7 +64,7 @@ class MacosOrchestrator(Orchestrator):
|
|||||||
label: str = ORCHESTRATOR_LABEL,
|
label: str = ORCHESTRATOR_LABEL,
|
||||||
port: int = DEFAULT_PORT,
|
port: int = DEFAULT_PORT,
|
||||||
control_network: str = CONTROL_NETWORK,
|
control_network: str = CONTROL_NETWORK,
|
||||||
repo_root: Path = _REPO_ROOT,
|
repo_root: Path | None = None,
|
||||||
db_volume: str = ORCHESTRATOR_DB_VOLUME,
|
db_volume: str = ORCHESTRATOR_DB_VOLUME,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.image_ref = image_ref
|
self.image_ref = image_ref
|
||||||
@@ -72,7 +72,9 @@ class MacosOrchestrator(Orchestrator):
|
|||||||
self.label = label
|
self.label = label
|
||||||
self.port = port
|
self.port = port
|
||||||
self.control_network = control_network
|
self.control_network = control_network
|
||||||
self._repo_root = repo_root
|
# Build context / bind-mount source: the repo root in a checkout, a
|
||||||
|
# staged copy from the installed wheel otherwise (bot_bottle.resources).
|
||||||
|
self._repo_root = repo_root if repo_root is not None else resources.build_root()
|
||||||
self._db_volume = db_volume
|
self._db_volume = db_volume
|
||||||
|
|
||||||
def url(self) -> str:
|
def url(self) -> str:
|
||||||
|
|||||||
@@ -62,7 +62,6 @@ GATEWAY_CA_GLOB = "mitmproxy-ca*"
|
|||||||
# that lands. Env override matches the backend's BOT_BOTTLE_GATEWAY_IMAGE.
|
# that lands. Env override matches the backend's BOT_BOTTLE_GATEWAY_IMAGE.
|
||||||
GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest")
|
GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest")
|
||||||
GATEWAY_DOCKERFILE = "Dockerfile.gateway"
|
GATEWAY_DOCKERFILE = "Dockerfile.gateway"
|
||||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
||||||
|
|
||||||
|
|
||||||
def rotate_gateway_ca(ca_dir: Path | None = None) -> list[Path]:
|
def rotate_gateway_ca(ca_dir: Path | None = None) -> list[Path]:
|
||||||
|
|||||||
@@ -0,0 +1,131 @@
|
|||||||
|
"""Locate build-time resources whether bot-bottle runs from a source
|
||||||
|
checkout or an installed wheel.
|
||||||
|
|
||||||
|
The gateway / infra / orchestrator images are built from a Docker (or Apple
|
||||||
|
`container`) build context that must contain the `bot_bottle` package,
|
||||||
|
`pyproject.toml`, and the root-level Dockerfiles as siblings. In a source
|
||||||
|
checkout that context is simply the repo root, one level above the package.
|
||||||
|
An installed wheel has no repo root: the same root-level files are shipped
|
||||||
|
inside the package under ``bot_bottle/_resources/`` (see ``setup.py``), and a
|
||||||
|
repo-root-shaped build context is staged on demand into the app-data dir.
|
||||||
|
|
||||||
|
``build_root()`` is the single source of truth — it returns a directory laid
|
||||||
|
out like a repo root (has ``bot_bottle/``, ``pyproject.toml``, the
|
||||||
|
Dockerfiles, ``nix/``, ``scripts/``). Every caller that needs a build
|
||||||
|
context, a Dockerfile path, the nix netpool module, or the netpool script
|
||||||
|
derives from it, so checkout and wheel installs share one downstream path.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import shutil
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from .paths import bot_bottle_root
|
||||||
|
|
||||||
|
_PKG = Path(__file__).resolve().parent # …/bot_bottle
|
||||||
|
_CHECKOUT_ROOT = _PKG.parent # repo root in a checkout
|
||||||
|
_BUNDLED = _PKG / "_resources" # wheel-shipped copies
|
||||||
|
|
||||||
|
# Root-level files bundled into the wheel under ``_resources/`` (paths are
|
||||||
|
# relative to the checkout root, and preserved verbatim under ``_resources/``
|
||||||
|
# and in the staged build root). ``setup.py`` copies exactly this set; keep
|
||||||
|
# the two lists in sync (``test_resources`` guards that every entry exists).
|
||||||
|
BUNDLED_RESOURCES: tuple[str, ...] = (
|
||||||
|
"pyproject.toml",
|
||||||
|
"Dockerfile.gateway",
|
||||||
|
"Dockerfile.orchestrator",
|
||||||
|
"Dockerfile.orchestrator.fc",
|
||||||
|
"nix/firecracker-netpool.nix",
|
||||||
|
"scripts/firecracker-netpool.sh",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Present at a checkout root, never in a bare installed package — the cheap
|
||||||
|
# tell for which layout we're in.
|
||||||
|
_CHECKOUT_MARKER = "Dockerfile.gateway"
|
||||||
|
|
||||||
|
|
||||||
|
class ResourceError(RuntimeError):
|
||||||
|
"""Build resources are missing from the install (corrupt/partial wheel)."""
|
||||||
|
|
||||||
|
|
||||||
|
def is_source_checkout() -> bool:
|
||||||
|
"""True when running from a source tree (the root Dockerfiles sit beside
|
||||||
|
the package); False from an installed wheel."""
|
||||||
|
return (_CHECKOUT_ROOT / _CHECKOUT_MARKER).is_file()
|
||||||
|
|
||||||
|
|
||||||
|
def build_root() -> Path:
|
||||||
|
"""A directory shaped like a repo root: ``bot_bottle/``, ``pyproject.toml``,
|
||||||
|
the root Dockerfiles, ``nix/``, and ``scripts/``.
|
||||||
|
|
||||||
|
A checkout returns the repo root itself (no copying). An installed wheel
|
||||||
|
returns a staged copy under the app-data dir, materialized once and reused
|
||||||
|
(keyed by distribution version)."""
|
||||||
|
if is_source_checkout():
|
||||||
|
return _CHECKOUT_ROOT
|
||||||
|
return _stage_build_root()
|
||||||
|
|
||||||
|
|
||||||
|
def dockerfile(name: str) -> Path:
|
||||||
|
"""Absolute path to a root-level Dockerfile, e.g. ``Dockerfile.gateway``."""
|
||||||
|
return build_root() / name
|
||||||
|
|
||||||
|
|
||||||
|
def nix_netpool_module() -> Path:
|
||||||
|
"""Absolute path to the firecracker netpool NixOS module."""
|
||||||
|
return build_root() / "nix" / "firecracker-netpool.nix"
|
||||||
|
|
||||||
|
|
||||||
|
def netpool_script() -> Path:
|
||||||
|
"""Absolute path to the firecracker netpool bring-up script."""
|
||||||
|
return build_root() / "scripts" / "firecracker-netpool.sh"
|
||||||
|
|
||||||
|
|
||||||
|
def _dist_version() -> str:
|
||||||
|
"""Installed distribution version, for keying the staged build root. Falls
|
||||||
|
back to ``dev`` when metadata is unavailable (e.g. running from a tree)."""
|
||||||
|
try:
|
||||||
|
from importlib.metadata import PackageNotFoundError, version
|
||||||
|
try:
|
||||||
|
return version("bot-bottle")
|
||||||
|
except PackageNotFoundError:
|
||||||
|
return "dev"
|
||||||
|
except Exception: # pragma: no cover - importlib.metadata is always present
|
||||||
|
return "dev"
|
||||||
|
|
||||||
|
|
||||||
|
def _stage_build_root() -> Path:
|
||||||
|
"""Materialize a repo-root-shaped build context from the installed wheel's
|
||||||
|
bundled resources. Idempotent and cached on disk by version."""
|
||||||
|
if not _BUNDLED.is_dir():
|
||||||
|
raise ResourceError(
|
||||||
|
"bot-bottle build resources are missing from this install "
|
||||||
|
f"(expected {_BUNDLED}). Reinstall the package."
|
||||||
|
)
|
||||||
|
dest = bot_bottle_root() / "build-root" / _dist_version()
|
||||||
|
if (dest / ".complete").is_file():
|
||||||
|
return dest
|
||||||
|
|
||||||
|
staging = dest.with_name(dest.name + ".staging")
|
||||||
|
shutil.rmtree(staging, ignore_errors=True)
|
||||||
|
staging.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# The package itself, minus caches and the bundled-resource copies, so the
|
||||||
|
# staged ``bot_bottle/`` matches a checkout's (keeps the firecracker
|
||||||
|
# infra-artifact content hash stable across checkout and wheel installs).
|
||||||
|
shutil.copytree(
|
||||||
|
_PKG,
|
||||||
|
staging / "bot_bottle",
|
||||||
|
ignore=shutil.ignore_patterns("__pycache__", "*.pyc", "_resources"),
|
||||||
|
)
|
||||||
|
# The bundled root files, restored to their checkout-relative layout.
|
||||||
|
for rel in BUNDLED_RESOURCES:
|
||||||
|
dst = staging / rel
|
||||||
|
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
shutil.copy2(_BUNDLED / rel, dst)
|
||||||
|
|
||||||
|
shutil.rmtree(dest, ignore_errors=True)
|
||||||
|
staging.rename(dest)
|
||||||
|
(dest / ".complete").write_text("")
|
||||||
|
return dest
|
||||||
@@ -67,10 +67,45 @@ bot-bottle = "bot_bottle.cli:main"
|
|||||||
```
|
```
|
||||||
|
|
||||||
`bot_bottle.cli:main` already exists (the `cli.py` shim calls it), so no
|
`bot_bottle.cli:main` already exists (the `cli.py` shim calls it), so no
|
||||||
refactor of the entry point is needed. `package-data` ships the container
|
refactor of the entry point is needed. `package-data` ships the non-Python
|
||||||
Dockerfiles, `egress_entrypoint.sh`, the firecracker netpool defaults, and
|
assets that live *inside* the package (`egress_entrypoint.sh`, the contrib
|
||||||
the macos-container init script so an installed wheel can still build its
|
Dockerfiles, the firecracker netpool defaults, the macos-container init
|
||||||
sidecar/agent images.
|
script).
|
||||||
|
|
||||||
|
### Self-contained wheel (build resources)
|
||||||
|
|
||||||
|
The gateway / infra / orchestrator images are built from a Docker (or Apple
|
||||||
|
`container`) build context that must contain the `bot_bottle` package,
|
||||||
|
`pyproject.toml`, and the **root-level** Dockerfiles as siblings. Several
|
||||||
|
modules used to locate that context by walking `__file__`'s parents to the
|
||||||
|
repo root (`_REPO_ROOT = Path(__file__)…parents[N]`) and reading
|
||||||
|
`Dockerfile.gateway`, `nix/firecracker-netpool.nix`, and
|
||||||
|
`scripts/firecracker-netpool.sh` from it. In an installed wheel the package
|
||||||
|
lives in `site-packages` with no repo root above it, so those reads fail —
|
||||||
|
`doctor` passes but `start` / backend setup breaks.
|
||||||
|
|
||||||
|
Fix: a single resolver, `bot_bottle/resources.py`.
|
||||||
|
|
||||||
|
- `build_root()` returns a directory shaped like a repo root (has
|
||||||
|
`bot_bottle/`, `pyproject.toml`, the Dockerfiles, `nix/`, `scripts/`).
|
||||||
|
In a **checkout** it's the repo root itself — unchanged behavior. From an
|
||||||
|
**installed wheel** it stages a copy under the app-data dir, once, keyed
|
||||||
|
by version.
|
||||||
|
- The root-level resources are shipped inside the wheel under
|
||||||
|
`bot_bottle/_resources/` by a `setup.py` `build_py` step (kept in sync
|
||||||
|
with `resources.BUNDLED_RESOURCES`); `MANIFEST.in` includes them in the
|
||||||
|
sdist.
|
||||||
|
- Every former `_REPO_ROOT` / `_REPO_DIR` call site now derives from
|
||||||
|
`resources`: the docker/macos agent-image launch, each backend's
|
||||||
|
`orchestrator` / `gateway` / `infra` service, firecracker `infra_vm` /
|
||||||
|
`infra_artifact` / `setup`, and the shared `gateway` build context. So
|
||||||
|
checkout and wheel installs share one downstream path.
|
||||||
|
|
||||||
|
Verification: `test_resources` exercises both layouts (including the staged
|
||||||
|
wheel context); `test_wheel_install` builds the wheel, installs it into an
|
||||||
|
isolated venv, and asserts `bot-bottle doctor` runs and `build_root()`
|
||||||
|
produces a valid context. Running `start` end-to-end still needs a
|
||||||
|
Docker/KVM host (CI), not a source checkout.
|
||||||
|
|
||||||
### `install.sh`
|
### `install.sh`
|
||||||
|
|
||||||
@@ -78,8 +113,11 @@ A POSIX `sh` bootstrapper that:
|
|||||||
|
|
||||||
1. Checks `python3` is present and ≥ 3.11; exits with a clear message
|
1. Checks `python3` is present and ≥ 3.11; exits with a clear message
|
||||||
otherwise.
|
otherwise.
|
||||||
2. Creates `~/.bot-bottle/{agents,bottles,contrib}`.
|
2. Checks `git` when installing a `git+` spec, and — when falling back to
|
||||||
3. Installs via `pipx` if available, else `python3 -m pip install --user`.
|
pip — that pip is usable and the interpreter isn't externally managed
|
||||||
|
(PEP 668), pointing at pipx otherwise.
|
||||||
|
3. Creates `~/.bot-bottle/{agents,bottles,contrib}`.
|
||||||
|
4. Installs via `pipx` if available, else `python3 -m pip install --user`.
|
||||||
The spec defaults to the git URL and is overridable via
|
The spec defaults to the git URL and is overridable via
|
||||||
`BOT_BOTTLE_INSTALL_SPEC` (used by tests / local installs).
|
`BOT_BOTTLE_INSTALL_SPEC` (used by tests / local installs).
|
||||||
4. Locates the `bot-bottle` entry point (PATH or `~/.local/bin`).
|
4. Locates the `bot-bottle` entry point (PATH or `~/.local/bin`).
|
||||||
|
|||||||
+35
@@ -40,6 +40,41 @@ want = (int(sys.argv[1]), int(sys.argv[2]))
|
|||||||
raise SystemExit(0 if sys.version_info[:2] >= want else 1)
|
raise SystemExit(0 if sys.version_info[:2] >= want else 1)
|
||||||
PY
|
PY
|
||||||
|
|
||||||
|
# Installing a `git+` spec (the default) shells out to git under the hood,
|
||||||
|
# whether via pipx or pip. Fail early with a clear message rather than deep
|
||||||
|
# inside the installer's output.
|
||||||
|
case "${PACKAGE_SPEC}" in
|
||||||
|
git+*|*.git)
|
||||||
|
command -v git >/dev/null 2>&1 || die \
|
||||||
|
"git is required to install from '${PACKAGE_SPEC}'. Install git, or set "\
|
||||||
|
"BOT_BOTTLE_INSTALL_SPEC to a non-git spec (e.g. a wheel path or a package index name)."
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
# The pip fallback needs a usable pip. Externally-managed interpreters
|
||||||
|
# (PEP 668, common on Debian/Ubuntu/Homebrew) reject `pip install --user`;
|
||||||
|
# pipx sidesteps that, so recommend it when pip can't be used.
|
||||||
|
if ! command -v pipx >/dev/null 2>&1; then
|
||||||
|
python3 -m pip --version >/dev/null 2>&1 || die \
|
||||||
|
"neither pipx nor a usable 'python3 -m pip' was found. Install pipx "\
|
||||||
|
"(recommended): 'python3 -m pip install --user pipx' or your OS package manager."
|
||||||
|
if python3 - <<'PY'
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import sysconfig
|
||||||
|
|
||||||
|
# PEP 668: an EXTERNALLY-MANAGED marker in the stdlib dir means pip refuses
|
||||||
|
# to install into this interpreter without --break-system-packages.
|
||||||
|
marker = os.path.join(sysconfig.get_path("stdlib"), "EXTERNALLY-MANAGED")
|
||||||
|
raise SystemExit(0 if os.path.exists(marker) else 1)
|
||||||
|
PY
|
||||||
|
then
|
||||||
|
die "this Python is externally managed (PEP 668), so 'pip install --user' is "\
|
||||||
|
"blocked. Install pipx and re-run: 'python3 -m pip install --user --break-system-packages pipx', "\
|
||||||
|
"then 'pipx ensurepath'."
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
# --- config directories ------------------------------------------------------
|
# --- config directories ------------------------------------------------------
|
||||||
|
|
||||||
mkdir -p \
|
mkdir -p \
|
||||||
|
|||||||
+1
-1
@@ -36,7 +36,7 @@ include = ["bot_bottle*"]
|
|||||||
# files shipped under bot_bottle/; test_pyproject.py asserts they exist.
|
# files shipped under bot_bottle/; test_pyproject.py asserts they exist.
|
||||||
[tool.setuptools.package-data]
|
[tool.setuptools.package-data]
|
||||||
bot_bottle = [
|
bot_bottle = [
|
||||||
"egress_entrypoint.sh",
|
"gateway/egress/entrypoint.sh",
|
||||||
"contrib/claude/Dockerfile",
|
"contrib/claude/Dockerfile",
|
||||||
"contrib/codex/Dockerfile",
|
"contrib/codex/Dockerfile",
|
||||||
"contrib/pi/Dockerfile",
|
"contrib/pi/Dockerfile",
|
||||||
|
|||||||
@@ -0,0 +1,46 @@
|
|||||||
|
"""Build shim. Project metadata lives in ``pyproject.toml``; this only adds a
|
||||||
|
build step that copies the root-level build resources (the Dockerfiles, the
|
||||||
|
nix netpool module, the netpool script, and ``pyproject.toml``) into
|
||||||
|
``bot_bottle/_resources/`` so an installed wheel is self-contained and can
|
||||||
|
build its gateway/infra/orchestrator images without a source checkout.
|
||||||
|
|
||||||
|
Kept in sync with ``bot_bottle.resources.BUNDLED_RESOURCES`` — the
|
||||||
|
``test_resources`` suite guards against drift between the two lists.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import shutil
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from setuptools import setup
|
||||||
|
from setuptools.command.build_py import build_py
|
||||||
|
|
||||||
|
_ROOT = Path(__file__).resolve().parent
|
||||||
|
|
||||||
|
# Must match bot_bottle.resources.BUNDLED_RESOURCES (paths relative to root).
|
||||||
|
_BUNDLED_RESOURCES = (
|
||||||
|
"pyproject.toml",
|
||||||
|
"Dockerfile.gateway",
|
||||||
|
"Dockerfile.orchestrator",
|
||||||
|
"Dockerfile.orchestrator.fc",
|
||||||
|
"nix/firecracker-netpool.nix",
|
||||||
|
"scripts/firecracker-netpool.sh",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _BundleResources(build_py):
|
||||||
|
"""Copy the root-level build resources into the built package tree so they
|
||||||
|
ship inside the wheel under ``bot_bottle/_resources/``."""
|
||||||
|
|
||||||
|
def run(self) -> None:
|
||||||
|
super().run()
|
||||||
|
pkg_resources = Path(self.build_lib) / "bot_bottle" / "_resources"
|
||||||
|
for rel in _BUNDLED_RESOURCES:
|
||||||
|
src = _ROOT / rel
|
||||||
|
dst = pkg_resources / rel
|
||||||
|
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
shutil.copy2(src, dst)
|
||||||
|
|
||||||
|
|
||||||
|
setup(cmdclass={"build_py": _BundleResources})
|
||||||
@@ -53,6 +53,21 @@ class TestInstallScript(unittest.TestCase):
|
|||||||
# Tests / local installs point BOT_BOTTLE_INSTALL_SPEC at a checkout.
|
# Tests / local installs point BOT_BOTTLE_INSTALL_SPEC at a checkout.
|
||||||
self.assertIn("BOT_BOTTLE_INSTALL_SPEC", self.text)
|
self.assertIn("BOT_BOTTLE_INSTALL_SPEC", self.text)
|
||||||
|
|
||||||
|
def test_requires_git_for_git_specs(self):
|
||||||
|
# A git+ / .git spec (the default) shells out to git; the script must
|
||||||
|
# gate on it rather than failing opaquely inside pipx/pip.
|
||||||
|
self.assertIn("command -v git", self.text)
|
||||||
|
self.assertIn("git+*|*.git", self.text)
|
||||||
|
|
||||||
|
def test_checks_pip_usable_before_fallback(self):
|
||||||
|
self.assertIn("python3 -m pip --version", self.text)
|
||||||
|
|
||||||
|
def test_detects_externally_managed_python(self):
|
||||||
|
# PEP 668: 'pip install --user' is blocked on externally-managed
|
||||||
|
# interpreters; the script must detect this and point at pipx.
|
||||||
|
self.assertIn("EXTERNALLY-MANAGED", self.text)
|
||||||
|
self.assertIn("pipx", self.text)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -283,7 +283,7 @@ class TestBuildOrLoadImages(unittest.TestCase):
|
|||||||
images = launch_mod.build_or_load_images(plan)
|
images = launch_mod.build_or_load_images(plan)
|
||||||
|
|
||||||
build.assert_called_once_with(
|
build.assert_called_once_with(
|
||||||
"agent:base", launch_mod._REPO_DIR, # pylint: disable=protected-access
|
"agent:base", str(launch_mod.resources.build_root()),
|
||||||
dockerfile="/repo/Dockerfile",
|
dockerfile="/repo/Dockerfile",
|
||||||
)
|
)
|
||||||
derived.assert_called_once_with("agent:base", build)
|
derived.assert_called_once_with("agent:base", build)
|
||||||
|
|||||||
@@ -0,0 +1,110 @@
|
|||||||
|
"""Unit: bot_bottle.resources — build-resource resolution for both a source
|
||||||
|
checkout and an installed wheel.
|
||||||
|
|
||||||
|
The checkout path is what the whole test suite already runs under; the wheel
|
||||||
|
path is exercised here by faking an installed layout (a package dir with a
|
||||||
|
bundled ``_resources/`` and no sibling Dockerfiles) and asserting that
|
||||||
|
``build_root()`` stages a repo-root-shaped context. See
|
||||||
|
``test_wheel_install.py`` for the end-to-end build+install check.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from bot_bottle import resources
|
||||||
|
|
||||||
|
from tests.unit import use_bottle_root
|
||||||
|
|
||||||
|
|
||||||
|
class TestCheckoutMode(unittest.TestCase):
|
||||||
|
"""The environment the suite runs in: a real source checkout."""
|
||||||
|
|
||||||
|
def test_is_source_checkout(self):
|
||||||
|
self.assertTrue(resources.is_source_checkout())
|
||||||
|
|
||||||
|
def test_build_root_is_repo_root(self):
|
||||||
|
root = resources.build_root()
|
||||||
|
self.assertTrue((root / "bot_bottle").is_dir())
|
||||||
|
self.assertTrue((root / "pyproject.toml").is_file())
|
||||||
|
self.assertTrue((root / "Dockerfile.gateway").is_file())
|
||||||
|
|
||||||
|
def test_resource_helpers_resolve(self):
|
||||||
|
self.assertTrue(resources.dockerfile("Dockerfile.gateway").is_file())
|
||||||
|
self.assertTrue(resources.nix_netpool_module().is_file())
|
||||||
|
self.assertTrue(resources.netpool_script().is_file())
|
||||||
|
|
||||||
|
def test_bundled_resources_all_exist_at_root(self):
|
||||||
|
# Drift guard: every path setup.py bundles must exist in the checkout.
|
||||||
|
root = resources.build_root()
|
||||||
|
for rel in resources.BUNDLED_RESOURCES:
|
||||||
|
self.assertTrue((root / rel).is_file(), f"missing bundled resource: {rel}")
|
||||||
|
|
||||||
|
|
||||||
|
class TestWheelMode(unittest.TestCase):
|
||||||
|
"""Fake an installed wheel: a package dir with _resources/ and no
|
||||||
|
checkout Dockerfiles beside it."""
|
||||||
|
|
||||||
|
def _fake_install(self, tmp: Path) -> Path:
|
||||||
|
pkg = tmp / "site-packages" / "bot_bottle"
|
||||||
|
(pkg / "cli").mkdir(parents=True)
|
||||||
|
(pkg / "__init__.py").write_text("")
|
||||||
|
(pkg / "cli" / "__init__.py").write_text("# module\n")
|
||||||
|
bundled = pkg / "_resources"
|
||||||
|
for rel in resources.BUNDLED_RESOURCES:
|
||||||
|
dst = bundled / rel
|
||||||
|
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
dst.write_text(f"# fake {rel}\n")
|
||||||
|
return pkg
|
||||||
|
|
||||||
|
def test_stage_and_resolve(self):
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmpname:
|
||||||
|
tmp = Path(tmpname)
|
||||||
|
pkg = self._fake_install(tmp)
|
||||||
|
appdata = tmp / "appdata"
|
||||||
|
restore = use_bottle_root(appdata)
|
||||||
|
self.addCleanup(restore)
|
||||||
|
with patch.object(resources, "_PKG", pkg), \
|
||||||
|
patch.object(resources, "_BUNDLED", pkg / "_resources"), \
|
||||||
|
patch.object(resources, "_CHECKOUT_ROOT", tmp / "no-checkout"):
|
||||||
|
self.assertFalse(resources.is_source_checkout())
|
||||||
|
|
||||||
|
root = resources.build_root()
|
||||||
|
# Staged context looks like a repo root.
|
||||||
|
self.assertTrue((root / "bot_bottle" / "__init__.py").is_file())
|
||||||
|
self.assertTrue((root / "bot_bottle" / "cli" / "__init__.py").is_file())
|
||||||
|
self.assertTrue((root / "pyproject.toml").is_file())
|
||||||
|
self.assertTrue((root / "Dockerfile.gateway").is_file())
|
||||||
|
self.assertTrue((root / "nix" / "firecracker-netpool.nix").is_file())
|
||||||
|
self.assertTrue((root / "scripts" / "firecracker-netpool.sh").is_file())
|
||||||
|
# The bundled-resource copies are NOT re-nested under the staged
|
||||||
|
# package (keeps it byte-identical to a checkout package).
|
||||||
|
self.assertFalse((root / "bot_bottle" / "_resources").exists())
|
||||||
|
# Helpers resolve off the staged root.
|
||||||
|
self.assertEqual(root / "Dockerfile.gateway",
|
||||||
|
resources.dockerfile("Dockerfile.gateway"))
|
||||||
|
# Idempotent: second call returns the same completed dir.
|
||||||
|
self.assertEqual(root, resources.build_root())
|
||||||
|
|
||||||
|
def test_missing_bundle_raises(self):
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmpname:
|
||||||
|
tmp = Path(tmpname)
|
||||||
|
pkg = tmp / "bot_bottle"
|
||||||
|
pkg.mkdir()
|
||||||
|
restore = use_bottle_root(tmp / "appdata")
|
||||||
|
self.addCleanup(restore)
|
||||||
|
with patch.object(resources, "_PKG", pkg), \
|
||||||
|
patch.object(resources, "_BUNDLED", pkg / "_resources"), \
|
||||||
|
patch.object(resources, "_CHECKOUT_ROOT", tmp / "no-checkout"):
|
||||||
|
with self.assertRaises(resources.ResourceError):
|
||||||
|
resources.build_root()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
"""Integration: build the wheel, install it into an isolated venv, and prove
|
||||||
|
the installed distribution is self-contained.
|
||||||
|
|
||||||
|
This is the boundary a source-tree existence test can't reach (issue #197
|
||||||
|
review): under an installed wheel the package lives in ``site-packages`` with
|
||||||
|
no repo root above it, so anything resolving Dockerfiles / nix / scripts from
|
||||||
|
``__file__``'s parents would break. Here we install for real and assert that
|
||||||
|
``bot-bottle doctor`` runs from the console script and that
|
||||||
|
``bot_bottle.resources`` stages a valid, repo-root-shaped build context.
|
||||||
|
|
||||||
|
It does NOT run `start` — building images needs a Docker/KVM host (CI). It
|
||||||
|
skips cleanly when the build/venv toolchain isn't available.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|
||||||
|
|
||||||
|
def _tooling_available() -> bool:
|
||||||
|
return importlib.util.find_spec("build") is not None
|
||||||
|
|
||||||
|
|
||||||
|
@unittest.skipUnless(_tooling_available(), "python 'build' module not installed")
|
||||||
|
class TestWheelInstall(unittest.TestCase):
|
||||||
|
_tmp: "tempfile.TemporaryDirectory[str]"
|
||||||
|
venv_py: Path
|
||||||
|
app_root: Path
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls._tmp = tempfile.TemporaryDirectory( # pylint: disable=consider-using-with
|
||||||
|
prefix="bb-wheel-")
|
||||||
|
tmp = Path(cls._tmp.name)
|
||||||
|
dist = tmp / "dist"
|
||||||
|
|
||||||
|
build = subprocess.run(
|
||||||
|
[sys.executable, "-m", "build", "--wheel", "--outdir", str(dist), str(REPO_ROOT)],
|
||||||
|
capture_output=True, text=True, check=False,
|
||||||
|
)
|
||||||
|
if build.returncode != 0:
|
||||||
|
raise unittest.SkipTest(f"wheel build unavailable:\n{build.stderr[-1500:]}")
|
||||||
|
wheels = list(dist.glob("*.whl"))
|
||||||
|
if not wheels:
|
||||||
|
raise unittest.SkipTest("no wheel produced")
|
||||||
|
|
||||||
|
venv = tmp / "venv"
|
||||||
|
made = subprocess.run([sys.executable, "-m", "venv", str(venv)],
|
||||||
|
capture_output=True, text=True, check=False)
|
||||||
|
if made.returncode != 0:
|
||||||
|
raise unittest.SkipTest(f"venv unavailable:\n{made.stderr[-1500:]}")
|
||||||
|
cls.venv_py = venv / "bin" / "python"
|
||||||
|
|
||||||
|
install = subprocess.run(
|
||||||
|
[str(cls.venv_py), "-m", "pip", "install", "--quiet", str(wheels[0])],
|
||||||
|
capture_output=True, text=True, check=False,
|
||||||
|
)
|
||||||
|
if install.returncode != 0:
|
||||||
|
raise unittest.SkipTest(f"pip install failed:\n{install.stderr[-1500:]}")
|
||||||
|
|
||||||
|
# Isolate the staged build root the wheel writes under the app-data dir.
|
||||||
|
cls.app_root = tmp / "appdata"
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls):
|
||||||
|
cls._tmp.cleanup()
|
||||||
|
|
||||||
|
def _run(self, tail: "list[str]") -> "subprocess.CompletedProcess[str]":
|
||||||
|
"""Run the installed venv's python with `tail` appended, from a neutral
|
||||||
|
cwd so the source checkout isn't on sys.path — we must import the
|
||||||
|
*installed* package, not the repo we built from."""
|
||||||
|
env = {"BOT_BOTTLE_ROOT": str(self.app_root), "PATH": "/usr/bin:/bin"}
|
||||||
|
return subprocess.run(
|
||||||
|
[str(self.venv_py), *tail],
|
||||||
|
capture_output=True, text=True, env=env, cwd=self._tmp.name, check=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_console_entry_point_installed(self):
|
||||||
|
# The `bot-bottle` script the wheel declares must exist in the venv.
|
||||||
|
script = self.venv_py.parent / "bot-bottle"
|
||||||
|
self.assertTrue(script.is_file(), "bot-bottle console script not installed")
|
||||||
|
|
||||||
|
def test_doctor_runs_from_installed_package(self):
|
||||||
|
proc = self._run(["-m", "bot_bottle.cli", "doctor"])
|
||||||
|
# doctor exits non-zero here (no backend), but it must RUN and report.
|
||||||
|
self.assertIn("python", proc.stdout)
|
||||||
|
self.assertIn(proc.returncode, (0, 1))
|
||||||
|
|
||||||
|
def test_installed_wheel_is_self_contained(self):
|
||||||
|
# From the installed layout (not a checkout), resources must resolve
|
||||||
|
# Dockerfiles and stage a repo-root-shaped build context.
|
||||||
|
script = (
|
||||||
|
"import bot_bottle.resources as r\n"
|
||||||
|
"assert not r.is_source_checkout(), 'should not look like a checkout'\n"
|
||||||
|
"assert r.dockerfile('Dockerfile.gateway').is_file()\n"
|
||||||
|
"assert r.nix_netpool_module().is_file()\n"
|
||||||
|
"assert r.netpool_script().is_file()\n"
|
||||||
|
"root = r.build_root()\n"
|
||||||
|
"assert (root / 'bot_bottle' / '__init__.py').is_file(), 'no package in context'\n"
|
||||||
|
"assert (root / 'pyproject.toml').is_file(), 'no pyproject in context'\n"
|
||||||
|
"assert (root / 'Dockerfile.gateway').is_file(), 'no Dockerfile in context'\n"
|
||||||
|
"print('SELF_CONTAINED_OK')\n"
|
||||||
|
)
|
||||||
|
proc = self._run(["-c", script])
|
||||||
|
self.assertIn("SELF_CONTAINED_OK", proc.stdout, msg=proc.stderr)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user