feat(firecracker): group-owned TAP pool for multi-user hosts
Add a `group` option to the netpool NixOS module + BOT_BOTTLE_FC_GROUP to the shell script: when set, the pool's TAP devices are owned by a group instead of a single user, so any group member can open them (the kernel lets a TAP's owning-group members attach). This lets an interactive user and, say, a CI-runner user share one pool. `owner`/`group` are mutually exclusive (asserted). Single-user `owner` remains the default. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WBMWTEtQdJ4W5UrWuLHCck
This commit is contained in:
@@ -77,10 +77,16 @@ let
|
|||||||
}
|
}
|
||||||
'';
|
'';
|
||||||
|
|
||||||
|
# A non-owner user can't open a user-owned TAP, but any member of a
|
||||||
|
# TAP's owning GROUP can (the kernel checks owner-uid OR owning-group).
|
||||||
|
# So `group` lets an interactive user and a CI-runner user share one
|
||||||
|
# pool; `owner` is the single-user default.
|
||||||
|
tapOwn = if cfg.group != null then "group ${cfg.group}" else "user ${cfg.owner}";
|
||||||
|
|
||||||
upScript = pkgs.writeShellScript "bot-bottle-fc-netpool-up" ''
|
upScript = pkgs.writeShellScript "bot-bottle-fc-netpool-up" ''
|
||||||
set -eu
|
set -eu
|
||||||
${lib.concatMapStringsSep "\n" (s: ''
|
${lib.concatMapStringsSep "\n" (s: ''
|
||||||
${ip} link show ${s.iface} >/dev/null 2>&1 || ${ip} tuntap add dev ${s.iface} mode tap user ${cfg.owner}
|
${ip} link show ${s.iface} >/dev/null 2>&1 || ${ip} tuntap add dev ${s.iface} mode tap ${tapOwn}
|
||||||
${ip} addr replace ${s.hostIp}/31 dev ${s.iface}
|
${ip} addr replace ${s.hostIp}/31 dev ${s.iface}
|
||||||
${ip} link set ${s.iface} up
|
${ip} link set ${s.iface} up
|
||||||
'') slots}
|
'') slots}
|
||||||
@@ -123,9 +129,26 @@ in
|
|||||||
};
|
};
|
||||||
|
|
||||||
owner = lib.mkOption {
|
owner = lib.mkOption {
|
||||||
type = lib.types.str;
|
type = lib.types.nullOr lib.types.str;
|
||||||
|
default = null;
|
||||||
example = "alice";
|
example = "alice";
|
||||||
description = "User that owns the TAP devices, so `./cli.py start` opens them without root.";
|
description = ''
|
||||||
|
User that owns the TAP devices (single-user hosts), so
|
||||||
|
`./cli.py start` opens them without root. Set exactly one of
|
||||||
|
`owner` or `group`.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
group = lib.mkOption {
|
||||||
|
type = lib.types.nullOr lib.types.str;
|
||||||
|
default = null;
|
||||||
|
example = "firecracker";
|
||||||
|
description = ''
|
||||||
|
Group that owns the TAP devices instead of a single user. Any
|
||||||
|
member of this group can open the pool, so an interactive user
|
||||||
|
and a CI-runner user can share one pool. Set exactly one of
|
||||||
|
`owner` or `group`.
|
||||||
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
tableName = lib.mkOption {
|
tableName = lib.mkOption {
|
||||||
@@ -151,6 +174,10 @@ in
|
|||||||
assertion = lib.mod baseInt 2 == 0;
|
assertion = lib.mod baseInt 2 == 0;
|
||||||
message = "services.bot-bottle-firecracker.ipBase must be /31-aligned (even final octet); got ${cfg.ipBase}.";
|
message = "services.bot-bottle-firecracker.ipBase must be /31-aligned (even final octet); got ${cfg.ipBase}.";
|
||||||
}
|
}
|
||||||
|
{
|
||||||
|
assertion = (cfg.owner != null) != (cfg.group != null);
|
||||||
|
message = "services.bot-bottle-firecracker: set exactly one of `owner` or `group`.";
|
||||||
|
}
|
||||||
];
|
];
|
||||||
|
|
||||||
# VM->sidecar traffic is DNAT'd and forwarded, so forwarding must be on.
|
# VM->sidecar traffic is DNAT'd and forwarded, so forwarding must be on.
|
||||||
|
|||||||
@@ -38,6 +38,10 @@
|
|||||||
# BOT_BOTTLE_FC_IP_BASE base IPv4 of the /31 pool (default 10.243.0.0)
|
# BOT_BOTTLE_FC_IP_BASE base IPv4 of the /31 pool (default 10.243.0.0)
|
||||||
# BOT_BOTTLE_FC_IFACE_PREFIX TAP name prefix (default bbfc)
|
# BOT_BOTTLE_FC_IFACE_PREFIX TAP name prefix (default bbfc)
|
||||||
# BOT_BOTTLE_FC_OWNER owning user (default $SUDO_USER or $USER)
|
# BOT_BOTTLE_FC_OWNER owning user (default $SUDO_USER or $USER)
|
||||||
|
# BOT_BOTTLE_FC_GROUP owning group; if set, TAPs are group-owned
|
||||||
|
# instead of user-owned, so any group member
|
||||||
|
# (e.g. an interactive user + a CI runner
|
||||||
|
# user) can open the pool. Overrides OWNER.
|
||||||
|
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
@@ -45,6 +49,7 @@ POOL_SIZE="${BOT_BOTTLE_FC_POOL_SIZE:-8}"
|
|||||||
IP_BASE="${BOT_BOTTLE_FC_IP_BASE:-10.243.0.0}"
|
IP_BASE="${BOT_BOTTLE_FC_IP_BASE:-10.243.0.0}"
|
||||||
PREFIX="${BOT_BOTTLE_FC_IFACE_PREFIX:-bbfc}"
|
PREFIX="${BOT_BOTTLE_FC_IFACE_PREFIX:-bbfc}"
|
||||||
OWNER="${BOT_BOTTLE_FC_OWNER:-${SUDO_USER:-$USER}}"
|
OWNER="${BOT_BOTTLE_FC_OWNER:-${SUDO_USER:-$USER}}"
|
||||||
|
GROUP="${BOT_BOTTLE_FC_GROUP:-}"
|
||||||
TABLE="bot_bottle_fc"
|
TABLE="bot_bottle_fc"
|
||||||
|
|
||||||
# Sidecar ports (must match the backend). egress=9099, supervise=9100,
|
# Sidecar ports (must match the backend). egress=9099, supervise=9100,
|
||||||
@@ -73,7 +78,14 @@ require_root() {
|
|||||||
|
|
||||||
cmd_up() {
|
cmd_up() {
|
||||||
require_root up
|
require_root up
|
||||||
echo "firecracker net pool: $POOL_SIZE slots, base $IP_BASE, owner $OWNER"
|
# Group ownership (any member can open the pool) overrides single-user
|
||||||
|
# ownership. The kernel lets a TAP's owning-group members attach.
|
||||||
|
if [ -n "$GROUP" ]; then
|
||||||
|
own_args=(group "$GROUP") ; own_desc="group=$GROUP"
|
||||||
|
else
|
||||||
|
own_args=(user "$OWNER") ; own_desc="owner=$OWNER"
|
||||||
|
fi
|
||||||
|
echo "firecracker net pool: $POOL_SIZE slots, base $IP_BASE, $own_desc"
|
||||||
|
|
||||||
# VM->sidecar traffic is DNAT'd to the sidecar container and
|
# VM->sidecar traffic is DNAT'd to the sidecar container and
|
||||||
# forwarded, so forwarding must be enabled (Docker also sets this).
|
# forwarded, so forwarding must be enabled (Docker also sets this).
|
||||||
@@ -86,10 +98,10 @@ cmd_up() {
|
|||||||
ip link set "$dev" down 2>/dev/null || true
|
ip link set "$dev" down 2>/dev/null || true
|
||||||
ip tuntap del dev "$dev" mode tap 2>/dev/null || true
|
ip tuntap del dev "$dev" mode tap 2>/dev/null || true
|
||||||
fi
|
fi
|
||||||
ip tuntap add dev "$dev" mode tap user "$OWNER"
|
ip tuntap add dev "$dev" mode tap "${own_args[@]}"
|
||||||
ip addr add "$host/31" dev "$dev"
|
ip addr add "$host/31" dev "$dev"
|
||||||
ip link set "$dev" up
|
ip link set "$dev" up
|
||||||
echo " $dev host=$host guest=$(guest_ip "$i") owner=$OWNER"
|
echo " $dev host=$host guest=$(guest_ip "$i") $own_desc"
|
||||||
done
|
done
|
||||||
|
|
||||||
_install_nft
|
_install_nft
|
||||||
|
|||||||
@@ -69,6 +69,8 @@ class TestNetpoolRenderers(unittest.TestCase):
|
|||||||
self.assertIn("bot-bottle-firecracker-netpool", mod) # the oneshot
|
self.assertIn("bot-bottle-firecracker-netpool", mod) # the oneshot
|
||||||
self.assertIn(netpool.NFT_TABLE, mod)
|
self.assertIn(netpool.NFT_TABLE, mod)
|
||||||
self.assertIn('iifname != "${cfg.ifacePrefix}*" return', mod)
|
self.assertIn('iifname != "${cfg.ifacePrefix}*" return', mod)
|
||||||
|
# Supports group-owned TAPs (shared pool for a multi-user host).
|
||||||
|
self.assertIn("group ${cfg.group}", mod)
|
||||||
|
|
||||||
def test_shell_setup_reflects_overrides(self):
|
def test_shell_setup_reflects_overrides(self):
|
||||||
with patch.dict(os.environ, {"BOT_BOTTLE_FC_POOL_SIZE": "3"}):
|
with patch.dict(os.environ, {"BOT_BOTTLE_FC_POOL_SIZE": "3"}):
|
||||||
|
|||||||
Reference in New Issue
Block a user