m***@gmx.de
2021-05-29 15:59:49 UTC
Hello everybody,
since a few weeks, my jails on a bhyve-vm, running current are not reachable via network, when configured with VNET. They can't even access the gateway. I don't remember when this problem started, but it's a few weeks.
The same jail.conf works on a 13.0 host, on a current system the network does not work. A configuration without VNET on the same jail works. Are there any changes, that i missed? Here is the configuration, maybe someone spots an error, or has an idea what's going on:
--
Martin
uname on bhyve vm:
------------------
***@fbsd14:~ # uname -a
FreeBSD fbsd14.fritz.box 14.0-CURRENT FreeBSD 14.0-CURRENT main-n247020-e0fa04e257c GENERIC-NODEBUG amd64
***@fbsd14:~ # freebsd-version -kru
14.0-CURRENT
14.0-CURRENT
14.0-CURRENT
jail.conf on bhyve vm:
----------------------
# set default configuration values
mount.devfs = true;
exec.clean = true;
allow.chflags = 1;
allow.raw_sockets = 1;
devfs_ruleset = 5;
exec.system_user = "root";
exec.jail_user = "root";
exec.timeout = 30;
stop.timeout = 30;
#########
# Jails #
#########
j1 {
# Hostname
host.hostname = "j1.fritz.box";
host.domainname = "fritz.box";
host.hostuuid = "68c2ad9b-b582-11eb-a925-589cfc0ac350";
osrelease = "14.0-CURRENT";
osreldate = "1400013";
# Network
vnet = 1;
vnet.interface = "epair2b";
exec.prestart += "ifconfig epair2 create up";
exec.prestart += "ifconfig epair2a description 'IFID=2 JAIL=j1'";
exec.prestart += "ifconfig bridge0 addm epair2a";
command = "ifconfig epair2b inet 192.168.1.101/22";
command += "route -n add -inet default 192.168.0.1";
exec.prestop = "ifconfig epair2b -vnet j1";
exec.poststop += "ifconfig bridge0 deletem epair2a";
exec.poststop += "ifconfig epair2a destroy";
sysvmsg = new;
sysvsem = new;
sysvshm = new;
path = "/jails/j1";
allow.mount.zfs = 1;
## Script execution
exec.timeout = 90;
# Pre-/Post-Scripts
exec.prestart += "logger trying to start jail j1 ...";
exec.poststart += "logger jail j1 has started";
exec.prestop += "logger shutting down jail j1";
exec.poststop += "logger jail j1 has shut down";
# Start Script
exec.start = "/bin/sh /etc/rc";
exec.stop = "/bin/sh /etc/rc.shutdown";
}
-----------------------------------
/etc/rc.conf on bhyve vm:
-------------------------
syslogd_flags="-ss"
sendmail_enable="NONE"
hostname="fbsd14.fritz.box"
ifconfig_vtnet0="inet 192.168.1.100 netmask 255.255.252.0"
defaultrouter="192.168.0.1"
local_unbound_enable="YES"
sshd_enable="YES"
ntpd_enable="YES"
# Set dumpdev to "AUTO" to enable crash dumps, "NO" to disable
dumpdev="AUTO"
zfs_enable="YES"
jail_enable="YES"
keymap="de"
cloned_interfaces="bridge0"
ifconfig_bridge0="addm vtnet0 up"
# NFS
rpc_lockd_enable="YES"
rpc_statd_enable="YES"
nfs_client_enable="YES"
nfsuserd_enable="YES"
-------------------------------------
ifconfig on bhyve vm:
---------------------
***@fbsd14:~ # ifconfig -f inet:cidr
vtnet0: flags=8863<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=80028<VLAN_MTU,JUMBO_MTU,LINKSTATE>
ether 58:9c:fc:0a:c3:50
inet 192.168.1.100/22 broadcast 192.168.3.255
media: Ethernet autoselect (10Gbase-T <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
options=680003<RXCSUM,TXCSUM,LINKSTATE,RXCSUM_IPV6,TXCSUM_IPV6>
inet6 ::1 prefixlen 128
inet6 fe80::1%lo0 prefixlen 64 scopeid 0x2
inet 127.0.0.1/8
groups: lo
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
ether 58:9c:fc:10:ff:bf
id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
member: epair2a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
ifmaxaddr 0 port 4 priority 128 path cost 2000
groups: bridge
nd6 options=9<PERFORMNUD,IFDISABLED>
epair2a: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
description: IFID=2 JAIL=j1
options=8<VLAN_MTU>
ether 02:b4:ee:59:b3:0a
groups: epair
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
-------------------------------
/etc/rc.conf in jail:
---------------------
syslogd_flags="-ss"
sendmail_enable="NO"
sendmail_submit_enable="NO"
sendmail_outbound_enable="NO"
sendmail_msp_queue_enable="NO"
sshd_enable="YES"
---------------------------
ifconfig in jail:
-----------------
***@j1:~ # ifconfig -f inet:cidr
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
options=680003<RXCSUM,TXCSUM,LINKSTATE,RXCSUM_IPV6,TXCSUM_IPV6>
inet6 ::1 prefixlen 128
inet6 fe80::1%lo0 prefixlen 64 scopeid 0x1
inet 127.0.0.1/8
groups: lo
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
epair2b: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=8<VLAN_MTU>
ether 02:b4:ee:59:b3:0b
inet 192.168.1.101/22 broadcast 192.168.3.255
groups: epair
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
------------------------------------
uname in jail:
--------------
***@j1:~ # uname -a
FreeBSD j1.fritz.box 14.0-CURRENT FreeBSD 14.0-CURRENT main-n247020-e0fa04e257c GENERIC-NODEBUG amd64
***@j1:~ # freebsd-version -ru
14.0-CURRENT
14.0-CURRENT
since a few weeks, my jails on a bhyve-vm, running current are not reachable via network, when configured with VNET. They can't even access the gateway. I don't remember when this problem started, but it's a few weeks.
The same jail.conf works on a 13.0 host, on a current system the network does not work. A configuration without VNET on the same jail works. Are there any changes, that i missed? Here is the configuration, maybe someone spots an error, or has an idea what's going on:
--
Martin
uname on bhyve vm:
------------------
***@fbsd14:~ # uname -a
FreeBSD fbsd14.fritz.box 14.0-CURRENT FreeBSD 14.0-CURRENT main-n247020-e0fa04e257c GENERIC-NODEBUG amd64
***@fbsd14:~ # freebsd-version -kru
14.0-CURRENT
14.0-CURRENT
14.0-CURRENT
jail.conf on bhyve vm:
----------------------
# set default configuration values
mount.devfs = true;
exec.clean = true;
allow.chflags = 1;
allow.raw_sockets = 1;
devfs_ruleset = 5;
exec.system_user = "root";
exec.jail_user = "root";
exec.timeout = 30;
stop.timeout = 30;
#########
# Jails #
#########
j1 {
# Hostname
host.hostname = "j1.fritz.box";
host.domainname = "fritz.box";
host.hostuuid = "68c2ad9b-b582-11eb-a925-589cfc0ac350";
osrelease = "14.0-CURRENT";
osreldate = "1400013";
# Network
vnet = 1;
vnet.interface = "epair2b";
exec.prestart += "ifconfig epair2 create up";
exec.prestart += "ifconfig epair2a description 'IFID=2 JAIL=j1'";
exec.prestart += "ifconfig bridge0 addm epair2a";
command = "ifconfig epair2b inet 192.168.1.101/22";
command += "route -n add -inet default 192.168.0.1";
exec.prestop = "ifconfig epair2b -vnet j1";
exec.poststop += "ifconfig bridge0 deletem epair2a";
exec.poststop += "ifconfig epair2a destroy";
sysvmsg = new;
sysvsem = new;
sysvshm = new;
path = "/jails/j1";
allow.mount.zfs = 1;
## Script execution
exec.timeout = 90;
# Pre-/Post-Scripts
exec.prestart += "logger trying to start jail j1 ...";
exec.poststart += "logger jail j1 has started";
exec.prestop += "logger shutting down jail j1";
exec.poststop += "logger jail j1 has shut down";
# Start Script
exec.start = "/bin/sh /etc/rc";
exec.stop = "/bin/sh /etc/rc.shutdown";
}
-----------------------------------
/etc/rc.conf on bhyve vm:
-------------------------
syslogd_flags="-ss"
sendmail_enable="NONE"
hostname="fbsd14.fritz.box"
ifconfig_vtnet0="inet 192.168.1.100 netmask 255.255.252.0"
defaultrouter="192.168.0.1"
local_unbound_enable="YES"
sshd_enable="YES"
ntpd_enable="YES"
# Set dumpdev to "AUTO" to enable crash dumps, "NO" to disable
dumpdev="AUTO"
zfs_enable="YES"
jail_enable="YES"
keymap="de"
cloned_interfaces="bridge0"
ifconfig_bridge0="addm vtnet0 up"
# NFS
rpc_lockd_enable="YES"
rpc_statd_enable="YES"
nfs_client_enable="YES"
nfsuserd_enable="YES"
-------------------------------------
ifconfig on bhyve vm:
---------------------
***@fbsd14:~ # ifconfig -f inet:cidr
vtnet0: flags=8863<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=80028<VLAN_MTU,JUMBO_MTU,LINKSTATE>
ether 58:9c:fc:0a:c3:50
inet 192.168.1.100/22 broadcast 192.168.3.255
media: Ethernet autoselect (10Gbase-T <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
options=680003<RXCSUM,TXCSUM,LINKSTATE,RXCSUM_IPV6,TXCSUM_IPV6>
inet6 ::1 prefixlen 128
inet6 fe80::1%lo0 prefixlen 64 scopeid 0x2
inet 127.0.0.1/8
groups: lo
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
ether 58:9c:fc:10:ff:bf
id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
member: epair2a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
ifmaxaddr 0 port 4 priority 128 path cost 2000
groups: bridge
nd6 options=9<PERFORMNUD,IFDISABLED>
epair2a: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
description: IFID=2 JAIL=j1
options=8<VLAN_MTU>
ether 02:b4:ee:59:b3:0a
groups: epair
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
-------------------------------
/etc/rc.conf in jail:
---------------------
syslogd_flags="-ss"
sendmail_enable="NO"
sendmail_submit_enable="NO"
sendmail_outbound_enable="NO"
sendmail_msp_queue_enable="NO"
sshd_enable="YES"
---------------------------
ifconfig in jail:
-----------------
***@j1:~ # ifconfig -f inet:cidr
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
options=680003<RXCSUM,TXCSUM,LINKSTATE,RXCSUM_IPV6,TXCSUM_IPV6>
inet6 ::1 prefixlen 128
inet6 fe80::1%lo0 prefixlen 64 scopeid 0x1
inet 127.0.0.1/8
groups: lo
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
epair2b: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=8<VLAN_MTU>
ether 02:b4:ee:59:b3:0b
inet 192.168.1.101/22 broadcast 192.168.3.255
groups: epair
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
------------------------------------
uname in jail:
--------------
***@j1:~ # uname -a
FreeBSD j1.fritz.box 14.0-CURRENT FreeBSD 14.0-CURRENT main-n247020-e0fa04e257c GENERIC-NODEBUG amd64
***@j1:~ # freebsd-version -ru
14.0-CURRENT
14.0-CURRENT