Network in VNET jail does not work on my FreeBSD current bhyve vm

From: <mj-mailinglist_at_gmx.de>
Date: Sat, 29 May 2021 17:59:49 +0200
Hello everybody,

since a few weeks, my jails on a bhyve-vm, running current are not reachable via network, when configured with VNET. They can't even access the gateway. I don't remember when this problem started, but it's a few weeks.
The same jail.conf works on a 13.0 host, on a current system the network does not work. A configuration without VNET on the same jail works. Are there any changes, that i missed? Here is the configuration, maybe someone spots an error, or has an idea what's going on:

--
Martin

uname on bhyve vm:
------------------
root_at_fbsd14:~ # uname -a
FreeBSD fbsd14.fritz.box 14.0-CURRENT FreeBSD 14.0-CURRENT main-n247020-e0fa04e257c GENERIC-NODEBUG  amd64

root_at_fbsd14:~ # freebsd-version -kru
14.0-CURRENT
14.0-CURRENT
14.0-CURRENT


jail.conf on bhyve vm:
----------------------
# set default configuration values
mount.devfs = true;
exec.clean = true;

allow.chflags = 1;
allow.raw_sockets = 1;

devfs_ruleset = 5;

exec.system_user  = "root";
exec.jail_user    = "root";

exec.timeout = 30;
stop.timeout = 30;

#########
# Jails #
#########
j1 {
    # Hostname
    host.hostname   = "j1.fritz.box";
    host.domainname = "fritz.box";
    host.hostuuid   = "68c2ad9b-b582-11eb-a925-589cfc0ac350";

    osrelease = "14.0-CURRENT";
    osreldate = "1400013";

    # Network
    vnet = 1;
    vnet.interface = "epair2b";

    exec.prestart += "ifconfig epair2 create up";
    exec.prestart += "ifconfig epair2a description 'IFID=2 JAIL=j1'";
    exec.prestart += "ifconfig bridge0 addm epair2a";

    command  = "ifconfig epair2b inet 192.168.1.101/22";
    command += "route -n add -inet default 192.168.0.1";

    exec.prestop   = "ifconfig epair2b -vnet j1";

    exec.poststop += "ifconfig bridge0 deletem epair2a";
    exec.poststop += "ifconfig epair2a destroy";

    sysvmsg = new;
    sysvsem = new;
    sysvshm = new;

    path = "/jails/j1";
    allow.mount.zfs = 1;

    ## Script execution
    exec.timeout = 90;

    # Pre-/Post-Scripts
    exec.prestart  += "logger trying to start jail j1 ...";
    exec.poststart += "logger jail j1 has started";
    exec.prestop   += "logger shutting down jail j1";
    exec.poststop  += "logger jail j1 has shut down";

    # Start Script
    exec.start  = "/bin/sh /etc/rc";
    exec.stop   = "/bin/sh /etc/rc.shutdown";
}
-----------------------------------



/etc/rc.conf on bhyve vm:
-------------------------
syslogd_flags="-ss"
sendmail_enable="NONE"
hostname="fbsd14.fritz.box"
ifconfig_vtnet0="inet 192.168.1.100 netmask 255.255.252.0"
defaultrouter="192.168.0.1"
local_unbound_enable="YES"
sshd_enable="YES"
ntpd_enable="YES"
# Set dumpdev to "AUTO" to enable crash dumps, "NO" to disable
dumpdev="AUTO"
zfs_enable="YES"
jail_enable="YES"
keymap="de"

cloned_interfaces="bridge0"
ifconfig_bridge0="addm vtnet0 up"

# NFS
rpc_lockd_enable="YES"
rpc_statd_enable="YES"
nfs_client_enable="YES"
nfsuserd_enable="YES"
-------------------------------------


ifconfig on bhyve vm:
---------------------
root_at_fbsd14:~ # ifconfig -f inet:cidr
vtnet0: flags=8863<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
        options=80028<VLAN_MTU,JUMBO_MTU,LINKSTATE>
        ether 58:9c:fc:0a:c3:50
        inet 192.168.1.100/22 broadcast 192.168.3.255
        media: Ethernet autoselect (10Gbase-T <full-duplex>)
        status: active
        nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
        options=680003<RXCSUM,TXCSUM,LINKSTATE,RXCSUM_IPV6,TXCSUM_IPV6>
        inet6 ::1 prefixlen 128
        inet6 fe80::1%lo0 prefixlen 64 scopeid 0x2
        inet 127.0.0.1/8
        groups: lo
        nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
        ether 58:9c:fc:10:ff:bf
        id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
        maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
        root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
        member: epair2a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
                ifmaxaddr 0 port 4 priority 128 path cost 2000
        groups: bridge
        nd6 options=9<PERFORMNUD,IFDISABLED>
epair2a: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
        description: IFID=2 JAIL=j1
        options=8<VLAN_MTU>
        ether 02:b4:ee:59:b3:0a
        groups: epair
        media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
        status: active
        nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
-------------------------------




/etc/rc.conf in jail:
---------------------
syslogd_flags="-ss"
sendmail_enable="NO"
sendmail_submit_enable="NO"
sendmail_outbound_enable="NO"
sendmail_msp_queue_enable="NO"
sshd_enable="YES"
---------------------------


ifconfig in jail:
-----------------
root_at_j1:~ # ifconfig -f inet:cidr
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
        options=680003<RXCSUM,TXCSUM,LINKSTATE,RXCSUM_IPV6,TXCSUM_IPV6>
        inet6 ::1 prefixlen 128
        inet6 fe80::1%lo0 prefixlen 64 scopeid 0x1
        inet 127.0.0.1/8
        groups: lo
        nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
epair2b: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
        options=8<VLAN_MTU>
        ether 02:b4:ee:59:b3:0b
        inet 192.168.1.101/22 broadcast 192.168.3.255
        groups: epair
        media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
        status: active
        nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
------------------------------------

uname in jail:
--------------
root_at_j1:~ # uname -a
FreeBSD j1.fritz.box 14.0-CURRENT FreeBSD 14.0-CURRENT main-n247020-e0fa04e257c GENERIC-NODEBUG  amd64

root_at_j1:~ # freebsd-version -ru
14.0-CURRENT
14.0-CURRENT
Received on Sat May 29 2021 - 15:59:49 UTC

Original text of this message