[Bug 248468] jail(8) host has no internet access when vnet & non-vnet jails running at same time
bugzilla-noreply at freebsd.org
bugzilla-noreply at freebsd.org
Tue Aug 4 15:06:22 UTC 2020
https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=248468
Bug ID: 248468
Summary: jail(8) host has no internet access when vnet &
non-vnet jails running at same time
Product: Base System
Version: 12.1-RELEASE
Hardware: Any
OS: Any
Status: New
Severity: Affects Many People
Priority: ---
Component: kern
Assignee: bugs at FreeBSD.org
Reporter: joeb1 at a1poweruser.com
Equipment. Real hardware, 12.1 release, amd64 dual cpu.
Summery Description;
non-vnet jails and vnet jails using the bridge/epair method can ping the public
internet when only non-vnet jails are started at a time or when only vnet jails
are started at a time. But when both non-vnet jails and vnet jails are started
together then neither one can ping the public internet. The order of the jails
definitions in the jail.conf file has no effect on changing what is happening.
The vnet jail doesn't run a firewall inside of it so the hosts firewall is
doing the NATing of the private addresses.
Bug description:
When non-vnet jails are started their ip addresses are added to the NIC facing
the public AFTER the public ip address and the non-vnet jail has access to the
public internet. But when both non-vnet jails and vnet jails are started at the
same time then the non-vnet jails ip addresses gets added BEFORE the public ip
address of the NIC facing the public internet causing the host to lose all
access to the public internet.
It makes no difference which command method is used to start and stop the
jails.
Service jail start jailname or jail –cv jailname
The following is a capture of the command sequence showing this bug. Follow the
re0 NIC public ip address xx.25.51.0 in the ifconfig -a listing.
Before any jails are started.
/root >ifconfig -a
snip ...
re0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu
1500
options=8209b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC,LINKSTATE>
ether 50:3e:aa:06:11:22
inet xx.25.51.0 netmask 0xfffff000 broadcast 255.255.255.255
media: Ethernet autoselect (1000baseT <full-duplex,master>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
ether 02:3e:ba:a7:58:00
id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
member: re0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
ifmaxaddr 0 port 2 priority 128 path cost 20000
groups: bridge
nd6 options=1<PERFORMNUD>
/root >cat /etc/jail.conf
# non-vnet jail
zdir20 {
host.hostname = "zdir20";
path = "/usr/jails/zdir20";
mount.fstab = "/usr/local/etc/fstab/zdir20";
exec.consolelog = "/var/log/zdir20.console.log";
mount.devfs;
ip4.addr = 10.0.22.5;
interface = "re0";
allow.raw_sockets;
devfs_ruleset = "4";
exec.start = "/bin/sh /etc/rc";
exec.stop = "/bin/sh /etc/rc.shutdown";
}
# vnet jail using the bridge/epair method
v0jail1 {
host.hostname = "v0jail1";
path = "/usr/jails/v0jail1";
mount.fstab = "/usr/local/etc/fstab/v0jail1";
exec.consolelog = "/var/log/v0jail1.console.log";
mount.devfs;
devfs_ruleset = "4";
vnet = "new";
vnet.interface = "epair55b";
exec.prestart = "ifconfig epair55 create up";
exec.prestart += "ifconfig bridge0 addm epair55a";
exec.prestart += "ifconfig epair55a descr vnet-v0jail1";
exec.prestart += "ifconfig bridge0 inet 10.0.48.2 netmask 255.255.255.0
alias";
exec.start = "/bin/sh /etc/rc";
exec.start += "ifconfig epair55b inet 10.0.48.1 netmask 255.255.255.0";
exec.start += "route add default 10.0.48.2";
exec.prestop = "ifconfig epair55b -vnet v0jail1";
exec.stop = "/bin/sh /etc/rc.shutdown";
exec.poststop = "ifconfig bridge0 deletem epair55a";
exec.poststop += "sleep 2";
exec.poststop += "ifconfig epair55a destroy";
exec.poststop += "ifconfig bridge0 inet 10.0.48.2 -alias";
}
/root >jls
JID IP Address Hostname Path
# start only the non-vnet jail
/root >service jail start zdir20
Starting jails: zdir20.
/root >jls
JID IP Address Hostname Path
18 10.0.22.5 zdir20 /usr/jails/zdir20
# Take notice that the non-vnet jails ip address follows the nic’s
# public ip address.
/root >ifconfig -a
re0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu
1500
options=8209b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC,LINKSTATE>
ether 50:3e:aa:06:11:22
inet xx.25.51.0 netmask 0xfffff000 broadcast 255.255.255.255
inet 10.0.22.5 netmask 0xffffffff broadcast 10.0.22.5
media: Ethernet autoselect (1000baseT <full-duplex,master>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
ether 02:3e:ba:a7:58:00
id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
member: re0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
ifmaxaddr 0 port 2 priority 128 path cost 20000
groups: bridge
nd6 options=1<PERFORMNUD>
# login to the non-vnet jail and ping the public
/root >jexec zdir20 login -f root
Last login: Sun Aug 2 11:30:40 on pts/0
FreeBSD 12.1-RELEASE-p6 GENERIC
Welcome to your FreeBSD jail.
zdir20 /root >
zdir20 /root >ping -c 2 freebsd.org
PING freebsd.org (96.47.72.84): 56 data bytes
64 bytes from 96.47.72.84: icmp_seq=0 ttl=48 time=44.426 ms
64 bytes from 96.47.72.84: icmp_seq=1 ttl=48 time=44.481 ms
--- freebsd.org ping statistics ---
2 packets transmitted, 2 packets received, 0.0% packet loss
round-trip min/avg/max/stddev = 44.426/44.453/44.481/0.027 ms
zdir20 /root >exit
logout
# stop the non-vnet jail and show that the network is back to
# starting condition.
/root >service jail stop zdir20
Stopping jails: zdir20.
/root >jls
JID IP Address Hostname Path
/root >ifconfig -a
re0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu
1500
options=8209b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC,LINKSTATE>
ether 50:3e:aa:06:11:22
inet xx.25.51.0 netmask 0xfffff000 broadcast 255.255.255.255
media: Ethernet autoselect (1000baseT <full-duplex,master>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
ether 02:3e:ba:a7:58:00
id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
member: re0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
ifmaxaddr 0 port 2 priority 128 path cost 20000
groups: bridge
nd6 options=1<PERFORMNUD>
# start only the vnet jail and see the bridge0
/root >service jail start v0jail1
Starting jails: v0jail1.
/root >jls
JID IP Address Hostname Path
19 v0jail1 /usr/jails/v0jail1
/root >ifconfig -a
re0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu
1500
options=82099<RXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC,LINKSTATE>
ether 50:3e:aa:06:11:22
inet xx.25.51.0 netmask 0xfffff000 broadcast 255.255.255.255
media: Ethernet autoselect (1000baseT <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
ether 02:3e:ba:a7:58:00
inet 10.0.48.2 netmask 0xffffff00 broadcast 10.0.48.255
id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
member: epair55a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
ifmaxaddr 0 port 5 priority 128 path cost 2000
member: re0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
ifmaxaddr 0 port 2 priority 128 path cost 20000
groups: bridge
nd6 options=1<PERFORMNUD>
epair55a: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0
mtu 1500
description: vnet-v0jail1
options=8<VLAN_MTU>
ether 02:eb:be:f5:15:0a
inet6 fe80::eb:beff:fef5:150a%epair55a prefixlen 64 scopeid 0x5
groups: epair
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
# login to the vnet jail and ping the public internet.
/root >jexec v0jail1 login -f root
Last login: Sun Aug 2 11:29:41 on pts/0
FreeBSD 12.1-RELEASE-p6 GENERIC
Welcome to your FreeBSD jail.
v0jail1 /root >ping -c 2 freebsd.org
PING freebsd.org (96.47.72.84): 56 data bytes
64 bytes from 96.47.72.84: icmp_seq=0 ttl=47 time=46.745 ms
64 bytes from 96.47.72.84: icmp_seq=1 ttl=47 time=43.930 ms
--- freebsd.org ping statistics ---
2 packets transmitted, 2 packets received, 0.0% packet loss
round-trip min/avg/max/stddev = 43.930/45.337/46.745/1.407 ms
v0jail1 /root >exit
logout
# close the vnet jail and return to starting condition.
/root >service jail stop v0jail1
Stopping jails: v0jail1.
/root >jls
JID IP Address Hostname Path
/root >ifconfig -a
re0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu
1500
options=8209b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC,LINKSTATE>
ether 50:3e:aa:06:11:22
inet xx.25.51.0 netmask 0xfffff000 broadcast 255.255.255.255
media: Ethernet autoselect (1000baseT <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
ether 02:3e:ba:a7:58:00
id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
member: re0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
ifmaxaddr 0 port 2 priority 128 path cost 20000
groups: bridge
nd6 options=1<PERFORMNUD>
# Start both the non-vnet jail and the vnet jail together.
/root >service jail start
Starting jails: zdir20 v0jail1.
# login to the non-vnet jail and it has no public access.
/root >jexec zdir20 login -f root [K
Last login: Sun Aug 2 11:36:34 on pts/0
FreeBSD 12.1-RELEASE-p6 GENERIC
Welcome to your FreeBSD jail.
zdir20 /root >ping -c 2 freebsd.org
ping: cannot resolve freebsd.org: Host name lookup failure
zdir20 /root >exit
logout
# login to the vnet jail and it has no public access.
/root >jexec v0jail1 login -f root
Last login: Sun Aug 2 11:38:56 on pts/0
FreeBSD 12.1-RELEASE-p6 GENERIC
Welcome to your FreeBSD jail.
v0jail1 /root >ping -c 2 freebsd.org
ping: cannot resolve freebsd.org: Host name lookup failure
v0jail1 /root >exit
logout
/root >jls
JID IP Address Hostname Path
20 10.0.22.5 zdir20 /usr/jails/zdir20
21 v0jail1 /usr/jails/v0jail1
# Here is the bug. See that the non-vnet jail ip address comes before the
# public address causing the host to lose access to the public internet.
/root >ifconfig -a
re0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu
1500
options=82099<RXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC,LINKSTATE>
ether 50:3e:aa:06:11:22
inet 10.0.22.5 netmask 0xffffffff broadcast 10.0.22.5
inet xx.25.51.0 netmask 0xfffff000 broadcast 255.255.255.255
media: Ethernet autoselect (1000baseT <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
ether 02:3e:ba:a7:58:00
inet 10.0.48.2 netmask 0xffffff00 broadcast 10.0.48.255
id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
member: epair55a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
ifmaxaddr 0 port 5 priority 128 path cost 2000
member: re0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
ifmaxaddr 0 port 2 priority 128 path cost 20000
groups: bridge
nd6 options=1<PERFORMNUD>
epair55a: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0
mtu 1500
description: vnet-v0jail1
options=8<VLAN_MTU>
ether 02:77:b8:5f:e4:0a
inet6 fe80::77:b8ff:fe5f:e40a%epair55a prefixlen 64 scopeid 0x5
groups: epair
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
# stop both jails and return to starting condition.
/root >service jail stop
Stopping jails: zdir20 v0jail1.
/root >jls
JID IP Address Hostname Path
/root >ifconfig -a
re0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu
1500
options=8209b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC,LINKSTATE>
ether 50:3e:aa:06:11:22
inet xx.25.51.0 netmask 0xfffff000 broadcast 255.255.255.255
media: Ethernet autoselect (1000baseT <full-duplex,master>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
ether 02:3e:ba:a7:58:00
id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
member: re0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
ifmaxaddr 0 port 2 priority 128 path cost 20000
groups: bridge
nd6 options=1<PERFORMNUD>
--
You are receiving this mail because:
You are the assignee for the bug.
More information about the freebsd-bugs
mailing list