jail(8) vimage epair bridge
Joe
fbsd8 at a1poweruser.com
Wed Apr 24 15:42:28 UTC 2013
Anders Hagman wrote:
> Hi
>
>> Hello list
>>
>> I am using jail(8) trying to get a functional vimage environment on my
>> 9.1-RELEASE system. My PC only has a single real NIC facing the public
>> internet.
>>
>> My goal is to be able to have multiple vimage jails, each with
>> their own epairXa epairXb and bridgeX where the "X" is the jails JID
>> number all having their traffic passing through the single rl0 real
>> interface. The vnet.start script shown below handles this nicely.
>>
>> The problem is after the first vimage jail is started the rl0 interface
>> gets marked as busy when the second vimage jail is started.
>>
>> How do I get all vnet jails to pass through the real rl0 interface?
>>
>> Thanks for you help
>>
>>
>>
>> # /root >cat /etc/jail.conf
>> vimage33 {
>> host.hostname = "vimage33";
>> path = "/usr/jails/vimage33";
>> mount.fstab = "/usr/local/etc/fstab/vimage33";
>> exec.start = "/bin/sh /etc/rc";
>> exec.stop = "/bin/sh /etc/rc.shutdown";
>> exec.consolelog = "/var/log/vimage33.console.log";
>> devfs_ruleset = "4";
>> allow.mount.devfs;
>> vnet;
>> exec.poststart="vnet.start vimage33 rl0";
>> exec.prestop="vnet.stop vimage33";
>> }
>>
>> # /root >cat /usr/local/bin/vnet.start
>> #!/bin/sh
>> jailname=$1
>> nicname=$2
>>
>> jid=`jls -j ${jailname} jid`
>>
>> if [ "${jid}" -gt "100" ]; then
>> echo " "
>> echo "The JID value is greater then 100."
>> echo "You must shutdown the host and reboot"
>> echo "to zero out the JID counter and recover"
>> echo "the lost memory from stopping vimage jails."
>> echo " "
>> exit 2
>> fi
>>
>> ifconfig bridge${jid} create > /dev/null 2> /dev/null
>> ifconfig bridge${jid} 10.${jid}.0.1
>> ifconfig bridge${jid} up
>> ifconfig epair${jid} create > /dev/null 2> /dev/null
>> ifconfig bridge${jid} addm ${nicname} addm epair${jid}a
>> ifconfig epair${jid}a up
>> ifconfig epair${jid}b vnet ${jid}
>>
>> jexec ${jailname} ifconfig epair${jid}b 10.${jid}.0.2
>> jexec ${jailname} route add default 10.${jid}.0.1 > /dev/null 2> /dev/null
>> jexec ${jailname} ifconfig lo0 127.0.0.1
>>
>>
>> # Display the hosts network view before starting any vnet jails
>> # /root >ifconfig
>> rl0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu
>> options=2008<VLAN_MTU,WOL_MAGIC>
>> ether 00:0c:6e:09:8b:74
>> inet 10.0.10.5 netmask 0xfffffff8 broadcast 10.0.10.7
>> nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
>> media: Ethernet autoselect (100baseTX <full-duplex>)
>> status: active
>> lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
>> options=600003<RXCSUM,TXCSUM,RXCSUM_IPV6,TXCSUM_IPV6>
>> inet6 ::1 prefixlen 128
>> inet6 fe80::1%lo0 prefixlen 64 scopeid 0x7
>> inet 127.0.0.1 netmask 0xff000000
>> nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
>>
>> # Start the first vnet jail
>> # /root >jail -f /etc/jail.conf -c vimage33
>> vimage33: created
>> bridge1: Ethernet address: 02:8f:94:84:0c:02
>> epair1a: Ethernet address: 02:c0:a4:00:0b:0a
>> epair1b: Ethernet address: 02:c0:a4:00:0c:0b
>>
>>
>> # /root >jls
>> JID IP Address Hostname Path
>> 1 - vimage33 /usr/jails/vimage33
>>
>>
>> # Lets display the hosts network after the first vnet jail has started
>> # /root >ifconfig
>> rl0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric
>> options=2008<VLAN_MTU,WOL_MAGIC>
>> ether 00:0c:6e:09:8b:74
>> inet 10.0.10.5 netmask 0xfffffff8 broadcast 10.0.10.7
>> nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
>> media: Ethernet autoselect (100baseTX <full-duplex>)
>> status: active
>>
>> lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
>> options=600003<RXCSUM,TXCSUM,RXCSUM_IPV6,TXCSUM_IPV6>
>> inet6 ::1 prefixlen 128
>> inet6 fe80::1%lo0 prefixlen 64 scopeid 0x7
>> inet 127.0.0.1 netmask 0xff000000
>> nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
>> bridge1: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric
>> ether 02:8f:94:84:0c:01
>> inet 10.1.0.1 netmask 0xff000000 broadcast 10.255.255.255
>> nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
>> id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
>> maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
>> root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
>> member: epair1a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
>> ifmaxaddr 0 port 9 priority 128 path cost 14183
>> member: rl0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
>> ifmaxaddr 0 port 5 priority 128 path cost 200000
>> epair1a: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST>
>> options=8<VLAN_MTU>
>> ether 02:c0:a4:00:09:0a
>> inet6 fe80::c0:a4ff:fe00:90a%epair1a prefixlen 64 scopeid 0x9
>> nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
>> media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
>> status: active
>>
>>
>> # Login to the vnet jail and display the jails view of the network
>> # /root >jexec vimage33 tcsh
>> vimage33 / >ifconfig
>> lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
>> options=600003<RXCSUM,TXCSUM,RXCSUM_IPV6,TXCSUM_IPV6>
>> inet 127.0.0.1 netmask 0xff000000
>> inet6 ::1 prefixlen 128
>> inet6 fe80::1%lo0 prefixlen 64 scopeid 0x1
>> nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
>> epair1b: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0
>> options=8<VLAN_MTU>
>> ether 02:c0:a4:00:0a:0b
>> inet 10.1.0.2 netmask 0xff000000 broadcast 10.255.255.255
>> inet6 fe80::c0:a4ff:fe00:a0b%epair1b prefixlen 64 scopeid 0x2
>> nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
>> media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
>> status: active
>>
>>
>> # Yes the vnet jail can reach the public network
>> vimage33 / >ping -c 4 8.8.178.135
>> PING 8.8.178.135 (8.8.178.135): 56 data bytes
>> 64 bytes from 8.8.178.135: icmp_seq=0 ttl=51 time=84.645 ms
>> 64 bytes from 8.8.178.135: icmp_seq=1 ttl=51 time=86.950 ms
>> 64 bytes from 8.8.178.135: icmp_seq=2 ttl=51 time=83.274 ms
>> 64 bytes from 8.8.178.135: icmp_seq=3 ttl=51 time=82.660 ms
>>
>> --- 8.8.178.135 ping statistics ---
>> 4 packets transmitted, 4 packets received, 0.0% packet loss
>> round-trip min/avg/max/stddev = 82.660/84.382/86.950/1.647 ms
>>
>> vimage33 / >exit
>> exit
>>
>>
>> # Lets start the second vnet jail
>> # /root >cat /etc/jail.conf.22
>> vimage22 {
>> host.hostname = "vimage22";
>> path = "/usr/jails/vimage22";
>> mount.fstab = "/usr/local/etc/fstab/vimage22";
>> exec.start = "/bin/sh /etc/rc";
>> exec.stop = "/bin/sh /etc/rc.shutdown";
>> exec.consolelog = "/var/log/vimage22.console.log";
>> devfs_ruleset = "4";
>> allow.mount.devfs;
>> vnet;
>> exec.poststart="vnet.start vimage22 rl0";
>> exec.prestop="vnet.stop vimage22";
>> }
>>
>>
>> # /root >jail -f /etc/jail.conf.22 -c vimage22
>> vimage22: created
>>
>> # Notice this message about rl0
>> ifconfig: BRDGADD rl0: Device busy
>>
>> bridge2: Ethernet address: 02:8f:94:84:0c:02
>> epair2a: Ethernet address: 02:c0:a4:00:0b:0a
>> epair2b: Ethernet address: 02:c0:a4:00:0c:0b
>>
>>
>>
>> # Lets check the hosts view of the network - no rl0 on bridge2
>> # /root >ifconfig
>> rl0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST>
>> options=2008<VLAN_MTU,WOL_MAGIC>
>> ether 00:0c:6e:09:8b:74
>> inet 10.0.10.5 netmask 0xfffffff8 broadcast 10.0.10.7
>> nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
>> media: Ethernet autoselect (100baseTX <full-duplex>)
>> status: active
>> lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
>> options=600003<RXCSUM,TXCSUM,RXCSUM_IPV6,TXCSUM_IPV6>
>> inet6 ::1 prefixlen 128
>> inet6 fe80::1%lo0 prefixlen 64 scopeid 0x7
>> inet 127.0.0.1 netmask 0xff000000
>> nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
>> bridge1: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0
>> ether 02:8f:94:84:0c:01
>> inet 10.1.0.1 netmask 0xff000000 broadcast 10.255.255.255
>> nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
>> id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
>> maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
>> root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
>> member: epair1a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
>> ifmaxaddr 0 port 9 priority 128 path cost 14183
>> member: rl0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
>> ifmaxaddr 0 port 5 priority 128 path cost 200000
>> epair1a: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST>
>> options=8<VLAN_MTU>
>> ether 02:c0:a4:00:09:0a
>> inet6 fe80::c0:a4ff:fe00:90a%epair1a prefixlen 64 scopeid 0x9
>> nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
>> media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
>> status: active
>> bridge2: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0
>> ether 02:8f:94:84:0c:02
>> inet 10.2.0.1 netmask 0xff000000 broadcast 10.255.255.255
>> nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
>> id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
>> maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
>> root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
>> epair2a: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0
>> options=8<VLAN_MTU>
>> ether 02:c0:a4:00:0b:0a
>> inet6 fe80::c0:a4ff:fe00:b0a%epair2a prefixlen 64 scopeid 0xb
>> nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
>> media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
>> status: active
>>
>>
>> # /root >jls
>> JID IP Address Hostname Path
>> 1 - vimage33 /usr/jails/vimage33
>> 2 - vimage22 /usr/jails/vimage22
>>
>> # login to second vnet jail and see if it has public internet connection
>> # /root >jexec vimage22 tcsh
>> vimage22 / >ifconfig
>> lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
>> options=600003<RXCSUM,TXCSUM,RXCSUM_IPV6,TXCSUM_IPV6>
>> inet 127.0.0.1 netmask 0xff000000
>> inet6 ::1 prefixlen 128
>> inet6 fe80::1%lo0 prefixlen 64 scopeid 0x1
>> nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
>> epair2b: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0
>> options=8<VLAN_MTU>
>> ether 02:c0:a4:00:0c:0b
>> inet 10.2.0.2 netmask 0xff000000 broadcast 10.255.255.255
>> inet6 fe80::c0:a4ff:fe00:c0b%epair2b prefixlen 64 scopeid 0x2
>> nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
>> media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
>> status: active
>>
>> vimage22 / >ping -c 4 8.8.178.135
>> PING 8.8.178.135 (8.8.178.135): 56 data bytes
>>
>> --- 8.8.178.135 ping statistics ---
>> 4 packets transmitted, 0 packets received, 100.0% packet loss
>> vimage22 / >exit
>> exit
>>
>>
>>
>> # Stop the second vnet jail
>> # /root >jail -f /etc/jail.conf.22 -r vimage22
>> vimage22: removed
>> Freed UMA keg was not empty (30 items). Lost 2 pages of memory.
>> Freed UMA keg was not empty (203 items). Lost 1 pages of memory.
>> Freed UMA keg was not empty (30 items). Lost 2 pages of memory.
>> Freed UMA keg was not empty (10 items). Lost 2 pages of memory.
>> Freed UMA keg was not empty (30 items). Lost 2 pages of memory.
>> hhook_vnet_uninit: hhook_head type=1, id=1 cleanup required
>> hhook_vnet_uninit: hhook_head type=1, id=0 cleanup required
>>
>>
>> # Stop the first vnet jail
>> # /root >jail -f /etc/jail.conf -r vimage33
>> vimage33: removed
>> Freed UMA keg was not empty (30 items). Lost 2 pages of memory.
>> Freed UMA keg was not empty (203 items). Lost 1 pages of memory.
>> Freed UMA keg was not empty (30 items). Lost 2 pages of memory.
>> Freed UMA keg was not empty (10 items). Lost 2 pages of memory.
>> Freed UMA keg was not empty (30 items). Lost 2 pages of memory.
>> hhook_vnet_uninit: hhook_head type=1, id=1 cleanup required
>> hhook_vnet_uninit: hhook_head type=1, id=0 cleanup required
>>
>>
>>
>
>
> You don't need more than one bridge.
> Only connect the rl0 interface to the bridge one time.
> Connect each jails epairXa to the bridge.
> Create alias for each jails ip to bridge.
> Put the epairXb in the right jail.
>
> If you want separation. Create vlan interfaces.
> Connect them to rl0 and put them inside the jail.
>
>
>
Thank you Anders, I was able to figure out the solution which I am
posting here for the archives.
#!/bin/sh
jailname=$1
nicname=$2
jid=`jls -j ${jailname} jid`
if [ "${jid}" -gt "100" ]; then
echo " "
echo "The JID value is greater then 100."
echo "You must shutdown the host and reboot"
echo "to zero out the JID counter and recover"
echo "the lost memory from stopping vimage jails."
echo " "
exit 2
fi
bridge=`ifconfig | grep -m 1 bridge | cut -f 1 -d :`
if [ -z ${bridge} ]; then
ifconfig bridge0 create > /dev/null 2> /dev/null
ifconfig bridge0 addm ${nicname}
ifconfig bridge0 up
fi
ifconfig bridge0 alias 10.${jid}.0.1
ifconfig epair${jid} create > /dev/null 2> /dev/null
ifconfig bridge0 addm epair${jid}a
ifconfig epair${jid}a up
ifconfig epair${jid}b vnet ${jid}
jexec ${jailname} ifconfig epair${jid}b 10.${jid}.0.2
jexec ${jailname} route add default 10.${jid}.0.1 > /dev/null 2> /dev/null
jexec ${jailname} ifconfig lo0 127.0.0.1
More information about the freebsd-jail
mailing list