[Bug 268976] Traffic will not route across two bridges on the same /8

https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=268976

            Bug ID: 268976
           Summary: Traffic will not route across two bridges on the same
                    /8
           Product: Base System
           Version: 13.1-RELEASE
          Hardware: Any
                OS: Any
            Status: New
          Severity: Affects Only Me
          Priority: ---
         Component: kern
          Assignee: bugs@FreeBSD.org
          Reporter: rtyler@brokenco.de

When setting up a network topology with FreeBSD vnet jails, I found that I was
unable to route traffic between jails attached to two different bridge
interfaces. It appears that if bridge0 and bridge1 share the same /8, traffic
will not route between them correctly.

Using the following topology as an example:

+-------+
| world |
+-------+
   |
  vtnet0
   |
  pf/nat

  +---------------+        +-------------------+
  | dmz (bridge0) |        | private (bridge1) |
  +---------------+        +-------------------+

  * http                           * db
  * git

When bridge0 is 10.10.1.1/24 and bridge0 is 10.200.2.1/24, traffic will *not*
route properly between the `http` and the `db` jails.

However, if bridge1 is `192.168.100.1/24`, then traffic will route properly
between the two jails. Basically any configuration of bridge1 to be under
10.xx.xx.xx resulted in traffic not routing properly.


Below are some configuration files from the test VM:

jail.conf
------------------------------------
persist;
mount.devfs;
path = "/jails/$name";
host.hostname = $name;

exec.start = "/bin/sh /etc/rc";
exec.stop = "/bin/sh /etc/rc.shutdown jail";
exec.clean;

vnet;

$dmz = "bridge0";
$dmz_gw = "10.10.1.1";
$private = "bridge1";
#$private_gw = "10.10.2.1";
$private_gw = "192.168.100.1";

http {
        $id = "0";
        $ip = "10.10.1.80";

        vnet.interface = "epair${id}b";

        exec.prestart = "ifconfig epair${id} create up";
        exec.prestart += "ifconfig epair${id}a up descr vnet-${name}";
        exec.prestart += "ifconfig ${dmz} addm epair${id}a up";

        exec.start = "/sbin/ifconfig epair${id}b ${ip}";
        exec.start += "/sbin/route add default ${dmz_gw}";
        exec.start += "/bin/sh /etc/rc";

        exec.poststop = "ifconfig ${dmz} deletem epair${id}a";
        exec.poststop += "ifconfig epair${id}a destroy";
}

db {    
        $id = "1";
        # For reproducing the bug
        #$ip = "10.10.2.32";
        $ip = "192.168.100.32";

        vnet.interface = "epair${id}b";

        exec.prestart = "ifconfig epair${id} create up";
        exec.prestart += "ifconfig epair${id}a up descr vnet-${name}";
        exec.prestart += "ifconfig ${private} addm epair${id}a up";

        exec.start = "/sbin/ifconfig epair${id}b ${ip}";
        exec.start += "/sbin/route add default ${private_gw}";
        exec.start += "/bin/sh /etc/rc";

        exec.poststop = "ifconfig ${private} deletem epair${id}a";
        exec.poststop += "ifconfig epair${id}a destroy";
}       

git {   
        $id = "2";
        $ip = "10.10.1.3";

        vnet.interface = "epair${id}b";

        exec.prestart = "ifconfig epair${id} create up";
        exec.prestart += "ifconfig epair${id}a up descr vnet-${name}";
        exec.prestart += "ifconfig ${dmz} addm epair${id}a up";

        exec.start = "/sbin/ifconfig epair${id}b ${ip}";
        exec.start += "/sbin/route add default ${dmz_gw}";
        exec.start += "/bin/sh /etc/rc";

        exec.poststop = "ifconfig ${dmz} deletem epair${id}a";
        exec.poststop += "ifconfig epair${id}a destroy";
}       
------------------------------------

rc.conf
------------------------------------
hostname="vnet-test"
ifconfig_vtnet0="DHCP"
#ifconfig_vtnet0_ipv6="inet6 accept_rtadv"
sshd_enable="YES"
ntpdate_enable="YES"
ntpd_enable="YES"
powerd_enable="YES"
# Set dumpdev to "AUTO" to enable crash dumps, "NO" to disable
dumpdev="AUTO"
zfs_enable="YES"
sendmail_enable="NONE"

# Networking and Jails
jail_enable="YES"
pf_enable="YES"
gateway_enable="YES"
cloned_interfaces="bridge0 bridge1"
ifconfig_bridge0="inet 10.10.1.1/24"
ifconfig_bridge1="inet 192.168.100.1/24"
# Using this network results in not being able to route
# Make sure to update /etc/jail.conf for the db jail when changing
#ifconfig_bridge1="inet 10.10.2.1/24"
------------------------------------

pf.conf
------------------------------------
extif="vtnet0"
dmz="bridge0"
private="bridge1"

scrub in all fragment reassemble

nat on $extif from $dmz:network to any -> ($extif)
nat on $extif from $private:network to any -> ($extif)
------------------------------------

-- 
You are receiving this mail because:
You are the assignee for the bug.