routing bug?

[Gabor]

I am experiencing some weird routing phenomena.
When I open a UDP socket and send datagrams to an address(172.30.1.1)
and then remove that route(route delete 172.30.1.1) then my packets
switch from going out the route specific interface(rl0) to going out
the default interface(fxp0).  This is as expected.  Then I add back
the route (route add 172.30.1.1 10.0.2.2) and the packets swing back
to the route specific interface(rl0).  However, if I bind my socket to
a source address(172.16.24.33), when I remove the route and then add
it back, the packets continue to go out the default interface(fxp0)
instead of going out the route specific interface(rl0).

This is on 4.9 STABLE.  I was unable to reproduce this on 5.2.1.  What
changes have there been that haven't been MFC'ed?

=0= udp-test # netstat -nr
Routing tables

Internet:
Destination        Gateway            Flags    Refs      Use  Netif Expire
default            192.168.43.1       UGSc        2     2440   fxp0
10.0.2/24          link#1             UC          2        0    rl0
10.0.2.1           00:50:fc:32:52:a7  UHLW        0        2    lo0
10.0.2.2           00:0e:0c:05:09:19  UHLW        1        3    rl0     69
172.0.0.1          172.0.0.1          UH          0        0    lo0
172.30.1.1         10.0.2.2           UGHS        0        0    rl0
192.168.7          link#1             UC          0        0    rl0
192.168.43         link#2             UC          4        0   fxp0
192.168.43.1       00:50:bf:33:63:70  UHLW        4    56111   fxp0   1079
192.168.43.31      link#2             UHLW        1      249   fxp0
192.168.43.157     00:a0:c9:4b:a5:f4  UHLW        6  2168880   fxp0    771
192.168.43.242     link#2             UHLW        1   718878   fxp0

=0= udp-test # ifconfig
rl0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> mtu 1500
        inet 192.168.7.2 netmask 0xffffff00 broadcast 192.168.7.255
        inet 10.0.2.1 netmask 0xffffff00 broadcast 10.0.2.255
        ether 00:50:fc:32:52:a7
        media: Ethernet 10baseT/UTP
        status: active
fxp0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> mtu 1500
        inet 192.168.43.26 netmask 0xffffff00 broadcast 192.168.43.255
        ether 00:01:80:3d:b4:4f
        media: Ethernet autoselect (100baseTX <full-duplex>)
        status: active
ppp0: flags=8010<POINTOPOINT,MULTICAST> mtu 1500
faith0: flags=8002<BROADCAST,MULTICAST> mtu 1500
ds0: flags=8008<LOOPBACK,MULTICAST> mtu 65532
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> mtu 16384
        inet 172.16.24.33 netmask 0xffff0000
        inet 172.0.0.1 netmask 0xffff0000
tun0: flags=8010<POINTOPOINT,MULTICAST> mtu 1500

=0= udp-test # cat udp-test.c
#include <sys/types.h>
#include <sys/time.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netdb.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>

int send_pkt(unsigned char *src, unsigned char *dest, unsigned short port);

int
main(int argc, char **argv)
{
    unsigned a, b, c, d, a2, b2, c2, d2;
    unsigned port;
    unsigned char src[4], dest[4];


    if (argc != 3) {
        fprintf(stderr,
                "Usage: %s <source> <dest>:<port>\n",
                argv[0]);
        return 1;
    }

    if (sscanf(argv[1], "%u.%u.%u.%u", &a, &b, &c, &d) == 4
            && a < 256 && b < 256 && c < 256 && d < 256
            && sscanf(argv[2], "%u.%u.%u.%u:%u", &a2, &b2, &c2, &d2, &port) == 5
            && a2 < 256 && b2 < 256 && c2 < 256 && d2 < 256 && port < 65536) {
        /* OK */
        src[0]  = a;
        src[1]  = b;
        src[2]  = c;
        src[3]  = d;
        dest[0] = a2;
        dest[1] = b2;
        dest[2] = c2;
        dest[3] = d2;
        send_pkt(src, dest, port);
    }
    else {
        fprintf(stderr,
                "Usage: %s <source> <dest>:<port>\n",
                argv[0]);
        return 1;
    }

    return 0;
}

int
send_pkt(unsigned char *src, unsigned char *dest, unsigned short port)
{
    int s, len, cnt, rc, on;
    struct protoent *proto;
    struct sockaddr_in to, from;
    char data[1024];

    if (!(proto = getprotobyname("udp"))) {
        perror("getprotobyname");
        return -1;
    }

    if ((s = socket(PF_INET, SOCK_DGRAM, proto->p_proto)) < 0) {
        perror("socket");
        return -1;
    }
    on = 1;

    memset(&from, 0, sizeof from);
    from.sin_family = AF_INET;
    from.sin_port = htons(0);
    memcpy(&from.sin_addr.s_addr, src, 4);
    fprintf(stderr,
            "bind:%d\n",
            bind(s, (struct sockaddr*)&from, sizeof from));

    memset(&to, 0, sizeof to);
    to.sin_family = AF_INET;
    to.sin_port = htons(port);
    memcpy(&to.sin_addr.s_addr, dest, 4);
    len = 58;
    cnt = 0;
    while (1) {
        memset(data, cnt, len);
        rc = sendto(s, data, len, 0, (struct sockaddr*)&to, sizeof to);
        if (rc < 0)
            perror("");
        fprintf(stderr, "%d %d\n", rc, cnt);
        sleep(5);
        ++cnt;
    }
    close(s);

    return 0;
}