svn commit: r189326 - in user/kmacy/HEAD_fast_net_merge/sys: net
netinet
Kip Macy
kmacy at FreeBSD.org
Tue Mar 3 16:56:49 PST 2009
Author: kmacy
Date: Wed Mar 4 00:56:48 2009
New Revision: 189326
URL: http://svn.freebsd.org/changeset/base/189326
Log:
IFC HEAD_fast_net
inpcb rtentry & llentry caching
186573, 186575, 186576, 186578, 186579, 186580
186573:
- Cache rtentry in inpcb for connected sockets
186575:
- Add cached llentry to inpcb
- reduce header pollution by removing net/route.h
from in_pcb.h
186576:
- install cached llentry in the inpcb and then pass
down to ether_output in the struct route
186578:
- don't LLE_FREE if llentry is unchanged
186579:
- cleanup upgrade / downgrade checks for route and lle
186580:
- don't check INP_WLOCKED if inp is NULL
Modified:
user/kmacy/HEAD_fast_net_merge/sys/net/if_var.h
user/kmacy/HEAD_fast_net_merge/sys/net/route.h
user/kmacy/HEAD_fast_net_merge/sys/netinet/if_ether.c
user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.c
user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.h
user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_output.c
user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_hostcache.c
user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_offload.c
user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_usrreq.c
Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_var.h
==============================================================================
--- user/kmacy/HEAD_fast_net_merge/sys/net/if_var.h Wed Mar 4 00:05:40 2009 (r189325)
+++ user/kmacy/HEAD_fast_net_merge/sys/net/if_var.h Wed Mar 4 00:56:48 2009 (r189326)
@@ -669,6 +669,8 @@ struct ifmultiaddr {
};
#ifdef _KERNEL
+#define LLTABLE(ifp) ((struct lltable *)(ifp)->if_afdata[AF_INET])
+
#define IFAFREE(ifa) \
do { \
IFA_LOCK(ifa); \
Modified: user/kmacy/HEAD_fast_net_merge/sys/net/route.h
==============================================================================
--- user/kmacy/HEAD_fast_net_merge/sys/net/route.h Wed Mar 4 00:05:40 2009 (r189325)
+++ user/kmacy/HEAD_fast_net_merge/sys/net/route.h Wed Mar 4 00:56:48 2009 (r189326)
@@ -46,8 +46,9 @@
* in their control blocks, e.g. inpcb.
*/
struct route {
- struct rtentry *ro_rt;
struct sockaddr ro_dst;
+ struct rtentry *ro_rt;
+ struct llentry *ro_lle;
};
/*
Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/if_ether.c
==============================================================================
--- user/kmacy/HEAD_fast_net_merge/sys/netinet/if_ether.c Wed Mar 4 00:05:40 2009 (r189325)
+++ user/kmacy/HEAD_fast_net_merge/sys/netinet/if_ether.c Wed Mar 4 00:56:48 2009 (r189326)
@@ -81,7 +81,6 @@ __FBSDID("$FreeBSD$");
#define SIN(s) ((struct sockaddr_in *)s)
#define SDL(s) ((struct sockaddr_dl *)s)
-#define LLTABLE(ifp) ((struct lltable *)(ifp)->if_afdata[AF_INET])
SYSCTL_DECL(_net_link_ether);
SYSCTL_NODE(_net_link_ether, PF_INET, inet, CTLFLAG_RW, 0, "");
Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.c
==============================================================================
--- user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.c Wed Mar 4 00:05:40 2009 (r189325)
+++ user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.c Wed Mar 4 00:56:48 2009 (r189326)
@@ -63,6 +63,7 @@ __FBSDID("$FreeBSD$");
#include <net/if.h>
#include <net/if_types.h>
+#include <net/if_llatbl.h>
#include <net/route.h>
#include <netinet/in.h>
@@ -497,6 +498,72 @@ in_pcbbind_setup(struct inpcb *inp, stru
return (0);
}
+void
+in_pcbrtalloc(struct inpcb *inp, in_addr_t faddr, struct route *sro)
+{
+ struct sockaddr_in *sin;
+ struct sockaddr *dst;
+ struct llentry *la;
+ struct rtentry *rt;
+ struct ifnet *ifp;
+ int flags = LLE_EXCLUSIVE;
+
+ INP_WLOCK_ASSERT(inp);
+ bzero(sro, sizeof(*sro));
+ sin = (struct sockaddr_in *)&sro->ro_dst;
+ sin->sin_family = AF_INET;
+ sin->sin_len = sizeof(struct sockaddr_in);
+ sin->sin_addr.s_addr = faddr;
+ /*
+ * If route is known our src addr is taken from the i/f,
+ * else punt.
+ *
+ * Find out route to destination.
+ */
+ if ((inp->inp_socket->so_options & SO_DONTROUTE) == 0) {
+#ifdef RADIX_MPATH
+ rtalloc_mpath_fib(sro, ntohl(faddr->s_addr),
+ inp->inp_inc.inc_fibnum);
+#else
+ in_rtalloc_ign(sro, 0, inp->inp_inc.inc_fibnum);
+#endif
+ }
+
+ rt = sro->ro_rt;
+ if (rt == NULL)
+ return;
+
+ inp->inp_rt = rt;
+ inp->inp_vflag |= INP_RT_VALID;
+
+ if (rt->rt_ifp == NULL)
+ return;
+
+ ifp = rt->rt_ifp;
+ dst = &sro->ro_dst;
+ if (rt->rt_flags & RTF_GATEWAY)
+ dst = rt->rt_gateway;
+
+ IF_AFDATA_RLOCK(ifp);
+ la = lla_lookup(LLTABLE(ifp), flags, dst);
+ IF_AFDATA_RUNLOCK(ifp);
+ if ((la == NULL) &&
+ (ifp->if_flags & (IFF_NOARP | IFF_STATICARP)) == 0) {
+ flags |= (LLE_CREATE | LLE_EXCLUSIVE);
+ IF_AFDATA_WLOCK(ifp);
+ la = lla_lookup(LLTABLE(ifp), flags, dst);
+ IF_AFDATA_WUNLOCK(ifp);
+ }
+ if (la == NULL)
+ return;
+
+ LLE_ADDREF(la);
+ LLE_WUNLOCK(la);
+
+ inp->inp_lle = la;
+ inp->inp_flags |= INP_LLE_VALID;
+}
+
/*
* Connect from a socket to a specified address.
* Both address and port must be specified in argument sin.
@@ -509,6 +576,7 @@ in_pcbconnect(struct inpcb *inp, struct
u_short lport, fport;
in_addr_t laddr, faddr;
int anonport, error;
+ struct route sro;
INP_INFO_WLOCK_ASSERT(inp->inp_pcbinfo);
INP_WLOCK_ASSERT(inp);
@@ -532,6 +600,7 @@ in_pcbconnect(struct inpcb *inp, struct
}
}
+ in_pcbrtalloc(inp, faddr, &sro);
/* Commit the remaining changes. */
inp->inp_lport = lport;
inp->inp_laddr.s_addr = laddr;
@@ -870,6 +939,17 @@ in_pcbdisconnect(struct inpcb *inp)
INP_INFO_WLOCK_ASSERT(inp->inp_pcbinfo);
INP_WLOCK_ASSERT(inp);
+ if (inp->inp_vflag & INP_RT_VALID) {
+ inp->inp_vflag &= ~INP_RT_VALID;
+ RTFREE(inp->inp_rt);
+ inp->inp_rt = NULL;
+ }
+ if (inp->inp_flags & INP_LLE_VALID) {
+ inp->inp_flags &= ~INP_LLE_VALID;
+ LLE_FREE(inp->inp_lle);
+ inp->inp_lle = NULL;
+ }
+
inp->inp_faddr.s_addr = INADDR_ANY;
inp->inp_fport = 0;
in_pcbrehash(inp);
@@ -907,6 +987,17 @@ in_pcbfree_internal(struct inpcb *inp)
INP_INFO_WLOCK_ASSERT(ipi);
INP_WLOCK_ASSERT(inp);
+ if (inp->inp_vflag & INP_RT_VALID) {
+ inp->inp_vflag &= ~INP_RT_VALID;
+ RTFREE(inp->inp_rt);
+ inp->inp_rt = NULL;
+ }
+ if (inp->inp_flags & INP_LLE_VALID) {
+ inp->inp_flags &= ~INP_LLE_VALID;
+ LLE_FREE(inp->inp_lle);
+ inp->inp_lle = NULL;
+ }
+
#ifdef IPSEC
if (inp->inp_sp != NULL)
ipsec_delete_pcbpolicy(inp);
Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.h
==============================================================================
--- user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.h Wed Mar 4 00:05:40 2009 (r189325)
+++ user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.h Wed Mar 4 00:56:48 2009 (r189326)
@@ -39,8 +39,6 @@
#include <sys/_mutex.h>
#include <sys/_rwlock.h>
-#include <net/route.h>
-
#ifdef _KERNEL
#include <sys/rwlock.h>
#endif
@@ -49,6 +47,10 @@
#define in6p_sp inp_sp /* for KAME src sync over BSD*'s */
struct inpcbpolicy;
+struct rtentry;
+struct llentry;
+struct route;
+
/*
* struct inpcb is the common protocol control block structure used in most
* IP transport protocols.
@@ -169,7 +171,8 @@ struct inpcb {
u_char inp_ip_minttl; /* (i) minimum TTL or drop */
uint32_t inp_ispare1; /* (x) connection id / queue id */
u_int inp_refcount; /* (i) refcount */
- void *inp_pspare[2]; /* (x) rtentry / general use */
+ struct llentry *inp_lle; /* L2 information */
+ struct rtentry *inp_rt; /* L3 information */
/* Local and foreign ports, local and foreign addr. */
struct in_conninfo inp_inc; /* (i/p) list for PCB's local port */
@@ -324,6 +327,9 @@ struct inpcbinfo {
#define INP_RLOCK_ASSERT(inp) rw_assert(&(inp)->inp_lock, RA_RLOCKED)
#define INP_WLOCK_ASSERT(inp) rw_assert(&(inp)->inp_lock, RA_WLOCKED)
#define INP_UNLOCK_ASSERT(inp) rw_assert(&(inp)->inp_lock, RA_UNLOCKED)
+#define INP_TRY_UPGRADE(inp) rw_try_upgrade(&(inp)->inp_lock)
+#define INP_DOWNGRADE(inp) rw_downgrade(&(inp)->inp_lock)
+#define INP_WLOCKED(inp) rw_wowned(&(inp)->inp_lock)
#ifdef _KERNEL
/*
@@ -395,6 +401,7 @@ void inp_4tuple_get(struct inpcb *inp,
#define INP_ONESBCAST 0x10 /* send all-ones broadcast */
#define INP_DROPPED 0x20 /* protocol drop flag */
#define INP_SOCKREF 0x40 /* strong socket reference */
+#define INP_RT_VALID 0x80 /* rtentry is set */
/*
* Flags for inp_flag.
@@ -412,7 +419,7 @@ void inp_4tuple_get(struct inpcb *inp,
#define INP_RECVTTL 0x400 /* receive incoming IP TTL */
#define INP_DONTFRAG 0x800 /* don't fragment packet */
#define INP_NONLOCALOK 0x1000 /* Allow bind to spoof any address */
- /* - requires options IP_NONLOCALBIND */
+#define INP_LLE_VALID 0x2000 /* L2 entry is set */
#define IN6P_IPV6_V6ONLY 0x008000 /* restrict AF_INET6 socket for v6 */
@@ -489,6 +496,7 @@ void in_pcbdisconnect(struct inpcb *);
void in_pcbdrop(struct inpcb *);
void in_pcbfree(struct inpcb *);
int in_pcbinshash(struct inpcb *);
+void in_pcbrtalloc(struct inpcb *inp, in_addr_t faddr, struct route *sro);
struct inpcb *
in_pcblookup_local(struct inpcbinfo *,
struct in_addr, u_short, int, struct ucred *);
Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_output.c
==============================================================================
--- user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_output.c Wed Mar 4 00:05:40 2009 (r189325)
+++ user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_output.c Wed Mar 4 00:56:48 2009 (r189326)
@@ -56,6 +56,7 @@ __FBSDID("$FreeBSD$");
#include <sys/vimage.h>
#include <net/if.h>
+#include <net/if_llatbl.h>
#include <net/netisr.h>
#include <net/pfil.h>
#include <net/route.h>
@@ -134,11 +135,13 @@ ip_output(struct mbuf *m, struct mbuf *o
int hlen = sizeof (struct ip);
int mtu;
int len, error = 0;
+ int neednewroute = 0, neednewlle = 0;
struct sockaddr_in *dst = NULL; /* keep compiler happy */
struct in_ifaddr *ia = NULL;
int isbroadcast, sw_csum;
struct route iproute;
struct in_addr odst;
+ struct sockaddr_in *sin;
#ifdef IPFIREWALL_FORWARD
struct m_tag *fwd_tag = NULL;
#endif
@@ -152,6 +155,22 @@ ip_output(struct mbuf *m, struct mbuf *o
if (inp != NULL) {
M_SETFIB(m, inp->inp_inc.inc_fibnum);
INP_LOCK_ASSERT(inp);
+ if ((ro == &iproute) && (inp->inp_vflag & INP_RT_VALID)) {
+ if (inp->inp_rt->rt_flags & RTF_UP) {
+ sin = (struct sockaddr_in *)&ro->ro_dst;
+ sin->sin_family = AF_INET;
+ sin->sin_len = sizeof(struct sockaddr_in);
+ sin->sin_addr.s_addr = inp->inp_faddr.s_addr;
+ ro->ro_rt = inp->inp_rt;
+ } else
+ neednewroute = 1;
+ }
+ if ((ro == &iproute) && (inp->inp_flags & INP_LLE_VALID)) {
+ if (inp->inp_lle->la_flags & LLE_VALID) {
+ ro->ro_lle = inp->inp_lle;
+ } else
+ neednewlle = 1;
+ }
}
if (opt) {
@@ -194,7 +213,8 @@ again:
if (ro->ro_rt && ((ro->ro_rt->rt_flags & RTF_UP) == 0 ||
dst->sin_family != AF_INET ||
dst->sin_addr.s_addr != ip->ip_dst.s_addr)) {
- RTFREE(ro->ro_rt);
+ if (inp == NULL || (ro->ro_rt != inp->inp_rt))
+ RTFREE(ro->ro_rt);
ro->ro_rt = (struct rtentry *)NULL;
}
#ifdef IPFIREWALL_FORWARD
@@ -637,9 +657,48 @@ passout:
V_ipstat.ips_fragmented++;
done:
- if (ro == &iproute && ro->ro_rt) {
- RTFREE(ro->ro_rt);
+ if (ro == &iproute && ro->ro_rt != NULL) {
+ int wlocked = 0;
+ struct llentry *la;
+
+ if (neednewlle || neednewroute) {
+ wlocked = INP_WLOCKED(inp);
+ if (!wlocked && INP_TRY_UPGRADE(inp) == 0)
+ return (error);
+ }
+
+ if (inp == NULL || (inp->inp_vflag & INP_RT_VALID) == 0)
+ RTFREE(ro->ro_rt);
+ else if (neednewroute && ro->ro_rt != inp->inp_rt) {
+ RTFREE(inp->inp_rt);
+ inp->inp_rt = ro->ro_rt;
+
+ }
+ if (neednewlle) {
+ IF_AFDATA_RLOCK(ifp);
+ la = lla_lookup(LLTABLE(ifp), LLE_EXCLUSIVE,
+ (struct sockaddr *)dst);
+ IF_AFDATA_RUNLOCK(ifp);
+ if ((la == NULL) &&
+ (ifp->if_flags & (IFF_NOARP | IFF_STATICARP)) == 0) {
+ IF_AFDATA_WLOCK(ifp);
+ la = lla_lookup(LLTABLE(ifp),
+ (LLE_CREATE | LLE_EXCLUSIVE),
+ (struct sockaddr *)dst);
+ IF_AFDATA_WUNLOCK(ifp);
+ }
+ if (la != NULL && (inp->inp_lle != la)) {
+ LLE_FREE(inp->inp_lle);
+ LLE_ADDREF(la);
+ LLE_WUNLOCK(la);
+ inp->inp_lle = la;
+ } else if (la != NULL)
+ LLE_WUNLOCK(la);
+ }
+ if ((neednewlle || neednewroute) && !wlocked)
+ INP_DOWNGRADE(inp);
}
+
return (error);
bad:
m_freem(m);
Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_hostcache.c
==============================================================================
--- user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_hostcache.c Wed Mar 4 00:05:40 2009 (r189325)
+++ user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_hostcache.c Wed Mar 4 00:56:48 2009 (r189326)
@@ -79,6 +79,7 @@ __FBSDID("$FreeBSD$");
#include <sys/vimage.h>
#include <net/if.h>
+#include <net/route.h>
#include <netinet/in.h>
#include <netinet/in_systm.h>
Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_offload.c
==============================================================================
--- user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_offload.c Wed Mar 4 00:05:40 2009 (r189325)
+++ user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_offload.c Wed Mar 4 00:56:48 2009 (r189326)
@@ -42,6 +42,7 @@ __FBSDID("$FreeBSD$");
#include <net/if.h>
#include <net/if_types.h>
#include <net/if_var.h>
+#include <net/route.h>
#include <netinet/in.h>
#include <netinet/in_systm.h>
Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_usrreq.c
==============================================================================
--- user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_usrreq.c Wed Mar 4 00:05:40 2009 (r189325)
+++ user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_usrreq.c Wed Mar 4 00:56:48 2009 (r189326)
@@ -1075,6 +1075,7 @@ tcp_connect(struct tcpcb *tp, struct soc
struct socket *so = inp->inp_socket;
INIT_VNET_INET(so->so_vnet);
struct in_addr laddr;
+ struct route sro;
u_short lport;
int error;
@@ -1103,6 +1104,7 @@ tcp_connect(struct tcpcb *tp, struct soc
inp->inp_laddr = laddr;
in_pcbrehash(inp);
+ in_pcbrtalloc(inp, inp->inp_faddr.s_addr, &sro);
/*
* Compute window scaling to request:
* Scale to fit into sweet spot. See tcp_syncache.c.
More information about the svn-src-user
mailing list