svn commit: r318793 - head/sys/netinet

[Gleb Smirnoff]

Author: glebius
Date: Wed May 24 17:47:16 2017
New Revision: 318793
URL: https://svnweb.freebsd.org/changeset/base/318793

Log:
  o Rearrange struct inpcb fields to optimize the TCP output code path
    considering cache line hits and misses.  Put the lock and hash list
    glue into the first cache line, put inp_refcount inp_flags inp_socket
    into the second cache line.
  o On allocation zero out entire structure except the lock and list entries,
    including inp_route inp_lle inp_gencnt.  When inp_route and inp_lle were
    introduced, they were added below inp_zero_size, resulting on not being
    cleared after free/alloc.  This definitely was a source of bugs with route
    caching.  Could be that r315956 has just fixed one of them.
    The inp_gencnt is reinitialized on every alloc, so it is safe to clear it.
  
  This has been proved to improve TCP performance at Netflix.
  
  Obtained from:		rrs
  Differential Revision:	D10686

Modified:
  head/sys/netinet/in_pcb.c
  head/sys/netinet/in_pcb.h

Modified: head/sys/netinet/in_pcb.c
==============================================================================
--- head/sys/netinet/in_pcb.c	Wed May 24 17:11:10 2017	(r318792)
+++ head/sys/netinet/in_pcb.c	Wed May 24 17:47:16 2017	(r318793)
@@ -300,7 +300,7 @@ in_pcballoc(struct socket *so, struct in
 	inp = uma_zalloc(pcbinfo->ipi_zone, M_NOWAIT);
 	if (inp == NULL)
 		return (ENOBUFS);
-	bzero(inp, inp_zero_size);
+	bzero(&inp->inp_start_zero, inp_zero_size);
 	inp->inp_pcbinfo = pcbinfo;
 	inp->inp_socket = so;
 	inp->inp_cred = crhold(so->so_cred);

Modified: head/sys/netinet/in_pcb.h
==============================================================================
--- head/sys/netinet/in_pcb.h	Wed May 24 17:11:10 2017	(r318792)
+++ head/sys/netinet/in_pcb.h	Wed May 24 17:47:16 2017	(r318793)
@@ -183,26 +183,29 @@ struct icmp6_filter;
 struct inpcbpolicy;
 struct m_snd_tag;
 struct inpcb {
+	/* Cache line #1 (amd64) */
 	LIST_ENTRY(inpcb) inp_hash;	/* (h/i) hash list */
 	LIST_ENTRY(inpcb) inp_pcbgrouphash;	/* (g/i) hash list */
-	LIST_ENTRY(inpcb) inp_list;	/* (p/l) list for all PCBs for proto */
-	                                /* (p[w]) for list iteration */
-	                                /* (p[r]/l) for addition/removal */
+	struct rwlock	inp_lock;
+	/* Cache line #2 (amd64) */
+#define	inp_start_zero	inp_refcount
+#define	inp_zero_size	(sizeof(struct inpcb) - \
+			    offsetof(struct inpcb, inp_start_zero))
+	u_int	inp_refcount;		/* (i) refcount */
+	int	inp_flags;		/* (i) generic IP/datagram flags */
+	int	inp_flags2;		/* (i) generic IP/datagram flags #2*/
 	void	*inp_ppcb;		/* (i) pointer to per-protocol pcb */
+	struct	socket *inp_socket;	/* (i) back pointer to socket */
 	struct	inpcbinfo *inp_pcbinfo;	/* (c) PCB list info */
 	struct	inpcbgroup *inp_pcbgroup; /* (g/i) PCB group list */
 	LIST_ENTRY(inpcb) inp_pcbgroup_wild; /* (g/i/h) group wildcard entry */
-	struct	socket *inp_socket;	/* (i) back pointer to socket */
 	struct	ucred	*inp_cred;	/* (c) cache of socket cred */
 	u_int32_t inp_flow;		/* (i) IPv6 flow information */
-	int	inp_flags;		/* (i) generic IP/datagram flags */
-	int	inp_flags2;		/* (i) generic IP/datagram flags #2*/
 	u_char	inp_vflag;		/* (i) IP version flag (v4/v6) */
 	u_char	inp_ip_ttl;		/* (i) time to live proto */
 	u_char	inp_ip_p;		/* (c) protocol proto */
 	u_char	inp_ip_minttl;		/* (i) minimum TTL or drop */
 	uint32_t inp_flowid;		/* (x) flow id / queue id */
-	u_int	inp_refcount;		/* (i) refcount */
 	struct m_snd_tag *inp_snd_tag;	/* (i) send tag for outgoing mbufs */
 	uint32_t inp_flowtype;		/* (x) M_HASHTYPE value */
 	uint32_t inp_rss_listen_bucket;	/* (x) overridden RSS listen bucket */
@@ -235,17 +238,16 @@ struct inpcb {
 	};
 	LIST_ENTRY(inpcb) inp_portlist;	/* (i/h) */
 	struct	inpcbport *inp_phd;	/* (i/h) head of this list */
-#define inp_zero_size offsetof(struct inpcb, inp_gencnt)
 	inp_gen_t	inp_gencnt;	/* (c) generation count */
 	struct llentry	*inp_lle;	/* cached L2 information */
-	struct rwlock	inp_lock;
 	rt_gen_t	inp_rt_cookie;	/* generation for route entry */
 	union {				/* cached L3 information */
-		struct route inpu_route;
-		struct route_in6 inpu_route6;
-	} inp_rtu;
-#define inp_route inp_rtu.inpu_route
-#define inp_route6 inp_rtu.inpu_route6
+		struct route inp_route;
+		struct route_in6 inp_route6;
+	};
+	LIST_ENTRY(inpcb) inp_list;	/* (p/l) list for all PCBs for proto */
+	                                /* (p[w]) for list iteration */
+	                                /* (p[r]/l) for addition/removal */
 };
 #endif	/* _KERNEL */