svn commit: r274853 - in head: sys/dev/iscsi usr.sbin/ctld usr.sbin/iscsid

[Navdeep Parhar]

On Sat, Nov 22, 2014 at 03:09:19PM +0000, Alexander Motin wrote:
> Author: mav
> Date: Sat Nov 22 15:09:18 2014
> New Revision: 274853
> URL: https://svnweb.freebsd.org/changeset/base/274853
> 
> Log:
>   For both iSCSI initiator and target increase socket buffer sizes before
>   establishing connection.
>   
>   This is a workaround for Chelsio TOE driver, that does not update socket
>   buffer size in hardware after connection established, and unless that is
>   done beforehand, kernel code will stuck, attempting to send/receive full
>   PDU at once.

cxgbe's TOE driver does update the rcv window in the hardware in
response to changes in the socket buffer size (see t4_rcvd in
cxgbe/tom/t4_cpl_io.c).

The bug is that the driver tries to avoid sending too many small updates
to the chip, and instead tries to build up a balance of credits before
an update.  This doesn't interact well with some applications that set
the sockbuf size to a high fixed value and then wait for the entire
buffer to fill up before draining it.  The driver ends up waiting for
the application to drain the socket's receive buffer so that pru_rcvd
(t4_rcvd) can then send a big batch of credits to the chip, the
application is waiting for the receive to reach its high water mark
before draining the buffer, but the TOE on the chip has shut down the
receive window because it's out of credits.  If you see the peer sending
zero window probes and the TOE refusing to budge, then the problem
you've observed is definitely the problem I just described.

I'll fix this in the TOE driver.  I can send you an early patch in case
you'd like to try out the fix before it gets committed.

Regards,
Navdeep

>   
>   MFC after:	1 week
> 
> Modified:
>   head/sys/dev/iscsi/icl.c
>   head/usr.sbin/ctld/ctld.c
>   head/usr.sbin/ctld/ctld.h
>   head/usr.sbin/iscsid/iscsid.c
>   head/usr.sbin/iscsid/iscsid.h
> 
> Modified: head/sys/dev/iscsi/icl.c
> ==============================================================================
> --- head/sys/dev/iscsi/icl.c	Sat Nov 22 14:59:50 2014	(r274852)
> +++ head/sys/dev/iscsi/icl.c	Sat Nov 22 15:09:18 2014	(r274853)
> @@ -1199,6 +1199,8 @@ icl_conn_start(struct icl_conn *ic)
>  		icl_conn_close(ic);
>  		return (error);
>  	}
> +	ic->ic_socket->so_snd.sb_flags |= SB_AUTOSIZE;
> +	ic->ic_socket->so_rcv.sb_flags |= SB_AUTOSIZE;
>  
>  	/*
>  	 * Disable Nagle.
> 
> Modified: head/usr.sbin/ctld/ctld.c
> ==============================================================================
> --- head/usr.sbin/ctld/ctld.c	Sat Nov 22 14:59:50 2014	(r274852)
> +++ head/usr.sbin/ctld/ctld.c	Sat Nov 22 15:09:18 2014	(r274853)
> @@ -1602,7 +1602,7 @@ conf_apply(struct conf *oldconf, struct 
>  	struct portal *oldp, *newp;
>  	struct isns *oldns, *newns;
>  	pid_t otherpid;
> -	int changed, cumulated_error = 0, error;
> +	int changed, cumulated_error = 0, error, sockbuf;
>  	int one = 1;
>  
>  	if (oldconf->conf_debug != newconf->conf_debug) {
> @@ -1899,6 +1899,16 @@ conf_apply(struct conf *oldconf, struct 
>  				cumulated_error++;
>  				continue;
>  			}
> +			sockbuf = SOCKBUF_SIZE;
> +			if (setsockopt(newp->p_socket, SOL_SOCKET, SO_RCVBUF,
> +			    &sockbuf, sizeof(sockbuf)) == -1)
> +				log_warn("setsockopt(SO_RCVBUF) failed "
> +				    "for %s", newp->p_listen);
> +			sockbuf = SOCKBUF_SIZE;
> +			if (setsockopt(newp->p_socket, SOL_SOCKET, SO_SNDBUF,
> +			    &sockbuf, sizeof(sockbuf)) == -1)
> +				log_warn("setsockopt(SO_SNDBUF) failed "
> +				    "for %s", newp->p_listen);
>  			error = setsockopt(newp->p_socket, SOL_SOCKET,
>  			    SO_REUSEADDR, &one, sizeof(one));
>  			if (error != 0) {
> 
> Modified: head/usr.sbin/ctld/ctld.h
> ==============================================================================
> --- head/usr.sbin/ctld/ctld.h	Sat Nov 22 14:59:50 2014	(r274852)
> +++ head/usr.sbin/ctld/ctld.h	Sat Nov 22 15:09:18 2014	(r274853)
> @@ -48,6 +48,7 @@
>  #define	MAX_NAME_LEN			223
>  #define	MAX_DATA_SEGMENT_LENGTH		(128 * 1024)
>  #define	MAX_BURST_LENGTH		16776192
> +#define	SOCKBUF_SIZE			1048576
>  
>  struct auth {
>  	TAILQ_ENTRY(auth)		a_next;
> 
> Modified: head/usr.sbin/iscsid/iscsid.c
> ==============================================================================
> --- head/usr.sbin/iscsid/iscsid.c	Sat Nov 22 14:59:50 2014	(r274852)
> +++ head/usr.sbin/iscsid/iscsid.c	Sat Nov 22 15:09:18 2014	(r274853)
> @@ -160,7 +160,7 @@ connection_new(unsigned int session_id, 
>  #ifdef ICL_KERNEL_PROXY
>  	struct iscsi_daemon_connect idc;
>  #endif
> -	int error;
> +	int error, sockbuf;
>  
>  	conn = calloc(1, sizeof(*conn));
>  	if (conn == NULL)
> @@ -237,6 +237,14 @@ connection_new(unsigned int session_id, 
>  		fail(conn, strerror(errno));
>  		log_err(1, "failed to create socket for %s", from_addr);
>  	}
> +	sockbuf = SOCKBUF_SIZE;
> +	if (setsockopt(conn->conn_socket, SOL_SOCKET, SO_RCVBUF,
> +	    &sockbuf, sizeof(sockbuf)) == -1)
> +		log_warn("setsockopt(SO_RCVBUF) failed");
> +	sockbuf = SOCKBUF_SIZE;
> +	if (setsockopt(conn->conn_socket, SOL_SOCKET, SO_SNDBUF,
> +	    &sockbuf, sizeof(sockbuf)) == -1)
> +		log_warn("setsockopt(SO_SNDBUF) failed");
>  	if (from_ai != NULL) {
>  		error = bind(conn->conn_socket, from_ai->ai_addr,
>  		    from_ai->ai_addrlen);
> 
> Modified: head/usr.sbin/iscsid/iscsid.h
> ==============================================================================
> --- head/usr.sbin/iscsid/iscsid.h	Sat Nov 22 14:59:50 2014	(r274852)
> +++ head/usr.sbin/iscsid/iscsid.h	Sat Nov 22 15:09:18 2014	(r274853)
> @@ -44,6 +44,7 @@
>  #define	CONN_DIGEST_CRC32C		1
>  
>  #define CONN_MUTUAL_CHALLENGE_LEN	1024
> +#define	SOCKBUF_SIZE			1048576
>  
>  struct connection {
>  	int			conn_iscsi_fd;
>