close: Socket is not connected

[Mikolaj Golub]

Hi,

We have an application that consists of two processes communicating via unix
socket (client connects, sends requests, reads response and close the
connection). Sometimes the error 'Socket is not connected' is observed when
client closes the socket.

We observed this problem on FreeBSD6.X and now have been observing it on 7.1.

To model the behaviour of the application I have written a simple test program
(see below) based on relevant parts from the application code. Using this test
program it requires from an a half of hour to several hours to reproduce the
error. It might fail on close() both in the parent (server) and the children
(client).

$ date; ./unixsocket ; date
Thu Dec 17 09:38:35 UTC 2009
unixsocket: parent: close error: 57
Thu Dec 17 14:13:07 UTC 2009
unixsocket: child: connect error 61

$ ./unixsocket
unixsocket: child: close error: 57

I can reproduce the error running the test on 8.0 too.

Does this indicate some bug in FreeBSD or may be just something is wrong with
our code?

#include <sys/types.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <strings.h>
#include <string.h>
#include <unistd.h>
#include <sys/select.h>
#include <err.h>

#define UNIXSTR_PATH "/tmp/mytest.socket"
#define BUFSIZE 557
#define USLEEP  10000
#define timeout 10

int waitData(int handle, int reading, int writing) {
	
	struct timeval tv;
	int    result;
	fd_set rset;
	fd_set wset;
	fd_set eset;
	
	FD_ZERO(&rset);
	FD_SET(handle, &rset);

	FD_ZERO(&wset);
	FD_SET(handle, &wset);

	FD_ZERO(&eset);
	FD_SET(handle, &eset);

	tv.tv_sec  = timeout;
	tv.tv_usec = 0;

	for (;;)
	{
		if (reading && !writing)
			result = select(handle + 1, &rset, 0, &eset, &tv);
		else
			if (!reading && writing)
				result = select(handle + 1, 0, &wset, &eset, &tv);
			else
				result = select(handle + 1, &rset, &wset, &eset, &tv);

		if (result == 0) /* timeout */
			return 0;

		if (result < 0)
		{
			if (errno == EINTR)
			{
				continue;
			}
			errx(1, "select: %d", errno);
		}

		if (FD_ISSET(handle, &rset) || FD_ISSET(handle, &wset))
		{
			int err = 0;
			socklen_t err_size = sizeof(err);
			if (getsockopt(handle, SOL_SOCKET, SO_ERROR, (char*)&err, &err_size) != 0)
			{
				errx(1, "getsockopts: %d", errno);
			}

			if (err != 0)
			{
				errx(1, "getsockopts: err: %d", err);
			}
		}
		else
		{
			errx(1, "select problems");
		}

		return 1; /* OK */
	}
}

void Write(int handle, const char* buf, size_t size) {
	    size_t left = size;
	    
	    while (left > 0)
	    {		    
		    while (1)
		    {
			    int sent = send(handle, buf, size, 0);
			    
			    if (sent <= 0)
			    {
				    if (errno == EAGAIN)
				    {
					    if (!waitData(handle, 0, 1))
						    errx(1, "Write: timeout");
					    continue;
				    }
				    errx(1, "Write: error: %d", errno);
			    }
			    left -= sent;
			    buf  += sent;
			    break;
		    }
	    }
}

void Read(int handle, char* buf, size_t size)
{
	while (size > 0)
	{
		int blockSize;

		if (!waitData(handle, 1, 0))
			errx(1, "Read: timeout");

		blockSize = recv(handle, buf, size, 0);
		
		if (blockSize <= 0)
		{
			errx(1, "Read: error: blockSize: %d", blockSize);
		}

		buf  += blockSize;
		size -= blockSize;
	}
}


int main(int argc, char **argv)
{
	int			listenfd, connfd, pid;
	struct sockaddr_un	servaddr;
	char			buf[BUFSIZE];

	memset(buf, 'X', sizeof(buf));
	
	pid = fork();
	if (-1 == pid)
		errx(1, "fork(): %d", errno);

	if (0 != pid) {
		/* parent */

		if ((listenfd = socket(AF_LOCAL, SOCK_STREAM, 0)) < 0)
			errx(1, "parent: socket error: %d", errno);

		unlink(UNIXSTR_PATH);
		bzero(&servaddr, sizeof(servaddr));
		servaddr.sun_family = AF_LOCAL;
		strcpy(servaddr.sun_path, UNIXSTR_PATH);

		if (bind(listenfd, (struct sockaddr *) &servaddr, sizeof(servaddr)) < 0)
			errx(1, "parent: bind error: %d", errno);

		if (listen(listenfd, 1024) < 0)
			errx(1, "parent: listen error: %d", errno);
		
		for ( ; ; ) {
			if ((connfd = accept(listenfd, (struct sockaddr *) NULL, NULL)) < 0)
				errx(1, "parent: accept error: %d", errno);

			if (fcntl(connfd, F_SETFL, O_NONBLOCK) == -1)
				errx(1, "parent: fcntl error: %d", errno);
			
			Read(connfd, buf, sizeof(buf));
			Write(connfd, buf, sizeof(buf));
			
	        	if (close(connfd) < 0)
				errx(1, "parent: close error: %d", errno);
		}
		
	} else {
		/* child */

		/* wait some time while parent has created socket */
		sleep(1);

		for ( ; ; ) {

			if ((connfd = socket(AF_LOCAL, SOCK_STREAM, 0)) < 0)
				errx(1, "child: socket error: %d", errno);

			if (fcntl(connfd, F_SETFL, O_NONBLOCK) == -1)
				errx(1, "child: fcntl error: %d", errno);
			
			bzero(&servaddr, sizeof(servaddr));
			servaddr.sun_family = AF_LOCAL;
			strcpy(servaddr.sun_path, UNIXSTR_PATH);

			if (connect(connfd, (struct sockaddr *) &servaddr, sizeof(servaddr)) < 0)
				errx(1, "child: connect error %d", errno);
			
			Write(connfd, buf, sizeof(buf));
			Read(connfd, buf, sizeof(buf));
	
			if (close(connfd) != 0) 
				errx(1, "child: close error: %d", errno);

			usleep(USLEEP);
		}
	}

	return 0;
}

-- 
Mikolaj Golub