resource leak in fifo_vnops.c: 6.x/7.x/8.x

[Dorr H. Clark]

We believe we have identified a significant resource leak
present in 6.x, 7.x, and 8.x.  We believe this is a regression
versus FreeBSD 4.x which appears to do the Right Thing (tm).

We have a test program (see below) which will run the system
out of sockets by repeated exercise of the failing code
path in the kernel.

Our proposed fix is applied to the file usr/src/sys/fs/fifofs/fifo_vnops.c


@@ -237,6 +237,8 @@
        if (ap->a_mode & FWRITE) {
                if ((ap->a_mode & O_NONBLOCK) && fip->fi_readers == 0) {
                        mtx_unlock(&fifo_mtx);
+                       /* Exclusive VOP lock is held - safe to clean */
+                       fifo_cleanup(vp);
                        return (ENXIO);
                }
                fip->fi_writers++;

Test program follows.

We welcome feedback on this proposed fix.

Chitti Nimmagadda
Engineer

Dorr H. Clark
Advisor

Graduate School of Engineering
Santa Clara University
Santa Clara, CA.

A test program which reveals the issue is presented here:

#include <stdio.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/sysctl.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <signal.h>


#define FIFOPATH "/tmp/fifobug.debug"

void getsysctl(name, ptr, len)
    const char *name;
    void *ptr;
    size_t len;
{
    size_t nlen = len;
    if (sysctlbyname(name, ptr, &nlen, NULL, 0) != 0) {
                perror("sysctl");
                printf("name: %s\n", name);
                exit(-1);
    }
    if (nlen != len) {
        printf("sysctl(%s...) expected %lu, got %lu", name,
            (unsigned long)len, (unsigned long)nlen);
                exit(-2);
    }
}

main(int argc, char *argv[])
{
	int acnt = 0, bcnt = 0, maxcnt;
	int fd;
	unsigned int maxiter;
	int notdone = 1;
	int i= 0;

	getsysctl("kern.ipc.maxsockets", &maxcnt, sizeof(maxcnt));
	if (argc == 2) {
		maxiter = atoi(argv[1]);
	} else {
		maxiter = maxcnt*2;
	}

	unlink(FIFOPATH);
	printf("Max sockets: %d\n", maxcnt);
	printf("FIFO %s will be created, opened, deleted %d times\n", 
		FIFOPATH, maxiter);

	getsysctl("kern.ipc.numopensockets", &bcnt, sizeof(bcnt));
	while(notdone && (i++ < maxiter)) {
		if (mkfifo(FIFOPATH, 0) == 0) {
			chmod(FIFOPATH, 
				S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH);
		}
		fd = open(FIFOPATH, O_WRONLY|O_NONBLOCK);
		if ((fd <= 0) && (errno != ENXIO)) {
			notdone = 0;
		} 
		unlink(FIFOPATH);
	}
	getsysctl("kern.ipc.numopensockets", &acnt, sizeof(acnt));
	printf("Open Sockets: Before Test: %d, After Test: %d, diff: %d\n",
		 bcnt, acnt, acnt - bcnt);
	if (notdone) {
		printf("FIFO/socket bug is fixed\n");
		exit(0);
	} else {
		printf("FIFO/socket bug is NOT fixed\n");
		exit(-1);
	}
}

http://www.cse.scu.edu/~dclark/coen_284_FreeBSD/fifo_socket_leak.txt