kern/172921: O_CLOEXEC support for libdb / dbopen

[Jukka Ukkonen]

>Number:         172921
>Category:       kern
>Synopsis:       O_CLOEXEC support for libdb / dbopen
>Confidential:   no
>Severity:       non-critical
>Priority:       low
>Responsible:    freebsd-bugs
>State:          open
>Quarter:        
>Keywords:       
>Date-Required:
>Class:          change-request
>Submitter-Id:   current-users
>Arrival-Date:   Sun Oct 21 07:00:01 UTC 2012
>Closed-Date:
>Last-Modified:
>Originator:     Jukka Ukkonen
>Release:        9.1-PRERELEASE
>Organization:
-----
>Environment:
FreeBSD sleipnir 9.1-PRERELEASE FreeBSD 9.1-PRERELEASE #0: Sun Oct 21 08:20:37 EEST 2012     root at sleipnir:/usr/obj/usr/src/sys/Sleipnir  amd64
>Description:
Traditionally libdb has used fcntl() and FD_CLOEXEC to mark the files to be
closed when exec() is called.
If used in a threaded code this leaves a minimal time window during which
another thread could call exec() while the FD_CLOEXEC flag has not yet been set.
Using O_CLOEXEC with open() can be used to close the window on system which
support O_CLOEXEC.
The attached path amends libdb (the default version bundled with libc) to use
O_CLOEXEC whenever possible.
The patch also changes dbopen to allow setting O_DIRECT on those systems which support the flag.

>How-To-Repeat:
In fact the little time window is extremely hard to abuse intentionally.

>Fix:
Apply the attached patch.


Patch attached with submission follows:

--- lib/libc/db/btree/bt_open.c.orig	2012-10-21 09:03:10.000000000 +0300
+++ lib/libc/db/btree/bt_open.c	2012-10-21 09:05:16.000000000 +0300
@@ -196,6 +196,10 @@
 			goto einval;
 		}
 
+#if defined(O_CLOEXEC)
+		flags |= O_CLOEXEC;
+#endif
+
 		if ((t->bt_fd = _open(fname, flags, mode)) < 0)
 			goto err;
 
@@ -207,7 +211,10 @@
 		F_SET(t, B_INMEM);
 	}
 
-	if (_fcntl(t->bt_fd, F_SETFD, 1) == -1)
+	/*
+	 * Needed for the tmp() case and for systems without O_CLOEXEC.
+	 */
+	if (_fcntl(t->bt_fd, F_SETFD, FD_CLOEXEC) == -1)
 		goto err;
 
 	if (_fstat(t->bt_fd, &sb))
--- lib/libc/db/hash/hash.c.orig	2012-10-21 09:08:49.000000000 +0300
+++ lib/libc/db/hash/hash.c	2012-10-21 09:10:30.000000000 +0300
@@ -108,6 +108,10 @@
 		return (NULL);
 	}
 
+#if defined(O_CLOEXEC)
+	flags |= O_CLOEXEC;
+#endif
+
 	if (!(hashp = (HTAB *)calloc(1, sizeof(HTAB))))
 		return (NULL);
 	hashp->fp = -1;
@@ -123,7 +127,9 @@
 	if (file) {
 		if ((hashp->fp = _open(file, flags, mode)) == -1)
 			RETURN_ERROR(errno, error0);
-		(void)_fcntl(hashp->fp, F_SETFD, 1);
+#if !defined(O_CLOEXEC)
+		(void)_fcntl(hashp->fp, F_SETFD, FD_CLOEXEC);
+#endif
 		new_table = _fstat(hashp->fp, &statbuf) == 0 &&
 		    statbuf.st_size == 0 && (flags & O_ACCMODE) != O_RDONLY;
 	} else
--- lib/libc/db/recno/rec_open.c.orig	2012-10-21 09:21:53.000000000 +0300
+++ lib/libc/db/recno/rec_open.c	2012-10-21 09:27:57.000000000 +0300
@@ -63,6 +63,10 @@
 	struct stat sb;
 	int rfd, sverrno;
 
+#if defined(O_CLOEXEC)
+	flags |= O_CLOEXEC;
+#endif
+
 	/* Open the user's file -- if this fails, we're done. */
 	if (fname != NULL && (rfd = _open(fname, flags, mode)) < 0)
 		return (NULL);
--- lib/libc/db/db/db.c.orig	2012-10-21 06:49:59.000000000 +0300
+++ lib/libc/db/db/db.c	2012-10-21 07:21:38.000000000 +0300
@@ -42,6 +42,20 @@
 
 #include <db.h>
 
+/*
+ * Just so that the exact same USE_OPEN_FLAGS
+ * will work also on systems which do not have
+ * support for some common open() flags.
+ */
+
+#if !defined(O_CLOEXEC)
+#  define O_CLOEXEC	0
+#endif
+
+#if !defined(O_DIRECT)
+#  define O_DIRECT	0
+#endif
+
 static int __dberr(void);
 
 DB *
@@ -51,7 +65,8 @@
 #define	DB_FLAGS	(DB_LOCK | DB_SHMEM | DB_TXN)
 #define	USE_OPEN_FLAGS							\
 	(O_CREAT | O_EXCL | O_EXLOCK | O_NOFOLLOW | O_NONBLOCK | 	\
-	 O_RDONLY | O_RDWR | O_SHLOCK | O_SYNC | O_TRUNC)
+	 O_RDONLY | O_RDWR | O_SHLOCK | O_SYNC | O_TRUNC | O_CLOEXEC |	\
+	 O_DIRECT )
 
 	if ((flags & ~(USE_OPEN_FLAGS | DB_FLAGS)) == 0)
 		switch (type) {


>Release-Note:
>Audit-Trail:
>Unformatted: