svn commit: r201999 - head/lib/libc/stdio

[Colin Percival]

Author: cperciva
Date: Sun Jan 10 14:30:30 2010
New Revision: 201999
URL: http://svn.freebsd.org/changeset/base/201999

Log:
  Give a less silly response to a silly request.
  
  Prior to this commit, fread/fwrite calls with size * nmemb > SIZE_MAX
  were handled by reading or writing (size_t)(size * nmemb) bytes; for
  example, on 32-bit platforms, fread(ptr, 641, 6700417, f) would read 1
  byte and indicate that the requested 6700417 blocks had been read.
  
  This commit adds a check for such integer overflows, and treats them as
  if an overly large request was passed to read/write; i.e., it sets errno
  to EINVAL, sets the error indicator on the file, and returns a short
  object count (0, to be specific).
  
  The overflow check involves an integer division, so as a performance
  optimization we check first to see if both size and nmemb are less than
  2^16; if they are, no overflow is possible and we avoid the division.
  We assume here that size_t is at least 32 bits; this appears to be true
  on all platforms FreeBSD supports.
  
  Although this commit fixes an integer overflow, it is not likely to have
  any security implications, since any program which would be affected by
  this bug fix is quite clearly already very confused.
  
  Reviewed by:	kib
  MFC after:	1 month

Modified:
  head/lib/libc/stdio/fread.c
  head/lib/libc/stdio/fwrite.c

Modified: head/lib/libc/stdio/fread.c
==============================================================================
--- head/lib/libc/stdio/fread.c	Sun Jan 10 13:30:45 2010	(r201998)
+++ head/lib/libc/stdio/fread.c	Sun Jan 10 14:30:30 2010	(r201999)
@@ -37,6 +37,8 @@ static char sccsid[] = "@(#)fread.c	8.2 
 __FBSDID("$FreeBSD$");
 
 #include "namespace.h"
+#include <errno.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <string.h>
 #include "un-namespace.h"
@@ -69,8 +71,27 @@ __fread(void * __restrict buf, size_t si
 	/*
 	 * ANSI and SUSv2 require a return value of 0 if size or count are 0.
 	 */
-	if ((resid = count * size) == 0)
+	if ((count == 0) || (size == 0))
 		return (0);
+
+	/*
+	 * Check for integer overflow.  As an optimization, first check that
+	 * at least one of {count, size} is at least 2^16, since if both
+	 * values are less than that, their product can't possible overflow
+	 * (size_t is always at least 32 bits on FreeBSD).
+	 */
+	if (((count | size) > 0xFFFF) &&
+	    (count > SIZE_MAX / size)) {
+		errno = EINVAL;
+		fp->_flags |= __SERR;
+		return (0);
+	}
+
+	/*
+	 * Compute the (now required to not overflow) number of bytes to
+	 * read and actually do the work.
+	 */
+	resid = count * size;
 	ORIENT(fp, -1);
 	if (fp->_r < 0)
 		fp->_r = 0;

Modified: head/lib/libc/stdio/fwrite.c
==============================================================================
--- head/lib/libc/stdio/fwrite.c	Sun Jan 10 13:30:45 2010	(r201998)
+++ head/lib/libc/stdio/fwrite.c	Sun Jan 10 14:30:30 2010	(r201999)
@@ -37,6 +37,8 @@ static char sccsid[] = "@(#)fwrite.c	8.1
 __FBSDID("$FreeBSD$");
 
 #include "namespace.h"
+#include <errno.h>
+#include <stdint.h>
 #include <stdio.h>
 #include "un-namespace.h"
 #include "local.h"
@@ -60,10 +62,24 @@ fwrite(buf, size, count, fp)
 	/*
 	 * ANSI and SUSv2 require a return value of 0 if size or count are 0.
 	 */
-	n = count * size;
-	if (n == 0)
+	if ((count == 0) || (size == 0))
 		return (0);
 
+	/*
+	 * Check for integer overflow.  As an optimization, first check that
+	 * at least one of {count, size} is at least 2^16, since if both
+	 * values are less than that, their product can't possible overflow
+	 * (size_t is always at least 32 bits on FreeBSD).
+	 */
+	if (((count | size) > 0xFFFF) &&
+	    (count > SIZE_MAX / size)) {
+		errno = EINVAL;
+		fp->_flags |= __SERR;
+		return (0);
+	}
+
+	n = count * size;
+
 	iov.iov_base = (void *)buf;
 	uio.uio_resid = iov.iov_len = n;
 	uio.uio_iov = &iov;