standards/166463: remquo[f] may return wrong sign in quo part and incorrect rem part on denormals

[Ilya Burylov]

>Number:         166463
>Category:       standards
>Synopsis:       remquo[f] may return wrong sign in quo part and incorrect rem part on denormals
>Confidential:   no
>Severity:       non-critical
>Priority:       low
>Responsible:    freebsd-standards
>State:          open
>Quarter:        
>Keywords:       
>Date-Required:
>Class:          sw-bug
>Submitter-Id:   current-users
>Arrival-Date:   Wed Mar 28 12:40:09 UTC 2012
>Closed-Date:
>Last-Modified:
>Originator:     Ilya Burylov
>Release:        Found in Android Bionic
>Organization:
Intel
>Environment:
Found in Android Bionic
>Description:
Currently we will get the wrong result as follows:

remquof(0x7bb33336, 0x63000000) = -671088640, 0x00000000 /* wrong sign in quo */
remquo(0xbff0000000000003, 0x3ff0000000000003) = 1, 0x8000000000000000 /* wrong sign in quo */
remquo(0x9120000000000001, 0x0000000000000005) = -1288490188, 0x0000000000000004 /* wrong quo and rem parts */

while [an example of] correct one should be:

remquof(0x7bb33336, 0x63000000) = 1476395008, 0x00000000
remquo(0xbff0000000000003, 0x3ff0000000000003) = -1, 0x8000000000000000
remquo(0x9120000000000001, 0x0000000000000005) = -1288490189, 0x0000000000000001

That does not fit into description in C99 standard: e.g. "In
the object pointed to by quo they store a value whose sign is the sign of x/y"

Fix attached

>How-To-Repeat:
Reproducible in description
>Fix:
Fix should be applied to 
\msun\src\s_remquo.c
\msun\src\s_remquof.c

Fix attached.

Patch attached with submission follows:

Index: src/s_remquof.c
===================================================================
--- src/s_remquof.c	(revision 233542)
+++ src/s_remquof.c	(working copy)
@@ -46,7 +46,7 @@
 	    q = 0;
 	    goto fixup;	/* |x|<|y| return x or x-y */
 	} else if(hx==hy) {
-	    *quo = 1;
+	    *quo = (sxy ? -1 : 1);
 	    return Zero[(u_int32_t)sx>>31];	/* |x|=|y| return x*0*/
 	}
 
@@ -88,6 +88,7 @@
 
     /* convert back to floating value and restore the sign */
 	if(hx==0) {				/* return sign(x)*0 */
+        q &= 0x7fffffff;
 	    *quo = (sxy ? -q : q);
 	    return Zero[(u_int32_t)sx>>31];
 	}
Index: src/s_remquo.c
===================================================================
--- src/s_remquo.c	(revision 233542)
+++ src/s_remquo.c	(working copy)
@@ -51,7 +51,7 @@
 		goto fixup;	/* |x|<|y| return x or x-y */
 	    }
 	    if(lx==ly) {
-		*quo = 1;
+		*quo = (sxy ? -1 : 1);
 		return Zero[(u_int32_t)sx>>31];	/* |x|=|y| return x*0*/
 	    }
 	}
@@ -114,6 +114,7 @@
 
     /* convert back to floating value and restore the sign */
 	if((hx|lx)==0) {			/* return sign(x)*0 */
+        q &= 0x7fffffff;
 	    *quo = (sxy ? -q : q);
 	    return Zero[(u_int32_t)sx>>31];
 	}
@@ -129,9 +130,9 @@
 		lx = (lx>>n)|((u_int32_t)hx<<(32-n));
 		hx >>= n;
 	    } else if (n<=31) {
-		lx = (hx<<(32-n))|(lx>>n); hx = sx;
+		lx = (hx<<(32-n))|(lx>>n); hx = 0;
 	    } else {
-		lx = hx>>(n-32); hx = sx;
+		lx = hx>>(n-32); hx = 0;
 	    }
 	}
 fixup:


>Release-Note:
>Audit-Trail:
>Unformatted: