git: 3501eec9dd39 - main - libc: Guard mergesort() allocation size arithmetic

The branch main has been updated by fuz:

URL: https://cgit.FreeBSD.org/src/commit/?id=3501eec9dd39b527a46e82de53480968d283b90e

commit 3501eec9dd39b527a46e82de53480968d283b90e
Author:     Faraz Vahedi <kfv@kfv.io>
AuthorDate: 2026-05-28 13:50:45 +0000
Commit:     Robert Clausecker <fuz@FreeBSD.org>
CommitDate: 2026-06-07 20:59:18 +0000

    libc: Guard mergesort() allocation size arithmetic
    
    Signed-off-by:  Faraz Vahedi <kfv@kfv.io>
    Pull Request:   https://github.com/freebsd/freebsd-src/pull/2243
    Reviewed by:    fuz
    MFC after:      1 week
---
 lib/libc/stdlib/merge.c | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/lib/libc/stdlib/merge.c b/lib/libc/stdlib/merge.c
index e70938088589..e07a3947e741 100644
--- a/lib/libc/stdlib/merge.c
+++ b/lib/libc/stdlib/merge.c
@@ -49,6 +49,7 @@
 #include <sys/param.h>
 
 #include <errno.h>
+#include <stdckdint.h>
 #include <stdlib.h>
 #include <string.h>
 
@@ -109,7 +110,7 @@ mergesort_b(void *base, size_t nmemb, size_t size, cmp_t cmp)
 mergesort(void *base, size_t nmemb, size_t size, cmp_t cmp)
 #endif
 {
-	size_t i;
+	size_t i, nbytes, asize;
 	int sense;
 	int big, iflag;
 	u_char *f1, *f2, *t, *b, *tp2, *q, *l1, *l2;
@@ -123,16 +124,21 @@ mergesort(void *base, size_t nmemb, size_t size, cmp_t cmp)
 	if (nmemb == 0)
 		return (0);
 
+	if (ckd_mul(&nbytes, nmemb, size) || ckd_add(&asize, nbytes, PSIZE)) {
+		errno = EINVAL;
+		return (-1);
+	}
+
 	iflag = 0;
 	if (__is_aligned(size, ISIZE) && __is_aligned(base, ISIZE))
 		iflag = 1;
 
-	if ((list2 = malloc(nmemb * size + PSIZE)) == NULL)
+	if ((list2 = malloc(asize)) == NULL)
 		return (-1);
 
 	list1 = base;
 	setup(list1, list2, nmemb, size, cmp);
-	last = list2 + nmemb * size;
+	last = list2 + nbytes;
 	i = big = 0;
 	while (*EVAL(list2) != last) {
 	    l2 = list1;
@@ -227,10 +233,10 @@ COPY:	    			b = t;
 	    tp2 = list1;	/* swap list1, list2 */
 	    list1 = list2;
 	    list2 = tp2;
-	    last = list2 + nmemb*size;
+	    last = list2 + nbytes;
 	}
 	if (base == list2) {
-		memmove(list2, list1, nmemb*size);
+		memmove(list2, list1, nbytes);
 		list2 = list1;
 	}
 	free(list2);