git: 95dc52422893 - stable/11 - openssl: MFC: Fix a bug in BN_mod_sqrt() that can cause it to loop forever.

The branch stable/11 has been updated by eugen:

URL: https://cgit.FreeBSD.org/src/commit/?id=95dc524228934e434236bb58d7e945703d2e037d

commit 95dc524228934e434236bb58d7e945703d2e037d
Author:     Eugene Grosbein <eugen@FreeBSD.org>
AuthorDate: 2022-03-27 15:13:52 +0000
Commit:     Eugene Grosbein <eugen@FreeBSD.org>
CommitDate: 2022-03-27 15:13:52 +0000

    openssl: MFC: Fix a bug in BN_mod_sqrt() that can cause it to loop forever.
    
    Obtained from:  OpenSSL Project
    Security:       CVE-2022-0778
    Security:       SA-22:03
    
    (cherry picked from commit fdc418f15e92732a3551832bcb625ba9b47242df)
---
 crypto/openssl/crypto/bn/bn_sqrt.c   | 30 ++++++++++++++++++------------
 crypto/openssl/doc/crypto/BN_add.pod | 15 +++++++++++++--
 2 files changed, 31 insertions(+), 14 deletions(-)

diff --git a/crypto/openssl/crypto/bn/bn_sqrt.c b/crypto/openssl/crypto/bn/bn_sqrt.c
index 232af99a216d..3504952c3116 100644
--- a/crypto/openssl/crypto/bn/bn_sqrt.c
+++ b/crypto/openssl/crypto/bn/bn_sqrt.c
@@ -64,7 +64,8 @@ BIGNUM *BN_mod_sqrt(BIGNUM *in, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
 /*
  * Returns 'ret' such that ret^2 == a (mod p), using the Tonelli/Shanks
  * algorithm (cf. Henri Cohen, "A Course in Algebraic Computational Number
- * Theory", algorithm 1.5.1). 'p' must be prime!
+ * Theory", algorithm 1.5.1). 'p' must be prime, otherwise an error or
+ * an incorrect "result" will be returned.
  */
 {
     BIGNUM *ret = in;
@@ -350,18 +351,23 @@ BIGNUM *BN_mod_sqrt(BIGNUM *in, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
             goto vrfy;
         }
 
-        /* find smallest  i  such that  b^(2^i) = 1 */
-        i = 1;
-        if (!BN_mod_sqr(t, b, p, ctx))
-            goto end;
-        while (!BN_is_one(t)) {
-            i++;
-            if (i == e) {
-                BNerr(BN_F_BN_MOD_SQRT, BN_R_NOT_A_SQUARE);
-                goto end;
+        /* Find the smallest i, 0 < i < e, such that b^(2^i) = 1. */
+        for (i = 1; i < e; i++) {
+            if (i == 1) {
+                if (!BN_mod_sqr(t, b, p, ctx))
+                    goto end;
+
+            } else {
+                if (!BN_mod_mul(t, t, t, p, ctx))
+                    goto end;
             }
-            if (!BN_mod_mul(t, t, t, p, ctx))
-                goto end;
+            if (BN_is_one(t))
+                break;
+        }
+        /* If not found, a is not a square or p is not prime. */
+        if (i >= e) {
+            BNerr(BN_F_BN_MOD_SQRT, BN_R_NOT_A_SQUARE);
+            goto end;
         }
 
         /* t := y^2^(e - i - 1) */
diff --git a/crypto/openssl/doc/crypto/BN_add.pod b/crypto/openssl/doc/crypto/BN_add.pod
index 02e548641663..4b4fd9e1a0fe 100644
--- a/crypto/openssl/doc/crypto/BN_add.pod
+++ b/crypto/openssl/doc/crypto/BN_add.pod
@@ -3,7 +3,7 @@
 =head1 NAME
 
 BN_add, BN_sub, BN_mul, BN_sqr, BN_div, BN_mod, BN_nnmod, BN_mod_add,
-BN_mod_sub, BN_mod_mul, BN_mod_sqr, BN_exp, BN_mod_exp, BN_gcd -
+BN_mod_sub, BN_mod_mul, BN_mod_sqr, BN_mod_sqrt, BN_exp, BN_mod_exp, BN_gcd -
 arithmetic operations on BIGNUMs
 
 =head1 SYNOPSIS
@@ -36,6 +36,8 @@ arithmetic operations on BIGNUMs
 
  int BN_mod_sqr(BIGNUM *r, BIGNUM *a, const BIGNUM *m, BN_CTX *ctx);
 
+ BIGNUM *BN_mod_sqrt(BIGNUM *in, BIGNUM *a, const BIGNUM *p, BN_CTX *ctx);
+
  int BN_exp(BIGNUM *r, BIGNUM *a, BIGNUM *p, BN_CTX *ctx);
 
  int BN_mod_exp(BIGNUM *r, BIGNUM *a, const BIGNUM *p,
@@ -86,6 +88,12 @@ L<BN_mod_mul_reciprocal(3)|BN_mod_mul_reciprocal(3)>.
 BN_mod_sqr() takes the square of I<a> modulo B<m> and places the
 result in I<r>.
 
+BN_mod_sqrt() returns the modular square root of I<a> such that
+C<in^2 = a (mod p)>. The modulus I<p> must be a
+prime, otherwise an error or an incorrect "result" will be returned.
+The result is stored into I<in> which can be NULL. The result will be
+newly allocated in that case.
+
 BN_exp() raises I<a> to the I<p>-th power and places the result in I<r>
 (C<r=a^p>). This function is faster than repeated applications of
 BN_mul().
@@ -107,7 +115,10 @@ the arguments.
 
 =head1 RETURN VALUES
 
-For all functions, 1 is returned for success, 0 on error. The return
+The BN_mod_sqrt() returns the result (possibly incorrect if I<p> is
+not a prime), or NULL.
+
+For all remaining functions, 1 is returned for success, 0 on error. The return
 value should always be checked (e.g., C<if (!BN_add(r,a,b)) goto err;>).
 The error codes can be obtained by L<ERR_get_error(3)|ERR_get_error(3)>.