From 91ddeba0f2269b017dc06c46c993a788974b1aa5 Mon Sep 17 00:00:00 2001
From: Tomas Mraz <tomas@openssl.org>
Date: Fri, 21 Jul 2023 11:39:41 +0200
Subject: [PATCH] DH_check(): Do not try checking q properties if it is
 obviously invalid

If  |q| >= |p| then the q value is obviously wrong as q
is supposed to be a prime divisor of p-1.

We check if p is overly large so this added test implies that
q is not large either when performing subsequent tests using that
q value.

Otherwise if it is too large these additional checks of the q value
such as the primality test can then trigger DoS by doing overly long
computations.

Fixes CVE-2023-3817

Reviewed-by: Paul Dale <pauli@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/21551)

diff -wpruN --no-dereference '--exclude=*.orig' a~/crypto/dh/dh_check.c a/crypto/dh/dh_check.c
--- a~/crypto/dh/dh_check.c	1970-01-01 00:00:00
+++ a/crypto/dh/dh_check.c	1970-01-01 00:00:00
@@ -73,7 +73,7 @@
 
 int DH_check(const DH *dh, int *ret)
 {
-    int ok = 0;
+    int ok = 0, q_good = 0;
     BN_CTX *ctx = NULL;
     BN_ULONG l;
     BIGNUM *t1 = NULL, *t2 = NULL;
@@ -96,7 +96,14 @@ int DH_check(const DH *dh, int *ret)
     if (t2 == NULL)
         goto err;
 
-    if (dh->q) {
+    if (dh->q != NULL) {
+        if (BN_ucmp(dh->p, dh->q) > 0)
+            q_good = 1;
+        else
+            *ret |= DH_CHECK_INVALID_Q_VALUE;
+    }
+
+    if (q_good) {
         if (BN_cmp(dh->g, BN_value_one()) <= 0)
             *ret |= DH_NOT_SUITABLE_GENERATOR;
         else if (BN_cmp(dh->g, dh->p) >= 0)