Speedup DTLS max fragment size calculation for MTU limits

src/internal.c

@@ -3706,14 +3706,17 @@ void InitSuites(Suites* suites, ProtocolVersion pv, int keySz, word16 haveRSA,
 #endif
 
 #ifdef BUILD_TLS_SM4_GCM_SM3
-    if (tls1_3) {
+    /* RFC 8998 registers TLS_SM4_GCM_SM3 with DTLS-OK: No and provides no
+     * record-number-mask construction; RFC 9147 Section 4.2.3 forbids using
+     * non-AES / non-ChaCha20 ciphers over DTLS without one. */
+    if (tls1_3 && !dtls) {
         suites->suites[idx++] = CIPHER_BYTE;
         suites->suites[idx++] = TLS_SM4_GCM_SM3;
     }
 #endif
 
 #ifdef BUILD_TLS_SM4_CCM_SM3
-    if (tls1_3) {
+    if (tls1_3 && !dtls) {
         suites->suites[idx++] = CIPHER_BYTE;
         suites->suites[idx++] = TLS_SM4_CCM_SM3;
     }
@@ -4647,21 +4650,22 @@ void InitSuites(Suites* suites, ProtocolVersion pv, int keySz, word16 haveRSA,
 #endif /* BUILD_TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256 */
 
 #ifdef BUILD_TLS_ECDHE_ECDSA_WITH_SM4_CBC_SM3
-    if (tls && haveECC) {
+    /* RFC 8998 registers the SM4 cipher suites with DTLS-OK: No. */
+    if (tls && !dtls && haveECC) {
         suites->suites[idx++] = SM_BYTE;
         suites->suites[idx++] = TLS_ECDHE_ECDSA_WITH_SM4_CBC_SM3;
     }
 #endif
 
 #ifdef BUILD_TLS_ECDHE_ECDSA_WITH_SM4_GCM_SM3
-    if (tls && haveECC) {
+    if (tls && !dtls && haveECC) {
         suites->suites[idx++] = SM_BYTE;
         suites->suites[idx++] = TLS_ECDHE_ECDSA_WITH_SM4_GCM_SM3;
     }
 #endif
 
 #ifdef BUILD_TLS_ECDHE_ECDSA_WITH_SM4_CCM_SM3
-    if (tls && haveECC) {
+    if (tls && !dtls && haveECC) {
         suites->suites[idx++] = SM_BYTE;
         suites->suites[idx++] = TLS_ECDHE_ECDSA_WITH_SM4_CCM_SM3;
     }
@@ -37283,6 +37287,22 @@ static int AddPSKtoPreMasterSecret(WOLFSSL* ssl)
         first   = suites->suites[idx];
         second  = suites->suites[idx+1];
 
+#ifdef WOLFSSL_DTLS
+        /* RFC 8998 registers the SM4 cipher suites with DTLS-OK: No.
+         * RFC 9147 Section 4.2.3 forbids using non-AES / non-ChaCha20 ciphers
+         * over DTLS without a defined record-number-mask construction, which
+         * RFC 8998 does not provide. */
+        if (ssl->options.dtls) {
+            if ((first == CIPHER_BYTE && (second == TLS_SM4_GCM_SM3 ||
+                                          second == TLS_SM4_CCM_SM3)) ||
+                first == SM_BYTE) {
+                WOLFSSL_MSG("SM cipher suite not allowed over DTLS "
+                            "(RFC 8998)");
+                return 0;
+            }
+        }
+#endif /* WOLFSSL_DTLS */
+
 #ifdef WOLFSSL_TLS13
         /* When negotiating TLS 1.3, reject non-TLS 1.3 cipher suites */
         if (IsAtLeastTLSv1_3(ssl->version) &&
@@ -42252,30 +42272,100 @@ int wolfSSL_AsyncPush(WOLFSSL* ssl, WC_ASYNC_DEV* asyncDev)
  */
 int wolfssl_local_GetRecordSize(WOLFSSL *ssl, int payloadSz, int isEncrypted)
 {
-    int recordSz;
+    int sz;
+    int headerSz;
+    int digestSz;
+    int ivSz;
+#ifdef WOLFSSL_DTLS_CID
+    byte cidSz;
+#endif
+#ifndef WOLFSSL_AEAD_ONLY
+    int blockSz;
+    int pad;
+#endif
 
     if (ssl == NULL)
         return BAD_FUNC_ARG;
 
-    if (isEncrypted) {
-        recordSz = BuildMessage(ssl, NULL, 0, NULL, payloadSz, application_data,
-             0, 1, 0, CUR_ORDER);
-        /* use a safe upper bound in case of error */
-        if (recordSz < 0) {
-            recordSz = payloadSz + RECORD_HEADER_SZ
-                + cipherExtraData(ssl) + COMP_EXTRA;
-            if (ssl->options.dtls) {
-                recordSz += DTLS_RECORD_EXTRA;
-            }
-        }
+    if (!isEncrypted) {
+        sz = payloadSz + RECORD_HEADER_SZ;
+        if (ssl->options.dtls)
+            sz += DTLS_RECORD_EXTRA;
+        return sz;
     }
-    else {
-        recordSz = payloadSz + RECORD_HEADER_SZ;
-        if (ssl->options.dtls) {
-            recordSz += DTLS_RECORD_EXTRA;
+
+#ifdef WOLFSSL_TLS13
+    if (ssl->options.tls1_3) {
+    #ifdef WOLFSSL_DTLS13
+        if (ssl->options.dtls)
+            headerSz = Dtls13GetRlHeaderLength(ssl, 1);
+        else
+    #endif
+            headerSz = RECORD_HEADER_SZ;
+        sz = payloadSz + headerSz + 1 /* inner type */
+                + ssl->specs.aead_mac_size;
+    #ifdef WOLFSSL_DTLS13
+        if (ssl->options.dtls && sz < Dtls13MinimumRecordLength(ssl))
+            sz = Dtls13MinimumRecordLength(ssl);
+    #endif
+        return sz;
+    }
+#endif
+
+    /* TLS 1.2 / TLS 1.1 / DTLS 1.2 path. Mirror BuildMessage's size
+     * calculation so the result matches exactly. */
+    headerSz = RECORD_HEADER_SZ;
+    sz = payloadSz + RECORD_HEADER_SZ;
+
+    if (ssl->options.dtls) {
+        sz       += DTLS_RECORD_EXTRA;
+        headerSz += DTLS_RECORD_EXTRA;
+    #ifdef WOLFSSL_DTLS_CID
+        cidSz = DtlsGetCidTxSize(ssl);
+        if (cidSz > 0) {
+            sz       += cidSz;
+            headerSz += cidSz;
+            sz++; /* real_type byte appended */
         }
+    #endif
+    }
+
+    digestSz = (int)ssl->specs.hash_size;
+#ifdef HAVE_TRUNCATED_HMAC
+    if (ssl->truncated_hmac)
+        digestSz = min(TRUNCATED_HMAC_SZ, digestSz);
+#endif
+    sz += digestSz;
+
+#ifndef WOLFSSL_AEAD_ONLY
+    if (ssl->specs.cipher_type == block) {
+        blockSz = (int)ssl->specs.block_size;
+
+        if (ssl->options.tls1_1)
+            sz += blockSz; /* explicit IV */
+        sz += 1; /* pad-length byte */
+
+    #if defined(HAVE_ENCRYPT_THEN_MAC) && !defined(WOLFSSL_AEAD_ONLY)
+        if (ssl->options.startedETMWrite)
+            pad = blockSz != 0 ?
+                (sz - headerSz - digestSz) % blockSz : 0;
+        else
+    #endif
+            pad = blockSz != 0 ? (sz - headerSz) % blockSz : 0;
+        if (pad != 0)
+            pad = blockSz - pad;
+        sz += pad;
+    }
+    else
+#endif /* WOLFSSL_AEAD_ONLY */
+    if (ssl->specs.cipher_type == aead) {
+        ivSz = 0;
+        if (ssl->specs.bulk_cipher_algorithm != wolfssl_chacha)
+            ivSz = AESGCM_EXP_IV_SZ;
+        sz += ivSz + (int)ssl->specs.aead_mac_size - digestSz;
     }
-    return recordSz;
+
+    return sz;
 }
 #endif