Reject SM cipher suites over DTLS and exercise them in record-size test

RFC 8998 registers TLS_SM4_GCM_SM3 / TLS_SM4_CCM_SM3 (and the TLS 1.2
SM4 suites) with DTLS-OK: No and defines no record-number-mask
construction for SM4. RFC 9147 Section 4.2.3 requires any non-AES /
non-ChaCha20 cipher to define its own record sequence number encryption
to be usable with DTLS. Drop SM cipher suites from the negotiable list
in InitSuites when DTLS, and reject them defensively in
VerifyServerSuite for the case where the user pins a cipher list
explicitly. Update test_record_size_matches_build_message to set up an
SM2 cert chain and SM2 key share for SM ciphers (so they actually
handshake over TLS 1.2 / TLS 1.3) and to skip the DTLS variants with
the RFC reason.

Author: julek-wolfssl

src/internal.c

@@ -3706,14 +3706,17 @@ void InitSuites(Suites* suites, ProtocolVersion pv, int keySz, word16 haveRSA,
 #endif
 
 #ifdef BUILD_TLS_SM4_GCM_SM3
-    if (tls1_3) {
+    /* RFC 8998 registers TLS_SM4_GCM_SM3 with DTLS-OK: No and provides no
+     * record-number-mask construction; RFC 9147 Section 4.2.3 forbids using
+     * non-AES / non-ChaCha20 ciphers over DTLS without one. */
+    if (tls1_3 && !dtls) {
         suites->suites[idx++] = CIPHER_BYTE;
         suites->suites[idx++] = TLS_SM4_GCM_SM3;
     }
 #endif
 
 #ifdef BUILD_TLS_SM4_CCM_SM3
-    if (tls1_3) {
+    if (tls1_3 && !dtls) {
         suites->suites[idx++] = CIPHER_BYTE;
         suites->suites[idx++] = TLS_SM4_CCM_SM3;
     }
@@ -4647,21 +4650,22 @@ void InitSuites(Suites* suites, ProtocolVersion pv, int keySz, word16 haveRSA,
 #endif /* BUILD_TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256 */
 
 #ifdef BUILD_TLS_ECDHE_ECDSA_WITH_SM4_CBC_SM3
-    if (tls && haveECC) {
+    /* RFC 8998 registers the SM4 cipher suites with DTLS-OK: No. */
+    if (tls && !dtls && haveECC) {
         suites->suites[idx++] = SM_BYTE;
         suites->suites[idx++] = TLS_ECDHE_ECDSA_WITH_SM4_CBC_SM3;
     }
 #endif
 
 #ifdef BUILD_TLS_ECDHE_ECDSA_WITH_SM4_GCM_SM3
-    if (tls && haveECC) {
+    if (tls && !dtls && haveECC) {
         suites->suites[idx++] = SM_BYTE;
         suites->suites[idx++] = TLS_ECDHE_ECDSA_WITH_SM4_GCM_SM3;
     }
 #endif
 
 #ifdef BUILD_TLS_ECDHE_ECDSA_WITH_SM4_CCM_SM3
-    if (tls && haveECC) {
+    if (tls && !dtls && haveECC) {
         suites->suites[idx++] = SM_BYTE;
         suites->suites[idx++] = TLS_ECDHE_ECDSA_WITH_SM4_CCM_SM3;
     }
@@ -37283,6 +37287,22 @@ static int AddPSKtoPreMasterSecret(WOLFSSL* ssl)
         first   = suites->suites[idx];
         second  = suites->suites[idx+1];
 
+#ifdef WOLFSSL_DTLS
+        /* RFC 8998 registers the SM4 cipher suites with DTLS-OK: No.
+         * RFC 9147 Section 4.2.3 forbids using non-AES / non-ChaCha20 ciphers
+         * over DTLS without a defined record-number-mask construction, which
+         * RFC 8998 does not provide. */
+        if (ssl->options.dtls) {
+            if ((first == CIPHER_BYTE && (second == TLS_SM4_GCM_SM3 ||
+                                          second == TLS_SM4_CCM_SM3)) ||
+                first == SM_BYTE) {
+                WOLFSSL_MSG("SM cipher suite not allowed over DTLS "
+                            "(RFC 8998)");
+                return 0;
+            }
+        }
+#endif /* WOLFSSL_DTLS */
+
 #ifdef WOLFSSL_TLS13
         /* When negotiating TLS 1.3, reject non-TLS 1.3 cipher suites */
         if (IsAtLeastTLSv1_3(ssl->version) &&

tests/api/test_tls.c

@@ -31,6 +31,9 @@
 #include <tests/utils.h>
 #include <tests/api/test_tls.h>
 #include <wolfssl/internal.h>
+#if defined(WOLFSSL_SM2) && defined(WOLFSSL_SM3) && defined(WOLFSSL_SM4)
+#include <wolfssl/certs_test_sm.h>
+#endif
 
 
 int test_utils_memio_move_message(void)
@@ -1184,12 +1187,34 @@ int test_record_size_matches_build_message(void)
             struct test_memio_ctx test_ctx;
             const char* name = allCiphers[i].name;
             int isTls13Cipher = (XSTRSTR(name, "TLS13-") != NULL);
+            int isSmCipher = (XSTRSTR(name, "SM4") != NULL);
             int handshakeRet;
 
             XMEMSET(&test_ctx, 0, sizeof(test_ctx));
 
-            ExpectIntEQ(test_memio_setup(&test_ctx, &ctx_c, &ctx_s, &ssl_c,
-                    &ssl_s, versions[v].client, versions[v].server), 0);
+            if (isSmCipher) {
+#if defined(WOLFSSL_SM2) && defined(WOLFSSL_SM3) && defined(WOLFSSL_SM4)
+                ExpectIntEQ(test_memio_setup_ex(&test_ctx, &ctx_c, &ctx_s,
+                        &ssl_c, &ssl_s, versions[v].client, versions[v].server,
+                        (byte*)ca_sm2_der, (int)sizeof_ca_sm2_der,
+                        (byte*)server_sm2_der, (int)sizeof_server_sm2_der,
+                        (byte*)server_sm2_priv_der,
+                        (int)sizeof_server_sm2_priv_der), 0);
+                if (versions[v].is_tls13) {
+                    ExpectIntEQ(wolfSSL_UseKeyShare(ssl_c,
+                            WOLFSSL_ECC_SM2P256V1), 1);
+                }
+#else
+                fprintf(stderr,
+                        "  [SKIP %-12s %-40s] SM build not enabled\n",
+                        versions[v].label, name);
+                goto next_iter;
+#endif
+            }
+            else {
+                ExpectIntEQ(test_memio_setup(&test_ctx, &ctx_c, &ctx_s, &ssl_c,
+                        &ssl_s, versions[v].client, versions[v].server), 0);
+            }
 
             /* Skip ciphers that aren't valid for this version/build. */
             if (wolfSSL_set_cipher_list(ssl_c, name) != 1 ||
@@ -1200,6 +1225,14 @@ int test_record_size_matches_build_message(void)
                 goto next_iter;
             }
 
+            if (isSmCipher &&
+                    XSTRNCMP(versions[v].label, "DTLS", 4) == 0) {
+                fprintf(stderr,
+                        "  [SKIP %-12s %-40s] RFC 8998 forbids SM suites over DTLS\n",
+                        versions[v].label, name);
+                goto next_iter;
+            }
+
 #ifdef WOLFSSL_DTLS_CID
             if (versions[v].use_cid) {
                 unsigned char cid_c[] = { 0, 1, 2, 3 };
@@ -1225,7 +1258,7 @@ int test_record_size_matches_build_message(void)
                     expected = 1;
                     reason = "version mismatch";
                 }
-                else if (XSTRSTR(name, "ECDSA") != NULL) {
+                else if (XSTRSTR(name, "ECDSA") != NULL && !isSmCipher) {
                     expected = 1;
                     reason = "no ECDSA cert";
                 }