Skip to content

Commit a36f908

Browse files
dgarskedgarske
committed
Fix for compat wolfSSL_RSA_sign and wolfSSL_RSA_verify to support RSA PSS with custom salt and mgf1 hash type. Adds compat API's for i2d_PrivateKey_bio , BN_ucmp and X509v3_get_ext_by_NID. ZD 20059
1 parent 388eea3 commit a36f908

14 files changed

Lines changed: 563 additions & 131 deletions

File tree

‎src/pk.c‎

Lines changed: 137 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -3534,9 +3534,42 @@ int wolfSSL_RSA_generate_key_ex(WOLFSSL_RSA* rsa, int bits, WOLFSSL_BIGNUM* e,
35343534
* RSA padding APIs
35353535
*/
35363536

3537-
#if defined(WC_RSA_PSS) && (defined(OPENSSL_ALL) || defined(WOLFSSL_ASIO) || \
3538-
defined(WOLFSSL_HAPROXY) || defined(WOLFSSL_NGINX))
3539-
#if !defined(HAVE_FIPS) || FIPS_VERSION_GT(2,0)
3537+
#ifdef WC_RSA_PSS
3538+
3539+
#if defined(OPENSSL_EXTRA) && !defined(HAVE_SELFTEST)
3540+
static int rsa_pss_calc_salt(int saltLen, int hashLen, int emLen)
3541+
{
3542+
/* Calculate the salt length to use for special cases. */
3543+
switch (saltLen) {
3544+
/* Negative saltLen values are treated differently. */
3545+
case WC_RSA_PSS_SALTLEN_DIGEST:
3546+
saltLen = hashLen;
3547+
break;
3548+
case WC_RSA_PSS_SALTLEN_MAX_SIGN:
3549+
case WC_RSA_PSS_SALTLEN_MAX:
3550+
#ifdef WOLFSSL_PSS_LONG_SALT
3551+
saltLen = emLen - hashLen - 2;
3552+
#else
3553+
saltLen = hashLen;
3554+
(void)emLen;
3555+
#endif
3556+
break;
3557+
default:
3558+
break;
3559+
}
3560+
if (saltLen < 0) {
3561+
/* log invalid salt, let wolfCrypt handle error */
3562+
WOLFSSL_ERROR_MSG("invalid saltLen");
3563+
saltLen = -3; /* for wolfCrypt to produce error must be < -2 */
3564+
}
3565+
return saltLen;
3566+
}
3567+
#endif /* OPENSSL_EXTRA && !HAVE_SELFTEST */
3568+
3569+
#if (defined(OPENSSL_ALL) || defined(WOLFSSL_ASIO) || \
3570+
defined(WOLFSSL_HAPROXY) || defined(WOLFSSL_NGINX)) && \
3571+
(!defined(HAVE_FIPS) || FIPS_VERSION_GT(2,0))
3572+
35403573
/* Add PKCS#1 PSS padding to hash.
35413574
*
35423575
*
@@ -3654,28 +3687,7 @@ int wolfSSL_RSA_padding_add_PKCS1_PSS_mgf1(WOLFSSL_RSA *rsa, unsigned char *em,
36543687
}
36553688

36563689
if (ret == 1) {
3657-
/* Calculate the salt length to use for special cases. */
3658-
/* TODO: use special case wolfCrypt values? */
3659-
switch (saltLen) {
3660-
/* Negative saltLen values are treated differently. */
3661-
case RSA_PSS_SALTLEN_DIGEST:
3662-
saltLen = hashLen;
3663-
break;
3664-
case RSA_PSS_SALTLEN_MAX_SIGN:
3665-
case RSA_PSS_SALTLEN_MAX:
3666-
#ifdef WOLFSSL_PSS_LONG_SALT
3667-
saltLen = emLen - hashLen - 2;
3668-
#else
3669-
saltLen = hashLen;
3670-
#endif
3671-
break;
3672-
default:
3673-
if (saltLen < 0) {
3674-
/* No other negative values implemented. */
3675-
WOLFSSL_ERROR_MSG("invalid saltLen");
3676-
ret = 0;
3677-
}
3678-
}
3690+
saltLen = rsa_pss_calc_salt(saltLen, hashLen, emLen);
36793691
}
36803692

36813693
if (ret == 1) {
@@ -3767,31 +3779,7 @@ int wolfSSL_RSA_verify_PKCS1_PSS_mgf1(WOLFSSL_RSA *rsa,
37673779
}
37683780

37693781
if (ret == 1) {
3770-
/* Calculate the salt length to use for special cases. */
3771-
switch (saltLen) {
3772-
/* Negative saltLen values are treated differently */
3773-
case RSA_PSS_SALTLEN_DIGEST:
3774-
saltLen = hashLen;
3775-
break;
3776-
case RSA_PSS_SALTLEN_AUTO:
3777-
#ifdef WOLFSSL_PSS_SALT_LEN_DISCOVER
3778-
saltLen = RSA_PSS_SALT_LEN_DISCOVER;
3779-
break;
3780-
#endif
3781-
case RSA_PSS_SALTLEN_MAX:
3782-
#ifdef WOLFSSL_PSS_LONG_SALT
3783-
saltLen = emLen - hashLen - 2;
3784-
#else
3785-
saltLen = hashLen;
3786-
#endif
3787-
break;
3788-
default:
3789-
if (saltLen < 0) {
3790-
/* No other negative values implemented. */
3791-
WOLFSSL_ERROR_MSG("invalid saltLen");
3792-
ret = 0;
3793-
}
3794-
}
3782+
saltLen = rsa_pss_calc_salt(saltLen, hashLen, emLen);
37953783
}
37963784

37973785
if (ret == 1) {
@@ -3856,18 +3844,23 @@ int wolfSSL_RSA_verify_PKCS1_PSS(WOLFSSL_RSA *rsa, const unsigned char *mHash,
38563844
return wolfSSL_RSA_verify_PKCS1_PSS_mgf1(rsa, mHash, hashAlg, NULL, em,
38573845
saltLen);
38583846
}
3859-
#endif /* !HAVE_FIPS || FIPS_VERSION_GT(2,0) */
3860-
#endif /* WC_RSA_PSS && (OPENSSL_ALL || WOLFSSL_ASIO || WOLFSSL_HAPROXY ||
3861-
* WOLFSSL_NGINX) */
3847+
#endif /* (!HAVE_FIPS || FIPS_VERSION_GT(2,0)) && \
3848+
(OPENSSL_ALL || WOLFSSL_ASIO || WOLFSSL_HAPROXY || WOLFSSL_NGINX) */
3849+
#endif /* WC_RSA_PSS */
38623850

38633851
/*
38643852
* RSA sign/verify APIs
38653853
*/
38663854

3867-
#ifndef WOLFSSL_PSS_SALT_LEN_DISCOVER
3868-
#define DEF_PSS_SALT_LEN RSA_PSS_SALT_LEN_DEFAULT
3855+
#if defined(WC_RSA_PSS) && !defined(HAVE_SELFTEST) && \
3856+
(!defined(HAVE_FIPS) || FIPS_VERSION_GE(5,1))
3857+
#ifndef WOLFSSL_PSS_SALT_LEN_DISCOVER
3858+
#define DEF_PSS_SALT_LEN RSA_PSS_SALT_LEN_DEFAULT
3859+
#else
3860+
#define DEF_PSS_SALT_LEN RSA_PSS_SALT_LEN_DISCOVER
3861+
#endif
38693862
#else
3870-
#define DEF_PSS_SALT_LEN RSA_PSS_SALT_LEN_DISCOVER
3863+
#define DEF_PSS_SALT_LEN 0 /* not used */
38713864
#endif
38723865

38733866
#if defined(OPENSSL_EXTRA)
@@ -3980,6 +3973,14 @@ int wolfSSL_RSA_sign_ex(int hashAlg, const unsigned char* hash,
39803973
return ret;
39813974
}
39823975

3976+
int wolfSSL_RSA_sign_generic_padding(int hashAlg, const unsigned char* hash,
3977+
unsigned int hLen, unsigned char* sigRet, unsigned int* sigLen,
3978+
WOLFSSL_RSA* rsa, int flag, int padding)
3979+
{
3980+
return wolfSSL_RSA_sign_mgf(hashAlg, hash, hLen, sigRet, sigLen, rsa, flag,
3981+
padding, hashAlg, DEF_PSS_SALT_LEN);
3982+
}
3983+
39833984
/**
39843985
* Sign a message hash with the chosen message digest, padding, and RSA key.
39853986
*
@@ -3998,12 +3999,14 @@ int wolfSSL_RSA_sign_ex(int hashAlg, const unsigned char* hash,
39983999
* @param [in] padding Padding to use. Only RSA_PKCS1_PSS_PADDING and
39994000
* WC_RSA_PKCS1_PADDING are currently supported for
40004001
* signing.
4002+
* @param [in] mgf1Hash MGF1 Hash NID
4003+
* @param [in] saltLen Length of RSA PSS salt
40014004
* @return 1 on success.
40024005
* @return 0 on failure.
40034006
*/
4004-
int wolfSSL_RSA_sign_generic_padding(int hashAlg, const unsigned char* hash,
4007+
int wolfSSL_RSA_sign_mgf(int hashAlg, const unsigned char* hash,
40054008
unsigned int hLen, unsigned char* sigRet, unsigned int* sigLen,
4006-
WOLFSSL_RSA* rsa, int flag, int padding)
4009+
WOLFSSL_RSA* rsa, int flag, int padding, int mgf1Hash, int saltLen)
40074010
{
40084011
int ret = 1;
40094012
word32 outLen = 0;
@@ -4020,8 +4023,7 @@ int wolfSSL_RSA_sign_generic_padding(int hashAlg, const unsigned char* hash,
40204023
#endif
40214024
unsigned int encSz = 0;
40224025

4023-
4024-
WOLFSSL_ENTER("wolfSSL_RSA_sign_generic_padding");
4026+
WOLFSSL_ENTER("wolfSSL_RSA_sign_mgf");
40254027

40264028
if (flag == 0) {
40274029
/* Only encode message. */
@@ -4088,7 +4090,7 @@ int wolfSSL_RSA_sign_generic_padding(int hashAlg, const unsigned char* hash,
40884090
case WC_RSA_NO_PAD:
40894091
if ((signSz = wc_RsaDirect(encodedSig, encSz, sigRet, &outLen,
40904092
(RsaKey*)rsa->internal, RSA_PRIVATE_ENCRYPT, rng)) <= 0) {
4091-
WOLFSSL_ERROR_MSG("Bad Rsa Sign no pad");
4093+
WOLFSSL_ERROR_MSG("Bad RSA Sign no pad");
40924094
ret = 0;
40934095
}
40944096
break;
@@ -4097,17 +4099,20 @@ int wolfSSL_RSA_sign_generic_padding(int hashAlg, const unsigned char* hash,
40974099
(!defined(HAVE_FIPS) || FIPS_VERSION_GE(5,1))
40984100
case WC_RSA_PKCS1_PSS_PADDING:
40994101
{
4100-
enum wc_HashType hType =
4101-
wc_OidGetHash((int)nid2oid(hashAlg, oidHashType));
4102-
#ifndef WOLFSSL_PSS_SALT_LEN_DISCOVER
4103-
WOLFSSL_MSG("Using RSA-PSS with hash length salt. "
4104-
"OpenSSL uses max length by default.");
4105-
#endif
4102+
RsaKey* key = (RsaKey*)rsa->internal;
4103+
enum wc_HashType mgf1, hType;
4104+
hType = wc_OidGetHash((int)nid2oid(hashAlg, oidHashType));
4105+
if (mgf1Hash == WC_NID_undef)
4106+
mgf1Hash = hashAlg;
4107+
mgf1 = wc_OidGetHash((int)nid2oid(mgf1Hash, oidHashType));
4108+
/* handle compat layer salt special cases */
4109+
saltLen = rsa_pss_calc_salt(saltLen, wc_HashGetDigestSize(hType),
4110+
wolfSSL_RSA_size(rsa));
4111+
41064112
/* Create RSA PSS signature. */
41074113
if ((signSz = wc_RsaPSS_Sign_ex(encodedSig, encSz, sigRet, outLen,
4108-
hType, wc_hash2mgf(hType), DEF_PSS_SALT_LEN,
4109-
(RsaKey*)rsa->internal, rng)) <= 0) {
4110-
WOLFSSL_ERROR_MSG("Bad Rsa Sign");
4114+
hType, wc_hash2mgf(mgf1), saltLen, key, rng)) <= 0) {
4115+
WOLFSSL_ERROR_MSG("Bad RSA PSS Sign");
41114116
ret = 0;
41124117
}
41134118
break;
@@ -4126,13 +4131,15 @@ int wolfSSL_RSA_sign_generic_padding(int hashAlg, const unsigned char* hash,
41264131
/* Sign (private encrypt) PKCS#1 encoded signature. */
41274132
if ((signSz = wc_RsaSSL_Sign(encodedSig, encSz, sigRet, outLen,
41284133
(RsaKey*)rsa->internal, rng)) <= 0) {
4129-
WOLFSSL_ERROR_MSG("Bad Rsa Sign");
4134+
WOLFSSL_ERROR_MSG("Bad PKCS1 RSA Sign");
41304135
ret = 0;
41314136
}
41324137
break;
41334138
}
41344139
default:
41354140
WOLFSSL_ERROR_MSG("Unsupported padding");
4141+
(void)mgf1Hash;
4142+
(void)saltLen;
41364143
ret = 0;
41374144
break;
41384145
}
@@ -4153,7 +4160,7 @@ int wolfSSL_RSA_sign_generic_padding(int hashAlg, const unsigned char* hash,
41534160
XFREE(encodedSig, NULL, DYNAMIC_TYPE_SIGNATURE);
41544161
#endif
41554162

4156-
WOLFSSL_LEAVE("wolfSSL_RSA_sign_generic_padding", ret);
4163+
WOLFSSL_LEAVE("wolfSSL_RSA_sign_mgf", ret);
41574164
return ret;
41584165
}
41594166

@@ -4177,6 +4184,14 @@ int wolfSSL_RSA_verify(int hashAlg, const unsigned char* hash,
41774184
WC_RSA_PKCS1_PADDING);
41784185
}
41794186

4187+
int wolfSSL_RSA_verify_ex(int hashAlg, const unsigned char* hash,
4188+
unsigned int hLen, const unsigned char* sig, unsigned int sigLen,
4189+
WOLFSSL_RSA* rsa, int padding)
4190+
{
4191+
return wolfSSL_RSA_verify_mgf(hashAlg, hash, hLen, sig, sigLen, rsa,
4192+
padding, hashAlg, DEF_PSS_SALT_LEN);
4193+
}
4194+
41804195
/**
41814196
* Verify a message hash with the chosen message digest, padding, and RSA key.
41824197
*
@@ -4191,12 +4206,14 @@ int wolfSSL_RSA_verify(int hashAlg, const unsigned char* hash,
41914206
* @param [in] padding Padding to use. Only RSA_PKCS1_PSS_PADDING and
41924207
* WC_RSA_PKCS1_PADDING are currently supported for
41934208
* signing.
4209+
* @param [in] mgf1Hash MGF1 Hash NID
4210+
* @param [in] saltLen Length of RSA PSS salt
41944211
* @return 1 on success.
41954212
* @return 0 on failure.
41964213
*/
4197-
int wolfSSL_RSA_verify_ex(int hashAlg, const unsigned char* hash,
4214+
int wolfSSL_RSA_verify_mgf(int hashAlg, const unsigned char* hash,
41984215
unsigned int hLen, const unsigned char* sig, unsigned int sigLen,
4199-
WOLFSSL_RSA* rsa, int padding)
4216+
WOLFSSL_RSA* rsa, int padding, int mgf1Hash, int saltLen)
42004217
{
42014218
int ret = 1;
42024219
#ifdef WOLFSSL_SMALL_STACK
@@ -4211,7 +4228,7 @@ int wolfSSL_RSA_verify_ex(int hashAlg, const unsigned char* hash,
42114228
enum wc_HashType hType = WC_HASH_TYPE_NONE;
42124229
#endif
42134230

4214-
WOLFSSL_ENTER("wolfSSL_RSA_verify");
4231+
WOLFSSL_ENTER("wolfSSL_RSA_verify_mgf");
42154232

42164233
/* Validate parameters. */
42174234
if ((hash == NULL) || (sig == NULL) || (rsa == NULL)) {
@@ -4228,8 +4245,49 @@ int wolfSSL_RSA_verify_ex(int hashAlg, const unsigned char* hash,
42284245
ret = 0;
42294246
}
42304247
}
4248+
if (ret == 1 && padding == WC_RSA_PKCS1_PSS_PADDING) {
4249+
#if defined(WC_RSA_PSS) && !defined(HAVE_SELFTEST) && \
4250+
(!defined(HAVE_FIPS) || FIPS_VERSION_GE(5,1))
4251+
RsaKey* key = (RsaKey*)rsa->internal;
4252+
enum wc_HashType mgf1;
4253+
hType = wc_OidGetHash((int)nid2oid(hashAlg, oidHashType));
4254+
if (mgf1Hash == WC_NID_undef)
4255+
mgf1Hash = hashAlg;
4256+
mgf1 = wc_OidGetHash((int)nid2oid(mgf1Hash, oidHashType));
4257+
4258+
/* handle compat layer salt special cases */
4259+
saltLen = rsa_pss_calc_salt(saltLen, wc_HashGetDigestSize(hType),
4260+
wolfSSL_RSA_size(rsa));
4261+
4262+
verLen = wc_RsaPSS_Verify_ex((byte*)sig, sigLen, sigDec, sigLen,
4263+
hType, wc_hash2mgf(mgf1), saltLen, key);
4264+
if (verLen > 0) {
4265+
/* Check PSS padding is valid. */
4266+
if (wc_RsaPSS_CheckPadding_ex(hash, hLen, sigDec, (word32)verLen,
4267+
hType, saltLen, mp_count_bits(&key->n)) != 0) {
4268+
WOLFSSL_ERROR_MSG("wc_RsaPSS_CheckPadding_ex error");
4269+
ret = WOLFSSL_FAILURE;
4270+
}
4271+
else {
4272+
/* Success! Free resources and return early */
4273+
XFREE(sigDec, NULL, DYNAMIC_TYPE_TMP_BUFFER);
4274+
return WOLFSSL_SUCCESS;
4275+
}
4276+
}
4277+
else {
4278+
WOLFSSL_ERROR_MSG("wc_RsaPSS_Verify_ex failed!");
4279+
ret = WOLFSSL_FAILURE;
4280+
}
4281+
#else
4282+
(void)mgf1Hash;
4283+
(void)saltLen;
4284+
WOLFSSL_ERROR_MSG("RSA PSS not compiled in!");
4285+
ret = WOLFSSL_FAILURE;
4286+
#endif
4287+
}
4288+
42314289
#ifdef WOLFSSL_SMALL_STACK
4232-
if ((ret == 1) && (padding != WC_RSA_PKCS1_PSS_PADDING)) {
4290+
if (ret == 1) {
42334291
/* Allocate memory for encoded signature. */
42344292
encodedSig = (unsigned char *)XMALLOC(len, NULL,
42354293
DYNAMIC_TYPE_TMP_BUFFER);
@@ -4239,7 +4297,7 @@ int wolfSSL_RSA_verify_ex(int hashAlg, const unsigned char* hash,
42394297
}
42404298
}
42414299
#endif
4242-
if ((ret == 1) && (padding != WC_RSA_PKCS1_PSS_PADDING)) {
4300+
if (ret == 1) {
42434301
/* Make encoded signature to compare with decrypted signature. */
42444302
if (wolfssl_rsa_sig_encode(hashAlg, hash, hLen, encodedSig, &len,
42454303
padding) <= 0) {
@@ -4266,20 +4324,6 @@ int wolfSSL_RSA_verify_ex(int hashAlg, const unsigned char* hash,
42664324
#endif
42674325
}
42684326
if (ret == 1) {
4269-
#if defined(WC_RSA_PSS) && !defined(HAVE_SELFTEST) && \
4270-
(!defined(HAVE_FIPS) || FIPS_VERSION_GE(5, 1))
4271-
if (padding == WC_RSA_PKCS1_PSS_PADDING) {
4272-
/* Check PSS padding is valid. */
4273-
if (wc_RsaPSS_CheckPadding_ex(hash, hLen, sigDec, (word32)verLen,
4274-
hType, DEF_PSS_SALT_LEN,
4275-
mp_count_bits(&((RsaKey*)rsa->internal)->n)) != 0) {
4276-
WOLFSSL_ERROR_MSG("wc_RsaPSS_CheckPadding_ex error");
4277-
ret = 0;
4278-
}
4279-
}
4280-
else
4281-
#endif /* WC_RSA_PSS && !HAVE_SELFTEST && (!HAVE_FIPS ||
4282-
* FIPS_VERSION >= 5.1) */
42834327
/* Compare decrypted signature to encoded signature. */
42844328
if (((int)len != verLen) ||
42854329
(XMEMCMP(encodedSig, sigDec, (size_t)verLen) != 0)) {
@@ -4293,6 +4337,8 @@ int wolfSSL_RSA_verify_ex(int hashAlg, const unsigned char* hash,
42934337
XFREE(encodedSig, NULL, DYNAMIC_TYPE_TMP_BUFFER);
42944338
#endif
42954339
XFREE(sigDec, NULL, DYNAMIC_TYPE_TMP_BUFFER);
4340+
4341+
WOLFSSL_LEAVE("wolfSSL_RSA_verify_mgf", ret);
42964342
return ret;
42974343
}
42984344

0 commit comments

Comments
 (0)