openssl: RSA fixes for openssl-1.1.

In openssl-1.1 the RSA structure has been made opaque, causing
compilation of crypt.c to fail because the code accesses ->n and ->e
directly to set the modulus and the public exponent according to the
values read from the public ssh key.

With openssl-1.1 applications are supposed to call RSA_set0_key()
to set n and e. Unfortunately, this function does not exist in
openssl-1.0.2.

This patch adds a configure check which defines HAVE_RSA_SET0_KEY if
RSA_set0_key() is available. In crypt.c we either call the function
or set ->n and ->e directly, depending on whether HAVE_RSA_SET0_KEY
is defined. This results in code which works on both openssl-1.0.2
and openssl-1.1.0.

diff --git a/configure.ac b/configure.ac
index f9115fd652962881c1e7548c3cbac55e724305df..e502e0a637a283882be8b64cfbe8ea1d03253376 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -116,6 +116,10 @@ LIB_ARG_WITH([openssl], [-lssl -lcrypto])
 AC_CHECK_HEADER(openssl/ssl.h, [], [HAVE_OPENSSL=no])
 AC_CHECK_LIB([crypto], [RAND_bytes], [], [HAVE_OPENSSL=no])
 LIB_SUBST_FLAGS(openssl)
+if test $HAVE_OPENSSL = yes; then
+       AC_CHECK_LIB([crypto], [RSA_set0_key],
+               AC_DEFINE([HAVE_RSA_SET0_KEY], [1], [openssl-1.1]))
+fi
 UNSTASH_FLAGS
 ######################################################################### gcrypt
 STASH_FLAGS

diff --git a/crypt.c b/crypt.c
index 06ae6265010a946bd51a1e64b1c620f0e4c89fc3..e819e8be3063cbe151634dc037d30e623e27357a 100644 (file)
--- a/crypt.c
+++ b/crypt.c
@@ -133,18 +133,25 @@ static int read_rsa_bignums(const unsigned char *blob, int blen, RSA **result)
 {
        int ret;
        RSA *rsa;
+       BIGNUM *n, *e;
        const unsigned char *p = blob, *end = blob + blen;
 
        rsa = RSA_new();
        if (!rsa)
                return -E_BIGNUM;
-       ret = read_bignum(p, end - p, &rsa->e);
+       ret = read_bignum(p, end - p, &e);
        if (ret < 0)
                goto fail;
        p += ret;
-       ret = read_bignum(p, end - p, &rsa->n);
+       ret = read_bignum(p, end - p, &n);
        if (ret < 0)
                goto fail;
+#ifdef HAVE_RSA_SET0_KEY
+       RSA_set0_key(rsa, n, e, NULL);
+#else
+       rsa->n = n;
+       rsa->e = e;
+#endif
        *result = rsa;
        return 1;
 fail: