Troubleshooting TLS Handshake Failure after OpenSSL and OpenLDAP Upgrade

I have encountered an issue while upgrading OpenSSL and OpenLDAP on our Windows 2019 server.

The upgrade involved transitioning from OpenSSL version 1.1.1 to version 3.0.10 and OpenLDAP version 2.4.47 to version 2.5.16. However, I have encountered a problem with the TLS Handshake when attempting to connect the client with the LDAP server. It appears that the client certificates are not being picked up by the LDAP server.

To diagnose the issue, I created a console application and conducted the following scenarios:

Using the command prompt, I successfully connected the upgraded OpenSSL with the upgraded LDAP server by setting the client certificate as an environment variable. The client was able to establish a connection with the server and receive responses without encountering any errors.

Similarly, using the command prompt, I attempted to connect the upgraded LDAP server with the client “ldapsearch.exe” by setting the same client certificate as an environment variable. Once again, the client was able to establish a connection with the server and receive responses without any errors.

The TLS Handshake fails when the client attempts to connect with the LDAP server through our application.

Given Below the code snippet that produces the TLS Handshake failure.

Test1.cpp : Defines the entry point for the console application.

#include “stdafx.h”
#include “windows.h”
#include “winldap.h”
#include “stdio.h”
#include “wincrypt.h”
#include <openssl/ssl.h>
#include <openssl/err.h>
//#include “ldap.h”

#pragma comment(lib, “wldap32.lib”)
#pragma comment(lib, “crypt32.lib”)
#pragma comment(lib, “D:/Test1/libssl.lib”) //lib built using openssl3.0.10
#pragma comment(lib, “D:/Test1/libcrypto.lib”) //lib built using openssl3.0.10

//The following LDAP TLS options are mentioned in ldap.h

#define LDAP_OPT_X_TLS_REQUIRE_CERT 0x6006
#define LDAP_OPT_X_TLS_HARD 1
#define LDAP_OPT_X_TLS_ALLOW 3
#define LDAP_OPT_X_TLS_CACERTFILE 0x6002
#define LDAP_OPT_X_TLS_CACERTDIR 0x6003
#define LDAP_OPT_X_TLS_CERTFILE 0x6004
#define LDAP_OPT_X_TLS_KEYFILE 0x6005
#define LDAP_OPT_X_TLS_CIPHER 0x6014
#define LDAP_OPT_X_TLS_PROTOCOL_TLS1_3 ((3 << 8) + 4)

const size_t newsize = 100;
// Entry point for application

int main(int argc, char* argv)

{
PWCHAR hostName = NULL;
LDAP* pLdapConnection = NULL;
ULONG version = LDAP_VERSION3;
ULONG getOptSuccess = 0;
ULONG connectSuccess = 0;
INT returnCode = 0;

            PCCERT_CONTEXT cert_ctx = NULL;
            ULONG lv;
            const char *cert_path = "D:/Test1/c_usr";
            const char *cert_file = "c_usr.crt"; // Path to the client certificate file
            const char *key_file = "c_usr.key.pem"; // Path to the private key file associated with the client certificate
            const char *CA_file = "root_ca.crt";
            const char *cipher_tls = "TLS_AES_256_GCM_SHA384";

//  Pass the hostname.

if (argc > 1)
{
    //  Convert argv[] to a wchar_t*
    size_t origsize = strlen(argv[1]) + 1;
    size_t convertedChars = 0;
    wchar_t wcstring[newsize];
    mbstowcs_s(&convertedChars, wcstring, origsize, argv[1], _TRUNCATE);
    wcscat_s(wcstring, L" (wchar_t *)");
    hostName = wcstring;
}
else
{
    hostName = NULL;
}
//  Initialize a session. LDAP_PORT is the default port, 389.
           pLdapConnection=ldap_sslinit(hostName, 636, 1);
if (pLdapConnection == NULL)
{
    //  Set the HRESULT based on the Windows error code.
    char hr = HRESULT_FROM_WIN32(GetLastError());
    printf( "ldap_init failed with 0x%x.\n",hr);
    goto error_exit;
}

else
printf(“ldap_init succeeded \n”);
// Set the version to 3.0 .
returnCode = ldap_set_option(pLdapConnection,
LDAP_OPT_PROTOCOL_VERSION,
(void*)&version);
if(returnCode == LDAP_SUCCESS)
printf(“ldap_set_option succeeded - version set to 3\n”);
else
{
printf(“SetOption Error:%0X\n”, returnCode);
goto error_exit;
}
returnCode = ldap_set_option(pLdapConnection,
LDAP_OPT_X_TLS_CACERTDIR,
(void*)cert_path);
if(returnCode == LDAP_SUCCESS)
printf(“ldap_set_option succeeded - certificate path\n”);
else
{
printf(“SetOption Error:%0X\n”, returnCode);
goto error_exit;
}

// Set TLS/SSL options
 returnCode = ldap_set_option(pLdapConnection, LDAP_OPT_X_TLS_CIPHER, cipher_tls);
            if(returnCode == LDAP_SUCCESS)
    printf("ldap_set_option succeeded - cipher set to TLS_AES_256_GCM_SHA384\n");
 else
 {
    printf("SetOption Error:%0X\n", returnCode);
    goto error_exit;
}

returnCode = ldap_set_option(pLdapConnection, LDAP_OPT_X_TLS_REQUIRE_CERT, (void *)LDAP_OPT_X_TLS_ALLOW); // Require valid server certificate
            if(returnCode == LDAP_SUCCESS)
    printf("ldap_set_option succeeded - LDAP_OPT_X_TLS_ALLOW \n");
 else
 {
    printf("SetOption Error:%0X\n", returnCode);
    goto error_exit;
}

returnCode = ldap_set_option(pLdapConnection, LDAP_OPT_X_TLS_CERTFILE, cert_file); // Set client certificate file
            if(returnCode == LDAP_SUCCESS)
    printf("ldap_set_option succeeded - client certificate\n");
 else
 {
    printf("SetOption Error:%0X\n", returnCode);
    goto error_exit;
}

returnCode = ldap_set_option(pLdapConnection, LDAP_OPT_X_TLS_KEYFILE, key_file); // Set private key file
            if(returnCode == LDAP_SUCCESS)
    printf("ldap_set_option succeeded - certificate key file\n");
 else
 {
    printf("SetOption Error:%0X\n", returnCode);
    goto error_exit;
}

            returnCode = ldap_set_option(pLdapConnection, LDAP_OPT_X_TLS_CACERTFILE, CA_file); // Set client certificate file
            if(returnCode == LDAP_SUCCESS)
    printf("ldap_set_option succeeded - CA file \n");
 else
 {
    printf("SetOption Error:%0X\n", returnCode);
    goto error_exit;
}

 connectSuccess = ldap_connect(pLdapConnection, NULL);
if(connectSuccess == LDAP_SUCCESS)
    printf("ldap_connect succeeded \n");
else
{
    printf("ldap_connect failed with 0x%x.\n",connectSuccess);
    goto error_exit;
}


//  Bind with current credentials (login credentials). Be
//  aware that the password itself is never sent over the

// network, and encryption is not used.

printf("Binding ...\n");
returnCode = ldap_bind_s(pLdapConnection, NULL, NULL,
                         LDAP_AUTH_NEGOTIATE);
if (returnCode == LDAP_SUCCESS)
    printf("The bind was successful");
else
    goto error_exit;
//  Normal cleanup and exit.

ldap_unbind(pLdapConnection);
return 0;
//  On error cleanup and exit.
error_exit:
    ldap_unbind(pLdapConnection);
    return -1;

}

This topic was automatically closed 31 days after the last reply. New replies are no longer allowed.