[PATCH v1 3/3] Fetch the channel binding digest explicitly with OpenSSL 3.0 and later - Mailing list pgsql-hackers

be_tls_get_certificate_hash() and pgtls_get_peer_certificate_hash()
computed the tls-server-end-point channel binding hash (RFC 5929) using
the implicit EVP_sha256()/EVP_get_digestbynid() digests.

On OpenSSL 3.0 and newer, select the digest by name and fetch it with
EVP_MD_fetch() so the certificate hash is computed by the active provider.
The fetched EVP_MD is freed on every path, including the error paths: it
is not tracked by a resource owner, and the backend raises errors with
elog(ERROR), which does not return.  The implicit path is retained for
older OpenSSL and for LibreSSL.
---
 src/backend/libpq/be-secure-openssl.c    | 73 +++++++++++++++-----
 src/interfaces/libpq/fe-secure-openssl.c | 86 ++++++++++++++++++------
 2 files changed, 125 insertions(+), 34 deletions(-)

diff --git a/src/backend/libpq/be-secure-openssl.c b/src/backend/libpq/be-secure-openssl.c
index 6a99a3d7f9..b50d733493 100644
--- a/src/backend/libpq/be-secure-openssl.c
+++ b/src/backend/libpq/be-secure-openssl.c
@@ -2270,7 +2270,6 @@ be_tls_get_certificate_hash(Port *port, size_t *len)
 {
     X509       *server_cert;
     char       *cert_hash;
-    const EVP_MD *algo_type = NULL;
     unsigned char hash[EVP_MAX_MD_SIZE];    /* size for SHA-512 */
     unsigned int hash_size;
     int            algo_nid;
@@ -2299,23 +2298,67 @@ be_tls_get_certificate_hash(Port *port, size_t *len)
      * (https://tools.ietf.org/html/rfc5929#section-4.1).  If something else
      * is used, the same hash as the signature algorithm is used.
      */
-    switch (algo_nid)
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+
+    /*
+     * On OpenSSL 3.0 and newer, explicitly fetch the digest by name so that it
+     * is served by the loaded provider.  The fetched EVP_MD must be freed, so
+     * do so on every path, including the error paths (elog(ERROR) does not
+     * return and the fetched object is not tracked by a resource owner).
+     */
     {
-        case NID_md5:
-        case NID_sha1:
-            algo_type = EVP_sha256();
-            break;
-        default:
-            algo_type = EVP_get_digestbynid(algo_nid);
-            if (algo_type == NULL)
-                elog(ERROR, "could not find digest for NID %s",
-                     OBJ_nid2sn(algo_nid));
-            break;
+        EVP_MD       *algo_type;
+        const char *algo_name;
+
+        switch (algo_nid)
+        {
+            case NID_md5:
+            case NID_sha1:
+                algo_name = "SHA256";
+                break;
+            default:
+                algo_name = OBJ_nid2sn(algo_nid);
+                if (algo_name == NULL)
+                    elog(ERROR, "could not determine digest for server certificate signature algorithm");
+                break;
+        }
+
+        algo_type = EVP_MD_fetch(NULL, algo_name, NULL);
+        if (algo_type == NULL)
+            elog(ERROR, "could not load digest \"%s\"", algo_name);
+
+        /* generate and save the certificate hash */
+        if (!X509_digest(server_cert, algo_type, hash, &hash_size))
+        {
+            EVP_MD_free(algo_type);
+            elog(ERROR, "could not generate server certificate hash");
+        }
+
+        EVP_MD_free(algo_type);
     }
+#else
+    {
+        const EVP_MD *algo_type = NULL;
 
-    /* generate and save the certificate hash */
-    if (!X509_digest(server_cert, algo_type, hash, &hash_size))
-        elog(ERROR, "could not generate server certificate hash");
+        switch (algo_nid)
+        {
+            case NID_md5:
+            case NID_sha1:
+                algo_type = EVP_sha256();
+                break;
+            default:
+                algo_type = EVP_get_digestbynid(algo_nid);
+                if (algo_type == NULL)
+                    elog(ERROR, "could not find digest for NID %s",
+                         OBJ_nid2sn(algo_nid));
+                break;
+        }
+
+        /* generate and save the certificate hash */
+        if (!X509_digest(server_cert, algo_type, hash, &hash_size))
+            elog(ERROR, "could not generate server certificate hash");
+    }
+#endif
 
     cert_hash = palloc(hash_size);
     memcpy(cert_hash, hash, hash_size);
diff --git a/src/interfaces/libpq/fe-secure-openssl.c b/src/interfaces/libpq/fe-secure-openssl.c
index 3e9b87940b..dc65a87ee7 100644
--- a/src/interfaces/libpq/fe-secure-openssl.c
+++ b/src/interfaces/libpq/fe-secure-openssl.c
@@ -371,7 +371,6 @@ char *
 pgtls_get_peer_certificate_hash(PGconn *conn, size_t *len)
 {
     X509       *peer_cert;
-    const EVP_MD *algo_type;
     unsigned char hash[EVP_MAX_MD_SIZE];    /* size for SHA-512 */
     unsigned int hash_size;
     int            algo_nid;
@@ -406,28 +405,77 @@ pgtls_get_peer_certificate_hash(PGconn *conn, size_t *len)
      * (https://tools.ietf.org/html/rfc5929#section-4.1).  If something else
      * is used, the same hash as the signature algorithm is used.
      */
-    switch (algo_nid)
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+
+    /*
+     * On OpenSSL 3.0 and newer, explicitly fetch the digest by name so that it
+     * is served by the loaded provider.  The fetched EVP_MD must be freed on
+     * every path, including the error paths.
+     */
     {
-        case NID_md5:
-        case NID_sha1:
-            algo_type = EVP_sha256();
-            break;
-        default:
-            algo_type = EVP_get_digestbynid(algo_nid);
-            if (algo_type == NULL)
-            {
-                libpq_append_conn_error(conn, "could not find digest for NID %s",
-                                        OBJ_nid2sn(algo_nid));
-                return NULL;
-            }
-            break;
-    }
+        EVP_MD       *algo_type;
+        const char *algo_name;
+
+        switch (algo_nid)
+        {
+            case NID_md5:
+            case NID_sha1:
+                algo_name = "SHA256";
+                break;
+            default:
+                algo_name = OBJ_nid2sn(algo_nid);
+                if (algo_name == NULL)
+                {
+                    libpq_append_conn_error(conn, "could not determine digest for server certificate signature
algorithm");
+                    return NULL;
+                }
+                break;
+        }
+
+        algo_type = EVP_MD_fetch(NULL, algo_name, NULL);
+        if (algo_type == NULL)
+        {
+            libpq_append_conn_error(conn, "could not load digest \"%s\"", algo_name);
+            return NULL;
+        }
 
-    if (!X509_digest(peer_cert, algo_type, hash, &hash_size))
+        if (!X509_digest(peer_cert, algo_type, hash, &hash_size))
+        {
+            EVP_MD_free(algo_type);
+            libpq_append_conn_error(conn, "could not generate peer certificate hash");
+            return NULL;
+        }
+
+        EVP_MD_free(algo_type);
+    }
+#else
     {
-        libpq_append_conn_error(conn, "could not generate peer certificate hash");
-        return NULL;
+        const EVP_MD *algo_type;
+
+        switch (algo_nid)
+        {
+            case NID_md5:
+            case NID_sha1:
+                algo_type = EVP_sha256();
+                break;
+            default:
+                algo_type = EVP_get_digestbynid(algo_nid);
+                if (algo_type == NULL)
+                {
+                    libpq_append_conn_error(conn, "could not find digest for NID %s",
+                                            OBJ_nid2sn(algo_nid));
+                    return NULL;
+                }
+                break;
+        }
+
+        if (!X509_digest(peer_cert, algo_type, hash, &hash_size))
+        {
+            libpq_append_conn_error(conn, "could not generate peer certificate hash");
+            return NULL;
+        }
     }
+#endif
 
     /* save result */
     cert_hash = malloc(hash_size);
-- 
2.43.0