[PATCH v1 2/3] Fetch digests explicitly for cryptohash with OpenSSL 3.0 and later - Mailing list pgsql-hackers

cryptohash_openssl.c initialized the EVP_MD_CTX with the implicit static
digest objects (EVP_sha256() and friends), which do not deterministically
dispatch through a loaded provider.

On OpenSSL 3.0 and newer, fetch the digest by name with EVP_MD_fetch(),
cache it in the context, and free it on teardown, so hashing is served by
the active provider.  The digest type is fixed for the lifetime of the
context, so the fetch is done once.  The implicit path is kept for older
OpenSSL and for LibreSSL.

All of PostgreSQL's authentication hashing (SCRAM, md5) rides pg_cryptohash,
so it follows the active provider automatically.  As a consequence, md5
authentication requires MD5 to be offered by the provider and is therefore
unavailable under FIPS; document that and point to scram-sha-256.
---
 doc/src/sgml/client-auth.sgml   | 11 +++++++
 src/common/cryptohash_openssl.c | 57 +++++++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+)

diff --git a/doc/src/sgml/client-auth.sgml b/doc/src/sgml/client-auth.sgml
index e4e65f8feb..a65894a7a5 100644
--- a/doc/src/sgml/client-auth.sgml
+++ b/doc/src/sgml/client-auth.sgml
@@ -1262,6 +1262,17 @@ omicron         bryanh                  guest1
        attacks.
       </para>
 
+      <para>
+       When <productname>PostgreSQL</productname> is built with
+       <productname>OpenSSL</productname>, <literal>md5</literal> authentication
+       relies on the MD5 implementation supplied by the active
+       <productname>OpenSSL</productname> provider.  MD5 is unavailable when
+       <productname>OpenSSL</productname> is operating in FIPS mode, or with any
+       provider that disables MD5, and <literal>md5</literal> authentication
+       will fail in that configuration; use <literal>scram-sha-256</literal>
+       instead.
+      </para>
+
       <para>
        To ease transition from the <literal>md5</literal> method to the newer
        SCRAM method, if <literal>md5</literal> is specified as a method
diff --git a/src/common/cryptohash_openssl.c b/src/common/cryptohash_openssl.c
index 51b7e04093..6772bcdbda 100644
--- a/src/common/cryptohash_openssl.c
+++ b/src/common/cryptohash_openssl.c
@@ -67,6 +67,9 @@ struct pg_cryptohash_ctx
     const char *errreason;
 
     EVP_MD_CTX *evpctx;
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+    EVP_MD       *algo;
+#endif
 
 #ifndef FRONTEND
     ResourceOwner resowner;
@@ -182,6 +185,56 @@ pg_cryptohash_init(pg_cryptohash_ctx *ctx)
     if (ctx == NULL)
         return -1;
 
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+
+    /*
+     * On OpenSSL 3.0 and newer, explicitly fetch the digest implementation so
+     * that it is served by the loaded provider.  This lets a third-party or
+     * FIPS provider service PostgreSQL's hashing, rather than relying on the
+     * implicit lookup done by EVP_md5()/EVP_sha*().  The fetched EVP_MD is
+     * cached in the context and released in pg_cryptohash_free().
+     */
+    {
+        const char *name = NULL;
+
+        switch (ctx->type)
+        {
+            case PG_MD5:
+                name = "MD5";
+                break;
+            case PG_SHA1:
+                name = "SHA1";
+                break;
+            case PG_SHA224:
+                name = "SHA224";
+                break;
+            case PG_SHA256:
+                name = "SHA256";
+                break;
+            case PG_SHA384:
+                name = "SHA384";
+                break;
+            case PG_SHA512:
+                name = "SHA512";
+                break;
+        }
+
+        /*
+         * ctx->type is fixed for the lifetime of the context, so the digest
+         * only needs to be fetched once; a second pg_cryptohash_init() on the
+         * same context reuses it.  Freeing and re-fetching here would drop the
+         * EVP_MD while the previous EVP_MD_CTX still references it, and would
+         * also repeat the relatively expensive provider lookup needlessly.
+         */
+        if (ctx->algo == NULL && name != NULL)
+            ctx->algo = EVP_MD_fetch(NULL, name, NULL);
+
+        if (ctx->algo != NULL)
+            status = EVP_DigestInit_ex(ctx->evpctx, ctx->algo, NULL);
+        else
+            status = 0;
+    }
+#else
     switch (ctx->type)
     {
         case PG_MD5:
@@ -203,6 +256,7 @@ pg_cryptohash_init(pg_cryptohash_ctx *ctx)
             status = EVP_DigestInit_ex(ctx->evpctx, EVP_sha512(), NULL);
             break;
     }
+#endif
 
     /* OpenSSL internals return 1 on success, 0 on failure */
     if (status <= 0)
@@ -329,6 +383,9 @@ pg_cryptohash_free(pg_cryptohash_ctx *ctx)
         return;
 
     EVP_MD_CTX_destroy(ctx->evpctx);
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+    EVP_MD_free(ctx->algo);
+#endif
 
 #ifndef FRONTEND
     if (ctx->resowner)
-- 
2.43.0