From f39c0fbca24541c5bcb56ba1de7d3e614b1f22cf Mon Sep 17 00:00:00 2001
From: David Christensen <david.christensen@crunchydata.com>
Date: Tue, 31 Oct 2023 15:21:14 -0400
Subject: [PATCH v3 2/5] Basic key management and cipher support

---
 src/common/Makefile             |   3 +
 src/common/cipher.c             | 138 +++++++++
 src/common/cipher_openssl.c     | 533 ++++++++++++++++++++++++++++++++
 src/common/kmgr_utils.c         | 467 ++++++++++++++++++++++++++++
 src/common/meson.build          |   3 +
 src/include/common/cipher.h     |  98 ++++++
 src/include/common/kmgr_utils.h | 100 ++++++
 7 files changed, 1342 insertions(+)
 create mode 100644 src/common/cipher.c
 create mode 100644 src/common/cipher_openssl.c
 create mode 100644 src/common/kmgr_utils.c
 create mode 100644 src/include/common/cipher.h
 create mode 100644 src/include/common/kmgr_utils.h

diff --git a/src/common/Makefile b/src/common/Makefile
index 70884be00c..14ca7f20f5 100644
--- a/src/common/Makefile
+++ b/src/common/Makefile
@@ -63,6 +63,7 @@ OBJS_COMMON = \
 	ip.o \
 	jsonapi.o \
 	keywords.o \
+	kmgr_utils.o \
 	kwlookup.o \
 	link-canary.o \
 	md5_common.o \
@@ -85,11 +86,13 @@ OBJS_COMMON = \
 
 ifeq ($(with_ssl),openssl)
 OBJS_COMMON += \
+	cipher_openssl.o \
 	cryptohash_openssl.o \
 	hmac_openssl.o \
 	protocol_openssl.o
 else
 OBJS_COMMON += \
+	cipher.o \
 	cryptohash.o \
 	hmac.o \
 	md5.o \
diff --git a/src/common/cipher.c b/src/common/cipher.c
new file mode 100644
index 0000000000..219bc6ea43
--- /dev/null
+++ b/src/common/cipher.c
@@ -0,0 +1,138 @@
+/*-------------------------------------------------------------------------
+ *
+ * cipher.c
+ *	  Shared frontend/backend for cryptographic functions
+ *
+ * This is the set of in-core functions used when there are no other
+ * alternative options like OpenSSL.
+ *
+ * Copyright (c) 2021, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/common/cipher.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef FRONTEND
+#include "postgres.h"
+#else
+#include "postgres_fe.h"
+#endif
+
+#include "common/cipher.h"
+
+static void cipher_failure(void) pg_attribute_noreturn();
+
+
+PgCipherCtx *
+pg_cipher_ctx_create(int cipher, unsigned char *key, int klen, bool enc)
+{
+	cipher_failure();
+	return NULL;				/* keep compiler quiet */
+}
+
+void
+pg_cipher_ctx_free(PgCipherCtx *ctx)
+{
+	cipher_failure();
+}
+
+int
+pg_cipher_blocksize(PgCipherCtx *ctx)
+{
+	cipher_failure();
+	return -1;					/* keep compiler quiet */
+}
+
+bool
+pg_cipher_encrypt(PgCipherCtx *ctx, const int cipher,
+				  const unsigned char *plaintext,
+				  const int inlen, unsigned char *ciphertext, int *outlen,
+				  const unsigned char *iv, const int ivlen,
+				  const unsigned char *aad, const int aadlen,
+				  unsigned char *outtag, const int taglen)
+{
+	cipher_failure();
+	return false;				/* keep compiler quiet */
+}
+
+bool
+pg_cipher_decrypt(PgCipherCtx *ctx, const int cipher,
+				  const unsigned char *ciphertext,
+				  const int inlen, unsigned char *plaintext, int *outlen,
+				  const unsigned char *iv, const int ivlen,
+				  const unsigned char *aad, const int aadlen,
+				  unsigned char *intag, const int taglen)
+{
+	cipher_failure();
+	return false;				/* keep compiler quiet */
+}
+
+/* handle incremental encryption; returns context */
+EncryptionHandle
+pg_cipher_incr_init(PgCipherCtx *ctx, const int cipher,
+					const unsigned char *iv, const int ivlen)
+{
+	cipher_failure();
+	return NULL;				/* keep compiler quiet */
+}
+
+/* add authenticated data; can be added multiple times */
+bool
+pg_cipher_incr_add_authenticated_data(EncryptionHandle incr, const unsigned char *aad, const int aadlen)
+{
+	cipher_failure();
+	return false;				/* keep compiler quiet */
+}
+
+/* add authenticated data; can be added multiple times */
+bool
+pg_cipher_incr_encrypt(EncryptionHandle incr,
+					   const unsigned char *plaintext,
+					   const int inlen,
+					   unsigned char *ciphertext, int *outlen)
+{
+	cipher_failure();
+	return false;				/* keep compiler quiet */
+}
+
+/* finish incremental encryption */
+bool
+pg_cipher_incr_finish(EncryptionHandle incr,
+					  unsigned char *ciphertext, int *outlen,
+					  unsigned char *tag, const int taglen)
+{
+	cipher_failure();
+	return false;				/* keep compiler quiet */
+}
+
+bool
+pg_cipher_keywrap(PgCipherCtx *ctx, const unsigned char *plaintext,
+				  const int inlen, unsigned char *ciphertext, int *outlen)
+{
+	cipher_failure();
+	return false;				/* keep compiler quiet */
+}
+
+bool
+pg_cipher_keyunwrap(PgCipherCtx *ctx, const unsigned char *ciphertext,
+					const int inlen, unsigned char *plaintext, int *outlen)
+{
+	cipher_failure();
+	return false;				/* keep compiler quiet */
+}
+
+static void
+cipher_failure(void)
+{
+#ifndef FRONTEND
+	ereport(ERROR,
+			(errcode(ERRCODE_CONFIG_FILE_ERROR),
+			 (errmsg("cluster file encryption is not supported because OpenSSL is not supported by this build"),
+			  errhint("Compile with --with-openssl to use this feature."))));
+#else
+	fprintf(stderr, _("cluster file encryption is not supported because OpenSSL is not supported by this build"));
+	exit(1);
+#endif
+}
diff --git a/src/common/cipher_openssl.c b/src/common/cipher_openssl.c
new file mode 100644
index 0000000000..d1d4581912
--- /dev/null
+++ b/src/common/cipher_openssl.c
@@ -0,0 +1,533 @@
+/*-------------------------------------------------------------------------
+ * cipher_openssl.c
+ *		Cryptographic function using OpenSSL
+ *
+ * This contains the common low-level functions needed in both frontend and
+ * backend, for implement the database encryption.
+ *
+ * Portions Copyright (c) 2021, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/common/cipher_openssl.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FRONTEND
+#include "postgres.h"
+#else
+#include "postgres_fe.h"
+#endif
+
+#include "common/cipher.h"
+#include <openssl/conf.h>
+#include <openssl/err.h>
+#include <openssl/evp.h>
+#include <openssl/ssl.h>
+
+/*
+ * prototype for the EVP functions that return an algorithm, e.g.
+ * EVP_aes_128_gcm().
+ */
+typedef const EVP_CIPHER *(*ossl_EVP_cipher_func) (void);
+
+static ossl_EVP_cipher_func get_evp_aes_gcm(int klen);
+static EVP_CIPHER_CTX *ossl_cipher_ctx_create(int cipher, unsigned char *key,
+											  int klen, bool enc);
+
+/*
+ * Return a newly created cipher context.  'cipher' specifies cipher algorithm
+ * by identifier like PG_CIPHER_XXX.
+ */
+PgCipherCtx *
+pg_cipher_ctx_create(int cipher, unsigned char *key, int klen, bool enc)
+{
+	PgCipherCtx *ctx = NULL;
+
+	if (cipher > PG_MAX_CIPHER_ID)
+		return NULL;
+
+	ctx = ossl_cipher_ctx_create(cipher, key, klen, enc);
+
+	return ctx;
+}
+
+void
+pg_cipher_ctx_free(PgCipherCtx *ctx)
+{
+	EVP_CIPHER_CTX_free(ctx);
+}
+
+int
+pg_cipher_blocksize(PgCipherCtx *ctx)
+{
+	Assert(ctx);
+
+	return EVP_CIPHER_CTX_block_size(ctx);
+}
+
+/*
+ * Encryption routine to encrypt data provided.
+ *
+ * ctx is the encryption context which must have been created previously.
+ *
+ * plaintext is the data we are going to encrypt
+ * inlen is the length of the data to encrypt
+ *
+ * ciphertext is the encrypted result
+ * outlen is the encrypted length
+ *
+ * iv is the IV to use.
+ * ivlen is the IV length to use.
+ *
+ * aad is a pointer to additional authenticated data.
+ * aadlen is the length of aad.
+ *
+ * aad is a pointer to additional authenticated data.
+ * aadlen is the length of aad.
+ *
+ * outtag is the resulting tag.
+ * taglen is the length of the tag.
+ */
+bool
+pg_cipher_encrypt(PgCipherCtx *ctx, int cipher,
+				  const unsigned char *plaintext, const int inlen,
+				  unsigned char *ciphertext, int *outlen,
+				  const unsigned char *iv, const int ivlen,
+				  const unsigned char *aad, const int aadlen,
+				  unsigned char *outtag, const int taglen)
+{
+	int			len;
+	int			enclen;
+
+	Assert(ctx != NULL);
+
+	/*
+	 * Here we are setting the IV for the context which was passed in.  Note
+	 * that we signal to OpenSSL that we are configuring a new value for the
+	 * context by passing in 'NULL' for the 2nd ('type') parameter.
+	 */
+
+	/* Set the GCM IV length first */
+	if (cipher == PG_CIPHER_AES_GCM &&
+		!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, ivlen, NULL))
+		return false;
+
+	/* Set the IV for this encryption. */
+	if (!EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv))
+		return false;
+
+    /*
+	 * Provide any AAD data. This can be called zero or more times as
+	 * required
+	 */
+	if (aad && aadlen && !EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen))
+		return false;
+
+	/*
+	 * This is the function which is actually performing the encryption for
+	 * us.
+	 */
+	if (!EVP_EncryptUpdate(ctx, ciphertext, &len, plaintext, inlen))
+		return false;
+
+	enclen = len;
+
+	/* Finalize the encryption, which could add more to output. */
+	if (!EVP_EncryptFinal_ex(ctx, ciphertext + enclen, &len))
+		return false;
+
+	*outlen = enclen + len;
+
+	/*
+	 * Once all of the encryption has been completed we grab the tag.
+	 */
+	if (cipher == PG_CIPHER_AES_GCM &&
+		!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, taglen, outtag))
+		return false;
+
+	return true;
+}
+
+/*
+ * Decryption routine
+ *
+ * ctx is the encryption context which must have been created previously.
+ *
+ * ciphertext is the data we are going to decrypt
+ * inlen is the length of the data to decrypt
+ *
+ * plaintext is the decrypted result
+ * outlen is the decrypted length
+ *
+ * iv is the IV to use.
+ * ivlen is the length of the IV.
+ *
+ * intag is the tag to use to verify.
+ * taglen is the length of the tag.
+ */
+bool
+pg_cipher_decrypt(PgCipherCtx *ctx, const int cipher,
+				  const unsigned char *ciphertext, const int inlen,
+				  unsigned char *plaintext, int *outlen,
+				  const unsigned char *iv, const int ivlen,
+				  const unsigned char *aad, const int aadlen,
+				  unsigned char *intag, const int taglen)
+{
+	int			declen;
+	int			len;
+
+	/*
+	 * Here we are setting the IV for the context which was passed in.  Note
+	 * that we signal to OpenSSL that we are configuring a new value for the
+	 * context by passing in 'NULL' for the 2nd ('type') parameter.
+	 */
+
+	/* Set the GCM IV length first */
+	if (cipher == PG_CIPHER_AES_GCM &&
+		!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, ivlen, NULL))
+		return false;
+
+	/* Set the IV for this decryption. */
+	if (!EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv))
+		return false;
+
+    /*
+	 * Provide any AAD data. This can be called zero or more times as
+	 * required
+	 */
+	if (aad && aadlen && !EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen))
+		return false;
+
+	/*
+	 * This is the function which is actually performing the decryption for
+	 * us.
+	 */
+	if (!EVP_DecryptUpdate(ctx, plaintext, &len, ciphertext, inlen))
+		return false;
+
+	declen = len;
+
+	/* Set the expected tag value. */
+	if (cipher == PG_CIPHER_AES_GCM &&
+		!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, taglen, intag))
+		return false;
+
+	/*
+	 * Finalize the decryption, which could add more to output, this is also
+	 * the step which checks the tag and we MUST fail if this indicates an
+	 * invalid tag!
+	 */
+	if (!EVP_DecryptFinal_ex(ctx, plaintext + declen, &len))
+		return false;
+
+	*outlen = declen + len;
+
+	return true;
+}
+
+/*
+ * current written in a way that assumes a single concurrent encryption state;
+ * may want to eventually allocate this dynamically in pg_cipher_incr_init()
+ */
+typedef struct IncrState {
+	PgCipherCtx *ctx;
+	int cipher;
+} IncrState;
+
+static IncrState incrState;
+
+/* handle incremental encryption; returns context */
+EncryptionHandle
+pg_cipher_incr_init(PgCipherCtx *ctx, const int cipher,
+					const unsigned char *iv, const int ivlen)
+{
+	bzero(&incrState, sizeof(IncrState));
+	incrState.ctx = ctx;
+	incrState.cipher = cipher;
+
+	if (cipher == PG_CIPHER_AES_GCM &&
+		!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, ivlen, NULL))
+		return NULL;
+
+	/* Set the IV for this encryption. */
+	if (!EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv))
+		return NULL;
+	
+	return (EncryptionHandle)&incrState;
+}
+
+bool
+pg_cipher_incr_add_authenticated_data(EncryptionHandle incr,
+									  const unsigned char *aad,
+									  const int aadlen)
+{
+	int len;
+
+	/* XXX: do we need to return `len` somehow here? */
+	if (aad && aadlen && !EVP_EncryptUpdate(((IncrState *)incr)->ctx, NULL, &len, aad, aadlen))
+		return false;
+	return true;
+}
+
+bool
+pg_cipher_incr_encrypt(EncryptionHandle incr,
+					   const unsigned char *plaintext,
+					   const int inlen,
+					   unsigned char *ciphertext, int *outlen)
+{
+	if (!EVP_EncryptUpdate(((IncrState *)incr)->ctx, ciphertext, outlen, plaintext, inlen))
+		return false;
+	return true;
+}
+
+bool
+pg_cipher_incr_finish(EncryptionHandle incr,
+					  unsigned char *ciphertext, int *outlen,
+					  unsigned char *tag, const int taglen)
+{
+	if (!EVP_EncryptFinal_ex(((IncrState *)incr)->ctx, ciphertext, outlen))
+		return false;
+
+	/*
+	 * Once all of the encryption has been completed we grab the tag.
+	 */
+	if (((IncrState *)incr)->cipher == PG_CIPHER_AES_GCM &&
+		!EVP_CIPHER_CTX_ctrl(((IncrState *)incr)->ctx, EVP_CTRL_GCM_GET_TAG, taglen, tag))
+		return false;
+
+	return true;
+}
+
+/*
+ * Routine to perform key wrapping for data provided.
+ *
+ * ctx is the encryption context which must have been created previously.
+ *
+ * plaintext is the data/key we are going to encrypt/wrap
+ * inlen is the length of the data
+ *
+ * ciphertext is the wrapped result
+ * outlen is the encrypted length (will be larger than input!)
+ */
+bool
+pg_cipher_keywrap(PgCipherCtx *ctx,
+				  const unsigned char *plaintext, const int inlen,
+				  unsigned char *ciphertext, int *outlen)
+{
+	int			len;
+	int			enclen;
+
+	Assert(ctx != NULL);
+
+	/*
+	 * This is the function which is actually performing the encryption for
+	 * us.
+	 */
+	if (!EVP_EncryptUpdate(ctx, ciphertext, &len, plaintext, inlen))
+		return false;
+
+	enclen = len;
+
+	/* Finalize the encryption, which could add more to output. */
+	if (!EVP_EncryptFinal_ex(ctx, ciphertext + enclen, &len))
+		return false;
+
+	*outlen = enclen + len;
+
+	return true;
+}
+
+/*
+ * Routine to perform key unwrapping of the data provided.
+ *
+ * ctx is the encryption context which must have been created previously.
+ *
+ * ciphertext is the wrapped key we are going to unwrap
+ * inlen is the length of the data to decrypt/unwrap
+ *
+ * plaintext is the decrypted result
+ * outlen is the decrypted length (will be smaller than input!)
+ */
+bool
+pg_cipher_keyunwrap(PgCipherCtx *ctx,
+					const unsigned char *ciphertext, const int inlen,
+					unsigned char *plaintext, int *outlen)
+{
+	int			declen;
+	int			len;
+
+	/*
+	 * This is the function which is actually performing the decryption for
+	 * us.
+	 */
+	if (!EVP_DecryptUpdate(ctx, plaintext, &len, ciphertext, inlen))
+		return false;
+
+	declen = len;
+
+	/*
+	 * Finalize the decryption, which could add more to output, this is also
+	 * the step which checks the tag and we MUST fail if this indicates an
+	 * invalid result!
+	 */
+	if (!EVP_DecryptFinal_ex(ctx, plaintext + declen, &len))
+		return false;
+
+	*outlen = declen + len;
+
+	return true;
+}
+
+/*
+ * Returns the correct GCM cipher functions for OpenSSL based
+ * on the key length requested.
+ */
+static ossl_EVP_cipher_func
+get_evp_aes_gcm(int klen)
+{
+	switch (klen)
+	{
+		case PG_AES128_KEY_LEN:
+			return EVP_aes_128_gcm;
+		case PG_AES192_KEY_LEN:
+			return EVP_aes_192_gcm;
+		case PG_AES256_KEY_LEN:
+			return EVP_aes_256_gcm;
+		default:
+			return NULL;
+	}
+}
+
+/*
+ * Returns the correct KWP cipher functions for OpenSSL based
+ * on the key length requested.
+ */
+static ossl_EVP_cipher_func
+get_evp_aes_kwp(int klen)
+{
+	switch (klen)
+	{
+		case PG_AES128_KEY_LEN:
+			return EVP_aes_128_wrap_pad;
+		case PG_AES192_KEY_LEN:
+			return EVP_aes_192_wrap_pad;
+		case PG_AES256_KEY_LEN:
+			return EVP_aes_256_wrap_pad;
+		default:
+			return NULL;
+	}
+}
+
+/*
+ * Returns the correct CTR cipher functions for OpenSSL based
+ * on the key length requested.
+ */
+static ossl_EVP_cipher_func
+get_evp_aes_ctr(int klen)
+{
+	switch (klen)
+	{
+		case PG_AES128_KEY_LEN:
+			return EVP_aes_128_ctr;
+		case PG_AES192_KEY_LEN:
+			return EVP_aes_192_ctr;
+		case PG_AES256_KEY_LEN:
+			return EVP_aes_256_ctr;
+		default:
+			return NULL;
+	}
+}
+
+/*
+ * Returns the correct XTS cipher functions for OpenSSL based
+ * on the key length requested.
+ */
+static ossl_EVP_cipher_func
+get_evp_aes_xts(int klen)
+{
+	switch (klen)
+	{
+		case PG_AES128_KEY_LEN:
+			return EVP_aes_128_xts;
+		case PG_AES192_KEY_LEN:
+			/* unsupported */
+			return NULL;
+		case PG_AES256_KEY_LEN:
+			return EVP_aes_256_xts;
+		default:
+			return NULL;
+	}
+}
+
+/*
+ * Initialize and return an EVP_CIPHER_CTX. Returns NULL if the given
+ * cipher algorithm is not supported or on failure.
+ */
+static EVP_CIPHER_CTX *
+ossl_cipher_ctx_create(int cipher, unsigned char *key, int klen, bool enc)
+{
+	EVP_CIPHER_CTX *ctx;
+	ossl_EVP_cipher_func func;
+	int			ret;
+
+	ctx = EVP_CIPHER_CTX_new();
+
+	switch (cipher)
+	{
+		case PG_CIPHER_AES_GCM:
+			func = get_evp_aes_gcm(klen);
+			if (!func)
+				goto failed;
+			break;
+		case PG_CIPHER_AES_KWP:
+			func = get_evp_aes_kwp(klen);
+
+			/*
+			 * Since wrapping will produce more output then input, and we have
+			 * to be ready for that, OpenSSL requires that we explicitly
+			 * enable wrapping for the context.
+			 */
+			EVP_CIPHER_CTX_set_flags(ctx, EVP_CIPHER_CTX_FLAG_WRAP_ALLOW);
+			if (!func)
+				goto failed;
+			break;
+		case PG_CIPHER_AES_CTR:
+			func = get_evp_aes_ctr(klen);
+			if (!func)
+				goto failed;
+			break;
+		case PG_CIPHER_AES_XTS:
+			func = get_evp_aes_xts(klen);
+			if (!func)
+				goto failed;
+			/* XTS uses twice as long a key than the bitsize implies */
+			klen *= 2;
+			break;
+		default:
+			goto failed;
+	}
+
+	/*
+	 * We create the context here based on the cipher requested and the
+	 * provided key.  Note that the IV will be provided in the actual
+	 * encryption call through another EVP_EncryptInit_ex call- this is fine
+	 * as long as 'type' is passed in as NULL!
+	 */
+	if (enc)
+		ret = EVP_EncryptInit_ex(ctx, (const EVP_CIPHER *) func(), NULL, key, NULL);
+	else
+		ret = EVP_DecryptInit_ex(ctx, (const EVP_CIPHER *) func(), NULL, key, NULL);
+
+	if (!ret)
+		goto failed;
+
+	/* Set the key length based on the key length requested. */
+	if (!EVP_CIPHER_CTX_set_key_length(ctx, klen))
+		goto failed;
+
+	return ctx;
+
+failed:
+	EVP_CIPHER_CTX_free(ctx);
+	return NULL;
+}
diff --git a/src/common/kmgr_utils.c b/src/common/kmgr_utils.c
new file mode 100644
index 0000000000..9597788647
--- /dev/null
+++ b/src/common/kmgr_utils.c
@@ -0,0 +1,467 @@
+/*-------------------------------------------------------------------------
+ *
+ * kmgr_utils.c
+ *	  Shared frontend/backend for cluster file encryption
+ *
+ * These functions handle reading the wrapped DEK files from the
+ * file system, and wrapping and unwrapping them.  It also handles
+ * running the cluster_key_command.
+ *
+ * Copyright (c) 2021-2023, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/common/kmgr_utils.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef FRONTEND
+#include "postgres.h"
+#else
+#include "postgres_fe.h"
+#endif
+
+#include <unistd.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#ifdef FRONTEND
+#include "common/logging.h"
+#endif
+#include "common/cryptohash.h"
+#include "common/file_perm.h"
+#include "common/string.h"
+#include "crypto/kmgr.h"
+#include "lib/stringinfo.h"
+#include "storage/fd.h"
+#include "postgres.h"
+#include "utils/builtins.h"
+
+#ifndef FRONTEND
+#include "pgstat.h"
+#include "storage/fd.h"
+#endif
+
+#define KMGR_PROMPT_MSG "Enter authentication needed to generate the cluster key: "
+
+#ifdef FRONTEND
+static FILE *open_pipe_stream(const char *command);
+static int	close_pipe_stream(FILE *file);
+#endif
+
+static void read_wrapped_data_key(const char *cryptoKeyDir, uint32 id, unsigned char **key_p, int *key_len);
+
+/*
+ * We need this array in frontend and backend code, so define it here.
+ * You can only add to the end of this array since the array index is
+ * stored in pg_control.
+ */
+const encryption_method encryption_methods[NUM_ENCRYPTION_METHODS] = {
+	{"", 0},
+	{"AES-128-XTS", 128, 32, PG_CIPHER_AES_XTS, 0},
+	{"AES-256-XTS", 256, 64, PG_CIPHER_AES_XTS, 0},
+	/* {"AES-128-GCM", 128, 16, PG_CIPHER_AES_GCM, 16}, */
+	/* {"AES-192-GCM", 128, 24, PG_CIPHER_AES_GCM, 16}, */
+	/* {"AES-256-GCM", 128, 32, PG_CIPHER_AES_GCM, 16}, */
+};
+
+/* This maps wrapped key filesnames to their array slots */
+char	   *wkey_filenames[KMGR_NUM_DATA_KEYS] = {
+	"relation",
+	"wal"
+};
+
+/*
+ * Wrap the given CryptoKey.
+ *
+ * Returns true and writes encrypted/wrapped/padded data to 'out', and the length
+ * of the result to outlen, if successful.
+ *
+ * Otherwise returns false. The caller must allocate sufficient space
+ * for cipher data calculated by using KmgrSizeOfCipherText(). Please note that
+ * this function modifies 'out' data even on failure.
+ */
+bool
+kmgr_wrap_data_key(PgCipherCtx *ctx, CryptoKey *in, unsigned char *out, int *outlen)
+{
+	Assert(ctx && in && out);
+
+	if (!pg_cipher_keywrap(ctx, (unsigned char *) in, sizeof(CryptoKey), out, outlen))
+		return false;
+
+	return true;
+}
+
+/*
+ * Decrypt the given data. Return true and set plain text data to `out` if
+ * successful.  Otherwise return false. The caller must allocate sufficient
+ * space for cipher data calculated by using KmgrSizeOfPlainText(). Please
+ * note that this function modifies 'out' data even on failure.
+ */
+bool
+kmgr_unwrap_data_key(PgCipherCtx *ctx, unsigned char *in, int inlen, CryptoKey *out)
+{
+	int			outlen;
+
+	Assert(ctx && in && out);
+
+	if (!pg_cipher_keyunwrap(ctx, in, inlen, (unsigned char *) out, &outlen))
+		return false;
+
+	Assert(outlen == sizeof(CryptoKey));
+
+	return true;
+}
+
+/*
+ * Verify the correctness of the given cluster key by unwrapping the given keys.
+ * If the given cluster key is correct we set unwrapped keys to out_keys and return
+ * true.  Otherwise return false.  Please note that this function changes the
+ * contents of out_keys even on failure.  Both in_keys and out_keys must be the
+ * same length.
+ */
+bool
+kmgr_verify_cluster_key(unsigned char *cluster_key,
+						unsigned char **in_keys, int *key_lens, CryptoKey *out_keys)
+{
+	PgCipherCtx *ctx;
+
+	/* Create decryption context with the KEK. */
+	ctx = pg_cipher_ctx_create(PG_CIPHER_AES_KWP, cluster_key,
+							   KMGR_KEK_KEY_LEN, false);
+
+	/* unwrap each DEK */
+	for (int i = 0; i < KMGR_NUM_DATA_KEYS; i++)
+	{
+		if (!kmgr_unwrap_data_key(ctx, in_keys[i], key_lens[i], &(out_keys[i])))
+		{
+			/* The cluster key is not correct */
+			pg_cipher_ctx_free(ctx);
+			return false;
+		}
+		explicit_bzero(in_keys[i], key_lens[i]);
+	}
+
+	/* The cluster key is correct, free the cipher context */
+	pg_cipher_ctx_free(ctx);
+
+	return true;
+}
+
+/*
+ * Run cluster key command.
+ *
+ * Substitute %d for directory, %p for prompt, %R for file descriptor.
+ *
+ * The result will be put in buffer buf, which is of size "size".
+ * The return value is the length of the actual result.
+ */
+int
+kmgr_run_cluster_key_command(char *cluster_key_command, char *buf,
+							 int size, char *dir, int terminal_fd)
+{
+	StringInfoData command;
+	const char *sp;
+	FILE	   *fh;
+	int			pclose_rc;
+	size_t		len = 0;
+
+	buf[0] = '\0';
+
+	Assert(size > 0);
+
+	/*
+	 * Build the command to be executed.
+	 */
+	initStringInfo(&command);
+
+	for (sp = cluster_key_command; *sp; sp++)
+	{
+		if (*sp == '%')
+		{
+			switch (sp[1])
+			{
+					/* directory */
+				case 'd':
+					{
+						char	   *nativePath;
+
+						sp++;
+
+						/*
+						 * This needs to use a placeholder to not modify the
+						 * input with the conversion done via
+						 * make_native_path().
+						 */
+						nativePath = pstrdup(dir);
+						make_native_path(nativePath);
+						appendStringInfoString(&command, nativePath);
+						pfree(nativePath);
+						break;
+					}
+					/* prompt string */
+				case 'p':
+					sp++;
+					appendStringInfoString(&command, KMGR_PROMPT_MSG);
+					break;
+					/* file descriptor number */
+				case 'R':
+					{
+						char		fd_str[20];
+
+						if (terminal_fd == -1)
+						{
+#ifdef FRONTEND
+							pg_fatal("cluster key command referenced %%R, but --authprompt not specified");
+#else
+							ereport(ERROR,
+									(errcode(ERRCODE_INTERNAL_ERROR),
+									 errmsg("cluster key command referenced %%R, but --authprompt not specified")));
+#endif
+						}
+
+						sp++;
+						snprintf(fd_str, sizeof(fd_str), "%d", terminal_fd);
+						appendStringInfoString(&command, fd_str);
+						break;
+					}
+					/* literal "%" */
+				case '%':
+					/* convert %% to a single % */
+					sp++;
+					appendStringInfoChar(&command, *sp);
+					break;
+				default:
+					/* otherwise treat the % as not special */
+					appendStringInfoChar(&command, *sp);
+					break;
+			}
+		}
+		else
+		{
+			appendStringInfoChar(&command, *sp);
+		}
+	}
+
+#ifdef FRONTEND
+	fh = open_pipe_stream(command.data);
+	if (fh == NULL)
+	{
+		pg_fatal("could not execute command \"%s\": %m",
+					 command.data);
+	}
+#else
+	fh = OpenPipeStream(command.data, "r");
+	if (fh == NULL)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not execute command \"%s\": %m",
+						command.data)));
+#endif
+
+	if (!fgets(buf, size, fh))
+	{
+		if (ferror(fh))
+		{
+#ifdef FRONTEND
+			pg_fatal("could not read from command \"%s\": %m",
+						 command.data);
+#else
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not read from command \"%s\": %m",
+							command.data)));
+#endif
+		}
+	}
+
+#ifdef FRONTEND
+	pclose_rc = close_pipe_stream(fh);
+#else
+	pclose_rc = ClosePipeStream(fh);
+#endif
+
+	if (pclose_rc == -1)
+	{
+#ifdef FRONTEND
+		pg_fatal("could not close pipe to external command: %m");
+#else
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close pipe to external command: %m")));
+#endif
+	}
+	else if (pclose_rc != 0)
+	{
+#ifdef FRONTEND
+		pg_fatal("command \"%s\" failed", command.data);
+#else
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("command \"%s\" failed",
+						command.data),
+				 errdetail_internal("%s", wait_result_to_str(pclose_rc))));
+#endif
+	}
+
+	/* strip trailing newline and carriage returns */
+	len = pg_strip_crlf(buf);
+
+	pfree(command.data);
+
+	return len;
+}
+
+#ifdef FRONTEND
+static FILE *
+open_pipe_stream(const char *command)
+{
+	FILE	   *res;
+
+#ifdef WIN32
+	size_t		cmdlen = strlen(command);
+	char	   *buf;
+	int			save_errno;
+
+	buf = malloc(cmdlen + 2 + 1);
+	if (buf == NULL)
+	{
+		errno = ENOMEM;
+		return NULL;
+	}
+	buf[0] = '"';
+	memcpy(&buf[1], command, cmdlen);
+	buf[cmdlen + 1] = '"';
+	buf[cmdlen + 2] = '\0';
+
+	res = _popen(buf, "r");
+
+	save_errno = errno;
+	free(buf);
+	errno = save_errno;
+#else
+	res = popen(command, "r");
+#endif							/* WIN32 */
+	return res;
+}
+
+static int
+close_pipe_stream(FILE *file)
+{
+#ifdef WIN32
+	return _pclose(file);
+#else
+	return pclose(file);
+#endif							/* WIN32 */
+}
+#endif							/* FRONTEND */
+
+/*
+ * Reads the keys at path.
+ *
+ * This routine simply reads in the raw encrypted/wrapped keys;
+ * it does not handle any decryption, see kmgr_key_unwrap().
+ *
+ * For each key returned, the key and key length are returned
+ * in the keys and key_lens arrays respectfully.
+ *
+ * Note that keys and key_lens must be allocated before calling
+ * this function as arrays of at least KMGR_NUM_DATA_KEYS length.
+ */
+void
+kmgr_read_wrapped_data_keys(const char *path, unsigned char **keys, int *key_lens)
+{
+/*	StaticAssertStmt(lengthof(wkey_filenames) == KMGR_NUM_DATA_KEYS,
+					 "wkey_filenames[] must match KMGR_NUM_DATA_KEYS");
+*/
+	StaticAssertStmt(1 == 1,
+					 "wkey_filenames[] must match KMGR_NUM_DATA_KEYS");
+
+	for (int id = 0; id < KMGR_NUM_DATA_KEYS; id++)
+		read_wrapped_data_key(path, id, &(keys[id]), &(key_lens[id]));
+
+	return;
+}
+
+/* Read a wrapped DEK file */
+static void
+read_wrapped_data_key(const char *cryptoKeyDir, uint32 id, unsigned char **key_p, int *key_len)
+{
+	char		path[MAXPGPATH];
+	int			fd;
+	int			r;
+	struct stat st;
+
+	CryptoKeyFilePath(path, cryptoKeyDir, id);
+
+#ifndef FRONTEND
+	if ((fd = OpenTransientFile(path, O_RDONLY | PG_BINARY)) == -1)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not open file \"%s\" for reading: %m",
+						path)));
+	else if (fstat(fd, &st))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not stat file \"%s\": %m",
+						path)));
+#else
+	if ((fd = open(path, O_RDONLY | PG_BINARY, 0)) == -1)
+		pg_fatal("could not open file \"%s\" for reading: %m",
+					 path);
+	else if (fstat(fd, &st))
+		pg_fatal("could not stat file \"%s\": %m",
+					 path);
+#endif
+
+	*key_len = st.st_size;
+
+#ifndef FRONTEND
+	pgstat_report_wait_start(WAIT_EVENT_KEY_FILE_READ);
+#endif
+
+	*key_p = (unsigned char *) palloc0(*key_len);
+
+	/* Get key bytes */
+	r = read(fd, *key_p, *key_len);
+	if (r != *key_len)
+	{
+		if (r < 0)
+		{
+#ifndef FRONTEND
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not read file \"%s\": %m", path)));
+#else
+			pg_fatal("could not read file \"%s\": %m", path);
+#endif
+		}
+		else
+		{
+#ifndef FRONTEND
+			ereport(ERROR,
+					(errcode(ERRCODE_DATA_CORRUPTED),
+					 errmsg("could not read file \"%s\": read %d of %u",
+							path, r, *key_len)));
+#else
+			pg_fatal("could not read file \"%s\": read %d of %u",
+						 path, r, *key_len);
+#endif
+		}
+	}
+
+#ifndef FRONTEND
+	pgstat_report_wait_end();
+#endif
+
+#ifndef FRONTEND
+	if (CloseTransientFile(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close file \"%s\": %m",
+						path)));
+#else
+	if (close(fd) != 0)
+		pg_fatal("could not close file \"%s\": %m", path);
+#endif
+}
diff --git a/src/common/meson.build b/src/common/meson.build
index ae05ac63cf..ff1633a708 100644
--- a/src/common/meson.build
+++ b/src/common/meson.build
@@ -15,6 +15,7 @@ common_sources = files(
   'ip.c',
   'jsonapi.c',
   'keywords.c',
+  'kmgr_utils.c',
   'kwlookup.c',
   'link-canary.c',
   'md5_common.c',
@@ -38,12 +39,14 @@ common_sources = files(
 
 if ssl.found()
   common_sources += files(
+    'cipher_openssl.c',
     'cryptohash_openssl.c',
     'hmac_openssl.c',
     'protocol_openssl.c',
   )
 else
   common_sources += files(
+    'cipher.c',
     'cryptohash.c',
     'hmac.c',
     'md5.c',
diff --git a/src/include/common/cipher.h b/src/include/common/cipher.h
new file mode 100644
index 0000000000..41de383857
--- /dev/null
+++ b/src/include/common/cipher.h
@@ -0,0 +1,98 @@
+/*-------------------------------------------------------------------------
+ *
+ * cipher.h
+ *		Declarations for cryptographic functions
+ *
+ * Portions Copyright (c) 2021, PostgreSQL Global Development Group
+ *
+ * src/include/common/cipher.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef PG_CIPHER_H
+#define PG_CIPHER_H
+
+#ifdef USE_OPENSSL
+#include <openssl/evp.h>
+#include <openssl/conf.h>
+#include <openssl/err.h>
+#endif
+
+/*
+ * Supported symmetric encryption algorithm. These identifiers are passed
+ * to pg_cipher_ctx_create() function, and then actual encryption
+ * implementations need to initialize their context of the given encryption
+ * algorithm.
+ */
+#define PG_CIPHER_AES_GCM			0
+#define PG_CIPHER_AES_CTR			1
+#define PG_CIPHER_AES_KWP			2
+#define PG_CIPHER_AES_XTS			3
+#define PG_MAX_CIPHER_ID			3
+#define SizeOfEncryptionTag(m) encryption_methods[m].authtag_len
+#define EncryptionAlgorithm(m) encryption_methods[m].algorithm
+#define EncryptionKeyLength(m) encryption_methods[m].key_length
+#define EncryptionBlockLength(m) (encryption_methods[m].block_length/8)
+
+/* AES128/192/256 various length definitions */
+#define PG_AES128_KEY_LEN			(128 / 8)
+#define PG_AES192_KEY_LEN			(192 / 8)
+#define PG_AES256_KEY_LEN			(256 / 8)
+
+/*
+ * The encrypted data is a series of blocks of size. Initialization
+ * vector(IV) is the same size of cipher block.
+ */
+#define PG_AES_BLOCK_SIZE			16
+#define PG_AES_IV_SIZE				(PG_AES_BLOCK_SIZE)
+
+#ifdef USE_OPENSSL
+typedef EVP_CIPHER_CTX PgCipherCtx;
+#else
+typedef void PgCipherCtx;
+#endif
+
+typedef Pointer EncryptionHandle;
+
+extern PgCipherCtx *pg_cipher_ctx_create(int cipher, unsigned char *key, int klen,
+										 bool enc);
+
+extern void pg_cipher_ctx_free(PgCipherCtx *ctx);
+extern bool pg_cipher_encrypt(PgCipherCtx *ctx, int cipher,
+							  const unsigned char *plaintext, const int inlen,
+							  unsigned char *ciphertext, int *outlen,
+							  const unsigned char *iv, const int ivlen,
+							  const unsigned char *aad, const int aadlen,
+							  unsigned char *tag, const int taglen);
+extern bool pg_cipher_decrypt(PgCipherCtx *ctx, const int cipher,
+							  const unsigned char *ciphertext, const int inlen,
+							  unsigned char *plaintext, int *outlen,
+							  const unsigned char *iv, const int ivlen,
+							  const unsigned char *aad, const int aadlen,
+							  unsigned char *intag, const int taglen);
+
+
+/* handle incremental encryption; returns context */
+extern EncryptionHandle pg_cipher_incr_init(PgCipherCtx *ctx, const int cipher,
+											const unsigned char *iv, const int ivlen);
+extern bool pg_cipher_incr_add_authenticated_data(EncryptionHandle incr,
+												  const unsigned char *aad,
+												  const int aadlen);
+extern bool pg_cipher_incr_encrypt(EncryptionHandle incr,
+								   const unsigned char *plaintext,
+								   const int inlen,
+								   unsigned char *ciphertext, int *outlen);
+extern bool pg_cipher_incr_finish(EncryptionHandle incr,
+								  unsigned char *ciphertext, int *outlen,
+								  unsigned char *tag, const int taglen);
+
+extern bool pg_cipher_keywrap(PgCipherCtx *ctx,
+							  const unsigned char *plaintext, const int inlen,
+							  unsigned char *ciphertext, int *outlen);
+extern bool pg_cipher_keyunwrap(PgCipherCtx *ctx,
+								const unsigned char *ciphertext, const int inlen,
+								unsigned char *plaintext, int *outlen);
+
+extern int	pg_cipher_blocksize(PgCipherCtx *ctx);
+
+#endif							/* PG_CIPHER_H */
diff --git a/src/include/common/kmgr_utils.h b/src/include/common/kmgr_utils.h
new file mode 100644
index 0000000000..5d3e2bb912
--- /dev/null
+++ b/src/include/common/kmgr_utils.h
@@ -0,0 +1,100 @@
+/*-------------------------------------------------------------------------
+ *
+ * kmgr_utils.h
+ *		Declarations for utility function for file encryption key
+ *
+ * Portions Copyright (c) 2021, PostgreSQL Global Development Group
+ *
+ * src/include/common/kmgr_utils.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef KMGR_UTILS_H
+#define KMGR_UTILS_H
+
+#include "common/cipher.h"
+
+/* Current version number */
+#define KMGR_VERSION 1
+
+/*
+ * Directories where cluster file encryption keys reside within PGDATA.
+ */
+#define KMGR_DIR			"pg_cryptokeys"
+#define KMGR_DIR_PID		KMGR_DIR"/pg_alterckey.pid"
+#define LIVE_KMGR_DIR		KMGR_DIR"/live"
+/* used during cluster key rotation */
+#define NEW_KMGR_DIR		KMGR_DIR"/new"
+#define OLD_KMGR_DIR		KMGR_DIR"/old"
+
+/* CryptoKey file name is keys id */
+#define CryptoKeyFilePath(path, dir, id) \
+	snprintf((path), MAXPGPATH, "%s/%s.wkey", (dir), (wkey_filenames[id]))
+
+/*
+ * Identifiers of internal keys.
+ */
+#define KMGR_KEY_ID_REL 		0
+#define KMGR_KEY_ID_WAL 		1
+#define KMGR_NUM_DATA_KEYS	2
+
+/* When using XTS, our cluster key length is twice the AES size; this is a max size */
+#define KMGR_CLUSTER_KEY_MAX_LEN 	PG_AES256_KEY_LEN * 2
+#define KMGR_CLUSTER_KEY_LEN(m) 	(encryption_methods[m].key_length)
+#define KMGR_KEK_KEY_LEN 	PG_AES256_KEY_LEN
+
+/* double for hex format, plus some for spaces, \r,\n, and null byte */
+#define ALLOC_KMGR_CLUSTER_KEY_MAX_LEN	(KMGR_CLUSTER_KEY_MAX_LEN * 2 + 10 + 2 + 1)
+#define ALLOC_KMGR_CLUSTER_KEY_LEN(m)	(KMGR_CLUSTER_KEY_LEN(m) * 2 + 10 + 2 + 1)
+
+/* Maximum length of key the key manager can store */
+#define KMGR_MAX_KEY_LEN			KMGR_CLUSTER_KEY_MAX_LEN * 8
+#define KMGR_MAX_KEY_LEN_BYTES		KMGR_MAX_KEY_LEN / 8
+
+
+/*
+ * Cryptographic key data structure.
+ *
+ * This is the structure we use to write out the encrypted keys and
+ * which we use to store the keys in shared memory.
+ *
+ * Note that wrapping this structure results in an encrypted byte
+ * string which is what we actually write and then read back in.
+ *
+ * klen is the key length in bytes
+ * key is the encryption key of klen length
+ */
+typedef struct CryptoKey
+{
+	int			klen;			/* key length in bytes */
+	unsigned char key[KMGR_MAX_KEY_LEN_BYTES];
+} CryptoKey;
+
+/* Encryption method array */
+typedef struct encryption_method
+{
+	const char *name;
+	const int	block_length;
+	const int   key_length;
+	const int	algorithm;
+	const int	authtag_len;
+} encryption_method;
+
+#define NUM_ENCRYPTION_METHODS	3
+#define DISABLED_ENCRYPTION_METHOD 0
+#define DEFAULT_ENABLED_ENCRYPTION_METHOD 1
+
+extern const encryption_method encryption_methods[NUM_ENCRYPTION_METHODS];
+extern char *wkey_filenames[KMGR_NUM_DATA_KEYS];
+
+extern bool kmgr_wrap_data_key(PgCipherCtx *ctx, CryptoKey *in, unsigned char *out, int *outlen);
+extern bool kmgr_unwrap_data_key(PgCipherCtx *ctx, unsigned char *in, int inlen, CryptoKey *out);
+extern bool kmgr_verify_cluster_key(unsigned char *cluster_key,
+									unsigned char **in_keys, int *klens, CryptoKey *out_keys);
+extern int	kmgr_run_cluster_key_command(char *cluster_key_command,
+										 char *buf, int size, char *dir,
+										 int terminal_fd);
+extern void kmgr_read_wrapped_data_keys(const char *path, unsigned char **keys,
+										int *key_lens);
+
+#endif							/* KMGR_UTILS_H */
-- 
2.40.1

