Refactor MD5 implementations according to new cryptohash infrastructure

This commit heavily reorganizes the MD5 implementations that exist in
the tree in various aspects.

First, MD5 is added to the list of options available in cryptohash.c and
cryptohash_openssl.c.  This means that if building with OpenSSL, EVP is
used for MD5 instead of the fallback implementation that Postgres had
for ages.  With the recent refactoring work for cryptohash functions,
this change is straight-forward.  If not building with OpenSSL, a
fallback implementation internal to src/common/ is used.

Second, this reduces the number of MD5 implementations present in the
tree from two to one, by moving the KAME implementation from pgcrypto to
src/common/, and by removing the implementation that existed in
src/common/.  KAME was already structured with an init/update/final set
of routines by pgcrypto (see original pgcrypto/md5.h) for compatibility
with OpenSSL, so moving it to src/common/ has proved to be a
straight-forward move, requiring no actual manipulation of the internals
of each routine.  Some benchmarking has not shown any performance gap
between both implementations.

Similarly to the fallback implementation used for SHA2, the fallback
implementation of MD5 is moved to src/common/md5.c with an internal
header called md5_int.h for the init, update and final routines.  This
gets then consumed by cryptohash.c.

The original routines used for MD5-hashed passwords are moved to a
separate file called md5_common.c, also in src/common/, aimed at being
shared between all MD5 implementations as utility routines to keep
compatibility with any code relying on them.

Like the SHA2 changes, this commit had its round of tests on both Linux
and Windows, across all versions of OpenSSL supported on HEAD, with and
even without OpenSSL.

Author: Michael Paquier
Reviewed-by: Daniel Gustafsson
Discussion: https://postgr.es/m/20201106073434.GA4961@paquier.xyz

contrib/pgcrypto/Makefile

 # contrib/pgcrypto/Makefile
 
-INT_SRCS = md5.c sha1.c internal.c internal-sha2.c blf.c rijndael.c \
+INT_SRCS = sha1.c internal.c internal-sha2.c blf.c rijndael.c \
 		pgp-mpi-internal.c imath.c
 INT_TESTS = sha2
 

contrib/pgcrypto/internal.c

@@ -34,11 +34,13 @@
 #include <time.h>
 
 #include "blf.h"
-#include "md5.h"
 #include "px.h"
 #include "rijndael.h"
 #include "sha1.h"
 
+#include "common/cryptohash.h"
+#include "common/md5.h"
+
 #ifndef MD5_DIGEST_LENGTH
 #define MD5_DIGEST_LENGTH 16
 #endif
@@ -96,34 +98,33 @@ int_md5_block_len(PX_MD *h)
 static void
 int_md5_update(PX_MD *h, const uint8 *data, unsigned dlen)
 {
-	MD5_CTX    *ctx = (MD5_CTX *) h->p.ptr;
+	pg_cryptohash_ctx *ctx = (pg_cryptohash_ctx *) h->p.ptr;
 
-	MD5Update(ctx, data, dlen);
+	pg_cryptohash_update(ctx, data, dlen);
 }
 
 static void
 int_md5_reset(PX_MD *h)
 {
-	MD5_CTX    *ctx = (MD5_CTX *) h->p.ptr;
+	pg_cryptohash_ctx *ctx = (pg_cryptohash_ctx *) h->p.ptr;
 
-	MD5Init(ctx);
+	pg_cryptohash_init(ctx);
 }
 
 static void
 int_md5_finish(PX_MD *h, uint8 *dst)
 {
-	MD5_CTX    *ctx = (MD5_CTX *) h->p.ptr;
+	pg_cryptohash_ctx *ctx = (pg_cryptohash_ctx *) h->p.ptr;
 
-	MD5Final(dst, ctx);
+	pg_cryptohash_final(ctx, dst);
 }
 
 static void
 int_md5_free(PX_MD *h)
 {
-	MD5_CTX    *ctx = (MD5_CTX *) h->p.ptr;
+	pg_cryptohash_ctx *ctx = (pg_cryptohash_ctx *) h->p.ptr;
 
-	px_memset(ctx, 0, sizeof(*ctx));
-	pfree(ctx);
+	pg_cryptohash_free(ctx);
 	pfree(h);
 }
 
@@ -180,9 +181,9 @@ int_sha1_free(PX_MD *h)
 static void
 init_md5(PX_MD *md)
 {
-	MD5_CTX    *ctx;
+	pg_cryptohash_ctx *ctx;
 
-	ctx = palloc0(sizeof(*ctx));
+	ctx = pg_cryptohash_create(PG_MD5);
 
 	md->p.ptr = ctx;
 

src/common/Makefile

@@ -63,7 +63,7 @@ OBJS_COMMON = \
 	keywords.o \
 	kwlookup.o \
 	link-canary.o \
-	md5.o \
+	md5_common.o \
 	pg_get_line.o \
 	pg_lzcompress.o \
 	pgfnames.o \
@@ -86,6 +86,7 @@ OBJS_COMMON += \
 else
 OBJS_COMMON += \
 	cryptohash.o \
+	md5.o \
 	sha2.o
 endif
 

src/common/cryptohash.c

@@ -24,6 +24,7 @@
 #include <sys/param.h>
 
 #include "common/cryptohash.h"
+#include "md5_int.h"
 #include "sha2_int.h"
 
 /*
@@ -57,6 +58,9 @@ pg_cryptohash_create(pg_cryptohash_type type)
 
 	switch (type)
 	{
+		case PG_MD5:
+			ctx->data = ALLOC(sizeof(pg_md5_ctx));
+			break;
 		case PG_SHA224:
 			ctx->data = ALLOC(sizeof(pg_sha224_ctx));
 			break;
@@ -95,6 +99,9 @@ pg_cryptohash_init(pg_cryptohash_ctx *ctx)
 
 	switch (ctx->type)
 	{
+		case PG_MD5:
+			pg_md5_init((pg_md5_ctx *) ctx->data);
+			break;
 		case PG_SHA224:
 			pg_sha224_init((pg_sha224_ctx *) ctx->data);
 			break;
@@ -126,6 +133,9 @@ pg_cryptohash_update(pg_cryptohash_ctx *ctx, const uint8 *data, size_t len)
 
 	switch (ctx->type)
 	{
+		case PG_MD5:
+			pg_md5_update((pg_md5_ctx *) ctx->data, data, len);
+			break;
 		case PG_SHA224:
 			pg_sha224_update((pg_sha224_ctx *) ctx->data, data, len);
 			break;
@@ -157,6 +167,9 @@ pg_cryptohash_final(pg_cryptohash_ctx *ctx, uint8 *dest)
 
 	switch (ctx->type)
 	{
+		case PG_MD5:
+			pg_md5_final((pg_md5_ctx *) ctx->data, dest);
+			break;
 		case PG_SHA224:
 			pg_sha224_final((pg_sha224_ctx *) ctx->data, dest);
 			break;

src/common/cryptohash_openssl.c

@@ -135,6 +135,9 @@ pg_cryptohash_init(pg_cryptohash_ctx *ctx)
 
 	switch (ctx->type)
 	{
+		case PG_MD5:
+			status = EVP_DigestInit_ex(state->evpctx, EVP_md5(), NULL);
+			break;
 		case PG_SHA224:
 			status = EVP_DigestInit_ex(state->evpctx, EVP_sha224(), NULL);
 			break;

src/common/md5_common.c

+/*-------------------------------------------------------------------------
+ *
+ * md5_common.c
+ *	  Routines shared between all MD5 implementations used for encrypted
+ *	  passwords.
+ *
+ * Sverre H. Huseby <sverrehu@online.no>
+ *
+ * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ *	  src/common/md5_common.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef FRONTEND
+#include "postgres.h"
+#else
+#include "postgres_fe.h"
+#endif
+
+#include "common/cryptohash.h"
+#include "common/md5.h"
+
+static void
+bytesToHex(uint8 b[16], char *s)
+{
+	static const char *hex = "0123456789abcdef";
+	int			q,
+				w;
+
+	for (q = 0, w = 0; q < 16; q++)
+	{
+		s[w++] = hex[(b[q] >> 4) & 0x0F];
+		s[w++] = hex[b[q] & 0x0F];
+	}
+	s[w] = '\0';
+}
+
+/*
+ *	pg_md5_hash
+ *
+ *	Calculates the MD5 sum of the bytes in a buffer.
+ *
+ *	SYNOPSIS	  #include "md5.h"
+ *				  int pg_md5_hash(const void *buff, size_t len, char *hexsum)
+ *
+ *	INPUT		  buff	  the buffer containing the bytes that you want
+ *						  the MD5 sum of.
+ *				  len	  number of bytes in the buffer.
+ *
+ *	OUTPUT		  hexsum  the MD5 sum as a '\0'-terminated string of
+ *						  hexadecimal digits.  an MD5 sum is 16 bytes long.
+ *						  each byte is represented by two hexadecimal
+ *						  characters.  you thus need to provide an array
+ *						  of 33 characters, including the trailing '\0'.
+ *
+ *	RETURNS		  false on failure (out of memory for internal buffers
+ *				  or MD5 computation failure) or true on success.
+ *
+ *	STANDARDS	  MD5 is described in RFC 1321.
+ *
+ *	AUTHOR		  Sverre H. Huseby <sverrehu@online.no>
+ *
+ */
+
+bool
+pg_md5_hash(const void *buff, size_t len, char *hexsum)
+{
+	uint8		sum[16];
+	pg_cryptohash_ctx *ctx;
+
+	ctx = pg_cryptohash_create(PG_MD5);
+	if (ctx == NULL)
+		return false;
+
+	if (pg_cryptohash_init(ctx) < 0 ||
+		pg_cryptohash_update(ctx, buff, len) < 0 ||
+		pg_cryptohash_final(ctx, sum) < 0)
+	{
+		pg_cryptohash_free(ctx);
+		return false;
+	}
+
+	bytesToHex(sum, hexsum);
+	pg_cryptohash_free(ctx);
+	return true;
+}
+
+bool
+pg_md5_binary(const void *buff, size_t len, void *outbuf)
+{
+	pg_cryptohash_ctx *ctx;
+
+	ctx = pg_cryptohash_create(PG_MD5);
+	if (ctx == NULL)
+		return false;
+
+	if (pg_cryptohash_init(ctx) < 0 ||
+		pg_cryptohash_update(ctx, buff, len) < 0 ||
+		pg_cryptohash_final(ctx, outbuf) < 0)
+	{
+		pg_cryptohash_free(ctx);
+		return false;
+	}
+
+	pg_cryptohash_free(ctx);
+	return true;
+}
+
+
+/*
+ * Computes MD5 checksum of "passwd" (a null-terminated string) followed
+ * by "salt" (which need not be null-terminated).
+ *
+ * Output format is "md5" followed by a 32-hex-digit MD5 checksum.
+ * Hence, the output buffer "buf" must be at least 36 bytes long.
+ *
+ * Returns true if okay, false on error (out of memory).
+ */
+bool
+pg_md5_encrypt(const char *passwd, const char *salt, size_t salt_len,
+			   char *buf)
+{
+	size_t		passwd_len = strlen(passwd);
+
+	/* +1 here is just to avoid risk of unportable malloc(0) */
+	char	   *crypt_buf = malloc(passwd_len + salt_len + 1);
+	bool		ret;
+
+	if (!crypt_buf)
+		return false;
+
+	/*
+	 * Place salt at the end because it may be known by users trying to crack
+	 * the MD5 output.
+	 */
+	memcpy(crypt_buf, passwd, passwd_len);
+	memcpy(crypt_buf + passwd_len, salt, salt_len);
+
+	strcpy(buf, "md5");
+	ret = pg_md5_hash(crypt_buf, passwd_len + salt_len, buf + 3);
+
+	free(crypt_buf);
+
+	return ret;
+}

contrib/pgcrypto/md5.h → src/common/md5_int.h

-/*	contrib/pgcrypto/md5.h */
+/*-------------------------------------------------------------------------
+ *
+ * md5_int.h
+ *	  Internal headers for fallback implementation of MD5
+ *
+ * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ *		  src/common/md5_int.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
 /*	   $KAME: md5.h,v 1.3 2000/02/22 14:01:18 itojun Exp $	   */
 
 /*
@@ -30,11 +43,14 @@
  * SUCH DAMAGE.
  */
 
-#ifndef _NETINET6_MD5_H_
-#define _NETINET6_MD5_H_
+#ifndef PG_MD5_INT_H
+#define PG_MD5_INT_H
+
+#include "common/md5.h"
 
 #define MD5_BUFLEN 64
 
+/* Context data for MD5 */
 typedef struct
 {
 	union
@@ -59,21 +75,11 @@ typedef struct
 
 	unsigned int md5_i;
 	uint8		md5_buf[MD5_BUFLEN];
-} md5_ctxt;
-
-extern void md5_init(md5_ctxt *);
-extern void md5_loop(md5_ctxt *, const uint8 *, unsigned int);
-extern void md5_pad(md5_ctxt *);
-extern void md5_result(uint8 *, md5_ctxt *);
+} pg_md5_ctx;
 
-/* compatibility */
-#define MD5_CTX		md5_ctxt
-#define MD5Init(x)	md5_init((x))
-#define MD5Update(x, y, z)	md5_loop((x), (y), (z))
-#define MD5Final(x, y) \
-do {				\
-	md5_pad((y));		\
-	md5_result((x), (y));	\
-} while (0)
+/* Interface routines for MD5 */
+extern void pg_md5_init(pg_md5_ctx *ctx);
+extern void pg_md5_update(pg_md5_ctx *ctx, const uint8 *data, size_t len);
+extern void pg_md5_final(pg_md5_ctx *ctx, uint8 *dest);
 
-#endif							/* ! _NETINET6_MD5_H_ */
+#endif							/* PG_MD5_INT_H */

src/include/common/cryptohash.h

@@ -18,7 +18,8 @@
 /* Context Structures for each hash function */
 typedef enum
 {
-	PG_SHA224 = 0,
+	PG_MD5 = 0,
+	PG_SHA224,
 	PG_SHA256,
 	PG_SHA384,
 	PG_SHA512

src/include/common/md5.h

 /*-------------------------------------------------------------------------
  *
  * md5.h
- *	  Interface to libpq/md5.c
+ *	  Constants and common utilities related to MD5.
  *
  * These definitions are needed by both frontend and backend code to work
  * with MD5-encrypted passwords.
@@ -19,9 +19,10 @@
 #define MD5_PASSWD_CHARSET	"0123456789abcdef"
 #define MD5_PASSWD_LEN	35
 
+/* Utilities common to all the MD5 implementations, as of md5_common.c */
 extern bool pg_md5_hash(const void *buff, size_t len, char *hexsum);
 extern bool pg_md5_binary(const void *buff, size_t len, void *outbuf);
 extern bool pg_md5_encrypt(const char *passwd, const char *salt,
 						   size_t salt_len, char *buf);
 
-#endif
+#endif							/* PG_MD5_H */

src/tools/msvc/Mkvcbuild.pm

@@ -122,7 +122,7 @@ sub mkvcbuild
 	  archive.c base64.c checksum_helper.c
 	  config_info.c controldata_utils.c d2s.c encnames.c exec.c
 	  f2s.c file_perm.c file_utils.c hashfn.c ip.c jsonapi.c
-	  keywords.c kwlookup.c link-canary.c md5.c
+	  keywords.c kwlookup.c link-canary.c md5_common.c
 	  pg_get_line.c pg_lzcompress.c pgfnames.c psprintf.c relpath.c rmtree.c
 	  saslprep.c scram-common.c string.c stringinfo.c unicode_norm.c username.c
 	  wait_error.c wchar.c);
@@ -135,6 +135,7 @@ sub mkvcbuild
 	else
 	{
 		push(@pgcommonallfiles, 'cryptohash.c');
+		push(@pgcommonallfiles, 'md5.c');
 		push(@pgcommonallfiles, 'sha2.c');
 	}
 
@@ -464,11 +465,10 @@ sub mkvcbuild
 	else
 	{
 		$pgcrypto->AddFiles(
-			'contrib/pgcrypto', 'md5.c',
-			'sha1.c',           'internal.c',
-			'internal-sha2.c',  'blf.c',
-			'rijndael.c',       'pgp-mpi-internal.c',
-			'imath.c');
+			'contrib/pgcrypto',   'sha1.c',
+			'internal.c',         'internal-sha2.c',
+			'blf.c',              'rijndael.c',
+			'pgp-mpi-internal.c', 'imath.c');
 	}
 	$pgcrypto->AddReference($postgres);
 	$pgcrypto->AddLibrary('ws2_32.lib');

src/tools/pgindent/typedefs.list

@@ -3192,6 +3192,7 @@ pg_int64
 pg_local_to_utf_combined
 pg_locale_t
 pg_mb_radix_tree
+pg_md5_ctx
 pg_on_exit_callback
 pg_re_flags
 pg_saslprep_rc