Unify some tar functionality across different parts

Move some of the tar functionality that existed mostly duplicated
in both pg_dump and the walsender basebackup functionality into
port/tar.c instead, so it can be used from both. It will also be
used by pg_basebackup in the future, which would've caused a third
copy of it around.

Zoltan Boszormenyi and Magnus Hagander

src/backend/replication/basebackup.c

@@ -740,24 +740,6 @@ sendDir(char *path, int basepathlen, bool sizeonly)
  */
 
 
-/*
- * Utility routine to print possibly larger than 32 bit integers in a
- * portable fashion.  Filled with zeros.
- */
-static void
-print_val(char *s, uint64 val, unsigned int base, size_t len)
-{
-	int			i;
-
-	for (i = len; i > 0; i--)
-	{
-		int			digit = val % base;
-
-		s[i - 1] = '0' + digit;
-		val = val / base;
-	}
-}
-
 /*
  * Maximum file size for a tar member: The limit inherent in the
  * format is 2^33-1 bytes (nearly 8 GB).  But we don't want to exceed
@@ -765,24 +747,6 @@ print_val(char *s, uint64 val, unsigned int base, size_t len)
  */
 #define MAX_TAR_MEMBER_FILELEN (((int64) 1 << Min(33, sizeof(pgoff_t)*8 - 1)) - 1)
 
-static int
-_tarChecksum(char *header)
-{
-	int			i,
-				sum;
-
-	/*
-	 * Per POSIX, the checksum is the simple sum of all bytes in the header,
-	 * treating the bytes as unsigned, and treating the checksum field (at
-	 * offset 148) as though it contained 8 spaces.
-	 */
-	sum = 8 * ' ';				/* presumed value for checksum field */
-	for (i = 0; i < 512; i++)
-		if (i < 148 || i >= 156)
-			sum += 0xFF & header[i];
-	return sum;
-}
-
 /*
  * Given the member, write the TAR header & send the file.
  *
@@ -874,95 +838,9 @@ _tarWriteHeader(const char *filename, const char *linktarget,
 {
 	char		h[512];
 
-	/*
-	 * Note: most of the fields in a tar header are not supposed to be
-	 * null-terminated.  We use sprintf, which will write a null after the
-	 * required bytes; that null goes into the first byte of the next field.
-	 * This is okay as long as we fill the fields in order.
-	 */
-	memset(h, 0, sizeof(h));
-
-	/* Name 100 */
-	sprintf(&h[0], "%.99s", filename);
-	if (linktarget != NULL || S_ISDIR(statbuf->st_mode))
-	{
-		/*
-		 * We only support symbolic links to directories, and this is
-		 * indicated in the tar format by adding a slash at the end of the
-		 * name, the same as for regular directories.
-		 */
-		int			flen = strlen(filename);
-
-		flen = Min(flen, 99);
-		h[flen] = '/';
-		h[flen + 1] = '\0';
-	}
-
-	/* Mode 8 */
-	sprintf(&h[100], "%07o ", (int) statbuf->st_mode);
-
-	/* User ID 8 */
-	sprintf(&h[108], "%07o ", statbuf->st_uid);
-
-	/* Group 8 */
-	sprintf(&h[116], "%07o ", statbuf->st_gid);
-
-	/* File size 12 - 11 digits, 1 space; use print_val for 64 bit support */
-	if (linktarget != NULL || S_ISDIR(statbuf->st_mode))
-		/* Symbolic link or directory has size zero */
-		print_val(&h[124], 0, 8, 11);
-	else
-		print_val(&h[124], statbuf->st_size, 8, 11);
-	sprintf(&h[135], " ");
-
-	/* Mod Time 12 */
-	sprintf(&h[136], "%011o ", (int) statbuf->st_mtime);
-
-	/* Checksum 8 cannot be calculated until we've filled all other fields */
-
-	if (linktarget != NULL)
-	{
-		/* Type - Symbolic link */
-		sprintf(&h[156], "2");
-		/* Link Name 100 */
-		sprintf(&h[157], "%.99s", linktarget);
-	}
-	else if (S_ISDIR(statbuf->st_mode))
-		/* Type - directory */
-		sprintf(&h[156], "5");
-	else
-		/* Type - regular file */
-		sprintf(&h[156], "0");
-
-	/* Magic 6 */
-	sprintf(&h[257], "ustar");
-
-	/* Version 2 */
-	sprintf(&h[263], "00");
-
-	/* User 32 */
-	/* XXX: Do we need to care about setting correct username? */
-	sprintf(&h[265], "%.31s", "postgres");
-
-	/* Group 32 */
-	/* XXX: Do we need to care about setting correct group name? */
-	sprintf(&h[297], "%.31s", "postgres");
-
-	/* Major Dev 8 */
-	sprintf(&h[329], "%07o ", 0);
-
-	/* Minor Dev 8 */
-	sprintf(&h[337], "%07o ", 0);
-
-	/* Prefix 155 - not used, leave as nulls */
-
-	/*
-	 * We mustn't overwrite the next field while inserting the checksum.
-	 * Fortunately, the checksum can't exceed 6 octal digits, so we just write
-	 * 6 digits, a space, and a null, which is legal per POSIX.
-	 */
-	sprintf(&h[148], "%06o ", _tarChecksum(h));
+	tarCreateHeader(h, filename, linktarget, statbuf->st_size,
+					statbuf->st_mode, statbuf->st_uid, statbuf->st_gid,
+					statbuf->st_mtime);
 
-	/* Now send the completed header. */
 	pq_putmessage('d', h, 512);
 }

src/bin/pg_dump/pg_backup_tar.c

@@ -115,7 +115,6 @@ static char *tarGets(char *buf, size_t len, TAR_MEMBER *th);
 static int	tarPrintf(ArchiveHandle *AH, TAR_MEMBER *th, const char *fmt,...) __attribute__((format(PG_PRINTF_ATTRIBUTE, 3, 4)));
 
 static void _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th);
-static int	_tarChecksum(char *th);
 static TAR_MEMBER *_tarPositionTo(ArchiveHandle *AH, const char *filename);
 static size_t tarRead(void *buf, size_t len, TAR_MEMBER *th);
 static size_t tarWrite(const void *buf, size_t len, TAR_MEMBER *th);
@@ -1016,29 +1015,11 @@ tarPrintf(ArchiveHandle *AH, TAR_MEMBER *th, const char *fmt,...)
 	return cnt;
 }
 
-static int
-_tarChecksum(char *header)
-{
-	int			i,
-				sum;
-
-	/*
-	 * Per POSIX, the checksum is the simple sum of all bytes in the header,
-	 * treating the bytes as unsigned, and treating the checksum field (at
-	 * offset 148) as though it contained 8 spaces.
-	 */
-	sum = 8 * ' ';				/* presumed value for checksum field */
-	for (i = 0; i < 512; i++)
-		if (i < 148 || i >= 156)
-			sum += 0xFF & header[i];
-	return sum;
-}
-
 bool
 isValidTarHeader(char *header)
 {
 	int			sum;
-	int			chk = _tarChecksum(header);
+	int			chk = tarChecksum(header);
 
 	sscanf(&header[148], "%8o", &sum);
 
@@ -1251,7 +1232,7 @@ _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th)
 						  (unsigned long) len);
 
 		/* Calc checksum */
-		chk = _tarChecksum(h);
+		chk = tarChecksum(h);
 		sscanf(&h[148], "%8o", &sum);
 
 		/*
@@ -1305,92 +1286,12 @@ _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th)
 }
 
 
-/*
- * Utility routine to print possibly larger than 32 bit integers in a
- * portable fashion.  Filled with zeros.
- */
-static void
-print_val(char *s, uint64 val, unsigned int base, size_t len)
-{
-	int			i;
-
-	for (i = len; i > 0; i--)
-	{
-		int			digit = val % base;
-
-		s[i - 1] = '0' + digit;
-		val = val / base;
-	}
-}
-
-
 static void
 _tarWriteHeader(TAR_MEMBER *th)
 {
 	char		h[512];
 
-	/*
-	 * Note: most of the fields in a tar header are not supposed to be
-	 * null-terminated.  We use sprintf, which will write a null after the
-	 * required bytes; that null goes into the first byte of the next field.
-	 * This is okay as long as we fill the fields in order.
-	 */
-	memset(h, 0, sizeof(h));
-
-	/* Name 100 */
-	sprintf(&h[0], "%.99s", th->targetFile);
-
-	/* Mode 8 */
-	sprintf(&h[100], "0000600 ");
-
-	/* User ID 8 */
-	sprintf(&h[108], "0004000 ");
-
-	/* Group 8 */
-	sprintf(&h[116], "0002000 ");
-
-	/* File size 12 - 11 digits, 1 space; use print_val for 64 bit support */
-	print_val(&h[124], th->fileLen, 8, 11);
-	sprintf(&h[135], " ");
-
-	/* Mod Time 12 */
-	sprintf(&h[136], "%011o ", (int) time(NULL));
-
-	/* Checksum 8 cannot be calculated until we've filled all other fields */
-
-	/* Type - regular file */
-	sprintf(&h[156], "0");
-
-	/* Link Name 100 (leave as nulls) */
-
-	/* Magic 6 */
-	sprintf(&h[257], "ustar");
-
-	/* Version 2 */
-	sprintf(&h[263], "00");
-
-	/* User 32 */
-	/* XXX: Do we need to care about setting correct username? */
-	sprintf(&h[265], "%.31s", "postgres");
-
-	/* Group 32 */
-	/* XXX: Do we need to care about setting correct group name? */
-	sprintf(&h[297], "%.31s", "postgres");
-
-	/* Major Dev 8 */
-	sprintf(&h[329], "%07o ", 0);
-
-	/* Minor Dev 8 */
-	sprintf(&h[337], "%07o ", 0);
-
-	/* Prefix 155 - not used, leave as nulls */
-
-	/*
-	 * We mustn't overwrite the next field while inserting the checksum.
-	 * Fortunately, the checksum can't exceed 6 octal digits, so we just write
-	 * 6 digits, a space, and a null, which is legal per POSIX.
-	 */
-	sprintf(&h[148], "%06o ", _tarChecksum(h));
+	tarCreateHeader(h, th->targetFile, NULL, th->fileLen, 0600, 04000, 02000, time(NULL));
 
 	/* Now write the completed header. */
 	if (fwrite(h, 1, 512, th->tarFH) != 512)

src/include/port.h

@@ -465,4 +465,8 @@ extern int	pg_check_dir(const char *dir);
 /* port/pgmkdirp.c */
 extern int	pg_mkdir_p(char *path, int omode);
 
+/* port/tar.c */
+void		tarCreateHeader(char *h, const char *filename, const char *linktarget, size_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime);
+int			tarChecksum(char *header);
+
 #endif   /* PG_PORT_H */

src/port/Makefile

@@ -32,7 +32,7 @@ LIBS += $(PTHREAD_LIBS)
 
 OBJS = $(LIBOBJS) chklocale.o dirmod.o erand48.o exec.o fls.o inet_net_ntop.o \
 	noblock.o path.o pgcheckdir.o pg_crc.o pgmkdirp.o pgsleep.o \
-	pgstrcasecmp.o qsort.o qsort_arg.o sprompt.o thread.o
+	pgstrcasecmp.o qsort.o qsort_arg.o sprompt.o tar.o thread.o
 
 # foo_srv.o and foo.o are both built from foo.c, but only foo.o has -DFRONTEND
 OBJS_SRV = $(OBJS:%.o=%_srv.o)

src/port/tar.c

+#include "c.h"
+#include <sys/stat.h>
+
+/*
+ * Utility routine to print possibly larger than 32 bit integers in a
+ * portable fashion.  Filled with zeros.
+ */
+static void
+print_val(char *s, uint64 val, unsigned int base, size_t len)
+{
+	int			i;
+
+	for (i = len; i > 0; i--)
+	{
+		int			digit = val % base;
+
+		s[i - 1] = '0' + digit;
+		val = val / base;
+	}
+}
+
+
+/*
+ * Calculate the tar checksum for a header. The header is assumed to always
+ * be 512 bytes, per the tar standard.
+ */
+int
+tarChecksum(char *header)
+{
+	int			i,
+				sum;
+
+	/*
+	 * Per POSIX, the checksum is the simple sum of all bytes in the header,
+	 * treating the bytes as unsigned, and treating the checksum field (at
+	 * offset 148) as though it contained 8 spaces.
+	 */
+	sum = 8 * ' ';				/* presumed value for checksum field */
+	for (i = 0; i < 512; i++)
+		if (i < 148 || i >= 156)
+			sum += 0xFF & header[i];
+	return sum;
+}
+
+
+/*
+ * Fill in the buffer pointed to by h with a tar format header. This buffer
+ * must always have space for 512 characters, which is a requirement by
+ * the tar format.
+ */
+void
+tarCreateHeader(char *h, const char *filename, const char *linktarget,
+				size_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
+{
+	/*
+	 * Note: most of the fields in a tar header are not supposed to be
+	 * null-terminated.  We use sprintf, which will write a null after the
+	 * required bytes; that null goes into the first byte of the next field.
+	 * This is okay as long as we fill the fields in order.
+	 */
+	memset(h, 0, 512);			/* assume tar header size */
+
+	/* Name 100 */
+	sprintf(&h[0], "%.99s", filename);
+	if (linktarget != NULL || S_ISDIR(mode))
+	{
+		/*
+		 * We only support symbolic links to directories, and this is
+		 * indicated in the tar format by adding a slash at the end of the
+		 * name, the same as for regular directories.
+		 */
+		int			flen = strlen(filename);
+
+		flen = Min(flen, 99);
+		h[flen] = '/';
+		h[flen + 1] = '\0';
+	}
+
+	/* Mode 8 */
+	sprintf(&h[100], "%07o ", (int) mode);
+
+	/* User ID 8 */
+	sprintf(&h[108], "%07o ", uid);
+
+	/* Group 8 */
+	sprintf(&h[116], "%07o ", gid);
+
+	/* File size 12 - 11 digits, 1 space; use print_val for 64 bit support */
+	if (linktarget != NULL || S_ISDIR(mode))
+		/* Symbolic link or directory has size zero */
+		print_val(&h[124], 0, 8, 11);
+	else
+		print_val(&h[124], size, 8, 11);
+	sprintf(&h[135], " ");
+
+	/* Mod Time 12 */
+	sprintf(&h[136], "%011o ", (int) mtime);
+
+	/* Checksum 8 cannot be calculated until we've filled all other fields */
+
+	if (linktarget != NULL)
+	{
+		/* Type - Symbolic link */
+		sprintf(&h[156], "2");
+		/* Link Name 100 */
+		sprintf(&h[157], "%.99s", linktarget);
+	}
+	else if (S_ISDIR(mode))
+		/* Type - directory */
+		sprintf(&h[156], "5");
+	else
+		/* Type - regular file */
+		sprintf(&h[156], "0");
+
+	/* Magic 6 */
+	sprintf(&h[257], "ustar");
+
+	/* Version 2 */
+	sprintf(&h[263], "00");
+
+	/* User 32 */
+	/* XXX: Do we need to care about setting correct username? */
+	sprintf(&h[265], "%.31s", "postgres");
+
+	/* Group 32 */
+	/* XXX: Do we need to care about setting correct group name? */
+	sprintf(&h[297], "%.31s", "postgres");
+
+	/* Major Dev 8 */
+	sprintf(&h[329], "%07o ", 0);
+
+	/* Minor Dev 8 */
+	sprintf(&h[337], "%07o ", 0);
+
+	/* Prefix 155 - not used, leave as nulls */
+
+	/*
+	 * We mustn't overwrite the next field while inserting the checksum.
+	 * Fortunately, the checksum can't exceed 6 octal digits, so we just write
+	 * 6 digits, a space, and a null, which is legal per POSIX.
+	 */
+	sprintf(&h[148], "%06o ", tarChecksum(h));
+}