- Support for BLOB output from pg_dump and input via pg_restore

- Support for direct DB connection in pg_restore
- Fixes in support for --insert flag
- pg_dump now outputs in modified OID order
- various other bug fixes

src/bin/pg_dump/Makefile

@@ -4,7 +4,7 @@
 #
 # Copyright (c) 1994, Regents of the University of California
 #
-# $Header: /cvsroot/pgsql/src/bin/pg_dump/Makefile,v 1.19 2000/07/04 19:52:00 petere Exp $
+# $Header: /cvsroot/pgsql/src/bin/pg_dump/Makefile,v 1.20 2000/07/21 11:40:08 pjw Exp $
 #
 #-------------------------------------------------------------------------
 
@@ -12,8 +12,8 @@ subdir = src/bin/pg_dump
 top_builddir = ../../..
 include ../../Makefile.global
 
-OBJS= pg_backup_archiver.o pg_backup_custom.o pg_backup_files.o \
-       pg_backup_plain_text.o $(STRDUP)
+OBJS= pg_backup_archiver.o pg_backup_db.o pg_backup_custom.o pg_backup_files.o \
+       pg_backup_null.o pg_backup_tar.o $(STRDUP)
 
 CFLAGS+= -I$(LIBPQDIR)
 LIBS+= -lz

src/bin/pg_dump/README

 Notes on pg_dump
 ================
 
-pg_dump, by default, still outputs text files.
+1. pg_dump, by default, still outputs text files.
 
-pg_dumpall forces all pg_dump output to be text, since it also outputs text into the same output stream.
+2. pg_dumpall forces all pg_dump output to be text, since it also outputs text into the same output stream.
 
-The plain text output format can not be used as input into pg_restore.
+3. The plain text output format can not be used as input into pg_restore.
+
+4. pg_dump now dumps the items in a modified OID order to try to improve relaibility of default restores.
 
 
 To dump a database into the next custom format, type:
 
     pg_dump <db-name> -Fc > <backup-file>
 
+or, in TAR format
+
+	pg_dump <db-name> -Ft > <backup-file>
+
 To restore, try
  
    To list contents:
@@ -53,7 +59,37 @@ or, simply:
     pg_restore backup.bck --use=toc.lis | psql newdbname
 
 
-Philip Warner, 3-Jul-2000
+BLOBs
+=====
+
+To dump blobs you must use the custom archive format (-Fc) or TAR format (-Ft), and specify the 
+--blobs qualifier to the pg_dump command.
+
+To restore blobs you must use a direct database connection (--db=db-to-restore-to).
+
+eg.
+
+   pg_dump --blob -Fc db-to-backup -f backup.bck
+
+   pg_restore backup.bck --db=db-to-restore-into
+
+
+TAR
+===
+
+The TAR archive that pg_dump creates currently has a blank username & group for the files, 
+but should be otherwise valid. It also includes a 'restore.sql' script which is there for
+the benefit of humans. It is never used by pg_restore.
+
+Note: the TAR format archive can only be used as input into pg_restore if it is in TAR form.
+(ie. you should not extract the files then expect pg_restore to work). 
+
+You can extract, edit, and tar the files again, and it should work, but the 'toc'
+file should go at the start, the data files be in the order they are used, and
+the BLOB files at the end.
+
+
+Philip Warner, 16-Jul-2000
 pjw@rhyme.com.au
 
 

src/bin/pg_dump/pg_backup.h

@@ -30,12 +30,15 @@
 
 #define PG_BACKUP__
 
+#include "postgres.h"
+#include "libpq-fe.h"
+
 typedef enum _archiveFormat {
     archUnknown = 0,
     archCustom = 1,
     archFiles = 2,
     archTar = 3,
-    archPlainText = 4
+    archNull = 4
 } ArchiveFormat;
 
 /*
@@ -43,7 +46,8 @@ typedef enum _archiveFormat {
  *  time this gives us some abstraction and type checking.
  */
 typedef struct _Archive {
-    /* Nothing here */
+	int			verbose;
+    /* The rest is private */
 } Archive;
 
 typedef int     (*DataDumperPtr)(Archive* AH, char* oid, void* userArg);
@@ -73,6 +77,13 @@ typedef struct _restoreOptions {
 	char		*tableNames;
 	char		*triggerNames;
 
+	int			useDB;
+	char		*dbname;
+	char		*pgport;
+	char		*pghost;
+	int			ignoreVersion;
+	int			requirePassword;
+
 	int		*idWanted;
 	int		limitToList;
 	int		compression;
@@ -83,24 +94,42 @@ typedef struct _restoreOptions {
  * Main archiver interface.
  */
 
+extern void exit_horribly(Archive *AH, const char *fmt, ...);
+
+/* Lets the archibe know we have a DB connection to shutdown if it dies */
+
+PGconn* ConnectDatabase(Archive *AH,
+		const char* 	dbname,
+		const char*	pghost,
+		const char*	pgport,
+		const int	reqPwd,
+		const int	ignoreVersion);
+
+
 /* Called to add a TOC entry */
 extern void	ArchiveEntry(Archive* AH, const char* oid, const char* name,
 			const char* desc, const char* (deps[]), const char* defn,
-			const char* dropStmt, const char* owner, 
+			const char* dropStmt, const char* copyStmt, const char* owner, 
 			DataDumperPtr dumpFn, void* dumpArg);
 
 /* Called to write *data* to the archive */
 extern int	WriteData(Archive* AH, const void* data, int dLen);
 
+//extern int 	StartBlobs(Archive* AH);
+//extern int	EndBlobs(Archive* AH);
+extern int	StartBlob(Archive* AH, int oid);
+extern int	EndBlob(Archive* AH, int oid);
+
 extern void	CloseArchive(Archive* AH);
 
 extern void	RestoreArchive(Archive* AH, RestoreOptions *ropt);
 
 /* Open an existing archive */
-extern Archive* OpenArchive(const char* FileSpec, ArchiveFormat fmt);
+extern Archive* OpenArchive(const char* FileSpec, const ArchiveFormat fmt);
 
 /* Create a new archive */
-extern Archive* CreateArchive(const char* FileSpec, ArchiveFormat fmt, int compression);
+extern Archive* CreateArchive(const char* FileSpec, const ArchiveFormat fmt, 
+								const int compression);
 
 /* The --list option */
 extern void	PrintTOCSummary(Archive* AH, RestoreOptions *ropt);

src/bin/pg_dump/pg_backup_archiver.c

@@ -25,6 +25,8 @@
 
 #include "pg_backup.h"
 #include "pg_backup_archiver.h"
+#include "pg_backup_db.h"
+
 #include <string.h>
 #include <unistd.h> /* for dup */
 
@@ -32,10 +34,13 @@
 #include <stdlib.h>
 #include <stdarg.h>
 
+#include "pqexpbuffer.h"
+#include "libpq/libpq-fs.h"
+
 static void		_SortToc(ArchiveHandle* AH, TocSortCompareFn fn);
 static int		_tocSortCompareByOIDNum(const void *p1, const void *p2);
 static int		_tocSortCompareByIDNum(const void *p1, const void *p2);
-static ArchiveHandle* 	_allocAH(const char* FileSpec, ArchiveFormat fmt, 
+static ArchiveHandle* 	_allocAH(const char* FileSpec, const ArchiveFormat fmt, 
 				int compression, ArchiveMode mode);
 static int 		_printTocEntry(ArchiveHandle* AH, TocEntry* te, RestoreOptions *ropt);
 static int		_tocEntryRequired(TocEntry* te, RestoreOptions *ropt);
@@ -45,8 +50,14 @@ static TocEntry*	_getTocEntry(ArchiveHandle* AH, int id);
 static void		_moveAfter(ArchiveHandle* AH, TocEntry* pos, TocEntry* te);
 static void		_moveBefore(ArchiveHandle* AH, TocEntry* pos, TocEntry* te);
 static int		_discoverArchiveFormat(ArchiveHandle* AH);
+
 static char	*progname = "Archiver";
 
+static void		_die_horribly(ArchiveHandle *AH, const char *fmt, va_list ap);
+
+static int 		_canRestoreBlobs(ArchiveHandle *AH);
+
+
 /*
  *  Wrapper functions.
  * 
@@ -58,7 +69,9 @@ static char	*progname = "Archiver";
 
 /* Create a new archive */
 /* Public */
-Archive* CreateArchive(const char* FileSpec, ArchiveFormat fmt, int compression)
+Archive* CreateArchive(const char* FileSpec, const ArchiveFormat fmt,
+                                const int compression)
+
 {
     ArchiveHandle*	AH = _allocAH(FileSpec, fmt, compression, archModeWrite);
     return (Archive*)AH;
@@ -66,7 +79,7 @@ Archive* CreateArchive(const char* FileSpec, ArchiveFormat fmt, int compression)
 
 /* Open an existing archive */
 /* Public */
-Archive* OpenArchive(const char* FileSpec, ArchiveFormat fmt) 
+Archive* OpenArchive(const char* FileSpec, const ArchiveFormat fmt) 
 {
     ArchiveHandle*      AH = _allocAH(FileSpec, fmt, 0, archModeRead);
     return (Archive*)AH;
@@ -93,47 +106,155 @@ void RestoreArchive(Archive* AHX, RestoreOptions *ropt)
     int			reqs;
     OutputContext	sav;
 
+	AH->ropt = ropt;
+
+	/*
+	 * If we're using a DB connection, then connect it.
+	 */
+	if (ropt->useDB)
+	{
+		ahlog(AH, 1, "Connecting to database for restore\n");
+		if (AH->version < K_VERS_1_3)
+			die_horribly(AH, "Direct database connections are not supported in pre-1.3 archives");
+
+		ConnectDatabase(AHX, ropt->dbname, ropt->pghost, ropt->pgport, 
+							ropt->requirePassword, ropt->ignoreVersion);
+	}
+
+	/*
+	 *	Setup the output file if necessary.
+	 */
     if (ropt->filename || ropt->compression)
 		sav = SetOutput(AH, ropt->filename, ropt->compression);
 
     ahprintf(AH, "--\n-- Selected TOC Entries:\n--\n");
 
-    /* Drop the items at the start, in reverse order */
+    /*
+     * Drop the items at the start, in reverse order 
+	 */
     if (ropt->dropSchema) {
 		te = AH->toc->prev;
 		while (te != AH->toc) {
 			reqs = _tocEntryRequired(te, ropt);
-	    if ( (reqs & 1) && te->dropStmt) {  /* We want the schema */
+			if ( ( (reqs & 1) != 0) && te->dropStmt) {  /* We want the schema */
+				ahlog(AH, 1, "Dropping %s %s\n", te->desc, te->name);
 				ahprintf(AH, "%s", te->dropStmt);
 			}
 			te = te->prev;
 		}
     }
 
+	/*
+	 * Now process each TOC entry
+	 */
     te = AH->toc->next;
     while (te != AH->toc) {
+
+		/* Work out what, if anything, we want from this entry */
 		reqs = _tocEntryRequired(te, ropt);
 
-	if (reqs & 1) /* We want the schema */
+		if ( (reqs & 1) != 0) /* We want the schema */
+		{
+			ahlog(AH, 1, "Creating %s %s\n", te->desc, te->name);
 			_printTocEntry(AH, te, ropt);
+		}
 
+		/* 
+		 * If we want data, and it has data, then restore that too 
+		 */
 		if (AH->PrintTocDataPtr != NULL && (reqs & 2) != 0) {
 #ifndef HAVE_LIBZ
 			if (AH->compression != 0)
-		die_horribly("%s: Unable to restore data from a compressed archive\n", progname);
+				die_horribly(AH, "%s: Unable to restore data from a compressed archive\n", progname);
 #endif
+
+			ahlog(AH, 1, "Restoring data for %s \n", te->name);
+
+			ahprintf(AH, "--\n-- Data for TOC Entry ID %d (OID %s) %s %s\n--\n\n",
+						te->id, te->oid, te->desc, te->name);
+
+			/*
+			 * Maybe we can't do BLOBS, so check if this node is for BLOBS 
+			 */
+			if ((strcmp(te->desc,"BLOBS") == 0) && !_canRestoreBlobs(AH))
+			{
+				ahprintf(AH, "--\n-- SKIPPED \n--\n\n");
+				/*
+				 * This is a bit nasty - we assume, for the moment, that if a custom
+				 * output is used, then we don't want warnings.
+				 */
+				if (!AH->CustomOutPtr)
+					fprintf(stderr, "%s: WARNING - skipping BLOB restoration\n", progname);
+			} else {
+
 				_disableTriggers(AH, te, ropt);
+
+
+				/* If we have a copy statement, use it. As of V1.3, these are separate 
+				 * to allow easy import from withing a database connection. Pre 1.3 
+				 * archives can not use DB connections and are sent to output only.
+				 *
+				 * For V1.3+, the table data MUST have a copy statement so that 
+				 * we can go into appropriate mode with libpq.
+				 */
+				if (te->copyStmt && strlen(te->copyStmt) > 0)
+					ahprintf(AH, te->copyStmt);
+
 				(*AH->PrintTocDataPtr)(AH, te, ropt);
+
 				_enableTriggers(AH, te, ropt);
 			}
+		}
 		te = te->next;
     }
 
+	/*
+	 * Now use blobs_xref (if used) to fixup any refs for tables that we loaded
+	 */
+	if (_canRestoreBlobs(AH) && AH->createdBlobXref)
+	{
+		te = AH->toc->next;
+		while (te != AH->toc) {
+
+			/* Is it table data? */
+			if (strcmp(te->desc, "TABLE DATA") == 0) {
+
+				ahlog(AH, 2, "Checking if we loaded %s\n", te->name);
+
+				reqs = _tocEntryRequired(te, ropt);
+
+				if ( (reqs & 2) != 0) /* We loaded the data */
+				{
+					ahlog(AH, 1, "Fixing up BLOB ref for %s\n", te->name);
+					FixupBlobRefs(AH, te->name);
+				}
+			}
+			else
+			{
+				ahlog(AH, 2, "Ignoring BLOB xrefs for %s %s\n", te->desc, te->name);
+			}
+
+			te = te->next;
+		}
+	}
+
+	/*
+	 * Clean up & we're done.
+	 */
     if (ropt->filename)
 		ResetOutput(AH, sav);
 
+	if (ropt->useDB)
+	{
+		PQfinish(AH->connection);
+		AH->connection = NULL;
+	}
 }
 
+/*
+ * Allocate a new RestoreOptions block.
+ * This is mainly so we can initialize it, but also for future expansion,
+ */
 RestoreOptions*		NewRestoreOptions(void)
 {
 	RestoreOptions* opts;
@@ -145,6 +266,11 @@ RestoreOptions*		NewRestoreOptions(void)
 	return opts;
 }
 
+static int _canRestoreBlobs(ArchiveHandle *AH)
+{
+	return (AH->ropt->useDB && AH->connection);
+}
+
 static void _disableTriggers(ArchiveHandle *AH, TocEntry *te, RestoreOptions *ropt)
 {
     ahprintf(AH, "-- Disable triggers\n");
@@ -168,7 +294,7 @@ static void _enableTriggers(ArchiveHandle *AH, TocEntry *te, RestoreOptions *rop
 
 
 /*
- * This is a routine that is available to pg_dump, hence the 'Archive*' parameter.
+ * This is a routine that is part of the dumper interface, hence the 'Archive*' parameter.
  */
 
 /* Public */
@@ -176,6 +302,10 @@ int	WriteData(Archive* AHX, const void* data, int dLen)
 {
     ArchiveHandle*      AH = (ArchiveHandle*)AHX;
 
+	if (!AH->currToc)
+		die_horribly(AH, "%s: WriteData can not be called outside the context of "
+						"a DataDumper routine\n", progname);
+
     return (*AH->WriteDataPtr)(AH, data, dLen);
 }
 
@@ -187,7 +317,7 @@ int	WriteData(Archive* AHX, const void* data, int dLen)
 /* Public */
 void	ArchiveEntry(Archive* AHX, const char* oid, const char* name,
 			const char* desc, const char* (deps[]), const char* defn,
-                        const char* dropStmt, const char* owner,
+			const char* dropStmt, const char* copyStmt, const char* owner,
 			DataDumperPtr dumpFn, void* dumpArg)
 {
     ArchiveHandle*	AH = (ArchiveHandle*)AHX;
@@ -196,9 +326,9 @@ void	ArchiveEntry(Archive* AHX, const char* oid, const char* name,
     AH->lastID++;
     AH->tocCount++;
 
-    newToc = (TocEntry*)malloc(sizeof(TocEntry));
+    newToc = (TocEntry*)calloc(1, sizeof(TocEntry));
     if (!newToc)
-	die_horribly("Archiver: unable to allocate memory for TOC entry\n");
+	die_horribly(AH, "Archiver: unable to allocate memory for TOC entry\n");
 
     newToc->prev = AH->toc->prev;
     newToc->next = AH->toc;
@@ -212,6 +342,7 @@ void	ArchiveEntry(Archive* AHX, const char* oid, const char* name,
     newToc->desc = strdup(desc);
     newToc->defn = strdup(defn);
     newToc->dropStmt = strdup(dropStmt);
+	newToc->copyStmt = copyStmt ? strdup(copyStmt) : NULL;
     newToc->owner = strdup(owner);
     newToc->printed = 0;
     newToc->formatData = NULL;
@@ -233,10 +364,30 @@ void PrintTOCSummary(Archive* AHX, RestoreOptions *ropt)
     ArchiveHandle*	AH = (ArchiveHandle*) AHX;
     TocEntry		*te = AH->toc->next;
     OutputContext	sav;
+	char			*fmtName;
 
     if (ropt->filename)
 	sav = SetOutput(AH, ropt->filename, ropt->compression);
 
+	ahprintf(AH, ";\n; Archive created at %s", ctime(&AH->createDate));
+	ahprintf(AH, ";     dbname: %s\n;     TOC Entries: %d\n;     Compression: %d\n",
+				AH->archdbname, AH->tocCount, AH->compression);
+
+	switch (AH->format) {
+		case archFiles:
+			fmtName = "FILES";
+			break;
+		case archCustom:
+			fmtName = "CUSTOM";
+			break;
+		case archTar:
+			fmtName = "TAR";
+			break;
+		default:
+			fmtName = "UNKNOWN";
+	}
+	ahprintf(AH, ";     Format: %s\n;\n", fmtName);
+
     ahprintf(AH, ";\n; Selected TOC Entries:\n;\n");
 
     while (te != AH->toc) {
@@ -249,6 +400,79 @@ void PrintTOCSummary(Archive* AHX, RestoreOptions *ropt)
 	ResetOutput(AH, sav);
 }
 
+/***********
+ * BLOB Archival
+ ***********/
+
+/* Called by a dumper to signal start of a BLOB */
+int StartBlob(Archive* AHX, int oid)
+{
+    ArchiveHandle* AH = (ArchiveHandle*)AHX;
+
+    if (!AH->StartBlobPtr)
+		die_horribly(AH, "%s: BLOB output not supported in chosen format\n", progname);
+
+    (*AH->StartBlobPtr)(AH, AH->currToc, oid);
+
+    return 1;
+}
+
+/* Called by a dumper to signal end of a BLOB */
+int EndBlob(Archive* AHX, int oid)
+{
+    ArchiveHandle* AH = (ArchiveHandle*)AHX;
+
+    if (AH->EndBlobPtr)
+		(*AH->EndBlobPtr)(AH, AH->currToc, oid);
+
+    return 1;
+}
+
+/**********
+ * BLOB Restoration
+ **********/
+
+/*
+ * Called by a format handler to initiate restoration of a blob
+ */
+void StartRestoreBlob(ArchiveHandle* AH, int oid)
+{
+	int			loOid;
+
+	if (!AH->createdBlobXref)
+	{
+		if (!AH->connection)
+			die_horribly(AH, "%s: can not restore BLOBs without a database connection", progname);
+
+		CreateBlobXrefTable(AH);
+		AH->createdBlobXref = 1;
+	}
+
+	loOid = lo_creat(AH->connection, INV_READ | INV_WRITE);
+	if (loOid == 0)
+		die_horribly(AH, "%s: unable to create BLOB\n", progname);
+
+	ahlog(AH, 1, "Restoring BLOB oid %d as %d\n", oid, loOid);
+
+	StartTransaction(AH);
+
+	InsertBlobXref(AH, oid, loOid);
+
+	AH->loFd = lo_open(AH->connection, loOid, INV_WRITE);
+	if (AH->loFd == -1)
+		die_horribly(AH, "%s: unable to open BLOB\n", progname);
+
+    AH->writingBlob = 1;
+}
+
+void EndRestoreBlob(ArchiveHandle* AH, int oid)
+{
+    lo_close(AH->connection, AH->loFd);
+    AH->writingBlob = 0;
+
+	CommitTransaction(AH);
+}
+
 /***********
  * Sorting and Reordering
  ***********/
@@ -256,6 +480,7 @@ void PrintTOCSummary(Archive* AHX, RestoreOptions *ropt)
 /*
  * Move TOC entries of the specified type to the START of the TOC.
  */
+
 /* Public */
 void MoveToStart(Archive* AHX, char *oType) 
 {
@@ -356,7 +581,7 @@ void SortTocFromFile(Archive* AHX, RestoreOptions *ropt)
     /* Setup the file */
     fh = fopen(ropt->tocFile, PG_BINARY_R);
     if (!fh)
-	die_horribly("%s: could not open TOC file\n", progname);
+	die_horribly(AH, "%s: could not open TOC file\n", progname);
 
     while (fgets(buf, 1024, fh) != NULL)
     {
@@ -377,14 +602,14 @@ void SortTocFromFile(Archive* AHX, RestoreOptions *ropt)
 	id = strtol(buf, &endptr, 10);
 	if (endptr == buf)
 	{
-	    fprintf(stderr, "%s: warning - line ignored: %s\n", progname, buf);
+	    fprintf(stderr, "%s: WARNING - line ignored: %s\n", progname, buf);
 	    continue;
 	}
 
 	/* Find TOC entry */
 	te = _getTocEntry(AH, id);
 	if (!te) 
-	    die_horribly("%s: could not find entry for id %d\n",progname, id);
+	    die_horribly(AH, "%s: could not find entry for id %d\n",progname, id);
 
 	ropt->idWanted[id-1] = 1;
 
@@ -428,7 +653,7 @@ int archprintf(Archive* AH, const char *fmt, ...)
 	if ((p = malloc(bSize)) == NULL)
 	{
 	    va_end(ap);
-	    die_horribly("%s: could not allocate buffer for archprintf\n", progname);
+	    exit_horribly(AH, "%s: could not allocate buffer for archprintf\n", progname);
 	}
 	cnt = vsnprintf(p, bSize, fmt, ap);
     }
@@ -525,7 +750,7 @@ int ahprintf(ArchiveHandle* AH, const char *fmt, ...)
 		if (p == NULL)
 		{
 			va_end(ap);
-	    die_horribly("%s: could not allocate buffer for ahprintf\n", progname);
+			die_horribly(AH, "%s: could not allocate buffer for ahprintf\n", progname);
 		}
 		cnt = vsnprintf(p, bSize, fmt, ap);
     }
@@ -535,28 +760,82 @@ int ahprintf(ArchiveHandle* AH, const char *fmt, ...)
     return cnt;
 }
 
+void ahlog(ArchiveHandle* AH, int level, const char *fmt, ...)
+{
+	va_list		ap;
+
+	if (AH->debugLevel < level && (!AH->public.verbose || level > 1))
+		return;
+
+	va_start(ap, fmt);
+	vfprintf(stderr, fmt, ap);
+	va_end(ap);
+}
+
 /*
- *  Write buffer to the output file (usually stdout).
+ *  Write buffer to the output file (usually stdout). This is user for
+ *  outputting 'restore' scripts etc. It is even possible for an archive
+ * 	format to create a custom output routine to 'fake' a restore if it
+ *	wants to generate a script (see TAR output).
  */
 int ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle* AH)
 {
-    if (AH->gzOut)
+	int		res;
+
+    if (AH->writingBlob)
+	{
+		res = lo_write(AH->connection, AH->loFd, (void*)ptr, size * nmemb);
+		ahlog(AH, 5, "Wrote %d bytes of BLOB data (result = %d)\n", size * nmemb, res);
+		return res;
+	}
+    else if (AH->gzOut)
 		return GZWRITE((void*)ptr, size, nmemb, AH->OF);
+    else if (AH->CustomOutPtr)
+		return AH->CustomOutPtr(AH, ptr, size * nmemb);
+	else
+	{
+		/*
+		 * If we're doing a restore, and it's direct to DB, and we're connected
+	     * then send it to the DB.
+		 */	
+		if (AH->ropt && AH->ropt->useDB && AH->connection)
+			return ExecuteSqlCommandBuf(AH, (void*)ptr, size*nmemb);
 		else
 			return fwrite((void*)ptr, size, nmemb, AH->OF);
+	}
 }		
 
+/* Common exit code */
+static void _die_horribly(ArchiveHandle *AH, const char *fmt, va_list ap)
+{
+    vfprintf(stderr, fmt, ap);
 
-void die_horribly(const char *fmt, ...)
+    if (AH)
+	if (AH->connection)
+	    PQfinish(AH->connection);
+
+    exit(1);
+}
+
+/* External use */
+void exit_horribly(Archive *AH, const char *fmt, ...)
 {
     va_list     ap;
 
     va_start(ap, fmt);
-    vfprintf(stderr, fmt, ap);
-    va_end(ap);
-    exit(1);
+    _die_horribly((ArchiveHandle*)AH, fmt, ap);
 }
 
+/* Archiver use (just different arg declaration) */
+void die_horribly(ArchiveHandle *AH, const char *fmt, ...)
+{
+    va_list 	ap;
+
+    va_start(ap, fmt);
+    _die_horribly(AH, fmt, ap);
+}
+
+
 static void _moveAfter(ArchiveHandle* AH, TocEntry* pos, TocEntry* te)
 {
     te->prev->next = te->next;
@@ -638,18 +917,19 @@ int	WriteInt(ArchiveHandle* AH, int i)
 int    	ReadInt(ArchiveHandle* AH)
 {
     int res = 0;
-    int shft = 1;
     int bv, b;
     int	sign = 0; /* Default positive */
+	int bitShift = 0;
 
     if (AH->version > K_VERS_1_0)
 		/* Read a sign byte */
 		sign = (*AH->ReadBytePtr)(AH);
 
     for( b = 0 ; b < AH->intSize ; b++) {
-        bv = (*AH->ReadBytePtr)(AH);
-        res = res  + shft * bv;
-        shft *= 256;
+        bv = (*AH->ReadBytePtr)(AH) & 0xFF;
+		if (bv != 0)
+			res = res + (bv << bitShift);
+		bitShift += 8;
     }
 
     if (sign)
@@ -660,8 +940,17 @@ int    	ReadInt(ArchiveHandle* AH)
 
 int	WriteStr(ArchiveHandle* AH, char* c)
 {
-    int l = WriteInt(AH, strlen(c));
-    return (*AH->WriteBufPtr)(AH, c, strlen(c)) + l;
+    int res;
+
+	if (c)
+	{
+		res = WriteInt(AH, strlen(c));
+		res += (*AH->WriteBufPtr)(AH, c, strlen(c));
+	}
+	else
+		res = WriteInt(AH, -1);
+
+    return res;
 }
 
 char*	ReadStr(ArchiveHandle* AH)
@@ -670,12 +959,18 @@ char*	ReadStr(ArchiveHandle* AH)
     int		l;
 
     l = ReadInt(AH);
+	if (l == -1)
+		buf = NULL;
+	else
+	{
 		buf = (char*)malloc(l+1);
 		if (!buf)
-	die_horribly("Archiver: Unable to allocate sufficient memory in ReadStr\n");
+			die_horribly(AH, "%s: Unable to allocate sufficient memory in ReadStr - "													"requested %d (0x%x) bytes\n", progname, l, l);
 
 		(*AH->ReadBufPtr)(AH, (void*)buf, l);
 		buf[l] = '\0';
+	}
+
     return buf;
 }
 
@@ -686,6 +981,17 @@ int _discoverArchiveFormat(ArchiveHandle* AH)
     int		cnt;
     int		wantClose = 0;
 
+	/*
+	 * fprintf(stderr, "%s: Attempting to ascertain archive format\n", progname);
+	 */
+
+	if (AH->lookahead)
+		free(AH->lookahead);
+
+	AH->lookaheadSize = 512;
+	AH->lookahead = calloc(1, 512);
+	AH->lookaheadLen = 0;
+	AH->lookaheadPos = 0;
 
     if (AH->fSpec) {
 		wantClose = 1;
@@ -695,114 +1001,149 @@ int _discoverArchiveFormat(ArchiveHandle* AH)
     }
 
     if (!fh)
-	die_horribly("Archiver: could not open input file\n");
+		die_horribly(AH, "Archiver: could not open input file\n");
 
     cnt = fread(sig, 1, 5, fh);
 
     if (cnt != 5)
-        die_horribly("%s: input file is too short, or is unreadable\n", progname);
+        die_horribly(AH, "%s: input file is too short, or is unreadable\n", progname);
 
-    if (strncmp(sig, "PGDMP", 5) != 0)
-	die_horribly("%s: input file does not appear to be a valid archive\n", progname);
+	/* Save it, just in case we need it later*/
+	strncpy(&AH->lookahead[0], sig, 5);
+	AH->lookaheadLen = 5;
 
+    if (strncmp(sig, "PGDMP", 5) == 0)
+	{
 		AH->vmaj = fgetc(fh);
 		AH->vmin = fgetc(fh);
 
+		/* Save these too... */
+		AH->lookahead[AH->lookaheadLen++] = AH->vmaj;
+		AH->lookahead[AH->lookaheadLen++] = AH->vmin;
+
 		/* Check header version; varies from V1.0 */
 		if (AH->vmaj > 1 || ( (AH->vmaj == 1) && (AH->vmin > 0) ) ) /* Version > 1.0 */
+		{	
 			AH->vrev = fgetc(fh);
+			AH->lookahead[AH->lookaheadLen++] = AH->vrev;
+		}
 		else
 			AH->vrev = 0;
 
 		AH->intSize = fgetc(fh);
+		AH->lookahead[AH->lookaheadLen++] = AH->intSize;
+
 		AH->format = fgetc(fh);
+		AH->lookahead[AH->lookaheadLen++] = AH->format;
 
 		/* Make a convenient integer <maj><min><rev>00 */
 		AH->version = ( (AH->vmaj * 256 + AH->vmin) * 256 + AH->vrev ) * 256 + 0;
+	} else {
+		/*
+		 * *Maybe* we have a tar archive format file...
+		 * So, read first 512 byte header...
+		 */
+		cnt = fread(&AH->lookahead[AH->lookaheadLen], 1, 512 - AH->lookaheadLen, fh);
+		AH->lookaheadLen += cnt;
+
+		if (AH->lookaheadLen != 512)
+			die_horribly(AH, "%s: input file does not appear to be a valid archive (too short?)\n",
+							progname);
+
+		if (!isValidTarHeader(AH->lookahead))
+			die_horribly(AH, "%s: input file does not appear to be a valid archive\n", progname);
+
+		AH->format = archTar;
+	}
 
     /* If we can't seek, then mark the header as read */
     if (fseek(fh, 0, SEEK_SET) != 0) 
+	{
+		/*
+		 * NOTE: Formats that use the looahead buffer can unset this in their Init routine.
+		 */
 		AH->readHeader = 1;
+	}
+	else
+		AH->lookaheadLen = 0; /* Don't bother since we've reset the file */
+
+	/*
+	 *fprintf(stderr, "%s: read %d bytes into lookahead buffer\n", progname, AH->lookaheadLen);
+	 */
 
     /* Close the file */
     if (wantClose)
 		fclose(fh);
 
     return AH->format;
-
 }
 
 
 /*
  * Allocate an archive handle
  */
-static ArchiveHandle* _allocAH(const char* FileSpec, ArchiveFormat fmt, 
-				int compression, ArchiveMode mode) {
+static ArchiveHandle* _allocAH(const char* FileSpec, const ArchiveFormat fmt, 
+				const int compression, ArchiveMode mode) 
+{
     ArchiveHandle*	AH;
 
+	/*
+	 *fprintf(stderr, "%s: allocating AH for %s, format %d\n", progname, FileSpec, fmt); 
+	 */
+
     AH = (ArchiveHandle*)calloc(1, sizeof(ArchiveHandle));
     if (!AH) 
-	die_horribly("Archiver: Could not allocate archive handle\n");
+		die_horribly(AH, "Archiver: Could not allocate archive handle\n");
 
     AH->vmaj = K_VERS_MAJOR;
     AH->vmin = K_VERS_MINOR;
 
+	AH->createDate = time(NULL);
+
     AH->intSize = sizeof(int);
     AH->lastID = 0;
     if (FileSpec) {
 		AH->fSpec = strdup(FileSpec);
+		/*
+		 * Not used; maybe later....
+		 *
+		 * AH->workDir = strdup(FileSpec);
+		 * for(i=strlen(FileSpec) ; i > 0 ; i--)
+		 *   if (AH->workDir[i-1] == '/')
+		 */
     } else {
 		AH->fSpec = NULL;
     } 
-    AH->FH = NULL;
-    AH->formatData = NULL;
 
-    AH->currToc = NULL;
     AH->currUser = "";
 
     AH->toc = (TocEntry*)calloc(1, sizeof(TocEntry));
     if (!AH->toc)
-	die_horribly("Archiver: Could not allocate TOC header\n");
+		die_horribly(AH, "Archiver: Could not allocate TOC header\n");
 
-    AH->tocCount = 0;
     AH->toc->next = AH->toc;
     AH->toc->prev = AH->toc;
-    AH->toc->id  = 0;
-    AH->toc->oid  = NULL;
-    AH->toc->name = NULL; /* eg. MY_SPECIAL_FUNCTION */
-    AH->toc->desc = NULL; /* eg. FUNCTION */
-    AH->toc->defn = NULL; /* ie. sql to define it */
-    AH->toc->depOid = NULL;
     
     AH->mode = mode;
-    AH->format = fmt;
     AH->compression = compression;
 
-    AH->ArchiveEntryPtr = NULL;
-
-    AH->StartDataPtr = NULL;
-    AH->WriteDataPtr = NULL;
-    AH->EndDataPtr = NULL;
-
-    AH->WriteBytePtr = NULL;
-    AH->ReadBytePtr = NULL;
-    AH->WriteBufPtr = NULL;
-    AH->ReadBufPtr = NULL;
-    AH->ClosePtr = NULL;
-    AH->WriteExtraTocPtr = NULL;
-    AH->ReadExtraTocPtr = NULL;
-    AH->PrintExtraTocPtr = NULL;
-
-    AH->readHeader = 0;
+	AH->pgCopyBuf = createPQExpBuffer();
+	AH->sqlBuf = createPQExpBuffer();
 
     /* Open stdout with no compression for AH output handle */
     AH->gzOut = 0;
     AH->OF = stdout;
 
+	/*
+	 *fprintf(stderr, "%s: archive format is %d\n", progname, fmt);
+	 */
+
     if (fmt == archUnknown)
-	fmt = _discoverArchiveFormat(AH);
+		AH->format = _discoverArchiveFormat(AH);
+	else
+		AH->format = fmt;
 
-    switch (fmt) {
+    switch (AH->format) {
 
 		case archCustom:
 			InitArchiveFmt_Custom(AH);
@@ -812,12 +1153,16 @@ static ArchiveHandle* _allocAH(const char* FileSpec, ArchiveFormat fmt,
 			InitArchiveFmt_Files(AH);
 			break;
 
-	case archPlainText:
-	    InitArchiveFmt_PlainText(AH);
+		case archNull:
+			InitArchiveFmt_Null(AH);
+			break;
+
+		case archTar:
+			InitArchiveFmt_Tar(AH);
 			break;
 
 		default:
-	    die_horribly("Archiver: Unrecognized file format '%d'\n", fmt);
+			die_horribly(AH, "Archiver: Unrecognized file format '%d'\n", fmt);
     }
 
     return AH;
@@ -827,13 +1172,25 @@ static ArchiveHandle* _allocAH(const char* FileSpec, ArchiveFormat fmt,
 void WriteDataChunks(ArchiveHandle* AH)
 {
     TocEntry		*te = AH->toc->next;
+	StartDataPtr	startPtr;
+	EndDataPtr		endPtr;
 
     while (te != AH->toc) {
 	if (te->dataDumper != NULL) {
 	    AH->currToc = te;
 	    /* printf("Writing data for %d (%x)\n", te->id, te); */
-	    if (AH->StartDataPtr != NULL) {
-		(*AH->StartDataPtr)(AH, te);
+
+		if (strcmp(te->desc, "BLOBS") == 0)
+		{
+			startPtr = AH->StartBlobsPtr;
+			endPtr = AH->EndBlobsPtr;
+		} else {
+			startPtr = AH->StartDataPtr;
+			endPtr = AH->EndDataPtr;
+		}
+
+	    if (startPtr != NULL) {
+			(*startPtr)(AH, te);
 	    }
 
 	    /* printf("Dumper arg for %d is %x\n", te->id, te->dataDumperArg); */
@@ -842,8 +1199,8 @@ void WriteDataChunks(ArchiveHandle* AH)
 	     */
 	    (*te->dataDumper)((Archive*)AH, te->oid, te->dataDumperArg);
 
-	    if (AH->EndDataPtr != NULL) {
-		(*AH->EndDataPtr)(AH, te);
+	    if (endPtr != NULL) {
+			(*endPtr)(AH, te);
 	    }
 	    AH->currToc = NULL;
 	}
@@ -866,6 +1223,7 @@ void WriteToc(ArchiveHandle* AH)
 	WriteStr(AH, te->desc);
 	WriteStr(AH, te->defn);
 	WriteStr(AH, te->dropStmt);
+	WriteStr(AH, te->copyStmt);
 	WriteStr(AH, te->owner);
 	if (AH->WriteExtraTocPtr) {
 	    (*AH->WriteExtraTocPtr)(AH, te);
@@ -884,12 +1242,12 @@ void ReadToc(ArchiveHandle* AH)
 
     for( i = 0 ; i < AH->tocCount ; i++) {
 
-	te = (TocEntry*)malloc(sizeof(TocEntry));
+		te = (TocEntry*)calloc(1, sizeof(TocEntry));
 		te->id = ReadInt(AH);
 
 		/* Sanity check */
 		if (te->id <= 0 || te->id > AH->tocCount)
-	    die_horribly("Archiver: failed sanity check (bad entry id) - perhaps a corrupt TOC\n");
+			die_horribly(AH, "Archiver: failed sanity check (bad entry id) - perhaps a corrupt TOC\n");
 
 		te->hadDumper = ReadInt(AH);
 		te->oid = ReadStr(AH);
@@ -898,10 +1256,18 @@ void ReadToc(ArchiveHandle* AH)
 		te->desc = ReadStr(AH);
 		te->defn = ReadStr(AH);
 		te->dropStmt = ReadStr(AH);
+
+		if (AH->version >= K_VERS_1_3)
+			te->copyStmt = ReadStr(AH);
+
 		te->owner = ReadStr(AH);
+
 		if (AH->ReadExtraTocPtr) {
 			(*AH->ReadExtraTocPtr)(AH, te);
 		}
+
+		ahlog(AH, 3, "Read TOC entry %d (id %d) for %s %s\n", i, te->id, te->desc, te->name);
+
 		te->prev = AH->toc->prev;
 		AH->toc->prev->next = te;
 		AH->toc->prev = te;
@@ -911,7 +1277,7 @@ void ReadToc(ArchiveHandle* AH)
 
 static int _tocEntryRequired(TocEntry* te, RestoreOptions *ropt)
 {
-    int res = 3; /* Data and Schema */
+    int res = 3; /* Schema = 1, Data = 2, Both = 3 */
  
     /* If it's an ACL, maybe ignore it */
     if (ropt->aclsSkip && strcmp(te->desc,"ACL") == 0)
@@ -979,7 +1345,8 @@ static int _printTocEntry(ArchiveHandle* AH, TocEntry* te, RestoreOptions *ropt)
     ahprintf(AH, "--\n\n");
 
     if (te->owner && strlen(te->owner) != 0 && strcmp(AH->currUser, te->owner) != 0) {
-	ahprintf(AH, "\\connect - %s\n", te->owner);
+		//todo pjw - fix for db connection...
+		//ahprintf(AH, "\\connect - %s\n", te->owner);
 		AH->currUser = te->owner;
     }
 
@@ -990,6 +1357,8 @@ static int _printTocEntry(ArchiveHandle* AH, TocEntry* te, RestoreOptions *ropt)
 
 void WriteHead(ArchiveHandle* AH) 
 {
+	struct tm		crtm;
+
     (*AH->WriteBufPtr)(AH, "PGDMP", 5); 	/* Magic code */
     (*AH->WriteBytePtr)(AH, AH->vmaj);
     (*AH->WriteBytePtr)(AH, AH->vmin);
@@ -1003,26 +1372,35 @@ void WriteHead(ArchiveHandle* AH)
 		    "archive will be uncompressed \n", progname);
 
     AH->compression = 0;
-    (*AH->WriteBytePtr)(AH, 0);
-
-#else
-
-    (*AH->WriteBytePtr)(AH, AH->compression);
 
 #endif
+
+	WriteInt(AH, AH->compression);
+
+	crtm = *localtime(&AH->createDate);
+	WriteInt(AH, crtm.tm_sec);
+	WriteInt(AH, crtm.tm_min);
+	WriteInt(AH, crtm.tm_hour);
+	WriteInt(AH, crtm.tm_mday);
+	WriteInt(AH, crtm.tm_mon);
+	WriteInt(AH, crtm.tm_year);
+	WriteInt(AH, crtm.tm_isdst);
+	WriteStr(AH, AH->dbname);	
 }
 
 void ReadHead(ArchiveHandle* AH)
 {
     char		tmpMag[7];
     int			fmt;
+	struct tm	crtm;
 
+	/* If we haven't already read the header... */
     if (!AH->readHeader) {
 
 		(*AH->ReadBufPtr)(AH, tmpMag, 5);
 
 		if (strncmp(tmpMag,"PGDMP", 5) != 0)
-	    die_horribly("Archiver: Did not fing magic PGDMP in file header\n");
+			die_horribly(AH, "Archiver: Did not fing magic PGDMP in file header\n");
 
 		AH->vmaj = (*AH->ReadBytePtr)(AH);
 		AH->vmin = (*AH->ReadBytePtr)(AH);
@@ -1038,35 +1416,58 @@ void ReadHead(ArchiveHandle* AH)
 
 
 		if (AH->version < K_VERS_1_0 || AH->version > K_VERS_MAX)
-	    die_horribly("Archiver: unsupported version (%d.%d) in file header\n", AH->vmaj, AH->vmin);
+			die_horribly(AH, "%s: unsupported version (%d.%d) in file header\n", 
+					progname, AH->vmaj, AH->vmin);
 
 		AH->intSize = (*AH->ReadBytePtr)(AH);
 		if (AH->intSize > 32)
-	    die_horribly("Archiver: sanity check on integer size (%d) failes\n", AH->intSize);
+			die_horribly(AH, "Archiver: sanity check on integer size (%d) failes\n", AH->intSize);
 
 		if (AH->intSize > sizeof(int))
-	    fprintf(stderr, "\nWARNING: Backup file was made on a machine with larger integers, "
-			    "some operations may fail\n");
+			fprintf(stderr, "\n%s: WARNING - archive was made on a machine with larger integers, "
+					"some operations may fail\n", progname);
 
 		fmt = (*AH->ReadBytePtr)(AH);
 
 		if (AH->format != fmt)
-	    die_horribly("Archiver: expected format (%d) differs from format found in file (%d)\n", 
-			    AH->format, fmt);
+			die_horribly(AH, "%s: expected format (%d) differs from format found in file (%d)\n", 
+					progname, AH->format, fmt);
     }
 
     if (AH->version >= K_VERS_1_2)
     {
+		if (AH->version < K_VERS_1_4)
 			AH->compression = (*AH->ReadBytePtr)(AH);
+		else
+			AH->compression = ReadInt(AH);
     } else {
 		AH->compression = Z_DEFAULT_COMPRESSION;
     }
 
 #ifndef HAVE_LIBZ
     if (AH->compression != 0)
-	fprintf(stderr, "%s: WARNING - archive is compressed - any data will not be available\n", progname);
+		fprintf(stderr, "%s: WARNING - archive is compressed - any data will not be available\n", 
+					progname);
 #endif
 
+	if (AH->version >= K_VERS_1_4)
+	{
+		crtm.tm_sec = ReadInt(AH);
+		crtm.tm_min = ReadInt(AH);
+		crtm.tm_hour = ReadInt(AH);
+		crtm.tm_mday = ReadInt(AH);
+		crtm.tm_mon = ReadInt(AH);
+		crtm.tm_year = ReadInt(AH);
+		crtm.tm_isdst = ReadInt(AH);
+
+		AH->archdbname = ReadStr(AH);
+
+		AH->createDate = mktime(&crtm);
+
+		if (AH->createDate == (time_t)-1)
+			fprintf(stderr, "%s: WARNING - bad creation date in header\n", progname);
+	}
+
 }
 
 
@@ -1144,5 +1545,12 @@ static int	_tocSortCompareByIDNum(const void* p1, const void* p2)
     }
 }
 
-
+/*
+ * Maybe I can use this somewhere...
+ *
+ *create table pgdump_blob_path(p text);
+ *insert into pgdump_blob_path values('/home/pjw/work/postgresql-cvs/pgsql/src/bin/pg_dump_140');
+ *
+ *insert into dump_blob_xref select 12345,lo_import(p || '/q.q') from pgdump_blob_path;
+ */
 

src/bin/pg_dump/pg_backup_archiver.h

@@ -29,6 +29,10 @@
 #define __PG_BACKUP_ARCHIVE__
 
 #include <stdio.h>
+#include <time.h>
+
+#include "postgres.h"
+#include "pqexpbuffer.h"
 
 #ifdef HAVE_LIBZ
 #include <zlib.h>
@@ -51,15 +55,23 @@ typedef z_stream *z_streamp;
 #endif
 
 #include "pg_backup.h"
+#include "libpq-fe.h"
 
 #define K_VERS_MAJOR 1
-#define K_VERS_MINOR 2 
-#define K_VERS_REV 2
+#define K_VERS_MINOR 4 
+#define K_VERS_REV 3 
+
+/* Data block types */
+#define BLK_DATA 1
+#define BLK_BLOB 2
+#define BLK_BLOBS 3
 
 /* Some important version numbers (checked in code) */
 #define K_VERS_1_0 (( (1 * 256 + 0) * 256 + 0) * 256 + 0)
-#define K_VERS_1_2 (( (1 * 256 + 2) * 256 + 0) * 256 + 0)
-#define K_VERS_MAX (( (1 * 256 + 2) * 256 + 255) * 256 + 0)
+#define K_VERS_1_2 (( (1 * 256 + 2) * 256 + 0) * 256 + 0) /* Allow No ZLIB */
+#define K_VERS_1_3 (( (1 * 256 + 3) * 256 + 0) * 256 + 0) /* BLOBs */
+#define K_VERS_1_4 (( (1 * 256 + 4) * 256 + 0) * 256 + 0) /* Date & name in header */
+#define K_VERS_MAX (( (1 * 256 + 4) * 256 + 255) * 256 + 0)
 
 struct _archiveHandle;
 struct _tocEntry;
@@ -72,6 +84,11 @@ typedef void	(*StartDataPtr)		(struct _archiveHandle* AH, struct _tocEntry* te);
 typedef int 	(*WriteDataPtr)		(struct _archiveHandle* AH, const void* data, int dLen);
 typedef void	(*EndDataPtr)		(struct _archiveHandle* AH, struct _tocEntry* te);
 
+typedef void	(*StartBlobsPtr)	(struct _archiveHandle* AH, struct _tocEntry* te);
+typedef void    (*StartBlobPtr)		(struct _archiveHandle* AH, struct _tocEntry* te, int oid);
+typedef void	(*EndBlobPtr)		(struct _archiveHandle* AH, struct _tocEntry* te, int oid);
+typedef void	(*EndBlobsPtr)		(struct _archiveHandle* AH, struct _tocEntry* te);
+
 typedef int		(*WriteBytePtr)		(struct _archiveHandle* AH, const int i);
 typedef int    	(*ReadBytePtr)		(struct _archiveHandle* AH);
 typedef int		(*WriteBufPtr)		(struct _archiveHandle* AH, const void* c, int len);
@@ -83,6 +100,8 @@ typedef void	(*PrintExtraTocPtr)	(struct _archiveHandle* AH, struct _tocEntry* t
 typedef void	(*PrintTocDataPtr)	(struct _archiveHandle* AH, struct _tocEntry* te, 
 						RestoreOptions *ropt);
 
+typedef int		(*CustomOutPtr)		(struct _archiveHandle* AH, const void* buf, int len);
+
 typedef int	(*TocSortCompareFn)	(const void* te1, const void *te2); 
 
 typedef enum _archiveMode {
@@ -95,16 +114,44 @@ typedef struct _outputContext {
 	int		gzOut;
 } OutputContext;
 
+typedef enum { 
+	SQL_SCAN = 0,
+	SQL_IN_SQL_COMMENT,
+	SQL_IN_EXT_COMMENT,
+	SQL_IN_QUOTE} sqlparseState;
+	
+typedef struct {
+	int					backSlash;
+	sqlparseState		state;
+	char				lastChar;
+	char				quoteChar;
+} sqlparseInfo;
+
 typedef struct _archiveHandle {
+	Archive				public;				/* Public part of archive */
 	char				vmaj;				/* Version of file */
 	char				vmin;
 	char				vrev;
 	int					version;			/* Conveniently formatted version */
 
+	int					debugLevel;			/* Not used. Intended for logging */
 	int					intSize;			/* Size of an integer in the archive */
 	ArchiveFormat		format;				/* Archive format */
 
+	sqlparseInfo		sqlparse;
+	PQExpBuffer			sqlBuf;
+
+	time_t				createDate;			/* Date archive created */
+
+	/*
+	 * Fields used when discovering header.
+	 * A format can always get the previous read bytes from here...
+	 */
 	int					readHeader;			/* Used if file header has been read already */
+	char				*lookahead;			/* Buffer used when reading header to discover format */
+	int					lookaheadSize;		/* Size of allocated buffer */
+	int					lookaheadLen;		/* Length of data in lookahead */
+	int					lookaheadPos;		/* Current read position in lookahead buffer */
 
 	ArchiveEntryPtr		ArchiveEntryPtr;	/* Called for each metadata object */
 	StartDataPtr		StartDataPtr; 		/* Called when table data is about to be dumped */
@@ -121,6 +168,28 @@ typedef struct _archiveHandle {
 	PrintExtraTocPtr	PrintExtraTocPtr;	/* Extra TOC info for format */
 	PrintTocDataPtr		PrintTocDataPtr;
 
+	StartBlobsPtr		StartBlobsPtr;
+	EndBlobsPtr			EndBlobsPtr;
+	StartBlobPtr		StartBlobPtr;
+	EndBlobPtr			EndBlobPtr;
+
+	CustomOutPtr		CustomOutPtr;		/* Alternate script output routine */
+
+	/* Stuff for direct DB connection */
+	char				username[100];
+	char				*dbname;			/* Name of db for connection */
+	char				*archdbname;		/* DB name *read* from archive */
+	char				*pghost;
+	char				*pgport;
+	PGconn				*connection;
+	int					connectToDB;		/* Flag to indicate if direct DB connection is required */
+	int					pgCopyIn;			/* Currently in libpq 'COPY IN' mode. */
+	PQExpBuffer			pgCopyBuf;			/* Left-over data from incomplete lines in COPY IN */
+
+	int					loFd;				/* BLOB fd */
+	int					writingBlob;		/* Flag */
+	int					createdBlobXref;	/* Flag */
+
 	int					lastID;				/* Last internal ID for a TOC entry */
 	char*				fSpec;				/* Archive File Spec */
 	FILE				*FH;				/* General purpose file handle */
@@ -135,6 +204,7 @@ typedef struct _archiveHandle {
 	ArchiveMode				mode;			/* File mode - r or w */
 	void*					formatData;		/* Header data specific to file format */
 
+	RestoreOptions			*ropt;			/* Used to check restore options in ahwrite etc */
 } ArchiveHandle;
 
 typedef struct _tocEntry {
@@ -148,6 +218,7 @@ typedef struct _tocEntry {
 	char*				desc;
 	char*				defn;
 	char*				dropStmt;
+	char*				copyStmt;
 	char*				owner;
 	char**				depOid;
 	int					printed;		/* Indicates if entry defn has been dumped */
@@ -159,7 +230,8 @@ typedef struct _tocEntry {
 
 } TocEntry;
 
-extern void die_horribly(const char *fmt, ...);
+/* Used everywhere */
+extern void die_horribly(ArchiveHandle *AH, const char *fmt, ...);
 
 extern void WriteTOC(ArchiveHandle* AH);
 extern void ReadTOC(ArchiveHandle* AH);
@@ -180,9 +252,15 @@ extern int ReadInt(ArchiveHandle* AH);
 extern char* 			ReadStr(ArchiveHandle* AH);
 extern int 				WriteStr(ArchiveHandle* AH, char* s);
 
+extern void 			StartRestoreBlob(ArchiveHandle* AH, int oid);
+extern void 			EndRestoreBlob(ArchiveHandle* AH, int oid);
+
 extern void 			InitArchiveFmt_Custom(ArchiveHandle* AH);
 extern void 			InitArchiveFmt_Files(ArchiveHandle* AH);
-extern void InitArchiveFmt_PlainText(ArchiveHandle* AH);
+extern void 			InitArchiveFmt_Null(ArchiveHandle* AH);
+extern void 			InitArchiveFmt_Tar(ArchiveHandle* AH);
+
+extern int 				isValidTarHeader(char *header);
 
 extern OutputContext	SetOutput(ArchiveHandle* AH, char *filename, int compression);
 extern void 			ResetOutput(ArchiveHandle* AH, OutputContext savedContext);
@@ -190,4 +268,6 @@ extern void 		ResetOutput(ArchiveHandle* AH, OutputContext savedContext);
 int ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle* AH);
 int ahprintf(ArchiveHandle* AH, const char *fmt, ...);
 
+void ahlog(ArchiveHandle* AH, int level, const char *fmt, ...);
+
 #endif

src/bin/pg_dump/pg_backup_custom.c

@@ -4,6 +4,9 @@
  *
  *	Implements the custom output format.
  *
+ *  The comments with the routined in this code are a good place to 
+ * 	understand how to write a new format.
+ *
  *	See the headers to pg_restore for more details.
  *
  * Copyright (c) 2000, Philip Warner
@@ -11,7 +14,8 @@
  *      as this notice is not removed.
  *
  *	The author is not responsible for loss or damages that may
- *	result from it's use.
+ *	and any liability will be limited to the time taken to fix any
+ * 	related bug.
  *
  *
  * IDENTIFICATION
@@ -26,8 +30,12 @@
 #include <stdlib.h>
 #include "pg_backup.h"
 #include "pg_backup_archiver.h"
+#include <errno.h>
 
-extern int	errno;
+/*--------
+ * Routines in the format interface
+ *--------
+ */
 
 static void     _ArchiveEntry(ArchiveHandle* AH, TocEntry* te);
 static void		_StartData(ArchiveHandle* AH, TocEntry* te);
@@ -45,7 +53,19 @@ static void	_PrintExtraToc(ArchiveHandle* AH, TocEntry* te);
 
 static void		_PrintData(ArchiveHandle* AH);
 static void     _skipData(ArchiveHandle* AH);
-
+static void		_skipBlobs(ArchiveHandle* AH);
+
+static void		_StartBlobs(ArchiveHandle* AH, TocEntry* te);
+static void		_StartBlob(ArchiveHandle* AH, TocEntry* te, int oid);
+static void		_EndBlob(ArchiveHandle* AH, TocEntry* te, int oid);
+static void		_EndBlobs(ArchiveHandle* AH, TocEntry* te);
+static void		_LoadBlobs(ArchiveHandle* AH);
+
+/*------------
+ * Buffers used in zlib compression and extra data stored in archive and
+ * in TOC entries.
+ *------------
+ */
 #define zlibOutSize	4096
 #define zlibInSize	4096
 
@@ -64,12 +84,30 @@ typedef struct {
     int		dataLen;
 } lclTocEntry;
 
+
+/*------
+ * Static declarations 
+ *------
+ */
+static void		_readBlockHeader(ArchiveHandle *AH, int *type, int *id);
+static void     _StartDataCompressor(ArchiveHandle* AH, TocEntry* te);
+static void     _EndDataCompressor(ArchiveHandle* AH, TocEntry* te);
 static int		_getFilePos(ArchiveHandle* AH, lclContext* ctx);
+static int		_DoDeflate(ArchiveHandle* AH, lclContext* ctx, int flush);
 
 static char*	progname = "Archiver(custom)";
 
+
+
 /*
- *  Handler functions. 
+ *  Init routine required by ALL formats. This is a global routine
+ *	and should be declared in pg_backup_archiver.h
+ *
+ *	It's task is to create any extra archive context (using AH->formatData),
+ *	and to initialize the supported function pointers.
+ *
+ *	It should also prepare whatever it's input source is for reading/writing,
+ *	and in the case of a read mode connection, it should load the Header & TOC. 
  */
 void InitArchiveFmt_Custom(ArchiveHandle* AH) 
 {
@@ -90,17 +128,22 @@ void InitArchiveFmt_Custom(ArchiveHandle* AH)
     AH->WriteExtraTocPtr = _WriteExtraToc;
     AH->PrintExtraTocPtr = _PrintExtraToc;
 
+    AH->StartBlobsPtr = _StartBlobs;
+    AH->StartBlobPtr = _StartBlob;
+    AH->EndBlobPtr = _EndBlob;
+    AH->EndBlobsPtr = _EndBlobs;
+
     /*
      *	Set up some special context used in compressing data.
     */
     ctx = (lclContext*)malloc(sizeof(lclContext));
     if (ctx == NULL)
-	die_horribly("%s: Unable to allocate archive context",progname);
+		die_horribly(AH, "%s: Unable to allocate archive context",progname);
     AH->formatData = (void*)ctx;
 
     ctx->zp = (z_streamp)malloc(sizeof(z_stream));
     if (ctx->zp == NULL)
-	die_horribly("%s: unable to allocate zlib stream archive context",progname);
+	die_horribly(AH, "%s: unable to allocate zlib stream archive context",progname);
 
     ctx->zlibOut = (char*)malloc(zlibOutSize);
     ctx->zlibIn = (char*)malloc(zlibInSize);
@@ -108,12 +151,13 @@ void InitArchiveFmt_Custom(ArchiveHandle* AH)
     ctx->filePos = 0;
 
     if (ctx->zlibOut == NULL || ctx->zlibIn == NULL)
-	die_horribly("%s: unable to allocate buffers in archive context",progname);
+		die_horribly(AH, "%s: unable to allocate buffers in archive context",progname);
 
     /*
      * Now open the file
     */
     if (AH->mode == archModeWrite) {
+
 		if (AH->fSpec && strcmp(AH->fSpec,"") != 0) {
 			AH->FH = fopen(AH->fSpec, PG_BINARY_W);
 		} else {
@@ -121,18 +165,19 @@ void InitArchiveFmt_Custom(ArchiveHandle* AH)
 		}
 
 		if (!AH)
-	    die_horribly("%s: unable to open archive file %s",progname, AH->fSpec);
+			die_horribly(AH, "%s: unable to open archive file %s",progname, AH->fSpec);
 
 		ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0);
 
 	} else {
+
 		if (AH->fSpec && strcmp(AH->fSpec,"") != 0) {
 			AH->FH = fopen(AH->fSpec, PG_BINARY_R);
 		} else {
 			AH->FH = stdin;
 		}
 		if (!AH)
-	    die_horribly("%s: unable to open archive file %s",progname, AH->fSpec);
+			die_horribly(AH, "%s: unable to open archive file %s",progname, AH->fSpec);
 
 		ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0);
 
@@ -144,13 +189,17 @@ void InitArchiveFmt_Custom(ArchiveHandle* AH)
 }
 
 /*
- * - Start a new TOC entry
+ * Called by the Archiver when the dumper creates a new TOC entry.
+ *
+ * Optional.
+ *
+ * Set up extrac format-related TOC data.  
 */
 static void	_ArchiveEntry(ArchiveHandle* AH, TocEntry* te) 
 {
     lclTocEntry*	ctx;
 
-    ctx = (lclTocEntry*)malloc(sizeof(lclTocEntry));
+    ctx = (lclTocEntry*)calloc(1, sizeof(lclTocEntry));
     if (te->dataDumper) {
 		ctx->dataPos = -1;
     } else {
@@ -161,6 +210,15 @@ static void	_ArchiveEntry(ArchiveHandle* AH, TocEntry* te)
 
 }
 
+/*
+ * Called by the Archiver to save any extra format-related TOC entry
+ * data.
+ *
+ * Optional.
+ *
+ * Use the Archiver routines to write data - they are non-endian, and
+ * maintain other important file information.
+ */
 static void	_WriteExtraToc(ArchiveHandle* AH, TocEntry* te)
 {
     lclTocEntry*	ctx = (lclTocEntry*)te->formatData;
@@ -169,6 +227,14 @@ static void	_WriteExtraToc(ArchiveHandle* AH, TocEntry* te)
     WriteInt(AH, ctx->dataLen);
 }
 
+/*
+ * Called by the Archiver to read any extra format-related TOC data.
+ *
+ * Optional.
+ *
+ * Needs to match the order defined in _WriteExtraToc, and sould also
+ * use the Archiver input routines.
+ */
 static void	_ReadExtraToc(ArchiveHandle* AH, TocEntry* te)
 {
     lclTocEntry*	ctx = (lclTocEntry*)te->formatData;
@@ -180,9 +246,15 @@ static void	_ReadExtraToc(ArchiveHandle* AH, TocEntry* te)
 
     ctx->dataPos = ReadInt( AH );
     ctx->dataLen = ReadInt( AH );
-    
 }
 
+/*
+ * Called by the Archiver when restoring an archive to output a comment
+ * that includes useful information about the TOC entry.
+ *
+ * Optional.
+ *
+ */
 static void	_PrintExtraToc(ArchiveHandle* AH, TocEntry* te)
 {
     lclTocEntry*	ctx = (lclTocEntry*)te->formatData;
@@ -190,101 +262,40 @@ static void	_PrintExtraToc(ArchiveHandle* AH, TocEntry* te)
     ahprintf(AH, "-- Data Pos: %d (Length %d)\n", ctx->dataPos, ctx->dataLen);
 }
 
+/*
+ * Called by the archiver when saving TABLE DATA (not schema). This routine
+ * should save whatever format-specific information is needed to read
+ * the archive back.
+ *
+ * It is called just prior to the dumper's 'DataDumper' routine being called.
+ *
+ * Optional, but strongly recommended.
+ *
+ */
 static void	_StartData(ArchiveHandle* AH, TocEntry* te)
 {
     lclContext*		ctx = (lclContext*)AH->formatData;
-    z_streamp   	zp = ctx->zp;
     lclTocEntry*	tctx = (lclTocEntry*)te->formatData;
 
     tctx->dataPos = _getFilePos(AH, ctx);
 
+    _WriteByte(AH, BLK_DATA); /* Block type */
     WriteInt(AH, te->id); /* For sanity check */
 
-#ifdef HAVE_LIBZ
-
-    if (AH->compression < 0 || AH->compression > 9) {
-	AH->compression = Z_DEFAULT_COMPRESSION;
-    }
-
-    if (AH->compression != 0) {
-	zp->zalloc = Z_NULL;
-	zp->zfree = Z_NULL;
-	zp->opaque = Z_NULL;
-
-	if (deflateInit(zp, AH->compression) != Z_OK)
-	    die_horribly("%s: could not initialize compression library - %s\n",progname, zp->msg);
-    }
+    _StartDataCompressor(AH, te);
 
-#else
-
-    AH->compression = 0;
-
-#endif
-
-    /* Just be paranoid - maye End is called after Start, with no Write */
-    zp->next_out = ctx->zlibOut;
-    zp->avail_out = zlibOutSize;
 }
 
-static int	_DoDeflate(ArchiveHandle* AH, lclContext* ctx, int flush) 
-{
-    z_streamp   zp = ctx->zp;
-
-#ifdef HAVE_LIBZ
-    char*	out = ctx->zlibOut;
-    int		res = Z_OK;
-
-    if (AH->compression != 0) 
-    {
-	res = deflate(zp, flush);
-	if (res == Z_STREAM_ERROR)
-	    die_horribly("%s: could not compress data - %s\n",progname, zp->msg);
-
-	if 	(      ( (flush == Z_FINISH) && (zp->avail_out < zlibOutSize) )
-		|| (zp->avail_out == 0) 
-		|| (zp->avail_in != 0)
-	    ) 
-	{
-	    /*
-	     * Extra paranoia: avoid zero-length chunks since a zero 
-	     * length chunk is the EOF marker. This should never happen
-	     * but...
+/*
+ * Called by archiver when dumper calls WriteData. This routine is 
+ * called for both BLOB and TABLE data; it is the responsibility of
+ * the format to manage each kind of data using StartBlob/StartData.
+ *
+ * It should only be called from withing a DataDumper routine.
+ *
+ * Mandatory.
+ *
  */
-	    if (zp->avail_out < zlibOutSize) {
-		/* printf("Wrote %d byte deflated chunk\n", zlibOutSize - zp->avail_out); */
-		WriteInt(AH, zlibOutSize - zp->avail_out);
-		fwrite(out, 1, zlibOutSize - zp->avail_out, AH->FH);
-		ctx->filePos += zlibOutSize - zp->avail_out;
-	    }
-	    zp->next_out = out;
-	    zp->avail_out = zlibOutSize;
-	}
-    } else {
-#endif
-	if (zp->avail_in > 0)
-	{
-	    WriteInt(AH, zp->avail_in);
-	    fwrite(zp->next_in, 1, zp->avail_in, AH->FH);
-	    ctx->filePos += zp->avail_in;
-	    zp->avail_in = 0;
-	} else {
-#ifdef HAVE_LIBZ
-	    if (flush == Z_FINISH)
-		res = Z_STREAM_END;
-#endif
-	}
-
-
-#ifdef HAVE_LIBZ
-    }
-
-    return res;
-#else
-    return 1;
-#endif
-
-}
-
 static int	_WriteData(ArchiveHandle* AH, const void* data, int dLen)
 {
     lclContext*	ctx = (lclContext*)AH->formatData;
@@ -300,35 +311,82 @@ static int	_WriteData(ArchiveHandle* AH, const void* data, int dLen)
     return dLen;
 }
 
+/*
+ * Called by the archiver when a dumper's 'DataDumper' routine has
+ * finished.
+ *
+ * Optional.
+ *
+ */
 static void	_EndData(ArchiveHandle* AH, TocEntry* te)
 {
     lclContext*		ctx = (lclContext*)AH->formatData;
     lclTocEntry*	tctx = (lclTocEntry*) te->formatData;
 
-#ifdef HAVE_LIBZ
-    z_streamp		zp = ctx->zp;
-    int			res;
+    _EndDataCompressor(AH, te);
 
-    if (AH->compression != 0)
-    {
-	zp->next_in = NULL;
-	zp->avail_in = 0;
+    tctx->dataLen = _getFilePos(AH, ctx) - tctx->dataPos;
+}
 
-	do { 	
-	    /* printf("Ending data output\n"); */
-	    res = _DoDeflate(AH, ctx, Z_FINISH);
-	} while (res != Z_STREAM_END);
+/*
+ * Called by the archiver when starting to save all BLOB DATA (not schema). 
+ * This routine should save whatever format-specific information is needed
+ * to read the BLOBs back into memory. 
+ *
+ * It is called just prior to the dumper's DataDumper routine.
+ *
+ * Optional, but strongly recommended.
+ *
+ */
+static void	_StartBlobs(ArchiveHandle* AH, TocEntry* te)
+{
+    lclContext*		ctx = (lclContext*)AH->formatData;
+    lclTocEntry*	tctx = (lclTocEntry*)te->formatData;
 
-	if (deflateEnd(zp) != Z_OK)
-	    die_horribly("%s: error closing compression stream - %s\n", progname, zp->msg);
-    }
-#endif
+    tctx->dataPos = _getFilePos(AH, ctx);
 
-    /* Send the end marker */
-    WriteInt(AH, 0);
+    _WriteByte(AH, BLK_BLOBS); /* Block type */
+    WriteInt(AH, te->id); /* For sanity check */
 
-    tctx->dataLen = _getFilePos(AH, ctx) - tctx->dataPos;
+}
+
+/*
+ * Called by the archiver when the dumper calls StartBlob.
+ *
+ * Mandatory.
+ *
+ * Must save the passed OID for retrieval at restore-time.
+ */
+static void	_StartBlob(ArchiveHandle* AH, TocEntry* te, int oid)
+{
+    if (oid == 0) 
+	die_horribly(AH, "%s: illegal OID for BLOB (%d)\n", progname, oid);
+
+    WriteInt(AH, oid);
+    _StartDataCompressor(AH, te);
+}
 
+/*
+ * Called by the archiver when the dumper calls EndBlob.
+ *
+ * Optional.
+ *
+ */
+static void	_EndBlob(ArchiveHandle* AH, TocEntry* te, int oid)
+{
+    _EndDataCompressor(AH, te);
+}
+
+/*
+ * Called by the archiver when finishing saving all BLOB DATA. 
+ *
+ * Optional.
+ *
+ */
+static void	_EndBlobs(ArchiveHandle* AH, TocEntry* te)
+{
+	/* Write out a fake zero OID to mark end-of-blobs. */
+    WriteInt(AH, 0);
 }
 
 /*
@@ -339,37 +397,86 @@ static void	_PrintTocData(ArchiveHandle* AH, TocEntry* te, RestoreOptions *ropt)
     lclContext* 	ctx = (lclContext*)AH->formatData;
     int			id;
     lclTocEntry*	tctx = (lclTocEntry*) te->formatData;
+    int			blkType;
+    int			found = 0;
 
     if (tctx->dataPos == 0) 
 		return;
 
     if (!ctx->hasSeek || tctx->dataPos < 0) {
-	id = ReadInt(AH);
+
+	/* Skip over unnecessary blocks until we get the one we want. */
+
+	found = 0;
+
+	_readBlockHeader(AH, &blkType, &id);
 
 	while (id != te->id) {
-	    if (TocIDRequired(AH, id, ropt) & 2)
-		die_horribly("%s: Dumping a specific TOC data block out of order is not supported"
+
+	    if ( (TocIDRequired(AH, id, ropt) & 2) != 0)
+			die_horribly(AH, "%s: Dumping a specific TOC data block out of order is not supported"
 							" without on this input stream (fseek required)\n", progname);
+
+	    switch (blkType) {
+
+			case BLK_DATA:
+
 				_skipData(AH);
-	    id = ReadInt(AH);
+				break;
+
+			case BLK_BLOBS:
+
+				_skipBlobs(AH);
+				break;
+
+			default: /* Always have a default */
+
+				die_horribly(AH, "%s: unrecognized data block type while searching archive %d\n", 
+								progname, blkType);
+				break;
 	    }
+
+	    _readBlockHeader(AH, &blkType, &id);
+
+	}
+
     } else {
 
+	/* Grab it */
+
 		if (fseek(AH->FH, tctx->dataPos, SEEK_SET) != 0)
-	    die_horribly("%s: error %d in file seek\n",progname, errno);
+			die_horribly(AH, "%s: error %d in file seek\n",progname, errno);
 
-	id = ReadInt(AH);
+		_readBlockHeader(AH, &blkType, &id);
 
     }
 
+    /* Are we sane? */
     if (id != te->id)
-	die_horribly("%s: Found unexpected block ID (%d) when reading data - expected %d\n",
+		die_horribly(AH, "%s: Found unexpected block ID (%d) when reading data - expected %d\n",
 						progname, id, te->id);
 
-    ahprintf(AH, "--\n-- Data for TOC Entry ID %d (OID %s) %s %s\n--\n\n",
-		te->id, te->oid, te->desc, te->name);
+    switch (blkType) {
+
+		case BLK_DATA:
 
 			_PrintData(AH);
+			break;
+
+		case BLK_BLOBS:
+
+			if (!AH->connection)
+				die_horribly(AH, "%s: BLOBs can not be loaded without a database connection\n", progname);
+
+			_LoadBlobs(AH);
+			break;
+
+		default: /* Always have a default */
+
+			die_horribly(AH, "%s: unrecognized data block type %d while restoring archive\n", 
+							progname, blkType);
+			break;
+    }
 
     ahprintf(AH, "\n\n");
 }
@@ -384,11 +491,12 @@ static void	_PrintData(ArchiveHandle* AH)
     int		blkLen;
     char*	in = ctx->zlibIn;
     int		cnt;
-
 #ifdef HAVE_LIBZ
-
     int         res;
     char*	out = ctx->zlibOut;
+#endif
+
+#ifdef HAVE_LIBZ
 
     res = Z_OK;
 
@@ -398,26 +506,27 @@ static void	_PrintData(ArchiveHandle* AH)
 		zp->opaque = Z_NULL;
 
 		if (inflateInit(zp) != Z_OK)
-	    die_horribly("%s: could not initialize compression library - %s\n", progname, zp->msg);
+			die_horribly(AH, "%s: could not initialize compression library - %s\n", progname, zp->msg);
     }
 
 #endif
 
     blkLen = ReadInt(AH);
     while (blkLen != 0) {
-	if (blkLen > ctx->inSize) {
+		if (blkLen > (ctx->inSize - 1)) {
 			free(ctx->zlibIn);
 			ctx->zlibIn = NULL;
 			ctx->zlibIn = (char*)malloc(blkLen);
 			if (!ctx->zlibIn)
-		die_horribly("%s: failed to allocate decompression buffer\n", progname);
+				die_horribly(AH, "%s: failed to allocate decompression buffer\n", progname);
 
 			ctx->inSize = blkLen;
 			in = ctx->zlibIn;
 		}
+
 		cnt = fread(in, 1, blkLen, AH->FH);
 		if (cnt != blkLen) 
-	    die_horribly("%s: could not read data block - expected %d, got %d\n", progname, blkLen, cnt);
+			die_horribly(AH, "%s: could not read data block - expected %d, got %d\n", progname, blkLen, cnt);
 
 		ctx->filePos += blkLen;
 
@@ -433,13 +542,14 @@ static void	_PrintData(ArchiveHandle* AH)
 				zp->avail_out = zlibOutSize;
 				res = inflate(zp, 0);
 				if (res != Z_OK && res != Z_STREAM_END)
-		    die_horribly("%s: unable to uncompress data - %s\n", progname, zp->msg);
+					die_horribly(AH, "%s: unable to uncompress data - %s\n", progname, zp->msg);
 
 				out[zlibOutSize - zp->avail_out] = '\0';
 				ahwrite(out, 1, zlibOutSize - zp->avail_out, AH);
 			}
 		} else {
 #endif
+			in[zp->avail_in] = '\0';
 			ahwrite(in, 1, zp->avail_in, AH);
 			zp->avail_in = 0;
 
@@ -448,6 +558,7 @@ static void	_PrintData(ArchiveHandle* AH)
 #endif
 
 		blkLen = ReadInt(AH);
+
     }
 
 #ifdef HAVE_LIBZ
@@ -460,7 +571,7 @@ static void	_PrintData(ArchiveHandle* AH)
 			zp->avail_out = zlibOutSize;
 			res = inflate(zp, 0);
 			if (res != Z_OK && res != Z_STREAM_END)
-		die_horribly("%s: unable to uncompress data - %s\n", progname, zp->msg);
+				die_horribly(AH, "%s: unable to uncompress data - %s\n", progname, zp->msg);
 
 			out[zlibOutSize - zp->avail_out] = '\0';
 			ahwrite(out, 1, zlibOutSize - zp->avail_out, AH);
@@ -470,8 +581,42 @@ static void	_PrintData(ArchiveHandle* AH)
 
 }
 
+static void	_LoadBlobs(ArchiveHandle* AH)
+{
+    int		oid;
+
+    oid = ReadInt(AH);
+    while(oid != 0)
+    {
+		StartRestoreBlob(AH, oid);
+		_PrintData(AH);
+		EndRestoreBlob(AH, oid);
+		oid = ReadInt(AH);
+    }
+}
+
+/*
+ * Skip the BLOBs from the current file position.
+ * BLOBS are written sequentially as data blocks (see below).
+ * Each BLOB is preceded by it's original OID. 
+ * A zero OID indicated the end of the BLOBS
+ */
+static void	_skipBlobs(ArchiveHandle* AH)
+{
+    int		oid;
+
+    oid = ReadInt(AH);
+    while(oid != 0)
+    {
+	_skipData(AH);
+	oid = ReadInt(AH);
+    }
+}
+
 /*
  * Skip data from current file position.
+ * Data blocks are formatted as an integer length, followed by data. 
+ * A zero length denoted the end of the block.
 */
 static void	_skipData(ArchiveHandle* AH)
 {
@@ -490,7 +635,7 @@ static void	_skipData(ArchiveHandle* AH)
 	}
 	cnt = fread(in, 1, blkLen, AH->FH);
 	if (cnt != blkLen) 
-	    die_horribly("%s: could not read data block - expected %d, got %d\n", progname, blkLen, cnt);
+	    die_horribly(AH, "%s: could not read data block - expected %d, got %d\n", progname, blkLen, cnt);
 
 	ctx->filePos += blkLen;
 
@@ -499,6 +644,15 @@ static void	_skipData(ArchiveHandle* AH)
 
 }
 
+/*
+ * Write a byte of data to the archive.
+ *
+ * Mandatory.
+ *
+ * Called by the archiver to do integer & byte output to the archive.
+ * These routines are only used to read & write headers & TOC.
+ *
+ */
 static int	_WriteByte(ArchiveHandle* AH, const int i)
 {
     lclContext*		ctx = (lclContext*)AH->formatData;
@@ -511,6 +665,15 @@ static int	_WriteByte(ArchiveHandle* AH, const int i)
     return res;
 }
 
+/*
+ * Read a byte of data from the archive.
+ *
+ * Mandatory
+ *
+ * Called by the archiver to read bytes & integers from the archive.
+ * These routines are only used to read & write headers & TOC.
+ *
+ */
 static int    	_ReadByte(ArchiveHandle* AH)
 {
     lclContext*		ctx = (lclContext*)AH->formatData;
@@ -523,6 +686,15 @@ static int    	_ReadByte(ArchiveHandle* AH)
     return res;
 }
 
+/*
+ * Write a buffer of data to the archive.
+ *
+ * Mandatory.
+ *
+ * Called by the archiver to write a block of bytes to the archive.
+ * These routines are only used to read & write headers & TOC.
+ *
+ */
 static int	_WriteBuf(ArchiveHandle* AH, const void* buf, int len)
 {
     lclContext*		ctx = (lclContext*)AH->formatData;
@@ -532,6 +704,15 @@ static int	_WriteBuf(ArchiveHandle* AH, const void* buf, int len)
     return res;
 }
 
+/*
+ * Read a block of bytes from the archive.
+ * 
+ * Mandatory.
+ * 
+ * Called by the archiver to read a block of bytes from the archive
+ * These routines are only used to read & write headers & TOC.
+ *
+ */
 static int	_ReadBuf(ArchiveHandle* AH, void* buf, int len)
 {
     lclContext*		ctx = (lclContext*)AH->formatData;
@@ -541,6 +722,21 @@ static int	_ReadBuf(ArchiveHandle* AH, void* buf, int len)
     return res;
 }
 
+/*
+ * Close the archive.
+ *
+ * Mandatory.
+ *
+ * When writing the archive, this is the routine that actually starts 
+ * the process of saving it to files. No data should be written prior 
+ * to this point, since the user could sort the TOC after creating it.
+ * 
+ * If an archive is to be written, this toutine must call:
+ *		WriteHead			to save the archive header
+ * 		WriteToc 			to save the TOC entries
+ *		WriteDataChunks		to save all DATA & BLOBs.
+ *
+ */
 static void	_CloseArchive(ArchiveHandle* AH)
 {
     lclContext*		ctx = (lclContext*)AH->formatData;
@@ -567,6 +763,14 @@ static void	_CloseArchive(ArchiveHandle* AH)
     AH->FH = NULL; 
 }
 
+/*--------------------------------------------------
+ * END OF FORMAT CALLBACKS
+ *--------------------------------------------------
+ */
+
+/*
+ * Get the current position in the archive file.
+ */
 static int	_getFilePos(ArchiveHandle* AH, lclContext* ctx) 
 {
     int		pos;
@@ -581,4 +785,151 @@ static int	_getFilePos(ArchiveHandle* AH, lclContext* ctx)
     return pos;
 }
 
+/*
+ * Read a data block header. The format changed in V1.3, so we 
+ * put the code here for simplicity.
+ */
+static void _readBlockHeader(ArchiveHandle *AH, int *type, int *id)
+{
+    if (AH->version < K_VERS_1_3)
+		*type = BLK_DATA;
+    else
+		*type = _ReadByte(AH);;
+
+    *id = ReadInt(AH);
+}
+
+/*
+ * If zlib is available, then startit up. This is called from
+ * StartData & StartBlob. The buffers are setup in the Init routine.
+ *
+ */
+static void	_StartDataCompressor(ArchiveHandle* AH, TocEntry* te)
+{
+    lclContext*		ctx = (lclContext*)AH->formatData;
+    z_streamp   	zp = ctx->zp;
+
+#ifdef HAVE_LIBZ
+
+    if (AH->compression < 0 || AH->compression > 9) {
+		AH->compression = Z_DEFAULT_COMPRESSION;
+    }
+
+    if (AH->compression != 0) {
+	zp->zalloc = Z_NULL;
+	zp->zfree = Z_NULL;
+	zp->opaque = Z_NULL;
+
+	if (deflateInit(zp, AH->compression) != Z_OK)
+	    die_horribly(AH, "%s: could not initialize compression library - %s\n",progname, zp->msg);
+    }
+
+#else
+
+    AH->compression = 0;
+
+#endif
+
+    /* Just be paranoid - maye End is called after Start, with no Write */
+    zp->next_out = ctx->zlibOut;
+    zp->avail_out = zlibOutSize;
+}
+
+/*
+ * Send compressed data to the output stream (via ahwrite).
+ * Each data chunk is preceded by it's length.
+ * In the case of Z0, or no zlib, just write the raw data.
+ *
+ */
+static int	_DoDeflate(ArchiveHandle* AH, lclContext* ctx, int flush) 
+{
+    z_streamp   zp = ctx->zp;
+
+#ifdef HAVE_LIBZ
+    char*	out = ctx->zlibOut;
+    int		res = Z_OK;
+
+    if (AH->compression != 0) 
+    {
+		res = deflate(zp, flush);
+		if (res == Z_STREAM_ERROR)
+			die_horribly(AH, "%s: could not compress data - %s\n",progname, zp->msg);
+
+		if 	(      ( (flush == Z_FINISH) && (zp->avail_out < zlibOutSize) )
+			|| (zp->avail_out == 0) 
+			|| (zp->avail_in != 0)
+			) 
+		{
+			/*
+		   	 * Extra paranoia: avoid zero-length chunks since a zero 
+			 * length chunk is the EOF marker. This should never happen
+		     * but...
+			 */
+			if (zp->avail_out < zlibOutSize) {
+				/* printf("Wrote %d byte deflated chunk\n", zlibOutSize - zp->avail_out); */
+				WriteInt(AH, zlibOutSize - zp->avail_out);
+				fwrite(out, 1, zlibOutSize - zp->avail_out, AH->FH);
+				ctx->filePos += zlibOutSize - zp->avail_out;
+			}
+			zp->next_out = out;
+			zp->avail_out = zlibOutSize;
+		}
+    } else {
+#endif
+		if (zp->avail_in > 0)
+		{
+			WriteInt(AH, zp->avail_in);
+			fwrite(zp->next_in, 1, zp->avail_in, AH->FH);
+			ctx->filePos += zp->avail_in;
+			zp->avail_in = 0;
+		} else {
+#ifdef HAVE_LIBZ
+			if (flush == Z_FINISH)
+				res = Z_STREAM_END;
+#endif
+		}
+
+
+#ifdef HAVE_LIBZ
+    }
+
+    return res;
+#else
+    return 1;
+#endif
+
+}
+
+/*
+ * Terminate zlib context and flush it's buffers. If no zlib 
+ * then just return.
+ *
+ */
+static void	_EndDataCompressor(ArchiveHandle* AH, TocEntry* te)
+{
+
+#ifdef HAVE_LIBZ
+    lclContext*		ctx = (lclContext*)AH->formatData;
+    z_streamp		zp = ctx->zp;
+    int			res;
+
+    if (AH->compression != 0)
+    {
+		zp->next_in = NULL;
+		zp->avail_in = 0;
+
+		do { 	
+			/* printf("Ending data output\n"); */
+			res = _DoDeflate(AH, ctx, Z_FINISH);
+		} while (res != Z_STREAM_END);
+
+		if (deflateEnd(zp) != Z_OK)
+			die_horribly(AH, "%s: error closing compression stream - %s\n", progname, zp->msg);
+	}
+#endif
+
+	/* Send the end marker */
+	WriteInt(AH, 0);
+}
+
 

src/bin/pg_dump/pg_backup_files.c

@@ -47,10 +47,17 @@ static void	_WriteExtraToc(ArchiveHandle* AH, TocEntry* te);
 static void	_ReadExtraToc(ArchiveHandle* AH, TocEntry* te);
 static void	_PrintExtraToc(ArchiveHandle* AH, TocEntry* te);
 
+static void 	_StartBlobs(ArchiveHandle* AH, TocEntry* te);
+static void		_StartBlob(ArchiveHandle* AH, TocEntry* te, int oid);
+static void		_EndBlob(ArchiveHandle* AH, TocEntry* te, int oid);
+static void		_EndBlobs(ArchiveHandle* AH, TocEntry* te);
+
+#define K_STD_BUF_SIZE 1024
 
 typedef struct {
     int		hasSeek;
     int		filePos;
+	FILE	*blobToc;
 } lclContext;
 
 typedef struct {
@@ -62,6 +69,10 @@ typedef struct {
     char	*filename;
 } lclTocEntry;
 
+static char* progname = "Archiver(files)";
+static void _LoadBlobs(ArchiveHandle* AH, RestoreOptions *ropt);
+static void _getBlobTocEntry(ArchiveHandle* AH, int *oid, char *fname);
+
 /*
  *  Initializer
  */
@@ -84,6 +95,11 @@ void InitArchiveFmt_Files(ArchiveHandle* AH)
     AH->WriteExtraTocPtr = _WriteExtraToc;
     AH->PrintExtraTocPtr = _PrintExtraToc;
 
+    AH->StartBlobsPtr = _StartBlobs;
+    AH->StartBlobPtr = _StartBlob;
+    AH->EndBlobPtr = _EndBlob;
+    AH->EndBlobsPtr = _EndBlobs;
+
     /*
      *	Set up some special context used in compressing data.
     */
@@ -95,6 +111,13 @@ void InitArchiveFmt_Files(ArchiveHandle* AH)
      * Now open the TOC file
      */
     if (AH->mode == archModeWrite) {
+
+		fprintf(stderr, "\n*************************************************************\n"
+						"* WARNING: This format is for demonstration purposes. It is   *\n"
+						"*          not intended for general use. Files will be dumped *\n"
+						"*          into the current working directory.                *\n"
+						"***************************************************************\n\n");
+
 		if (AH->fSpec && strcmp(AH->fSpec,"") != 0) {
 			AH->FH = fopen(AH->fSpec, PG_BINARY_W);
 		} else {
@@ -107,7 +130,8 @@ void InitArchiveFmt_Files(ArchiveHandle* AH)
 		}
 
 
-    } else {
+    } else { /* Read Mode */
+
 		if (AH->fSpec && strcmp(AH->fSpec,"") != 0) {
 			AH->FH = fopen(AH->fSpec, PG_BINARY_R);
 		} else {
@@ -129,7 +153,7 @@ void InitArchiveFmt_Files(ArchiveHandle* AH)
 static void	_ArchiveEntry(ArchiveHandle* AH, TocEntry* te) 
 {
     lclTocEntry*	ctx;
-    char		fn[1024];
+    char			fn[K_STD_BUF_SIZE];
 
     ctx = (lclTocEntry*)malloc(sizeof(lclTocEntry));
     if (te->dataDumper) {
@@ -218,35 +242,98 @@ static void	_EndData(ArchiveHandle* AH, TocEntry* te)
 }
 
 /* 
- * Print data for a given TOC entry
-*/
-static void	_PrintTocData(ArchiveHandle* AH, TocEntry* te, RestoreOptions *ropt)
+ * Print data for a given file 
+ */
+static void	_PrintFileData(ArchiveHandle* AH, char *filename, RestoreOptions *ropt)
 {
-    lclTocEntry*	tctx = (lclTocEntry*) te->formatData;
     char		buf[4096];
     int			cnt;
 
-    if (!tctx->filename) 
+    if (!filename) 
 		return;
 
 #ifdef HAVE_LIBZ
-    AH->FH = gzopen(tctx->filename,"rb");
+    AH->FH = gzopen(filename,"rb");
 #else
-    AH->FH = fopen(tctx->filename,PG_BINARY_R);
+    AH->FH = fopen(filename,PG_BINARY_R);
 #endif
 
-    ahprintf(AH, "--\n-- Data for TOC Entry ID %d (OID %s) %s %s\n--\n\n",
-		te->id, te->oid, te->desc, te->name);
-
-    while ( (cnt = GZREAD(buf, 1, 4096, AH->FH)) > 0) {
+    while ( (cnt = GZREAD(buf, 1, 4095, AH->FH)) > 0) {
+		buf[cnt] = '\0';
 		ahwrite(buf, 1, cnt, AH);
     }
 
     GZCLOSE(AH->FH);
+}
+
+
+/*
+ * Print data for a given TOC entry
+*/
+static void	_PrintTocData(ArchiveHandle* AH, TocEntry* te, RestoreOptions *ropt)
+{
+    lclTocEntry*	tctx = (lclTocEntry*) te->formatData;
+
+    if (!tctx->filename) 
+		return;
+
+	if (strcmp(te->desc, "BLOBS") == 0)
+		_LoadBlobs(AH, ropt);
+	else
+	{
+		_PrintFileData(AH, tctx->filename, ropt);
+	}
+}
+
+static void _getBlobTocEntry(ArchiveHandle* AH, int *oid, char fname[K_STD_BUF_SIZE])
+{
+	lclContext*		ctx = (lclContext*)AH->formatData;
+	char			blobTe[K_STD_BUF_SIZE];
+	int				fpos;
+	int				eos;
+
+	if (fgets(&blobTe[0], K_STD_BUF_SIZE - 1, ctx->blobToc) != NULL)
+	{
+		*oid = atoi(blobTe);
+
+		fpos = strcspn(blobTe, " ");
 
-    ahprintf(AH, "\n\n");
+		strncpy(fname, &blobTe[fpos+1], K_STD_BUF_SIZE - 1);
+
+		eos = strlen(fname)-1;
+
+		if (fname[eos] == '\n')
+			fname[eos] = '\0';
+
+	} else {
+
+		*oid = 0;
+		fname[0] = '\0';
+	}
 }
 
+static void	_LoadBlobs(ArchiveHandle* AH, RestoreOptions *ropt)
+{
+    int				oid;
+	lclContext*		ctx = (lclContext*)AH->formatData;
+	char			fname[K_STD_BUF_SIZE];
+
+	ctx->blobToc = fopen("blobs.toc", PG_BINARY_R);
+
+	_getBlobTocEntry(AH, &oid, fname);
+
+    while(oid != 0)
+    {
+		StartRestoreBlob(AH, oid);
+		_PrintFileData(AH, fname, ropt);
+		EndRestoreBlob(AH, oid);
+		_getBlobTocEntry(AH, &oid, fname);
+    }
+
+	fclose(ctx->blobToc);
+}
+
+
 static int	_WriteByte(ArchiveHandle* AH, const int i)
 {
     lclContext*		ctx = (lclContext*)AH->formatData;
@@ -284,6 +371,7 @@ static int	_ReadBuf(ArchiveHandle* AH, void* buf, int len)
 {
     lclContext*		ctx = (lclContext*)AH->formatData;
     int			res;
+
     res = fread(buf, 1, len, AH->FH);
     ctx->filePos += res;
     return res;
@@ -301,3 +389,95 @@ static void	_CloseArchive(ArchiveHandle* AH)
     AH->FH = NULL; 
 }
 
+
+
+/*
+ * BLOB support
+ */
+
+/*
+ * Called by the archiver when starting to save all BLOB DATA (not schema). 
+ * This routine should save whatever format-specific information is needed
+ * to read the BLOBs back into memory. 
+ *
+ * It is called just prior to the dumper's DataDumper routine.
+ *
+ * Optional, but strongly recommended.
+ *
+ */
+static void	_StartBlobs(ArchiveHandle* AH, TocEntry* te)
+{
+    lclContext*		ctx = (lclContext*)AH->formatData;
+	char			fname[K_STD_BUF_SIZE];
+
+	sprintf(fname, "blobs.toc");
+	ctx->blobToc = fopen(fname, PG_BINARY_W);
+ 
+}
+
+/*
+ * Called by the archiver when the dumper calls StartBlob.
+ *
+ * Mandatory.
+ *
+ * Must save the passed OID for retrieval at restore-time.
+ */
+static void	_StartBlob(ArchiveHandle* AH, TocEntry* te, int oid)
+{
+	lclContext*		ctx = (lclContext*)AH->formatData;
+	lclTocEntry*	tctx = (lclTocEntry*)te->formatData;
+	char			fmode[10];
+	char			fname[255];
+	char			*sfx;
+
+    if (oid == 0) 
+		die_horribly(AH, "%s: illegal OID for BLOB (%d)\n", progname, oid);
+
+	if (AH->compression != 0)
+		sfx = ".gz";
+	else
+		sfx = "";
+
+    sprintf(fmode, "wb%d", AH->compression);
+	sprintf(fname, "blob_%d.dat%s", oid, sfx);
+
+	fprintf(ctx->blobToc, "%d %s\n", oid, fname);
+
+#ifdef HAVE_LIBZ
+    tctx->FH = gzopen(fname, fmode);
+#else
+    tctx->FH = fopen(fname, PG_BINARY_W);
+#endif
+
+}
+
+/*
+ * Called by the archiver when the dumper calls EndBlob.
+ *
+ * Optional.
+ *
+ */
+static void	_EndBlob(ArchiveHandle* AH, TocEntry* te, int oid)
+{
+	lclTocEntry*	tctx = (lclTocEntry*)te->formatData;
+
+	GZCLOSE(tctx->FH);
+}
+
+/*
+ * Called by the archiver when finishing saving all BLOB DATA. 
+ *
+ * Optional.
+ *
+ */
+static void	_EndBlobs(ArchiveHandle* AH, TocEntry* te)
+{
+	lclContext*		ctx = (lclContext*)AH->formatData;
+	/* Write out a fake zero OID to mark end-of-blobs. */
+    /* WriteInt(AH, 0); */
+
+	fclose(ctx->blobToc);
+
+}
+
+

src/bin/pg_dump/pg_dump.c

@@ -22,7 +22,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_dump.c,v 1.159 2000/07/17 03:05:20 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_dump.c,v 1.160 2000/07/21 11:40:08 pjw Exp $
  *
  * Modifications - 6/10/96 - dave@bensoft.com - version 1.13.dhb
  *
@@ -61,12 +61,19 @@
  *		 - Added a -Z option for compression level on compressed formats
  *		 - Restored '-f' in usage output
  *
-*-------------------------------------------------------------------------
+ *
+ * Modifications - 17-Jul-2000 - Philip Warner pjw@rhyme.com.au
+ *		 - Support for BLOB output.
+ *		 - Sort archive by OID, put some items at end (out of OID order)
+ *
+ *-------------------------------------------------------------------------
  */
 
 #include <unistd.h>				/* for getopt() */
 #include <ctype.h>
 
+#include "pg_backup.h"
+
 #include "postgres.h"
 
 #ifdef HAVE_GETOPT_H
@@ -84,6 +91,7 @@
 #include "catalog/pg_type.h"
 
 #include "libpq-fe.h"
+#include <libpq/libpq-fs.h>
 #ifndef HAVE_STRDUP
 #include "strdup.h"
 #endif
@@ -109,6 +117,9 @@ static void setMaxOid(Archive *fout);
 static void AddAcl(char *aclbuf, const char *keyword);
 static char *GetPrivileges(const char *s);
 
+static int dumpBlobs(Archive *AH, char*, void*);
+
+
 extern char *optarg;
 extern int	optind,
 			opterr;
@@ -268,14 +279,26 @@ dumpClasses_nodumpData(Archive *fout, char* oid, void *dctxv)
 
 	if (oids == true)
 	{
-		archprintf(fout, "COPY %s WITH OIDS FROM stdin;\n",
-				fmtId(classname, force_quotes));
+		/* 
+		 * archprintf(fout, "COPY %s WITH OIDS FROM stdin;\n",
+		 *		fmtId(classname, force_quotes));
+		 *
+		 *  - Not used as of V1.3 (needs to be in ArchiveEntry call)
+		 *
+		 */
+
 		sprintf(query, "COPY %s WITH OIDS TO stdout;\n",
 				fmtId(classname, force_quotes));
 	}
 	else
 	{
-		archprintf(fout, "COPY %s FROM stdin;\n", fmtId(classname, force_quotes));
+		/* 
+		 *archprintf(fout, "COPY %s FROM stdin;\n", fmtId(classname, force_quotes));
+		 *
+		 * - Not used as of V1.3 (needs to be in ArchiveEntry call)
+		 *
+		 */
+
 		sprintf(query, "COPY %s TO stdout;\n", fmtId(classname, force_quotes));
 	}
 	res = PQexec(g_conn, query);
@@ -452,19 +475,28 @@ dumpClasses_dumpData(Archive *fout, char* oid, void *dctxv)
  */
 static void
 dumpClasses(const TableInfo *tblinfo, const int numTables, Archive *fout,
-			const char *onlytable, const bool oids)
+			const char *onlytable, const bool oids, const bool force_quotes)
 {
 
 	int				i;
 	char	   		*all_only;
 	DataDumperPtr	dumpFn;
 	DumpContext		*dumpCtx;
+	char			*oidsPart;
+	char			copyBuf[512];
+	char			*copyStmt;
 
 	if (onlytable == NULL)
 		all_only = "all";
 	else
 		all_only = "only";
 
+	if (oids == true)
+		oidsPart = "WITH OIDS ";
+	else
+		oidsPart = "";
+
+
 	if (g_verbose)
 		fprintf(stderr, "%s dumping out the contents of %s %d table%s/sequence%s %s\n",
 				g_comment_start, all_only,
@@ -514,112 +546,28 @@ dumpClasses(const TableInfo *tblinfo, const int numTables, Archive *fout,
 			dumpCtx->tblidx = i;
 			dumpCtx->oids = oids;
 
-			if (!dumpData)
+			if (!dumpData) /* Dump/restore using COPY */
+			{
 				dumpFn = dumpClasses_nodumpData;
 				/* dumpClasses_nodumpData(fout, classname, oids); */
-			else
+				sprintf(copyBuf, "COPY %s %s FROM stdin;\n", fmtId(tblinfo[i].relname, force_quotes),
+						oidsPart);
+				copyStmt = copyBuf;
+			}
+			else /* Restore using INSERT */
+			{
 				dumpFn = dumpClasses_dumpData;
 				/* dumpClasses_dumpData(fout, classname); */
+				copyStmt = NULL;
+			}
 
 			ArchiveEntry(fout, tblinfo[i].oid, fmtId(tblinfo[i].relname, false),
-							"TABLE DATA", NULL, "", "", tblinfo[i].usename,
+							"TABLE DATA", NULL, "", "", copyStmt, tblinfo[i].usename,
 							dumpFn, dumpCtx);
 		}
 	}
 }
 
-static void
-prompt_for_password(char *username, char *password)
-{
-	char		buf[512];
-	int			length;
-
-#ifdef HAVE_TERMIOS_H
-	struct termios t_orig,
-				   t;
-#endif
-
-	fprintf(stderr, "Username: ");
-	fflush(stderr);
-	fgets(username, 100, stdin);
-	length = strlen(username);
-	/* skip rest of the line */
-	if (length > 0 && username[length - 1] != '\n')
-	{
-		do
-		{
-			fgets(buf, 512, stdin);
-		} while (buf[strlen(buf) - 1] != '\n');
-	}
-	if (length > 0 && username[length - 1] == '\n')
-		username[length - 1] = '\0';
-
-#ifdef HAVE_TERMIOS_H
-	tcgetattr(0, &t);
-	t_orig = t;
-	t.c_lflag &= ~ECHO;
-	tcsetattr(0, TCSADRAIN, &t);
-#endif
-	fprintf(stderr, "Password: ");
-	fflush(stderr);
-	fgets(password, 100, stdin);
-#ifdef HAVE_TERMIOS_H
-	tcsetattr(0, TCSADRAIN, &t_orig);
-#endif
-
-	length = strlen(password);
-	/* skip rest of the line */
-	if (length > 0 && password[length - 1] != '\n')
-	{
-		do
-		{
-			fgets(buf, 512, stdin);
-		} while (buf[strlen(buf) - 1] != '\n');
-	}
-	if (length > 0 && password[length - 1] == '\n')
-		password[length - 1] = '\0';
-
-	fprintf(stderr, "\n\n");
-}
-
-
-static void
-check_database_version(bool ignoreVersion)
-{
-	PGresult   *res;
-	double      myversion;
-	const char *remoteversion_str;
-	double      remoteversion;
-
-	myversion = strtod(PG_VERSION, NULL);
-	res = PQexec(g_conn, "SELECT version()");
-	if (!res ||
-		PQresultStatus(res) != PGRES_TUPLES_OK ||
-		PQntuples(res) != 1)
-	{
-		fprintf(stderr, "check_database_version(): command failed.  Explanation from backend: '%s'.\n", PQerrorMessage(g_conn));
-		exit_nicely(g_conn);
-	}
-
-	remoteversion_str = PQgetvalue(res, 0, 0);
-	remoteversion = strtod(remoteversion_str + 11, NULL);
-	if (myversion != remoteversion)
-	{
-		fprintf(stderr, "Database version: %s\npg_dump version: %s\n",
-				remoteversion_str, PG_VERSION);
-		if (ignoreVersion)
-			fprintf(stderr, "Proceeding despite version mismatch.\n");
-		else
-		{
-			fprintf(stderr, "Aborting because of version mismatch.\n"
-					"Use --ignore-version if you think it's safe to proceed anyway.\n");
-			exit_nicely(g_conn);
-		}
-	}
-	PQclear(res);
-}
-
-
 int
 main(int argc, char **argv)
 {
@@ -634,20 +582,19 @@ main(int argc, char **argv)
 	bool		oids = false;
 	TableInfo  *tblinfo;
 	int			numTables;
-	char		connect_string[512] = "";
-	char		tmp_string[128];
-	char		username[100];
-	char		password[100];
 	bool		use_password = false;
 	int			compressLevel = -1;
 	bool		ignore_version = false;
 	int			plainText = 0;
 	int			outputClean = 0;
+	int			outputBlobs = 0;
+
 	RestoreOptions	*ropt;
 
 #ifdef HAVE_GETOPT_LONG
 	static struct option long_options[] = {
 		{"data-only", no_argument, NULL, 'a'},
+		{"blobs", no_argument, NULL, 'b' },
 		{"clean", no_argument, NULL, 'c'},
 		{"file", required_argument, NULL, 'f'},
 		{"format", required_argument, NULL, 'F'},
@@ -686,6 +633,12 @@ main(int argc, char **argv)
 	else
 		progname = strrchr(argv[0], SEP_CHAR) + 1;
 
+	/* Set defaulty options based on progname */
+	if (strcmp(progname, "pg_backup") == 0)
+	{
+		format = "c";
+		outputBlobs = 1;
+	}
 
 #ifdef HAVE_GETOPT_LONG
 	while ((c = getopt_long(argc, argv, "acdDf:F:h:inNop:st:uvxzZ:V?", long_options, &optindex)) != -1)
@@ -698,6 +651,10 @@ main(int argc, char **argv)
 			case 'a':			/* Dump data only */
 				dataOnly = true;
 				break;
+			case 'b':			/* Dump blobs */
+				outputBlobs = true;
+				break;
+
 			case 'c':			/* clean (i.e., drop) schema prior to
  								 * create */
 				outputClean = 1;
@@ -843,7 +800,12 @@ main(int argc, char **argv)
 		case 'p':
 		case 'P':
 			plainText = 1;
-			g_fout = CreateArchive(filename, archPlainText, 0);
+			g_fout = CreateArchive(filename, archNull, 0);
+			break;
+
+		case 't':
+		case 'T':
+			g_fout = CreateArchive(filename, archTar, compressLevel);
 			break;
 
 		default:
@@ -860,53 +822,13 @@ main(int argc, char **argv)
 		exit(1);
 	}
 
-	/* find database */
-	if (!(dbname = argv[optind]) &&
-		!(dbname = getenv("PGDATABASE")))
-	{
-		fprintf(stderr, "%s: no database name specified\n", progname);
-		exit(1);
-	}
+	/* Let the archiver know how noisy to be */
+	g_fout->verbose = g_verbose;
 
-	/* g_conn = PQsetdb(pghost, pgport, NULL, NULL, dbname); */
-	if (pghost != NULL)
-	{
-		sprintf(tmp_string, "host=%s ", pghost);
-		strcat(connect_string, tmp_string);
-	}
-	if (pgport != NULL)
-	{
-		sprintf(tmp_string, "port=%s ", pgport);
-		strcat(connect_string, tmp_string);
-	}
-	if (dbname != NULL)
-	{
-		sprintf(tmp_string, "dbname=%s ", dbname);
-		strcat(connect_string, tmp_string);
-	}
-	if (use_password)
-	{
-		prompt_for_password(username, password);
-		strcat(connect_string, "authtype=password ");
-		sprintf(tmp_string, "user=%s ", username);
-		strcat(connect_string, tmp_string);
-		sprintf(tmp_string, "password=%s ", password);
-		strcat(connect_string, tmp_string);
-		MemSet(tmp_string, 0, sizeof(tmp_string));
-		MemSet(password, 0, sizeof(password));
-	}
-	g_conn = PQconnectdb(connect_string);
-	MemSet(connect_string, 0, sizeof(connect_string));
-	/* check to see that the backend connection was successfully made */
-	if (PQstatus(g_conn) == CONNECTION_BAD)
-	{
-		fprintf(stderr, "Connection to database '%s' failed.\n", dbname);
-		fprintf(stderr, "%s\n", PQerrorMessage(g_conn));
-		exit_nicely(g_conn);
-	}
+	dbname = argv[optind];
 
-	/* check for version mismatch */
-	check_database_version(ignore_version);
+	/* Open the database using the Archiver, so it knows about it. Errors mean death */
+	g_conn = ConnectDatabase(g_fout, dbname, pghost, pgport, use_password, ignore_version);
 
 	/*
 	 * Start serializable transaction to dump consistent data
@@ -916,17 +838,15 @@ main(int argc, char **argv)
 
 		res = PQexec(g_conn, "begin");
 		if (!res || PQresultStatus(res) != PGRES_COMMAND_OK)
-		{
-			fprintf(stderr, "BEGIN command failed.  Explanation from backend: '%s'.\n", PQerrorMessage(g_conn));
-			exit_nicely(g_conn);
-		}
+		    exit_horribly(g_fout, "BEGIN command failed. Explanation from backend: '%s'.\n",
+				PQerrorMessage(g_conn));
+
 		PQclear(res);
 		res = PQexec(g_conn, "set transaction isolation level serializable");
 		if (!res || PQresultStatus(res) != PGRES_COMMAND_OK)
-		{
-			fprintf(stderr, "SET TRANSACTION command failed.  Explanation from backend: '%s'.\n", PQerrorMessage(g_conn));
-			exit_nicely(g_conn);
-		}
+		    exit_horribly(g_fout, "SET TRANSACTION command failed. Explanation from backend: '%s'.\n",
+				PQerrorMessage(g_conn));
+
 		PQclear(res);
 	}
 
@@ -941,7 +861,12 @@ main(int argc, char **argv)
 	tblinfo = dumpSchema(g_fout, &numTables, tablename, aclsSkip, oids, schemaOnly, dataOnly);
 
 	if (!schemaOnly)
-	    dumpClasses(tblinfo, numTables, g_fout, tablename, oids);
+	{
+	    dumpClasses(tblinfo, numTables, g_fout, tablename, oids, force_quotes);
+	}
+
+	if (outputBlobs)
+		ArchiveEntry(g_fout, "0", "BLOBS", "BLOBS", NULL, "", "", "", "", dumpBlobs, 0); 
 
 	if (!dataOnly)				/* dump indexes and triggers at the end
 								 * for performance */
@@ -951,6 +876,15 @@ main(int argc, char **argv)
 		dumpRules(g_fout, tablename, tblinfo, numTables);
 	}
 
+	/* Now sort the output nicely */
+	SortTocByOID(g_fout);
+	MoveToEnd(g_fout, "TABLE DATA");
+	MoveToEnd(g_fout, "BLOBS");
+	MoveToEnd(g_fout, "INDEX");
+	MoveToEnd(g_fout, "TRIGGER");
+	MoveToEnd(g_fout, "RULE");
+	MoveToEnd(g_fout, "ACL");
+
 	if (plainText) 
 	{
 		ropt = NewRestoreOptions();
@@ -973,6 +907,92 @@ main(int argc, char **argv)
 	exit(0);
 }
 
+/*
+ * dumpBlobs:
+ *	dump all blobs
+ *
+ */
+
+#define loBufSize 16384 
+#define loFetchSize 1000
+
+static int 
+dumpBlobs(Archive *AH, char* junkOid, void *junkVal)
+{
+	PQExpBuffer		oidQry = createPQExpBuffer();
+	PQExpBuffer		oidFetchQry = createPQExpBuffer();
+	PGresult		*res;
+	int				i;
+	int				loFd;
+	char			buf[loBufSize];
+	int				cnt;
+	int				blobOid;
+
+	if (g_verbose)
+		fprintf(stderr, "%s saving BLOBs\n", g_comment_start);
+
+	/* Cursor to get all BLOB tables */
+    appendPQExpBuffer(oidQry, "Declare blobOid Cursor for SELECT oid from pg_class where relkind = 'l'");
+
+	res = PQexec(g_conn, oidQry->data);
+	if (!res || PQresultStatus(res) != PGRES_COMMAND_OK)
+	{
+		fprintf(stderr, "dumpBlobs(): Declare Cursor failed.  Explanation from backend: '%s'.\n", PQerrorMessage(g_conn));
+		exit_nicely(g_conn);
+	}
+
+	/* Fetch for cursor */
+	appendPQExpBuffer(oidFetchQry, "Fetch %d in blobOid", loFetchSize);
+
+	do {
+		/* Do a fetch */
+		PQclear(res);
+		res = PQexec(g_conn, oidFetchQry->data);
+
+		if (!res || PQresultStatus(res) != PGRES_TUPLES_OK)
+		{
+		    fprintf(stderr, "dumpBlobs(): Fetch Cursor failed.  Explanation from backend: '%s'.\n", PQerrorMessage(g_conn));
+		    exit_nicely(g_conn);
+		}
+
+		/* Process the tuples, if any */
+		for (i = 0; i < PQntuples(res); i++)
+		{
+			blobOid = atoi(PQgetvalue(res, i, 0));
+			/* Open the BLOB */
+			loFd = lo_open(g_conn, blobOid, INV_READ);
+			if (loFd == -1)
+			{
+				fprintf(stderr, "dumpBlobs(): Could not open large object.  "
+									"Explanation from backend: '%s'.\n", PQerrorMessage(g_conn));
+				exit_nicely(g_conn);
+			}
+
+			StartBlob(AH, blobOid);
+
+			/* Now read it in chunks, sending data to archive */
+			do {
+				cnt = lo_read(g_conn, loFd, buf, loBufSize);
+				if (cnt < 0) {
+					fprintf(stderr, "dumpBlobs(): Error reading large object. "
+								" Explanation from backend: '%s'.\n", PQerrorMessage(g_conn));
+					exit_nicely(g_conn);
+				}
+
+				WriteData(AH, buf, cnt); 
+
+			} while (cnt > 0);
+
+			lo_close(g_conn, loFd);
+
+			EndBlob(AH, blobOid);
+
+		}
+	} while (PQntuples(res) > 0);
+
+	return 1;
+}
+
 /*
  * getTypes:
  *	  read all base types in the system catalogs and return them in the
@@ -2409,7 +2429,7 @@ dumpComment(Archive *fout, const char *target, const char *oid)
 							target, checkForQuote(PQgetvalue(res, 0, i_description)));
 
 		ArchiveEntry(fout, oid, target, "COMMENT", NULL, query->data, "" /*Del*/,
-					   "" /*Owner*/, NULL, NULL);	
+					   "" /* Copy */, "" /*Owner*/, NULL, NULL);	
 
 	}
 
@@ -2542,7 +2562,7 @@ dumpTypes(Archive *fout, FuncInfo *finfo, int numFuncs,
 			appendPQExpBuffer(q, ");\n");
 
 		ArchiveEntry(fout, tinfo[i].oid, fmtId(tinfo[i].typname, force_quotes), "TYPE", NULL,
-						q->data, delq->data, tinfo[i].usename, NULL, NULL);
+						q->data, delq->data, "", tinfo[i].usename, NULL, NULL);
 
 		/*** Dump Type Comments ***/
 
@@ -2629,7 +2649,7 @@ dumpProcLangs(Archive *fout, FuncInfo *finfo, int numFuncs,
 				lancompiler);
 
 		ArchiveEntry(fout, PQgetvalue(res, i, i_oid), lanname, "PROCEDURAL LANGUAGE",
-			    NULL, defqry->data, delqry->data, "", NULL, NULL);
+			    NULL, defqry->data, delqry->data, "", "", NULL, NULL);
 
 		free(lanname);
 		free(lancompiler);
@@ -2669,8 +2689,8 @@ dumpOneFunc(Archive *fout, FuncInfo *finfo, int i,
 	PQExpBuffer fn = createPQExpBuffer();
 	PQExpBuffer delqry = createPQExpBuffer();
 	PQExpBuffer fnlist = createPQExpBuffer();
-	PQExpBuffer	asPart = createPQExpBuffer();
 	int			j;
+	PQExpBuffer asPart = createPQExpBuffer();
 	char		func_lang[NAMEDATALEN + 1];
 	PGresult   *res;
 	int			nlangs;
@@ -2751,7 +2771,7 @@ dumpOneFunc(Archive *fout, FuncInfo *finfo, int i,
 					  asPart->data, func_lang);
 
 	ArchiveEntry(fout, finfo[i].oid, fn->data, "FUNCTION", NULL, q->data, delqry->data,
-					finfo[i].usename, NULL, NULL);
+					"", finfo[i].usename, NULL, NULL);
 
 	/*** Dump Function Comments ***/
 
@@ -2870,7 +2890,7 @@ dumpOprs(Archive *fout, OprInfo *oprinfo, int numOperators,
 						  sort2->data);
 
 		ArchiveEntry(fout, oprinfo[i].oid, oprinfo[i].oprname, "OPERATOR", NULL,
-						q->data, delq->data, oprinfo[i].usename, NULL, NULL);
+						q->data, delq->data, "", oprinfo[i].usename, NULL, NULL);
 	}
 }
 
@@ -2927,7 +2947,7 @@ dumpAggs(Archive *fout, AggInfo *agginfo, int numAggs,
 						  details->data);
 
 		ArchiveEntry(fout, agginfo[i].oid, aggSig->data, "AGGREGATE", NULL,
-						q->data, delq->data, agginfo[i].usename, NULL, NULL);
+						q->data, delq->data, "", agginfo[i].usename, NULL, NULL);
 
 		/*** Dump Aggregate Comments ***/
 
@@ -3096,7 +3116,7 @@ dumpACL(Archive *fout, TableInfo tbinfo)
 
 	free(aclbuf);
 
-	ArchiveEntry(fout, tbinfo.oid, tbinfo.relname, "ACL", NULL, sql, "", "", NULL, NULL);
+	ArchiveEntry(fout, tbinfo.oid, tbinfo.relname, "ACL", NULL, sql, "", "", "", NULL, NULL);
 
 }
 
@@ -3274,7 +3294,7 @@ dumpTables(Archive *fout, TableInfo *tblinfo, int numTables,
 
 			if (!dataOnly) {
 				ArchiveEntry(fout, tblinfo[i].oid, fmtId(tblinfo[i].relname, false),
-								"TABLE", NULL, q->data, delq->data, tblinfo[i].usename,
+								"TABLE", NULL, q->data, delq->data, "", tblinfo[i].usename,
 								NULL, NULL);
 			}
 
@@ -3468,7 +3488,7 @@ dumpIndices(Archive *fout, IndInfo *indinfo, int numIndices,
 			/* Dump Index Comments */
 
 			ArchiveEntry(fout, tblinfo[tableInd].oid, id1->data, "INDEX", NULL, q->data, delq->data,
-							tblinfo[tableInd].usename, NULL, NULL);
+							"", tblinfo[tableInd].usename, NULL, NULL);
 
 			resetPQExpBuffer(q);
 			appendPQExpBuffer(q, "INDEX %s", id1->data);
@@ -3599,7 +3619,7 @@ setMaxOid(Archive *fout)
 	pos = pos + snprintf(sql+pos, 1024-pos, "\\.\n");
 	pos = pos + snprintf(sql+pos, 1024-pos, "DROP TABLE pg_dump_oid;\n");
 
-	ArchiveEntry(fout, "0", "Max OID", "<Init>", NULL, sql, "","", NULL, NULL);
+	ArchiveEntry(fout, "0", "Max OID", "<Init>", NULL, sql, "", "", "", NULL, NULL);
 }
 
 /*
@@ -3750,7 +3770,7 @@ dumpSequence(Archive *fout, TableInfo tbinfo)
 	}
 
 	ArchiveEntry(fout, tbinfo.oid, fmtId(tbinfo.relname, force_quotes), "SEQUENCE", NULL,
-					query->data, delqry->data, tbinfo.usename, NULL, NULL);
+					query->data, delqry->data, "", tbinfo.usename, NULL, NULL);
 
 	/* Dump Sequence Comments */
 
@@ -3779,7 +3799,7 @@ dumpTriggers(Archive *fout, const char *tablename,
 		for (j = 0; j < tblinfo[i].ntrig; j++)
 		{
 			ArchiveEntry(fout, tblinfo[i].triggers[j].oid, tblinfo[i].triggers[j].tgname,
-						"TRIGGER", NULL, tblinfo[i].triggers[j].tgsrc, "",
+						"TRIGGER", NULL, tblinfo[i].triggers[j].tgsrc, "", "", 
 						tblinfo[i].usename, NULL, NULL);
 			dumpComment(fout, tblinfo[i].triggers[j].tgcomment, tblinfo[i].triggers[j].oid);
 		}
@@ -3846,7 +3866,7 @@ dumpRules(Archive *fout, const char *tablename,
 
 			ArchiveEntry(fout, PQgetvalue(res, i, i_oid), PQgetvalue(res, i, i_rulename),
 							"RULE", NULL, PQgetvalue(res, i, i_definition),
-							"", "", NULL, NULL);
+							"", "", "", NULL, NULL);
 
 			/* Dump rule comments */
 

src/bin/pg_dump/pg_restore.c

@@ -84,19 +84,24 @@ typedef struct option optType;
 struct option cmdopts[] = {	
 				{ "clean", 0, NULL, 'c' },
 				{ "data-only", 0, NULL, 'a' },
+				{ "dbname", 1, NULL, 'd' },
 				{ "file", 1, NULL, 'f' },
 				{ "format", 1, NULL, 'F' },
-				{ "function", 2, NULL, 'p' },
-				{ "index", 2, NULL, 'i'},
+				{ "function", 2, NULL, 'P' },
+				{ "host", 1, NULL, 'h' },
+				{ "ignore-version", 0, NULL, 'i'},
+				{ "index", 2, NULL, 'I'},
 				{ "list", 0, NULL, 'l'},
 				{ "no-acl", 0, NULL, 'x' },
+				{ "port", 1, NULL, 'p' },
 				{ "oid-order", 0, NULL, 'o'},
 				{ "orig-order", 0, NULL, 'O' },
+				{ "password", 0, NULL, 'u' },
 				{ "rearrange", 0, NULL, 'r'},
 				{ "schema-only", 0, NULL, 's' },
 				{ "table", 2, NULL, 't'},
 				{ "trigger", 2, NULL, 'T' },
-				{ "use-list", 1, NULL, 'u'},
+				{ "use-list", 1, NULL, 'U'},
 				{ "verbose", 0, NULL, 'v' },
 				{ NULL, 0, NULL, 0}
 			    };
@@ -115,9 +120,9 @@ int main(int argc, char **argv)
 	progname = *argv;
 
 #ifdef HAVE_GETOPT_LONG
-	while ((c = getopt_long(argc, argv, "acf:F:i:loOp:st:T:u:vx", cmdopts, NULL)) != EOF)
+	while ((c = getopt_long(argc, argv, "acd:f:F:h:i:loOp:st:T:u:U:vx", cmdopts, NULL)) != EOF)
 #else
-	while ((c = getopt(argc, argv, "acf:F:i:loOp:st:T:u:vx")) != -1)
+	while ((c = getopt(argc, argv, "acd:f:F:h:i:loOp:st:T:u:U:vx")) != -1)
 #endif
 	{
 		switch (c)
@@ -129,6 +134,13 @@ int main(int argc, char **argv)
 								 * create */
 				opts->dropSchema = 1;
 				break;
+			case 'd':
+				if (strlen(optarg) != 0)
+				{
+					opts->dbname = strdup(optarg);
+					opts->useDB = 1;
+				}
+				break;
 			case 'f':			/* output file name */
 				opts->filename = strdup(optarg);
 				break;
@@ -136,22 +148,32 @@ int main(int argc, char **argv)
 				if (strlen(optarg) != 0) 
 				    opts->formatName = strdup(optarg);
 				break;
+			case 'h':
+				if (strlen(optarg) != 0)
+					opts->pghost = strdup(optarg);
+				break;
+			case 'i':
+				opts->ignoreVersion = 1;
+				break;
 			case 'o':
 				opts->oidOrder = 1;
 				break;
 			case 'O':
 				opts->origOrder = 1;
 				break;
+			case 'p':
+				if (strlen(optarg) != 0)
+					opts->pgport = strdup(optarg);
+				break;
 			case 'r':
 				opts->rearrange = 1;
 				break;
-
-			case 'p': /* Function */
+			case 'P': /* Function */
 				opts->selTypes = 1;
 				opts->selFunction = 1;
 				opts->functionNames = _cleanupName(optarg);
 				break;
-			case 'i': /* Index */
+			case 'I': /* Index */
 				opts->selTypes = 1;
 				opts->selIndex = 1;
 				opts->indexNames = _cleanupName(optarg);
@@ -173,7 +195,11 @@ int main(int argc, char **argv)
 				opts->tocSummary = 1;
 				break;
 
-			case 'u':			/* input TOC summary file name */
+			case 'u':
+				opts->requirePassword = 1;
+				break;
+
+			case 'U':			/* input TOC summary file name */
 				opts->tocFile = strdup(optarg);
 				break;
 
@@ -209,14 +235,23 @@ int main(int argc, char **argv)
 			opts->format = archFiles;
 			break;
 
+		case 't':
+		case 'T':
+			opts->format = archTar;
+			break;
+
 	    default:
-		fprintf(stderr, "%s: Unknown archive format '%s', please specify 'f' or 'c'\n", progname, opts->formatName);
+			fprintf(stderr, "%s: Unknown archive format '%s', please specify 't' or 'c'\n",
+						progname, opts->formatName);
 			exit (1);
 	}
     }
 
     AH = OpenArchive(fileSpec, opts->format);
 
+	/* Let the archiver know how noisy to be */
+	AH->verbose = opts->verbose;
+
     if (opts->tocFile)
 		SortTocFromFile(AH, opts);
 
@@ -226,7 +261,9 @@ int main(int argc, char **argv)
 		SortTocByID(AH);
 
     if (opts->rearrange) {
+		MoveToStart(AH, "<Init>");
 		MoveToEnd(AH, "TABLE DATA");
+		MoveToEnd(AH, "BLOBS");
 		MoveToEnd(AH, "INDEX");
 		MoveToEnd(AH, "TRIGGER");
 		MoveToEnd(AH, "RULE");
@@ -250,39 +287,48 @@ static void usage(const char *progname)
 	fprintf(stderr,
 	"usage:  %s [options] [backup file]\n"
 	    "  -a, --data-only             \t dump out only the data, no schema\n"
+		"  -d, --dbname <name>         \t specify database name\n"
 	    "  -c, --clean                 \t clean(drop) schema prior to create\n"
 	    "  -f filename                 \t script output filename\n"
 	    "  -F, --format {c|f}          \t specify backup file format\n"
-	    "  -p, --function[=name]       \t dump functions or named function\n"
+		"  -h, --host <hostname>       \t server host name\n"
 	    "  -i, --index[=name]          \t dump indexes or named index\n"
 	    "  -l, --list                  \t dump summarized TOC for this file\n"
 	    "  -o, --oid-order             \t dump in oid order\n"
 	    "  -O, --orig-order            \t dump in original dump order\n"
+		"  -p, --port <port>           \t server port number\n"
+		"  -P, --function[=name]       \t dump functions or named function\n"
 	    "  -r, --rearrange             \t rearrange output to put indexes etc at end\n"
 	    "  -s, --schema-only           \t dump out only the schema, no data\n"
 	    "  -t [table], --table[=table] \t dump for this table only\n"
 	    "  -T, --trigger[=name]        \t dump triggers or named trigger\n"
-	    "  -u, --use-list filename     \t use specified TOC for ordering output from this file\n"
-	    "  -v                          \t verbose\n"
+		"  -u, --password              \t use password authentication\n"
+	    "  -U, --use-list filename     \t use specified TOC for ordering output from this file\n"
+	    "  -v, --verbose               \t verbose\n"
 	    "  -x, --no-acl                \t skip dumping of ACLs (grant/revoke)\n"
 	    , progname);
+
 #else
 	fprintf(stderr,
 	"usage:  %s [options] [backup file]\n"
 	    "  -a                          \t dump out only the data, no schema\n"
+		"  -d,          <name>         \t specify database name\n"
 	    "  -c                          \t clean(drop) schema prior to create\n"
 	    "  -f filename NOT IMPLEMENTED \t script output filename\n"
 	    "  -F           {c|f}          \t specify backup file format\n"
-	    "  -p name                     \t dump functions or named function\n"
+		"  -h,        <hostname>       \t server host name\n"
 	    "  -i name                     \t dump indexes or named index\n"
 	    "  -l                          \t dump summarized TOC for this file\n"
 	    "  -o                          \t dump in oid order\n"
 	    "  -O                          \t dump in original dump order\n"
+		"  -p         <port>           \t server port number\n"
+		"  -P name                     \t dump functions or named function\n"
 	    "  -r                          \t rearrange output to put indexes etc at end\n"
 	    "  -s                          \t dump out only the schema, no data\n"
 	    "  -t name                     \t dump for this table only\n"
 	    "  -T name                     \t dump triggers or named trigger\n"
-	    "  -u filename                 \t use specified TOC for ordering output from this file\n"
+		"  -u                          \t use password authentication\n"
+	    "  -U filename                 \t use specified TOC for ordering output from this file\n"
 	    "  -v                          \t verbose\n"
 	    "  -x                          \t skip dumping of ACLs (grant/revoke)\n"
 	    , progname);