- Support for BLOB output from pg_dump and input via pg_restore

- Support for direct DB connection in pg_restore
- Fixes in support for --insert flag
- pg_dump now outputs in modified OID order
- various other bug fixes

src/bin/pg_dump/Makefile

@@ -4,7 +4,7 @@
 #
 # Copyright (c) 1994, Regents of the University of California
 #
-# $Header: /cvsroot/pgsql/src/bin/pg_dump/Makefile,v 1.19 2000/07/04 19:52:00 petere Exp $
+# $Header: /cvsroot/pgsql/src/bin/pg_dump/Makefile,v 1.20 2000/07/21 11:40:08 pjw Exp $
 #
 #-------------------------------------------------------------------------
 
@@ -12,8 +12,8 @@ subdir = src/bin/pg_dump
 top_builddir = ../../..
 include ../../Makefile.global
 
-OBJS= pg_backup_archiver.o pg_backup_custom.o pg_backup_files.o \
-       pg_backup_plain_text.o $(STRDUP)
+OBJS= pg_backup_archiver.o pg_backup_db.o pg_backup_custom.o pg_backup_files.o \
+       pg_backup_null.o pg_backup_tar.o $(STRDUP)
 
 CFLAGS+= -I$(LIBPQDIR)
 LIBS+= -lz

src/bin/pg_dump/README

 Notes on pg_dump
 ================
 
-pg_dump, by default, still outputs text files.
+1. pg_dump, by default, still outputs text files.
 
-pg_dumpall forces all pg_dump output to be text, since it also outputs text into the same output stream.
+2. pg_dumpall forces all pg_dump output to be text, since it also outputs text into the same output stream.
 
-The plain text output format can not be used as input into pg_restore.
+3. The plain text output format can not be used as input into pg_restore.
+
+4. pg_dump now dumps the items in a modified OID order to try to improve relaibility of default restores.
 
 
 To dump a database into the next custom format, type:
 
     pg_dump <db-name> -Fc > <backup-file>
 
+or, in TAR format
+
+	pg_dump <db-name> -Ft > <backup-file>
+
 To restore, try
  
    To list contents:
@@ -53,7 +59,37 @@ or, simply:
     pg_restore backup.bck --use=toc.lis | psql newdbname
 
 
-Philip Warner, 3-Jul-2000
+BLOBs
+=====
+
+To dump blobs you must use the custom archive format (-Fc) or TAR format (-Ft), and specify the 
+--blobs qualifier to the pg_dump command.
+
+To restore blobs you must use a direct database connection (--db=db-to-restore-to).
+
+eg.
+
+   pg_dump --blob -Fc db-to-backup -f backup.bck
+
+   pg_restore backup.bck --db=db-to-restore-into
+
+
+TAR
+===
+
+The TAR archive that pg_dump creates currently has a blank username & group for the files, 
+but should be otherwise valid. It also includes a 'restore.sql' script which is there for
+the benefit of humans. It is never used by pg_restore.
+
+Note: the TAR format archive can only be used as input into pg_restore if it is in TAR form.
+(ie. you should not extract the files then expect pg_restore to work). 
+
+You can extract, edit, and tar the files again, and it should work, but the 'toc'
+file should go at the start, the data files be in the order they are used, and
+the BLOB files at the end.
+
+
+Philip Warner, 16-Jul-2000
 pjw@rhyme.com.au
 
 

src/bin/pg_dump/pg_backup.h

-/*-------------------------------------------------------------------------
- *
- * pg_backup.h
- *
- *	Public interface to the pg_dump archiver routines.
- *
- *	See the headers to pg_restore for more details.
- *
- * Copyright (c) 2000, Philip Warner
- *      Rights are granted to use this software in any way so long
- *      as this notice is not removed.
- *
- *	The author is not responsible for loss or damages that may
- *	result from it's use.
- *
- *
- * IDENTIFICATION
- *
- * Modifications - 28-Jun-2000 - pjw@rhyme.com.au
- *
- *	Initial version. 
- *
- *-------------------------------------------------------------------------
- */
-
-#ifndef PG_BACKUP__
-
-#include "config.h"
-#include "c.h"
-
-#define PG_BACKUP__
-
-typedef enum _archiveFormat {
-    archUnknown = 0,
-    archCustom = 1,
-    archFiles = 2,
-    archTar = 3,
-    archPlainText = 4
-} ArchiveFormat;
-
-/*
- *  We may want to have so user-readbale data, but in the mean
- *  time this gives us some abstraction and type checking.
- */
-typedef struct _Archive {
-    /* Nothing here */
-} Archive;
-
-typedef int     (*DataDumperPtr)(Archive* AH, char* oid, void* userArg);
-
-typedef struct _restoreOptions {
-	int			dataOnly;
-	int			dropSchema;
-	char		*filename;
-	int			schemaOnly;
-	int			verbose;
-	int			aclsSkip;
-	int			tocSummary;
-	char		*tocFile;
-	int			oidOrder;
-	int			origOrder;
-	int			rearrange;
-	int			format;
-	char		*formatName;
-
-	int			selTypes;
-	int		selIndex;
-	int		selFunction;
-	int		selTrigger;
-	int		selTable;
-	char		*indexNames;
-	char		*functionNames;
-	char		*tableNames;
-	char		*triggerNames;
-
-	int		*idWanted;
-	int		limitToList;
-	int		compression;
-
-} RestoreOptions;
-
-/*
- * Main archiver interface.
- */
-
-/* Called to add a TOC entry */
-extern void	ArchiveEntry(Archive* AH, const char* oid, const char* name,
-			const char* desc, const char* (deps[]), const char* defn,
-			const char* dropStmt, const char* owner, 
-			DataDumperPtr dumpFn, void* dumpArg);
-
-/* Called to write *data* to the archive */
-extern int	WriteData(Archive* AH, const void* data, int dLen);
-
-extern void	CloseArchive(Archive* AH);
-
-extern void	RestoreArchive(Archive* AH, RestoreOptions *ropt);
-
-/* Open an existing archive */
-extern Archive* OpenArchive(const char* FileSpec, ArchiveFormat fmt);
-
-/* Create a new archive */
-extern Archive* CreateArchive(const char* FileSpec, ArchiveFormat fmt, int compression);
-
-/* The --list option */
-extern void	PrintTOCSummary(Archive* AH, RestoreOptions *ropt);
-
-extern RestoreOptions*		NewRestoreOptions(void);
-
-/* Rearrange TOC entries */
-extern void	MoveToStart(Archive* AH, char *oType);
-extern void 	MoveToEnd(Archive* AH, char *oType); 
-extern void	SortTocByOID(Archive* AH);
-extern void	SortTocByID(Archive* AH);
-extern void	SortTocFromFile(Archive* AH, RestoreOptions *ropt);
-
-/* Convenience functions used only when writing DATA */
-extern int archputs(const char *s, Archive* AH);
-extern int archputc(const char c, Archive* AH);
-extern int archprintf(Archive* AH, const char *fmt, ...);
-
-#endif
-
-
-
+/*-------------------------------------------------------------------------
+ *
+ * pg_backup.h
+ *
+ *	Public interface to the pg_dump archiver routines.
+ *
+ *	See the headers to pg_restore for more details.
+ *
+ * Copyright (c) 2000, Philip Warner
+ *      Rights are granted to use this software in any way so long
+ *      as this notice is not removed.
+ *
+ *	The author is not responsible for loss or damages that may
+ *	result from it's use.
+ *
+ *
+ * IDENTIFICATION
+ *
+ * Modifications - 28-Jun-2000 - pjw@rhyme.com.au
+ *
+ *	Initial version. 
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef PG_BACKUP__
+
+#include "config.h"
+#include "c.h"
+
+#define PG_BACKUP__
+
+#include "postgres.h"
+#include "libpq-fe.h"
+
+typedef enum _archiveFormat {
+    archUnknown = 0,
+    archCustom = 1,
+    archFiles = 2,
+    archTar = 3,
+    archNull = 4
+} ArchiveFormat;
+
+/*
+ *  We may want to have so user-readbale data, but in the mean
+ *  time this gives us some abstraction and type checking.
+ */
+typedef struct _Archive {
+	int			verbose;
+    /* The rest is private */
+} Archive;
+
+typedef int     (*DataDumperPtr)(Archive* AH, char* oid, void* userArg);
+
+typedef struct _restoreOptions {
+	int			dataOnly;
+	int			dropSchema;
+	char		*filename;
+	int			schemaOnly;
+	int			verbose;
+	int			aclsSkip;
+	int			tocSummary;
+	char		*tocFile;
+	int			oidOrder;
+	int			origOrder;
+	int			rearrange;
+	int			format;
+	char		*formatName;
+
+	int			selTypes;
+	int			selIndex;
+	int			selFunction;
+	int			selTrigger;
+	int			selTable;
+	char		*indexNames;
+	char		*functionNames;
+	char		*tableNames;
+	char		*triggerNames;
+
+	int			useDB;
+	char		*dbname;
+	char		*pgport;
+	char		*pghost;
+	int			ignoreVersion;
+	int			requirePassword;
+
+	int		*idWanted;
+	int		limitToList;
+	int		compression;
+
+} RestoreOptions;
+
+/*
+ * Main archiver interface.
+ */
+
+extern void exit_horribly(Archive *AH, const char *fmt, ...);
+
+/* Lets the archibe know we have a DB connection to shutdown if it dies */
+
+PGconn* ConnectDatabase(Archive *AH,
+		const char* 	dbname,
+		const char*	pghost,
+		const char*	pgport,
+		const int	reqPwd,
+		const int	ignoreVersion);
+
+
+/* Called to add a TOC entry */
+extern void	ArchiveEntry(Archive* AH, const char* oid, const char* name,
+			const char* desc, const char* (deps[]), const char* defn,
+			const char* dropStmt, const char* copyStmt, const char* owner, 
+			DataDumperPtr dumpFn, void* dumpArg);
+
+/* Called to write *data* to the archive */
+extern int	WriteData(Archive* AH, const void* data, int dLen);
+
+//extern int 	StartBlobs(Archive* AH);
+//extern int	EndBlobs(Archive* AH);
+extern int	StartBlob(Archive* AH, int oid);
+extern int	EndBlob(Archive* AH, int oid);
+
+extern void	CloseArchive(Archive* AH);
+
+extern void	RestoreArchive(Archive* AH, RestoreOptions *ropt);
+
+/* Open an existing archive */
+extern Archive* OpenArchive(const char* FileSpec, const ArchiveFormat fmt);
+
+/* Create a new archive */
+extern Archive* CreateArchive(const char* FileSpec, const ArchiveFormat fmt, 
+								const int compression);
+
+/* The --list option */
+extern void	PrintTOCSummary(Archive* AH, RestoreOptions *ropt);
+
+extern RestoreOptions*		NewRestoreOptions(void);
+
+/* Rearrange TOC entries */
+extern void	MoveToStart(Archive* AH, char *oType);
+extern void 	MoveToEnd(Archive* AH, char *oType); 
+extern void	SortTocByOID(Archive* AH);
+extern void	SortTocByID(Archive* AH);
+extern void	SortTocFromFile(Archive* AH, RestoreOptions *ropt);
+
+/* Convenience functions used only when writing DATA */
+extern int archputs(const char *s, Archive* AH);
+extern int archputc(const char c, Archive* AH);
+extern int archprintf(Archive* AH, const char *fmt, ...);
+
+#endif
+
+
+

src/bin/pg_dump/pg_backup_archiver.h

@@ -29,6 +29,10 @@
 #define __PG_BACKUP_ARCHIVE__
 
 #include <stdio.h>
+#include <time.h>
+
+#include "postgres.h"
+#include "pqexpbuffer.h"
 
 #ifdef HAVE_LIBZ
 #include <zlib.h>
@@ -51,15 +55,23 @@ typedef z_stream *z_streamp;
 #endif
 
 #include "pg_backup.h"
+#include "libpq-fe.h"
 
 #define K_VERS_MAJOR 1
-#define K_VERS_MINOR 2 
-#define K_VERS_REV 2
+#define K_VERS_MINOR 4 
+#define K_VERS_REV 3 
+
+/* Data block types */
+#define BLK_DATA 1
+#define BLK_BLOB 2
+#define BLK_BLOBS 3
 
 /* Some important version numbers (checked in code) */
 #define K_VERS_1_0 (( (1 * 256 + 0) * 256 + 0) * 256 + 0)
-#define K_VERS_1_2 (( (1 * 256 + 2) * 256 + 0) * 256 + 0)
-#define K_VERS_MAX (( (1 * 256 + 2) * 256 + 255) * 256 + 0)
+#define K_VERS_1_2 (( (1 * 256 + 2) * 256 + 0) * 256 + 0) /* Allow No ZLIB */
+#define K_VERS_1_3 (( (1 * 256 + 3) * 256 + 0) * 256 + 0) /* BLOBs */
+#define K_VERS_1_4 (( (1 * 256 + 4) * 256 + 0) * 256 + 0) /* Date & name in header */
+#define K_VERS_MAX (( (1 * 256 + 4) * 256 + 255) * 256 + 0)
 
 struct _archiveHandle;
 struct _tocEntry;
@@ -72,10 +84,15 @@ typedef void	(*StartDataPtr)		(struct _archiveHandle* AH, struct _tocEntry* te);
 typedef int 	(*WriteDataPtr)		(struct _archiveHandle* AH, const void* data, int dLen);
 typedef void	(*EndDataPtr)		(struct _archiveHandle* AH, struct _tocEntry* te);
 
-typedef int	(*WriteBytePtr)		(struct _archiveHandle* AH, const int i);
+typedef void	(*StartBlobsPtr)	(struct _archiveHandle* AH, struct _tocEntry* te);
+typedef void    (*StartBlobPtr)		(struct _archiveHandle* AH, struct _tocEntry* te, int oid);
+typedef void	(*EndBlobPtr)		(struct _archiveHandle* AH, struct _tocEntry* te, int oid);
+typedef void	(*EndBlobsPtr)		(struct _archiveHandle* AH, struct _tocEntry* te);
+
+typedef int		(*WriteBytePtr)		(struct _archiveHandle* AH, const int i);
 typedef int    	(*ReadBytePtr)		(struct _archiveHandle* AH);
-typedef int	(*WriteBufPtr)		(struct _archiveHandle* AH, const void* c, int len);
-typedef int	(*ReadBufPtr)		(struct _archiveHandle* AH, void* buf, int len);
+typedef int		(*WriteBufPtr)		(struct _archiveHandle* AH, const void* c, int len);
+typedef int		(*ReadBufPtr)		(struct _archiveHandle* AH, void* buf, int len);
 typedef void	(*SaveArchivePtr)	(struct _archiveHandle* AH);
 typedef void 	(*WriteExtraTocPtr)	(struct _archiveHandle* AH, struct _tocEntry* te);
 typedef void	(*ReadExtraTocPtr)	(struct _archiveHandle* AH, struct _tocEntry* te);
@@ -83,6 +100,8 @@ typedef void	(*PrintExtraTocPtr)	(struct _archiveHandle* AH, struct _tocEntry* t
 typedef void	(*PrintTocDataPtr)	(struct _archiveHandle* AH, struct _tocEntry* te, 
 						RestoreOptions *ropt);
 
+typedef int		(*CustomOutPtr)		(struct _archiveHandle* AH, const void* buf, int len);
+
 typedef int	(*TocSortCompareFn)	(const void* te1, const void *te2); 
 
 typedef enum _archiveMode {
@@ -95,16 +114,44 @@ typedef struct _outputContext {
 	int		gzOut;
 } OutputContext;
 
+typedef enum { 
+	SQL_SCAN = 0,
+	SQL_IN_SQL_COMMENT,
+	SQL_IN_EXT_COMMENT,
+	SQL_IN_QUOTE} sqlparseState;
+	
+typedef struct {
+	int					backSlash;
+	sqlparseState		state;
+	char				lastChar;
+	char				quoteChar;
+} sqlparseInfo;
+
 typedef struct _archiveHandle {
+	Archive				public;				/* Public part of archive */
 	char				vmaj;				/* Version of file */
 	char				vmin;
 	char				vrev;
 	int					version;			/* Conveniently formatted version */
 
+	int					debugLevel;			/* Not used. Intended for logging */
 	int					intSize;			/* Size of an integer in the archive */
 	ArchiveFormat		format;				/* Archive format */
 
+	sqlparseInfo		sqlparse;
+	PQExpBuffer			sqlBuf;
+
+	time_t				createDate;			/* Date archive created */
+
+	/*
+	 * Fields used when discovering header.
+	 * A format can always get the previous read bytes from here...
+	 */
 	int					readHeader;			/* Used if file header has been read already */
+	char				*lookahead;			/* Buffer used when reading header to discover format */
+	int					lookaheadSize;		/* Size of allocated buffer */
+	int					lookaheadLen;		/* Length of data in lookahead */
+	int					lookaheadPos;		/* Current read position in lookahead buffer */
 
 	ArchiveEntryPtr		ArchiveEntryPtr;	/* Called for each metadata object */
 	StartDataPtr		StartDataPtr; 		/* Called when table data is about to be dumped */
@@ -121,11 +168,33 @@ typedef struct _archiveHandle {
 	PrintExtraTocPtr	PrintExtraTocPtr;	/* Extra TOC info for format */
 	PrintTocDataPtr		PrintTocDataPtr;
 
-	int			lastID;						/* Last internal ID for a TOC entry */
-	char*		fSpec;						/* Archive File Spec */
-	FILE		*FH;						/* General purpose file handle */
-	void		*OF;
-	int		gzOut;						/* Output file */
+	StartBlobsPtr		StartBlobsPtr;
+	EndBlobsPtr			EndBlobsPtr;
+	StartBlobPtr		StartBlobPtr;
+	EndBlobPtr			EndBlobPtr;
+
+	CustomOutPtr		CustomOutPtr;		/* Alternate script output routine */
+
+	/* Stuff for direct DB connection */
+	char				username[100];
+	char				*dbname;			/* Name of db for connection */
+	char				*archdbname;		/* DB name *read* from archive */
+	char				*pghost;
+	char				*pgport;
+	PGconn				*connection;
+	int					connectToDB;		/* Flag to indicate if direct DB connection is required */
+	int					pgCopyIn;			/* Currently in libpq 'COPY IN' mode. */
+	PQExpBuffer			pgCopyBuf;			/* Left-over data from incomplete lines in COPY IN */
+
+	int					loFd;				/* BLOB fd */
+	int					writingBlob;		/* Flag */
+	int					createdBlobXref;	/* Flag */
+
+	int					lastID;				/* Last internal ID for a TOC entry */
+	char*				fSpec;				/* Archive File Spec */
+	FILE				*FH;				/* General purpose file handle */
+	void				*OF;
+	int					gzOut;				/* Output file */
 
 	struct _tocEntry*		toc;			/* List of TOC entries */
 	int						tocCount;		/* Number of TOC entries */
@@ -135,6 +204,7 @@ typedef struct _archiveHandle {
 	ArchiveMode				mode;			/* File mode - r or w */
 	void*					formatData;		/* Header data specific to file format */
 
+	RestoreOptions			*ropt;			/* Used to check restore options in ahwrite etc */
 } ArchiveHandle;
 
 typedef struct _tocEntry {
@@ -148,6 +218,7 @@ typedef struct _tocEntry {
 	char*				desc;
 	char*				defn;
 	char*				dropStmt;
+	char*				copyStmt;
 	char*				owner;
 	char**				depOid;
 	int					printed;		/* Indicates if entry defn has been dumped */
@@ -159,7 +230,8 @@ typedef struct _tocEntry {
 
 } TocEntry;
 
-extern void die_horribly(const char *fmt, ...);
+/* Used everywhere */
+extern void die_horribly(ArchiveHandle *AH, const char *fmt, ...);
 
 extern void WriteTOC(ArchiveHandle* AH);
 extern void ReadTOC(ArchiveHandle* AH);
@@ -175,19 +247,27 @@ extern int TocIDRequired(ArchiveHandle* AH, int id, RestoreOptions *ropt);
  * Mandatory routines for each supported format
  */
 
-extern int WriteInt(ArchiveHandle* AH, int i);
-extern int ReadInt(ArchiveHandle* AH);
-extern char* ReadStr(ArchiveHandle* AH);
-extern int WriteStr(ArchiveHandle* AH, char* s);
+extern int 				WriteInt(ArchiveHandle* AH, int i);
+extern int 				ReadInt(ArchiveHandle* AH);
+extern char* 			ReadStr(ArchiveHandle* AH);
+extern int 				WriteStr(ArchiveHandle* AH, char* s);
+
+extern void 			StartRestoreBlob(ArchiveHandle* AH, int oid);
+extern void 			EndRestoreBlob(ArchiveHandle* AH, int oid);
 
-extern void InitArchiveFmt_Custom(ArchiveHandle* AH);
-extern void InitArchiveFmt_Files(ArchiveHandle* AH);
-extern void InitArchiveFmt_PlainText(ArchiveHandle* AH);
+extern void 			InitArchiveFmt_Custom(ArchiveHandle* AH);
+extern void 			InitArchiveFmt_Files(ArchiveHandle* AH);
+extern void 			InitArchiveFmt_Null(ArchiveHandle* AH);
+extern void 			InitArchiveFmt_Tar(ArchiveHandle* AH);
+
+extern int 				isValidTarHeader(char *header);
 
 extern OutputContext	SetOutput(ArchiveHandle* AH, char *filename, int compression);
-extern void 		ResetOutput(ArchiveHandle* AH, OutputContext savedContext);
+extern void 			ResetOutput(ArchiveHandle* AH, OutputContext savedContext);
 
 int ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle* AH);
 int ahprintf(ArchiveHandle* AH, const char *fmt, ...);
 
+void ahlog(ArchiveHandle* AH, int level, const char *fmt, ...);
+
 #endif