Massive overhaul of pg_dump: make use of dependency information from

pg_depend to determine a safe dump order.  Defaults and check constraints
can be emitted either as part of a table or domain definition, or
separately if that's needed to break a dependency loop.  Lots of old
half-baked code for controlling dump order removed.

doc/src/sgml/ref/pg_restore.sgml

-<!-- $PostgreSQL: pgsql/doc/src/sgml/ref/pg_restore.sgml,v 1.44 2003/11/29 19:51:39 pgsql Exp $ -->
+<!-- $PostgreSQL: pgsql/doc/src/sgml/ref/pg_restore.sgml,v 1.45 2003/12/06 03:00:10 tgl Exp $ -->
 
 <refentry id="APP-PGRESTORE">
  <refmeta>
@@ -227,35 +227,6 @@
       </listitem>
      </varlistentry>
 
-     <varlistentry>
-      <term><option>-N</option></term>
-      <term><option>--orig-order</option></term>
-      <listitem>
-       <para>
-        Restore items in the order they were originally generated within
-        <application>pg_dump</application>.  This option has no known
-	practical use, since <application>pg_dump</application> generates
-	the items in an order convenient to it, which is unlikely to be a
-	safe order for restoring them.  (This is <emphasis>not</> the order
-	in which the items are ultimately listed in the archive's table of
-	contents.)  See also <option>-r</>.
-       </para>
-      </listitem>
-     </varlistentry>
-
-     <varlistentry>
-      <term><option>-o</option></term>
-      <term><option>--oid-order</option></term>
-      <listitem>
-       <para>
-        Restore items in order by OID.  This option is of limited usefulness,
-	since OID is only an approximate indication of original creation
-	order.  This option overrides <option>-N</> if both are specified.
-	See also <option>-r</>.
-       </para>
-      </listitem>
-     </varlistentry>
-
      <varlistentry>
       <term><option>-O</option></term>
       <term><option>--no-owner</option></term>
@@ -287,31 +258,6 @@
       </listitem>
      </varlistentry>
 
-     <varlistentry>
-      <term><option>-r</option></term>
-      <term><option>--rearrange</option></term>
-      <listitem>
-       <para>
-        Rearrange items by object type (this occurs after the sorting
-	specified by <option>-N</option> or <option>-o</option>, if
-	given).  The rearrangement is intended to give the best possible
-	restore performance.
-       </para>
-
-       <para>
-        When none of <option>-N</option>, <option>-o</option>, and
-	<option>-r</> appear, <application>pg_restore</application> restores
-	items in the order they appear in the dump's table of contents,
-	or in the order they appear in the  <REPLACEABLE
-        CLASS="PARAMETER">list-file</REPLACEABLE> if <option>-L</> is
-	given.  The combination of <option>-o</> and <option>-r</>
-	duplicates the sorting done by <application>pg_dump</application>
-	before creating the dump's table of contents,
-	and so it is normally unnecessary to specify it.
-       </para>
-      </listitem>
-     </varlistentry>
-
      <varlistentry>
       <term><option>-R</option></term>
       <term><option>--no-reconnect</option></term>

src/bin/pg_dump/Makefile

@@ -5,7 +5,7 @@
 # Portions Copyright (c) 1996-2002, PostgreSQL Global Development Group
 # Portions Copyright (c) 1994, Regents of the University of California
 #
-# $PostgreSQL: pgsql/src/bin/pg_dump/Makefile,v 1.41 2003/11/29 19:52:04 pgsql Exp $
+# $PostgreSQL: pgsql/src/bin/pg_dump/Makefile,v 1.42 2003/12/06 03:00:11 tgl Exp $
 #
 #-------------------------------------------------------------------------
 
@@ -24,8 +24,8 @@ override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS) -DBINDIR=\"$(bindir)\"
 
 all: submake-libpq submake-libpgport submake-backend pg_dump pg_restore pg_dumpall
 
-pg_dump: pg_dump.o common.o $(OBJS) $(libpq_builddir)/libpq.a 
-	$(CC) $(CFLAGS) pg_dump.o common.o $(OBJS) $(EXTRA_OBJS) $(libpq) $(LDFLAGS) $(LIBS) -o $@
+pg_dump: pg_dump.o common.o pg_dump_sort.o $(OBJS) $(libpq_builddir)/libpq.a 
+	$(CC) $(CFLAGS) pg_dump.o common.o pg_dump_sort.o $(OBJS) $(EXTRA_OBJS) $(libpq) $(LDFLAGS) $(LIBS) -o $@
 
 pg_restore: pg_restore.o $(OBJS) $(libpq_builddir)/libpq.a
 	$(CC) $(CFLAGS) pg_restore.o $(OBJS) $(EXTRA_OBJS) $(libpq) $(LDFLAGS) $(LIBS) -o $@
@@ -50,4 +50,4 @@ uninstall:
 	rm -f $(addprefix $(DESTDIR)$(bindir)/, pg_dump$(X) pg_restore$(X) pg_dumpall$(X))
 
 clean distclean maintainer-clean:
-	rm -f pg_dump$(X) pg_restore$(X) pg_dumpall$(X) $(OBJS) pg_dump.o common.o pg_restore.o pg_dumpall.o
+	rm -f pg_dump$(X) pg_restore$(X) pg_dumpall$(X) $(OBJS) pg_dump.o common.o pg_dump_sort.o pg_restore.o pg_dumpall.o

src/bin/pg_dump/pg_backup.h

@@ -15,19 +15,22 @@
  *
  *
  * IDENTIFICATION
- *		$PostgreSQL: pgsql/src/bin/pg_dump/pg_backup.h,v 1.27 2003/11/29 19:52:05 pgsql Exp $
+ *		$PostgreSQL: pgsql/src/bin/pg_dump/pg_backup.h,v 1.28 2003/12/06 03:00:11 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
 
-#ifndef PG_BACKUP__
-#define PG_BACKUP__
+#ifndef PG_BACKUP_H
+#define PG_BACKUP_H
 
 #include "postgres_fe.h"
 
+#include "pg_dump.h"
+
 #include "libpq-fe.h"
 #include "pqexpbuffer.h"
 
+
 #define atooid(x)  ((Oid) strtoul((x), NULL, 10))
 #define oidcmp(x,y) ( ((x) < (y) ? -1 : ((x) > (y)) ?  1 : 0) )
 #define oideq(x,y) ( (x) == (y) )
@@ -45,7 +48,7 @@ typedef enum _archiveFormat
 } ArchiveFormat;
 
 /*
- *	We may want to have so user-readbale data, but in the mean
+ *	We may want to have some more user-readable data, but in the mean
  *	time this gives us some abstraction and type checking.
  */
 typedef struct _Archive
@@ -57,7 +60,7 @@ typedef struct _Archive
 	/* The rest is private */
 } Archive;
 
-typedef int (*DataDumperPtr) (Archive *AH, char *oid, void *userArg);
+typedef int (*DataDumperPtr) (Archive *AH, void *userArg);
 
 typedef struct _restoreOptions
 {
@@ -74,9 +77,6 @@ typedef struct _restoreOptions
 	int			aclsSkip;
 	int			tocSummary;
 	char	   *tocFile;
-	int			oidOrder;
-	int			origOrder;
-	int			rearrange;
 	int			format;
 	char	   *formatName;
 
@@ -98,8 +98,8 @@ typedef struct _restoreOptions
 	int			ignoreVersion;
 	int			requirePassword;
 
-	int		   *idWanted;
-	int			limitToList;
+	bool	   *idWanted;
+	bool		limitToList;
 	int			compression;
 
 	int			suppressDumpWarnings;	/* Suppress output of WARNING
@@ -127,11 +127,13 @@ PGconn *ConnectDatabase(Archive *AH,
 
 
 /* Called to add a TOC entry */
-extern void ArchiveEntry(Archive *AHX, const char *oid, const char *tag,
+extern void ArchiveEntry(Archive *AHX,
+			 CatalogId catalogId, DumpId dumpId,
+			 const char *tag,
 			 const char *namespace, const char *owner,
-			 const char *desc, const char *((*deps)[]),
-			 const char *defn, const char *dropStmt,
-			 const char *copyStmt,
+			 const char *desc, const char *defn,
+			 const char *dropStmt, const char *copyStmt,
+			 const DumpId *deps, int nDeps,
 			 DataDumperPtr dumpFn, void *dumpArg);
 
 /* Called to write *data* to the archive */
@@ -161,19 +163,13 @@ extern void PrintTOCSummary(Archive *AH, RestoreOptions *ropt);
 extern RestoreOptions *NewRestoreOptions(void);
 
 /* Rearrange TOC entries */
-extern void MoveToStart(Archive *AH, const char *oType);
-extern void MoveToEnd(Archive *AH, const char *oType);
-extern void SortTocByObjectType(Archive *AH);
-extern void SortTocByOID(Archive *AH);
-extern void SortTocByID(Archive *AH);
 extern void SortTocFromFile(Archive *AH, RestoreOptions *ropt);
 
 /* Convenience functions used only when writing DATA */
 extern int	archputs(const char *s, Archive *AH);
-extern int	archputc(const char c, Archive *AH);
 extern int
 archprintf(Archive *AH, const char *fmt,...)
 /* This extension allows gcc to check the format string */
 __attribute__((format(printf, 2, 3)));
 
-#endif
+#endif /* PG_BACKUP_H */

src/bin/pg_dump/pg_backup_archiver.h

@@ -17,7 +17,7 @@
  *
  *
  * IDENTIFICATION
- *		$PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_archiver.h,v 1.53 2003/11/29 19:52:05 pgsql Exp $
+ *		$PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_archiver.h,v 1.54 2003/12/06 03:00:11 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -59,7 +59,7 @@ typedef z_stream *z_streamp;
 #include "libpq-fe.h"
 
 #define K_VERS_MAJOR 1
-#define K_VERS_MINOR 7
+#define K_VERS_MINOR 8
 #define K_VERS_REV 0
 
 /* Data block types */
@@ -76,7 +76,9 @@ typedef z_stream *z_streamp;
 #define K_VERS_1_6 (( (1 * 256 + 6) * 256 + 0) * 256 + 0)		/* Schema field in TOCs */
 #define K_VERS_1_7 (( (1 * 256 + 7) * 256 + 0) * 256 + 0)		/* File Offset size in
 																 * header */
-#define K_VERS_MAX (( (1 * 256 + 7) * 256 + 255) * 256 + 0)
+#define K_VERS_1_8 (( (1 * 256 + 8) * 256 + 0) * 256 + 0)		/* change interpretation of ID numbers and dependencies */
+
+#define K_VERS_MAX (( (1 * 256 + 8) * 256 + 255) * 256 + 0)
 
 /* No of BLOBs to restore in 1 TX */
 #define BLOB_BATCH_SIZE 100
@@ -114,8 +116,6 @@ typedef void (*PrintTocDataPtr) (struct _archiveHandle * AH, struct _tocEntry *
 
 typedef size_t (*CustomOutPtr) (struct _archiveHandle * AH, const void *buf, size_t len);
 
-typedef int (*TocSortCompareFn) (const void *te1, const void *te2);
-
 typedef enum _archiveMode
 {
 	archModeWrite,
@@ -222,7 +222,6 @@ typedef struct _archiveHandle
 	int			createdBlobXref;	/* Flag */
 	int			blobCount;		/* # of blobs restored */
 
-	int			lastID;			/* Last internal ID for a TOC entry */
 	char	   *fSpec;			/* Archive File Spec */
 	FILE	   *FH;				/* General purpose file handle */
 	void	   *OF;
@@ -230,6 +229,8 @@ typedef struct _archiveHandle
 
 	struct _tocEntry *toc;		/* List of TOC entries */
 	int			tocCount;		/* Number of TOC entries */
+	DumpId		maxDumpId;		/* largest DumpId among all TOC entries */
+
 	struct _tocEntry *currToc;	/* Used when dumping data */
 	int			compression;	/* Compression requested on open */
 	ArchiveMode mode;			/* File mode - r or w */
@@ -252,8 +253,9 @@ typedef struct _tocEntry
 {
 	struct _tocEntry *prev;
 	struct _tocEntry *next;
-	int			id;
-	int			hadDumper;		/* Archiver was passed a dumper routine
+	CatalogId	catalogId;
+	DumpId		dumpId;
+	bool		hadDumper;		/* Archiver was passed a dumper routine
 								 * (used in restore) */
 	char	   *tag;			/* index tag */
 	char	   *namespace;		/* null or empty string if not in a schema */
@@ -262,23 +264,17 @@ typedef struct _tocEntry
 	char	   *defn;
 	char	   *dropStmt;
 	char	   *copyStmt;
-	char	   *oid;			/* Oid of source of entry */
-	Oid			oidVal;			/* Value of above */
-	const char *((*depOid)[]);
-	Oid			maxDepOidVal;	/* Value of largest OID in deps */
-	Oid			maxOidVal;		/* Max of entry OID and max dep OID */
+	DumpId	   *dependencies;	/* dumpIds of objects this one depends on */
+	int			nDeps;			/* number of dependencies */
 
-	int			printed;		/* Indicates if entry defn has been dumped */
 	DataDumperPtr dataDumper;	/* Routine to dump data for object */
 	void	   *dataDumperArg;	/* Arg for above routine */
 	void	   *formatData;		/* TOC Entry data specific to file format */
-
-	int			_moved;			/* Marker used when rearranging TOC */
-
 } TocEntry;
 
 /* Used everywhere */
 extern const char *progname;
+
 extern void die_horribly(ArchiveHandle *AH, const char *modulename, const char *fmt,...) __attribute__((format(printf, 3, 4)));
 extern void write_msg(const char *modulename, const char *fmt,...) __attribute__((format(printf, 2, 3)));
 
@@ -290,7 +286,7 @@ extern void WriteToc(ArchiveHandle *AH);
 extern void ReadToc(ArchiveHandle *AH);
 extern void WriteDataChunks(ArchiveHandle *AH);
 
-extern int	TocIDRequired(ArchiveHandle *AH, int id, RestoreOptions *ropt);
+extern int	TocIDRequired(ArchiveHandle *AH, DumpId id, RestoreOptions *ropt);
 extern bool checkSeek(FILE *fp);
 
 /*

src/bin/pg_dump/pg_backup_custom.c

@@ -19,7 +19,7 @@
  *
  *
  * IDENTIFICATION
- *		$PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_custom.c,v 1.27 2003/11/29 19:52:05 pgsql Exp $
+ *		$PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_custom.c,v 1.28 2003/12/06 03:00:11 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -303,7 +303,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
 	tctx->dataState = K_OFFSET_POS_SET;
 
 	_WriteByte(AH, BLK_DATA);	/* Block type */
-	WriteInt(AH, te->id);		/* For sanity check */
+	WriteInt(AH, te->dumpId);	/* For sanity check */
 
 	_StartDataCompressor(AH, te);
 }
@@ -371,7 +371,7 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
 	tctx->dataState = K_OFFSET_POS_SET;
 
 	_WriteByte(AH, BLK_BLOBS);	/* Block type */
-	WriteInt(AH, te->id);		/* For sanity check */
+	WriteInt(AH, te->dumpId);	/* For sanity check */
 }
 
 /*
@@ -439,7 +439,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te, RestoreOptions *ropt)
 
 		_readBlockHeader(AH, &blkType, &id);
 
-		while (id != te->id)
+		while (id != te->dumpId)
 		{
 			if ((TocIDRequired(AH, id, ropt) & 2) != 0)
 				die_horribly(AH, modulename,
@@ -475,9 +475,9 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te, RestoreOptions *ropt)
 	}
 
 	/* Are we sane? */
-	if (id != te->id)
+	if (id != te->dumpId)
 		die_horribly(AH, modulename, "found unexpected block ID (%d) when reading data -- expected %d\n",
-					 id, te->id);
+					 id, te->dumpId);
 
 	switch (blkType)
 	{
@@ -863,7 +863,7 @@ _readBlockHeader(ArchiveHandle *AH, int *type, int *id)
 	if (AH->version < K_VERS_1_3)
 		*type = BLK_DATA;
 	else
-		*type = _ReadByte(AH);;
+		*type = _ReadByte(AH);
 
 	*id = ReadInt(AH);
 }

src/bin/pg_dump/pg_backup_files.c

@@ -20,7 +20,7 @@
  *
  *
  * IDENTIFICATION
- *		$PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_files.c,v 1.23 2003/11/29 19:52:05 pgsql Exp $
+ *		$PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_files.c,v 1.24 2003/12/06 03:00:11 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -172,11 +172,11 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
 	{
 #ifdef HAVE_LIBZ
 		if (AH->compression == 0)
-			sprintf(fn, "%d.dat", te->id);
+			sprintf(fn, "%d.dat", te->dumpId);
 		else
-			sprintf(fn, "%d.dat.gz", te->id);
+			sprintf(fn, "%d.dat.gz", te->dumpId);
 #else
-		sprintf(fn, "%d.dat", te->id);
+		sprintf(fn, "%d.dat", te->dumpId);
 #endif
 		ctx->filename = strdup(fn);
 	}

src/bin/pg_dump/pg_backup_null.c

@@ -17,7 +17,7 @@
  *
  *
  * IDENTIFICATION
- *		$PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_null.c,v 1.12 2003/11/29 19:52:05 pgsql Exp $
+ *		$PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_null.c,v 1.13 2003/12/06 03:00:11 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -91,7 +91,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te, RestoreOptions *ropt)
 	if (te->dataDumper)
 	{
 		AH->currToc = te;
-		(*te->dataDumper) ((Archive *) AH, te->oid, te->dataDumperArg);
+		(*te->dataDumper) ((Archive *) AH, te->dataDumperArg);
 		AH->currToc = NULL;
 	}
 }

src/bin/pg_dump/pg_backup_tar.c

@@ -16,7 +16,7 @@
  *
  *
  * IDENTIFICATION
- *		$PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_tar.c,v 1.39 2003/11/29 19:52:05 pgsql Exp $
+ *		$PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_tar.c,v 1.40 2003/12/06 03:00:11 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -259,11 +259,11 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
 	{
 #ifdef HAVE_LIBZ
 		if (AH->compression == 0)
-			sprintf(fn, "%d.dat", te->id);
+			sprintf(fn, "%d.dat", te->dumpId);
 		else
-			sprintf(fn, "%d.dat.gz", te->id);
+			sprintf(fn, "%d.dat.gz", te->dumpId);
 #else
-		sprintf(fn, "%d.dat", te->id);
+		sprintf(fn, "%d.dat", te->dumpId);
 #endif
 		ctx->filename = strdup(fn);
 	}

src/bin/pg_dump/pg_restore.c

@@ -34,7 +34,7 @@
  *
  *
  * IDENTIFICATION
- *		$PostgreSQL: pgsql/src/bin/pg_dump/pg_restore.c,v 1.54 2003/11/29 19:52:05 pgsql Exp $
+ *		$PostgreSQL: pgsql/src/bin/pg_dump/pg_restore.c,v 1.55 2003/12/06 03:00:16 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -101,10 +101,7 @@ main(int argc, char **argv)
 		{"no-owner", 0, NULL, 'O'},
 		{"no-reconnect", 0, NULL, 'R'},
 		{"port", 1, NULL, 'p'},
-		{"oid-order", 0, NULL, 'o'},
-		{"orig-order", 0, NULL, 'N'},
 		{"password", 0, NULL, 'W'},
-		{"rearrange", 0, NULL, 'r'},
 		{"schema-only", 0, NULL, 's'},
 		{"superuser", 1, NULL, 'S'},
 		{"table", 1, NULL, 't'},
@@ -147,7 +144,7 @@ main(int argc, char **argv)
 		}
 	}
 
-	while ((c = getopt_long(argc, argv, "acCd:f:F:h:iI:lL:NoOp:P:rRsS:t:T:uU:vWxX:",
+	while ((c = getopt_long(argc, argv, "acCd:f:F:h:iI:lL:Op:P:RsS:t:T:uU:vWxX:",
 							cmdopts, NULL)) != -1)
 	{
 		switch (c)
@@ -188,12 +185,6 @@ main(int argc, char **argv)
 				opts->tocFile = strdup(optarg);
 				break;
 
-			case 'N':
-				opts->origOrder = 1;
-				break;
-			case 'o':
-				opts->oidOrder = 1;
-				break;
 			case 'O':
 				opts->noOwner = 1;
 				break;
@@ -201,9 +192,6 @@ main(int argc, char **argv)
 				if (strlen(optarg) != 0)
 					opts->pgport = strdup(optarg);
 				break;
-			case 'r':
-				opts->rearrange = 1;
-				break;
 			case 'R':
 				/* no-op, still accepted for backwards compatibility */
 				break;
@@ -338,19 +326,6 @@ main(int argc, char **argv)
 	if (opts->tocFile)
 		SortTocFromFile(AH, opts);
 
-	if (opts->oidOrder)
-		SortTocByOID(AH);
-	else if (opts->origOrder)
-		SortTocByID(AH);
-
-	if (opts->rearrange)
-		SortTocByObjectType(AH);
-	else
-	{
-		/* Database MUST be at start (see also SortTocByObjectType) */
-		MoveToStart(AH, "DATABASE");
-	}
-
 	if (opts->tocSummary)
 		PrintTOCSummary(AH, opts);
 	else
@@ -385,12 +360,9 @@ usage(const char *progname)
 	printf(_("  -I, --index=NAME         restore named index\n"));
 	printf(_("  -L, --use-list=FILENAME  use specified table of contents for ordering\n"
 			 "                           output from this file\n"));
-	printf(_("  -N, --orig-order         restore in original dump order\n"));
-	printf(_("  -o, --oid-order          restore in OID order\n"));
 	printf(_("  -O, --no-owner           do not output commands to set object ownership\n"));
 	printf(_("  -P, --function=NAME(args)\n"
 			 "                           restore named function\n"));
-	printf(_("  -r, --rearrange          rearrange output to put indexes etc. at end\n"));
 	printf(_("  -s, --schema-only        restore only the schema, no data\n"));
 	printf(_("  -S, --superuser=NAME     specify the superuser user name to use for\n"
 			 "                           disabling triggers\n"));