Move pg_upgrade to /contrib. Still need to make Peter's portability

changes.

contrib/README

@@ -131,6 +131,10 @@ pg_resetxlog -
 	Reset the WAL log (pg_xlog) to recover from crash or format change
 	by Tom Lane <tgl@sss.pgh.pa.us>
 
+pg_upgrade -
+	Upgrade from previous PostgreSQL version without pg_dump/reload
+	by Bruce Momjian <pgman@candle.pha.pa.us>
+
 pgbench -
 	TPC-B like benchmarking tool
 	by Tatsuo Ishii <t-ishii@sra.co.jp>

contrib/pg_upgrade/pg_upgrade

+#!/bin/sh
+#
+# pg_upgrade: update a database without needing a full dump/reload cycle.
+# CAUTION: Read the manual page before trying to use this!
+
+# $Header: /cvsroot/pgsql/contrib/pg_upgrade/Attic/pg_upgrade,v 1.1 2002/01/14 17:32:07 momjian Exp $
+#
+# NOTE: we must be sure to update the version-checking code a few dozen lines
+# below for each new PostgreSQL release.
+
+#set -x
+
+# Set this to "Y" to enable this program
+ENABLE="N"
+
+if [ "$ENABLE" != "Y" ]
+then
+	echo "Sorry, $0 cannot upgrade database
+version $SRC_VERSION to $DST_VERSION." 1>&2
+	echo "The on-disk structure of tables has changed." 1>&2
+	echo "You will need to dump and restore using pg_dumpall." 1>&2
+	exit 1
+fi
+
+
+# UPGRADE_VERSION is the expected old database version
+UPGRADE_VERSION="7.1"
+CUR_VERSION="7.2"
+
+trap "rm -f /tmp/$$.*" 0 1 2 3 15
+
+PHASE=""
+if [ "$#" -eq 1 ]
+then
+	if [ "X$1" = "X-1" ]
+	then	PHASE="1"
+		shift
+	elif [ "X$1" = "X-2" ]
+	then	PHASE="2"
+		shift
+	fi
+fi
+
+if [ "$PHASE" = "" ]
+then	echo "You must run $0 in either mode 1 or mode 2." 1>&2
+	echo "Usage:  $0 [ -1 | -2 ]" 1>&2
+	exit 1
+fi
+
+if [ ! -e data ]
+then	echo "$0 must be run from the directory above your /data directory.
+$0 aborted." 1>&2
+	if [ "$PHASE" -eq 2 ]
+	then	echo "You must run initdb to create a template1 database." 1>&2
+	fi
+	exit 1
+fi
+
+INFODIR="pg_upgrade_info"
+OLDDIR="$INFODIR/data"
+
+make_dbobjoidmap()
+{
+	psql -d template1 -At -c "SELECT datname FROM pg_database" |
+	grep -v '^template0$' |
+	while read DB
+	do	
+		QUERY="`echo \"	SELECT 	relname, oid
+				FROM 	pg_class
+				WHERE 	relkind = 'r' OR
+					relkind = 'i' OR
+					relkind = 't'\"`"
+		# Don't move over 7.1 int4 sequences;  use setval() file.
+		# Sequence XIDs changed from 7.2beta4 to beta5;  don't copy them.
+		if [ "$SRC_VERSION" != "7.1" -a \
+		     "$SRC_VERSION" != "7.2" ]
+		then	QUERY="$QUERY OR relkind = 'S';";
+			QUERY="$QUERY;"
+		fi
+
+		psql -d "$DB" -At -F'	' -c "$QUERY" |
+		while read RELNAME_OID
+		do	
+			echo "$DB	$RELNAME_OID"
+		done
+	done
+}
+
+
+make_dboidmap()
+{
+	psql -d template1 -At -F'	' -c \
+		'SELECT datname, oid FROM pg_database;' |
+	grep -v '^template0$'
+}
+
+
+move_objfiles()
+{
+	# Test to make sure there is a matching file in each place
+
+	if [ ! -e "$OLDDIR"/base/"$SRC_DBOID"/"$SRC_OID" ]
+	then	echo "Move of database $DB, OID $SRC_OID, object $OBJ failed.
+File not found;  exiting" 1>&2
+		exit 1
+	fi
+
+	if [ ! -e data/base/"$DST_DBOID"/"$DST_OID" ]
+	then	echo "Move of database $DB, OID $DST_OID, object $OBJ failed.
+File not found;  exiting" 1>&2
+		exit 1
+	fi
+
+	# Move files
+
+	mv -f "$OLDDIR"/base/"$SRC_DBOID"/"$SRC_OID" data/base/"$DST_DBOID"/"$DST_OID"
+	if [ "$?" -ne 0 ]
+	then	echo "Move of database $DB, OID $SRC_OID, object $OBJ
+to $DST_OID failed.;  exiting" 1>&2
+		exit 1
+	fi
+
+	# handle table extents
+
+	ls "$OLDDIR"/base/"$SRC_DBOID"/"$SRC_OID".* 2>/dev/null | while read FILE
+	do
+		EXT=`basename "$FILE" | sed 's/^.*\.\(.*\)$/\1/'`
+		mv -f "$FILE" data/base/"$DST_DBOID"/"$DST_OID"."$EXT"
+		if [ "$?" -ne 0 ]
+		then	echo "Move of database $DB, OID $SRC_OID, object $OBJ
+to $DST_OID failed.;  exiting" 1>&2
+			exit 1
+		fi
+	done
+}
+
+if [ "$PHASE" -eq 1 ]
+then
+
+	##########################
+	#  Phase 1 starts here   #
+	##########################
+
+
+	if [ ! -d data/base/1 ]
+	then	echo "There is no database template1 in data/base." 1>&2
+		exit 1
+	fi
+
+	# get version
+	SRC_VERSION="`cat data/PG_VERSION`"
+	if [ "$SRC_VERSION" = "" ]
+	then	echo "$0 can not find PostgreSQL version file 'data/PG_VERSION'.
+$0 aborted." 1>&2
+		exit 1
+	fi
+
+	if [ 	"$SRC_VERSION" != "$CUR_VERSION" -a \
+		"$SRC_VERSION" != "$UPGRADE_VERSION" ]
+	then	echo 
+	"$0 supports versions $UPGRADE_VERSION and $CUR_VERSION only." 1>&2
+		echo 
+	"However, your database is version $SRC_VERSION;
+$0 aborted." 1>&2
+		echo "You will need to dump and restore using pg_dumpall." 1>&2
+		exit 1
+	fi
+
+	# Start server, if needed, so we can do some work.
+	if ! pg_ctl status | head -1 | grep -q "is running"
+	then	pg_ctl -w start
+		if [ $? -ne 0 ]
+		then	echo "Can not start server.
+$0 aborted." 1>&2
+			exit 1
+		fi
+	fi
+
+	# create directory for our data
+	rm -rf "$INFODIR"
+	mkdir "$INFODIR"
+	chmod og-rwx "$INFODIR"
+
+	# Dump schema
+	pg_dumpall -s |
+	awk -F'  *' '
+	{
+		# Modify sequences with int8 maximums if we are upgrading from 7.1.
+		if ("'"$SRC_VERSION"'" == "7.1" &&
+		    $1 == "CREATE" &&
+		    $2 == "SEQUENCE" &&
+		    # handle OS rounding
+		    $9 >= 2147483646 &&
+		    $9 <= 2147483648)
+		{
+			for(i=1; i <= NF; i++)
+			{
+				if (i != 9)
+					printf "%s ", $i;
+				else
+					printf "%s ", "9223372036854775807";
+			}
+			print "";
+		}
+		else    print $0;
+	}' > "$INFODIR"/schema
+	if [ $? -ne 0 ]
+	then	echo "Can not dump schema.
+$0 aborted." 1>&2
+		exit 1
+	fi
+	
+	# Generate mappings for database
+	make_dboidmap > "$INFODIR"/dboidmap
+	make_dbobjoidmap > "$INFODIR"/dbobjoidmap
+
+	# Generate setval() script for 7.1 because it has int4 sequences
+	# Sequence XIDs changed from 7.2beta4 to beta5;  we have to recreate them.
+	if [ "$SRC_VERSION" = "7.1" -o \
+	     "$SRC_VERSION" = "7.2" ]
+	then
+		psql -d template1 -At -c "SELECT datname FROM pg_database" |
+		grep -v '^template0$' |
+		while read DB
+		do
+			# We use awk as a portable way to output a backslash
+			awk 'BEGIN {print "\\connect '"$DB"'"}'
+			psql -d "$DB" -At -c "
+				SELECT 	relname
+				FROM 	pg_class
+				WHERE 	relkind = 'S';" |
+			while read SEQUENCE
+			do
+				VALUE=`psql -d "$DB" -At -c "SELECT last_value 
+								 FROM \"$SEQUENCE\";"`
+				echo "SELECT setval ('$SEQUENCE', $VALUE, true);"
+			done
+		done > "$INFODIR"/setval
+	else	rm -f "$INFODIR"/setval
+	fi
+
+	# Vacuum all databases to remove exipired rows.
+	# We will lose our transaction log file during the upgrade so we
+	# have to do this.
+
+	vacuumdb -a
+	if [ $? -ne 0 ]
+	then	echo "Can not vacuum server.
+$0 aborted." 1>&2
+		exit 1
+	fi
+
+	# Stop server so we can move the directory.
+	pg_ctl -w stop
+	if [ $? -ne 0 ]
+	then	echo "Can not stop server.
+$0 aborted." 1>&2
+		exit 1
+	fi
+
+	mv data "$INFODIR"
+	if [ $? -ne 0 ]
+	then	echo "Can not move old /data out of the way.
+$0 aborted." 1>&2
+		exit 1
+	fi
+	echo
+	echo
+	echo "Plase 1 completed.
+Continue with the steps outlined in the $0 manual page."
+	exit 0
+fi
+
+
+	##########################
+	#  Phase 2 starts here   #
+	##########################
+
+# check things
+
+if [ ! -d "$INFODIR" ]
+then	echo "There is no '$INFODIR' directory from a phase 1 run of $0." 1>&2
+	exit 1	
+fi
+
+if [ ! -e "$OLDDIR" ]
+then	echo "There is no '$OLDDIR' directory from the phase 1 run of $0." 1>&2
+	exit 1	
+fi
+
+if [ ! -f "$OLDDIR/PG_VERSION" ]
+then	echo "Cannot read '$OLDDIR/PG_VERSION' --- something is wrong." 1>&2
+	exit 1
+fi
+
+if [ ! -f "data/PG_VERSION" ]
+then	echo "Cannot read 'data/PG_VERSION' --- something is wrong." 1>&2
+	exit 1
+fi
+
+if [ ! -d "data/base/1" ]
+then	echo "Cannot find database template1 in 'data/base'." 1>&2
+	echo "Are you running $0 as the postgres superuser?" 1>&2
+	exit 1
+fi
+
+# Get the actual versions seen in the data dirs.
+
+SRC_VERSION=`cat "$OLDDIR"/PG_VERSION`
+DST_VERSION=`cat data/PG_VERSION`
+
+# Check for version compatibility.
+# This code will need to be updated/reviewed for each new PostgreSQL release.
+
+if [ "$DST_VERSION" != "$CUR_VERSION" ]
+then	echo "$0 is for PostgreSQL version $CUR_VERSION
+but data/PG_VERSION contains $DST_VERSION." 1>&2
+	echo "Did you run initdb for version $UPGRADE_VERSION by mistake?" 1>&2
+	exit 1
+fi
+
+# Stop server for pg_resetxlog use
+
+if pg_ctl status | head -1 | grep -q "is running"
+then	pg_ctl -w stop
+	if [ $? -ne 0 ]
+	then	echo "Can no start server.
+$0 aborted." 1>&2
+		exit 1
+	fi
+fi
+
+# check for proper pg_resetxlog version
+
+pg_resetxlog 2>/dev/null
+# file not found status is normally 127, not 1
+if [ "$?" -ne 1 ]
+then	echo "Unable to find pg_resetxlog in path.
+Install it from pgsql/contrib/pg_resetxlog and continue.;  exiting" 1>&2
+	exit 1
+fi
+
+if ! pg_resetxlog -x 2>&1 | grep -q 'xid'
+then	echo "Old version of pg_resetxlog found in path.
+Install a newer version from pgsql/contrib/pg_resetxlog.;  exiting" 1>&2
+	exit 1
+fi
+
+# If the XID is > 2 billion, 7.1 database will have non-frozen XID's in 
+# low numbers, and 7.2 will think they are in the future --- bad.
+
+SRC_XID=`pg_resetxlog -n "$OLDDIR" | grep "NextXID" | awk -F'  *' '{print $4}'`
+if [ "$SRC_VERSION" = "7.1" -a "$SRC_XID" -gt 2000000000 ]
+then	echo "XID too high for $0.;  exiting" 1>&2
+	exit 1
+fi
+DST_XID=`pg_resetxlog -n data | grep "NextXID" | awk -F'  *' '{print $4}'`
+
+# compare locales to make sure they match
+
+pg_resetxlog -n "$OLDDIR" | grep "^LC_" > /tmp/$$.0
+pg_resetxlog -n data | grep "^LC_" > /tmp/$$.1
+if ! diff /tmp/$$.0 /tmp/$$.1 >/dev/null
+then	echo "Locales do not match between the two versions.;  exiting" 1>&2
+	exit 1
+fi
+
+# Restart postmaster
+
+pg_ctl -w start
+if [ $? -ne 0 ]
+then	echo "Can not start server.
+$0 aborted." 1>&2
+	exit 1
+fi
+
+
+###################################
+# Checking done.  Ready to proceed.
+###################################
+
+
+# Execute the schema script to create everything
+
+psql template1 < "$INFODIR"/schema
+if [ $? -ne 0 ]
+then	echo "There were errors in the input script.
+$0 aborted." 1>&2
+	exit 1
+fi
+
+echo "Input script complete, fixing row commit statuses..."
+
+# XXX do we still need this?
+# Now vacuum each result database because our movement of transaction log
+# causes some committed transactions to appear as non-committed
+
+vacuumdb -a
+if [ $? -ne 0 ]
+then	echo "There were errors during VACUUM.
+$0 aborted." 1>&2
+	exit 1
+fi
+
+# Generate mappings for database
+make_dboidmap > /tmp/$$.dboidmap
+make_dbobjoidmap > /tmp/$$.dbobjoidmap
+
+# we are done with SQL database access
+# shutdown forces buffers to disk
+
+pg_ctl -w stop
+if [ "$?" -ne 0 ]
+then	echo "Unable to stop database server.;  exiting" 1>&2
+	exit 1
+fi
+
+echo "Commit fixes complete, moving data files..."
+
+# Move table/index/sequence files
+
+cat "$INFODIR"/dbobjoidmap | while read LINE
+do
+	DB=`echo "$LINE" | awk '{print $1}'`
+	OBJ=`echo "$LINE" | awk '{print $2}'`
+
+	# Skip system tables, except for pg_largeobject
+	# pg_toast tables are handled later as part of the
+	# base table move
+	if [ `expr "$OBJ" : 'pg_'` -eq 3 -a \
+	     `expr "$OBJ" : 'pg_largeobject'` -ne 14 ]
+	then	continue
+	fi
+
+	SRC_OID=`echo "$LINE" | awk '{print $3}'`
+	SRC_DBOID=`grep "^$DB	" "$INFODIR"/dboidmap | awk '{print $2}'`
+	DST_DBOID=`grep "^$DB	" /tmp/$$.dboidmap | awk '{print $2}'`
+	DST_OID=`grep "^$DB	$OBJ	" /tmp/$$.dbobjoidmap | awk '{print $3}'`
+
+	move_objfiles
+
+	# Handle TOAST files if they exist
+	if grep -q "^$DB	pg_toast_$SRC_OID	" "$INFODIR"/dbobjoidmap
+	then	# toast heap
+		SAVE_SRC_OID="$SRC_OID"
+		SAVE_DST_OID="$DST_OID"
+		SRC_OID=`grep "^$DB	pg_toast_$SAVE_SRC_OID	" \
+			"$INFODIR"/dbobjoidmap | awk '{print $3}'`
+		DST_OID=`grep "^$DB	pg_toast_$SAVE_DST_OID	" \
+			/tmp/$$.dbobjoidmap | awk '{print $3}'`
+		move_objfiles
+		# toast index
+		SRC_OID=`grep "^$DB	pg_toast_${SAVE_SRC_OID}_idx	" \
+			"$INFODIR"/dbobjoidmap | awk '{print $3}'`
+		DST_OID=`grep "^$DB	pg_toast_${SAVE_DST_OID}_idx	" \
+			/tmp/$$.dbobjoidmap | awk '{print $3}'`
+		move_objfiles
+	fi
+done
+								   
+
+# Set this so future backends don't think these tuples are their own
+# because it matches their own XID.
+# Commit status already updated by vacuum above
+# Set to maximum XID just in case SRC wrapped around recently and
+# is lower than DST's database
+
+if [ "$SRC_XID" -gt "$DST_XID" ]
+then	MAX_XID="$SRC_XID"
+else	MAX_XID="$DST_XID"
+fi
+
+pg_resetxlog -x "$MAX_XID" data
+if [ "$?" -ne 0 ]
+then	echo "Unable to set new XID.;  exiting" 1>&2
+	exit 1
+fi
+
+# Move over old WAL
+
+rm -r data/pg_xlog
+mv -f "$OLDDIR"/pg_xlog data
+
+# Set last log file id and segment from old database
+
+LOG_ID=`pg_resetxlog -n "$OLDDIR" | grep "Current log file id:" |
+	awk -F'  *' '{print $5}'`
+if [ "$LOG_ID" = "" ]
+then	echo "Unable to get old log file id.;  exiting" 1>&2
+	exit 1
+fi
+SEG_ID=`pg_resetxlog -n "$OLDDIR" | grep "Next log file segment:" |
+	awk -F'  *' '{print $5}'`
+if [ "$SEG_ID" = "" ]
+then	echo "Unable to get old log segment id.;  exiting" 1>&2
+	exit 1
+fi
+
+# Set checkpoint location of new database
+
+pg_resetxlog -l "$LOG_ID" "$SEG_ID" data
+if [ "$?" -ne 0 ]
+then	echo "Unable to set new log file/segment id.;  exiting" 1>&2
+	exit 1
+fi
+
+# Restart server with moved data
+
+pg_ctl -w start
+if [ "$?" -ne 0 ]
+then	echo "Unable to restart database server.;  exiting" 1>&2
+	exit 1
+fi
+
+# Set sequence values for 7.1-version sequences, which were int4.
+
+if [ "$SRC_VERSION" = "7.1" -o \
+     "$SRC_VERSION" = "7.2" ]
+then	echo "Set sequence values..."
+	psql -d template1 -At < "$INFODIR"/setval
+	if [ $? -ne 0 ]
+	then	echo "There were errors during int4 sequence restore.
+$0 aborted." 1>&2
+		exit 1
+	fi
+fi
+
+echo
+echo
+echo "You may remove the old database files with 'rm -r pg_upgrade'."
+
+exit 0

doc/src/sgml/ref/pg_upgrade.sgml

-<!--
-$Header: /cvsroot/pgsql/doc/src/sgml/ref/Attic/pg_upgrade.sgml,v 1.20 2002/01/13 01:13:59 momjian Exp $
-PostgreSQL documentation
--->
-
-<refentry id="APP-PG-UPGRADE">
- <refmeta>
-  <refentrytitle id="APP-PG-UPGRADE-TITLE">
-   <application>pg_upgrade</application>
-  </refentrytitle>
-  <manvolnum>1</manvolnum>
-  <refmiscinfo>Application</refmiscinfo>
- </refmeta>
- <refnamediv>
-  <refname>
-   <application>pg_upgrade</application>
-  </refname>
-  <refpurpose>
-   Allows upgrade from a previous release without reloading data
-  </refpurpose>
- </refnamediv>
- <refsynopsisdiv>
-  <refsynopsisdivinfo>
-   <date>1999-07-31</date>
-  </refsynopsisdivinfo>
-  <cmdsynopsis>
-   <command>pg_upgrade</command>
-    <group choice="plain"><arg>-1</arg><arg>-2</arg></group>
-  </cmdsynopsis>
- </refsynopsisdiv>
-
- <refsect1 id="R1-APP-PG-UPGRADE-1">
-  <refsect1info>
-   <date>1999-07-31</date>
-  </refsect1info>
-  <title>
-   Description
-  </title>
-
-  <para>
-   <application>pg_upgrade</application>
-   is a utility for upgrading from a previous
-   <productname>PostgreSQL</productname> release without reloading all the data.
-   Not all <productname>PostgreSQL</productname> release transitions can be
-   handled this way.  Check the release notes for details on your installation.
-  </para>
-
-  <procedure>
-   <title>Upgrading <productname>PostgreSQL</productname> with pg_upgrade</title>
-
-   <step performance="required">
-    <para>
-     Back up your existing data directory, preferably by making a
-     complete dump with pg_dumpall.
-    </para>
-   </step>
-
-   <step performance="required">
-    <para>
-     Copy the program <filename>pgsql/src/bin/pg_dump/pg_upgrade</filename>
-     from the current PostgreSQL distribution into somewhere in your path.
-    </para>
-   </step>
-
-   <step performance="required">
-    <para>
-     Change your working directory to the 
-     pgsql main directory, and type:
-     <programlisting>
-$ pg_upgrade -1
-     </programlisting>
-     to collect information about the old database needed for the
-     upgrade.
-    </para>
-   </step>
-
-   <step performance="required">
-    <para>
-     Do:
-     <programlisting>
-$ make install
-     </programlisting>
-     to install the new binaries.
-    </para>
-   </step>
-
-   <step performance="required">
-    <para>
-     Do:
-     <programlisting>
-$ cd pgsql/contrib/pg_resetxlog
-$ make install
-     </programlisting>
-     to install the /contrib/pg_resetxlog utility 
-     which will be used by pg_upgrade.
-    </para>
-   </step>
-
-   <step performance="required">
-    <para>
-     Run <application>initdb</application> to create a new template1 database
-     containing the system tables for the new release.  Make sure you use
-     settings similar to those used in your previous version.
-    </para>
-   </step>
-
-   <step performance="required">
-    <para>
-     Start the new postmaster.  (Note: it is critical that no users connect
-     to the server until the upgrade is complete.  You may wish to start
-     the postmaster without -i and/or alter <filename>pg_hba.conf</filename>
-     temporarily.)
-    </para>
-   </step>
-
-   <step performance="required">
-    <para>
-     Change your working directory to the 
-     pgsql main directory, and type:
-     <programlisting>
-$ pg_upgrade -2
-     </programlisting>
-     The program will do some checking to make sure everything is properly
-     configured, and will run your db.out script to recreate all the databases
-     and tables you had, but with no data.  It will then physically move the
-     data files containing non-system tables and indexes from
-     <filename>data.old/</filename> into the proper
-     <filename>data/</filename> subdirectories, replacing the empty data files
-     created during the db.out script.
-    </para>
-   </step>
-
-   <step performance="required">
-    <para>
-     Restore your old <filename>pg_hba.conf</filename> if needed to allow
-     user logins.
-    </para>
-   </step>
-
-   <step performance="required">
-    <para>
-     <emphasis>Carefully</emphasis> examine the contents of the upgraded
-     databases.  If you detect problems, you'll need to recover by restoring
-     from your full pg_dump backup.
-     You can delete the <filename>data.old/</filename> directory when you
-     are satisfied.
-    </para>
-   </step>
-
-   <step performance="required">
-    <para>
-     The upgraded databases will be in an un-vacuumed state.  You will probably
-     want to run a <command>VACUUM ANALYZE</command> before beginning
-     production work.
-    </para>
-   </step>
-  </procedure>
- </refsect1>
-</refentry>
-
-<!-- Keep this comment at the end of the file
-Local variables:
-mode: sgml
-sgml-omittag:nil
-sgml-shorttag:t
-sgml-minimize-attributes:nil
-sgml-always-quote-attributes:t
-sgml-indent-step:1
-sgml-indent-data:t
-sgml-parent-document:nil
-sgml-default-dtd-file:"../reference.ced"
-sgml-exposed-tags:nil
-sgml-local-catalogs:"/usr/lib/sgml/catalog"
-sgml-local-ecat-files:nil
-End:
--->