Rework PostgresNode's psql method

This makes the psql() method much more capable: it captures both stdout
and stderr; it now returns the psql exit code rather than stdout; a
timeout can now be specified, as can ON_ERROR_STOP behavior; it gained a
new "on_error_die" (defaulting to off) parameter to raise an exception
if there's any problem.  Finally, additional parameters to psql can be
passed if there's need for further tweaking.

For convenience, a new safe_psql() method retains much of the old
behavior of psql(), except that it uses on_error_die on, so that
problems like syntax errors in SQL commands can be detected more easily.

Many existing TAP test files now use safe_psql, which is what is really
wanted.  A couple of ->psql() calls are now added in the commit_ts
tests, which verify that the right thing is happening on certain errors.
Some ->command_fails() calls in recovery tests that were verifying that
psql failed also became ->psql() calls now.

Author: Craig Ringer. Some tweaks by Álvaro Herrera
Reviewed-By: Michaël Paquier

src/bin/pg_basebackup/t/010_pg_basebackup.pl

@@ -112,9 +112,9 @@ SKIP:
 	symlink "$tempdir", $shorter_tempdir;
 
 	mkdir "$tempdir/tblspc1";
-	$node->psql('postgres',
+	$node->safe_psql('postgres',
 		"CREATE TABLESPACE tblspc1 LOCATION '$shorter_tempdir/tblspc1';");
-	$node->psql('postgres', "CREATE TABLE test1 (a int) TABLESPACE tblspc1;");
+	$node->safe_psql('postgres', "CREATE TABLE test1 (a int) TABLESPACE tblspc1;");
 	$node->command_ok([ 'pg_basebackup', '-D', "$tempdir/tarbackup2", '-Ft' ],
 		'tar format with tablespaces');
 	ok(-f "$tempdir/tarbackup2/base.tar", 'backup tar was created');
@@ -140,9 +140,9 @@ SKIP:
 	closedir $dh;
 
 	mkdir "$tempdir/tbl=spc2";
-	$node->psql('postgres', "DROP TABLE test1;");
-	$node->psql('postgres', "DROP TABLESPACE tblspc1;");
-	$node->psql('postgres',
+	$node->safe_psql('postgres', "DROP TABLE test1;");
+	$node->safe_psql('postgres', "DROP TABLESPACE tblspc1;");
+	$node->safe_psql('postgres',
 		"CREATE TABLESPACE tblspc2 LOCATION '$shorter_tempdir/tbl=spc2';");
 	$node->command_ok(
 		[   'pg_basebackup', '-D', "$tempdir/backup3", '-Fp',
@@ -150,15 +150,15 @@ SKIP:
 		'mapping tablespace with = sign in path');
 	ok(-d "$tempdir/tbackup/tbl=spc2",
 		'tablespace with = sign was relocated');
-	$node->psql('postgres', "DROP TABLESPACE tblspc2;");
+	$node->safe_psql('postgres', "DROP TABLESPACE tblspc2;");
 
 	mkdir "$tempdir/$superlongname";
-	$node->psql('postgres',
+	$node->safe_psql('postgres',
 		"CREATE TABLESPACE tblspc3 LOCATION '$tempdir/$superlongname';");
 	$node->command_ok(
 		[ 'pg_basebackup', '-D', "$tempdir/tarbackup_l3", '-Ft' ],
 		'pg_basebackup tar with long symlink target');
-	$node->psql('postgres', "DROP TABLESPACE tblspc3;");
+	$node->safe_psql('postgres', "DROP TABLESPACE tblspc3;");
 }
 
 $node->command_ok([ 'pg_basebackup', '-D', "$tempdir/backupR", '-R' ],
@@ -199,9 +199,9 @@ $node->command_fails(
 		'slot1' ],
 	'pg_basebackup fails with nonexistent replication slot');
 
-$node->psql('postgres',
+$node->safe_psql('postgres',
 	q{SELECT * FROM pg_create_physical_replication_slot('slot1')});
-my $lsn = $node->psql('postgres',
+my $lsn = $node->safe_psql('postgres',
 	q{SELECT restart_lsn FROM pg_replication_slots WHERE slot_name = 'slot1'}
 );
 is($lsn, '', 'restart LSN of new slot is null');
@@ -209,7 +209,7 @@ $node->command_ok(
 	[   'pg_basebackup', '-D', "$tempdir/backupxs_sl", '-X',
 		'stream',        '-S', 'slot1' ],
 	'pg_basebackup -X stream with replication slot runs');
-$lsn = $node->psql('postgres',
+$lsn = $node->safe_psql('postgres',
 	q{SELECT restart_lsn FROM pg_replication_slots WHERE slot_name = 'slot1'}
 );
 like($lsn, qr!^0/[0-9A-Z]{7,8}$!, 'restart LSN of slot has advanced');

src/bin/pgbench/t/001_pgbench.pl

@@ -12,7 +12,7 @@ use Test::More tests => 3;
 my $node = get_new_node('main');
 $node->init;
 $node->start;
-$node->psql('postgres',
+$node->safe_psql('postgres',
 	    'CREATE UNLOGGED TABLE oid_tbl () WITH OIDS; '
 	  . 'ALTER TABLE oid_tbl ADD UNIQUE (oid);');
 my $script = $node->basedir . '/pgbench_script';

src/bin/scripts/t/010_clusterdb.pl

@@ -21,7 +21,7 @@ $node->issues_sql_like(
 $node->command_fails([ 'clusterdb', '-t', 'nonexistent' ],
 	'fails with nonexistent table');
 
-$node->psql('postgres',
+$node->safe_psql('postgres',
 'CREATE TABLE test1 (a int); CREATE INDEX test1x ON test1 (a); CLUSTER test1 USING test1x'
 );
 $node->issues_sql_like(

src/bin/scripts/t/030_createlang.pl

@@ -16,7 +16,7 @@ $node->start;
 $node->command_fails([ 'createlang', 'plpgsql' ],
 	'fails if language already exists');
 
-$node->psql('postgres', 'DROP EXTENSION plpgsql');
+$node->safe_psql('postgres', 'DROP EXTENSION plpgsql');
 $node->issues_sql_like(
 	[ 'createlang', 'plpgsql' ],
 	qr/statement: CREATE EXTENSION "plpgsql"/,

src/bin/scripts/t/050_dropdb.pl

@@ -13,7 +13,7 @@ my $node = get_new_node('main');
 $node->init;
 $node->start;
 
-$node->psql('postgres', 'CREATE DATABASE foobar1');
+$node->safe_psql('postgres', 'CREATE DATABASE foobar1');
 $node->issues_sql_like(
 	[ 'dropdb', 'foobar1' ],
 	qr/statement: DROP DATABASE foobar1/,

src/bin/scripts/t/070_dropuser.pl

@@ -13,7 +13,7 @@ my $node = get_new_node('main');
 $node->init;
 $node->start;
 
-$node->psql('postgres', 'CREATE ROLE foobar1');
+$node->safe_psql('postgres', 'CREATE ROLE foobar1');
 $node->issues_sql_like(
 	[ 'dropuser', 'foobar1' ],
 	qr/statement: DROP ROLE foobar1/,

src/bin/scripts/t/090_reindexdb.pl

@@ -20,7 +20,7 @@ $node->issues_sql_like(
 	qr/statement: REINDEX DATABASE postgres;/,
 	'SQL REINDEX run');
 
-$node->psql('postgres',
+$node->safe_psql('postgres',
 	'CREATE TABLE test1 (a int); CREATE INDEX test1x ON test1 (a);');
 $node->issues_sql_like(
 	[ 'reindexdb', '-t', 'test1', 'postgres' ],

src/test/modules/commit_ts/t/001_base.pl

@@ -13,17 +13,17 @@ $node->append_conf('postgresql.conf', 'track_commit_timestamp = on');
 $node->start;
 
 # Create a table, compare "now()" to the commit TS of its xmin
-$node->psql('postgres', 'create table t as select now from (select now(), pg_sleep(1)) f');
-my $true = $node->psql('postgres',
+$node->safe_psql('postgres', 'create table t as select now from (select now(), pg_sleep(1)) f');
+my $true = $node->safe_psql('postgres',
 	'select t.now - ts.* < \'1s\' from t, pg_class c, pg_xact_commit_timestamp(c.xmin) ts where relname = \'t\'');
 is($true, 't', 'commit TS is set');
-my $ts = $node->psql('postgres',
+my $ts = $node->safe_psql('postgres',
 	'select ts.* from pg_class, pg_xact_commit_timestamp(xmin) ts where relname = \'t\'');
 
 # Verify that we read the same TS after crash recovery
 $node->stop('immediate');
 $node->start;
 
-my $recovered_ts = $node->psql('postgres',
+my $recovered_ts = $node->safe_psql('postgres',
 	'select ts.* from pg_class, pg_xact_commit_timestamp(xmin) ts where relname = \'t\'');
 is($recovered_ts, $ts, 'commit TS remains after crash recovery');

src/test/modules/commit_ts/t/002_standby.pl

@@ -4,7 +4,7 @@ use strict;
 use warnings;
 
 use TestLib;
-use Test::More tests => 2;
+use Test::More tests => 4;
 use PostgresNode;
 
 my $bkplabel = 'backup';
@@ -25,31 +25,33 @@ $standby->start;
 
 for my $i (1 .. 10)
 {
-	$master->psql('postgres', "create table t$i()");
+	$master->safe_psql('postgres', "create table t$i()");
 }
-my $master_ts = $master->psql('postgres',
+my $master_ts = $master->safe_psql('postgres',
 	qq{SELECT ts.* FROM pg_class, pg_xact_commit_timestamp(xmin) AS ts WHERE relname = 't10'});
-my $master_lsn = $master->psql('postgres',
+my $master_lsn = $master->safe_psql('postgres',
 	'select pg_current_xlog_location()');
 $standby->poll_query_until('postgres',
 	qq{SELECT '$master_lsn'::pg_lsn <= pg_last_xlog_replay_location()})
 	or die "slave never caught up";
 
-my $standby_ts = $standby->psql('postgres',
+my $standby_ts = $standby->safe_psql('postgres',
 	qq{select ts.* from pg_class, pg_xact_commit_timestamp(xmin) ts where relname = 't10'});
 is($master_ts, $standby_ts, "standby gives same value as master");
 
 $master->append_conf('postgresql.conf', 'track_commit_timestamp = off');
 $master->restart;
-$master->psql('postgres', 'checkpoint');
-$master_lsn = $master->psql('postgres',
+$master->safe_psql('postgres', 'checkpoint');
+$master_lsn = $master->safe_psql('postgres',
 	'select pg_current_xlog_location()');
 $standby->poll_query_until('postgres',
 	qq{SELECT '$master_lsn'::pg_lsn <= pg_last_xlog_replay_location()})
 	or die "slave never caught up";
-$standby->psql('postgres', 'checkpoint');
+$standby->safe_psql('postgres', 'checkpoint');
 
 # This one should raise an error now
-$standby_ts = $standby->psql('postgres',
+my ($ret, $standby_ts_stdout, $standby_ts_stderr) = $standby->psql('postgres',
 	'select ts.* from pg_class, pg_xact_commit_timestamp(xmin) ts where relname = \'t10\'');
-is($standby_ts, '', "standby gives no value when master turned feature off");
+is($ret, 3, 'standby errors when master turned feature off');
+is($standby_ts_stdout, '', "standby gives no value when master turned feature off");
+like($standby_ts_stderr, qr/could not get commit timestamp data/, 'expected error when master turned feature off');

src/test/modules/commit_ts/t/003_standby_2.pl

@@ -4,7 +4,7 @@ use strict;
 use warnings;
 
 use TestLib;
-use Test::More tests => 2;
+use Test::More tests => 4;
 use PostgresNode;
 
 my $bkplabel = 'backup';
@@ -24,23 +24,25 @@ $standby->start;
 
 for my $i (1 .. 10)
 {
-	$master->psql('postgres', "create table t$i()");
+	$master->safe_psql('postgres', "create table t$i()");
 }
 $master->append_conf('postgresql.conf', 'track_commit_timestamp = off');
 $master->restart;
-$master->psql('postgres', 'checkpoint');
-my $master_lsn = $master->psql('postgres',
+$master->safe_psql('postgres', 'checkpoint');
+my $master_lsn = $master->safe_psql('postgres',
 	'select pg_current_xlog_location()');
 $standby->poll_query_until('postgres',
 	qq{SELECT '$master_lsn'::pg_lsn <= pg_last_xlog_replay_location()})
 	or die "slave never caught up";
 
-$standby->psql('postgres', 'checkpoint');
+$standby->safe_psql('postgres', 'checkpoint');
 $standby->restart;
 
-my $standby_ts = $standby->psql('postgres',
+my ($psql_ret, $standby_ts_stdout, $standby_ts_stderr) = $standby->psql('postgres',
 	qq{SELECT ts.* FROM pg_class, pg_xact_commit_timestamp(xmin) AS ts WHERE relname = 't10'});
-is($standby_ts, '', "standby does not return a value after restart");
+is($psql_ret, 3, 'expect error when getting commit timestamp after restart');
+is($standby_ts_stdout, '', "standby does not return a value after restart");
+like($standby_ts_stderr, qr/could not get commit timestamp data/, 'expected err msg after restart');
 
 $master->append_conf('postgresql.conf', 'track_commit_timestamp = on');
 $master->restart;
@@ -50,7 +52,7 @@ $master->restart;
 system_or_bail('pg_ctl', '-w', '-D', $standby->data_dir, 'promote');
 $standby->poll_query_until('postgres', "SELECT pg_is_in_recovery() <> true");
 
-$standby->psql('postgres', "create table t11()");
-$standby_ts = $standby->psql('postgres',
+$standby->safe_psql('postgres', "create table t11()");
+my $standby_ts = $standby->safe_psql('postgres',
 	qq{SELECT ts.* FROM pg_class, pg_xact_commit_timestamp(xmin) AS ts WHERE relname = 't11'});
 isnt($standby_ts, '', "standby gives valid value ($standby_ts) after promotion");

src/test/perl/PostgresNode.pm

@@ -21,9 +21,24 @@ PostgresNode - class representing PostgreSQL server instance
   $node->append_conf('postgresql.conf', 'hot_standby = on');
   $node->restart('fast');
 
-  # run a query with psql
-  # like: psql -qAXt postgres -c 'SELECT 1;'
-  $psql_stdout = $node->psql('postgres', 'SELECT 1');
+  # run a query with psql, like:
+  #   echo 'SELECT 1' | psql -qAXt postgres -v ON_ERROR_STOP=1
+  $psql_stdout = $node->safe_psql('postgres', 'SELECT 1');
+
+  # Run psql with a timeout, capturing stdout and stderr
+  # as well as the psql exit code. Pass some extra psql
+  # options. If there's an error from psql raise an exception.
+  my ($stdout, $stderr, $timed_out);
+  my $cmdret = $node->psql('postgres', 'SELECT pg_sleep(60)',
+	  stdout => \$stdout, stderr => \$stderr,
+	  timeout => 30, timed_out => \$timed_out,
+	  extra_params => ['--single-transaction'],
+	  on_error_die => 1)
+  print "Sleep timed out" if $timed_out;
+
+  # Similar thing, more convenient in common cases
+  my ($cmdret, $stdout, $stderr) =
+      $node->psql('postgres', 'SELECT 1');
 
   # run query every second until it returns 't'
   # or times out
@@ -70,6 +85,7 @@ use IPC::Run;
 use RecursiveCopy;
 use Test::More;
 use TestLib ();
+use Scalar::Util qw(blessed);
 
 our @EXPORT = qw(
   get_new_node
@@ -780,41 +796,255 @@ sub teardown_node
 
 =pod
 
-=item $node->psql(dbname, sql)
+=item $node->safe_psql($dbname, $sql) => stdout
 
-Run a query with psql and return stdout, or on error print stderr.
+Invoke B<psql> to run B<sql> on B<dbname> and return its stdout on success.
+Die if the SQL produces an error. Runs with B<ON_ERROR_STOP> set.
 
-Executes a query/script with psql and returns psql's standard output.  psql is
-run in unaligned tuples-only quiet mode with psqlrc disabled so simple queries
-will just return the result row(s) with fields separated by commas.
+Takes optional extra params like timeout and timed_out parameters with the same
+options as psql.
 
 =cut
 
-sub psql
+sub safe_psql
 {
-	my ($self, $dbname, $sql) = @_;
+	my ($self, $dbname, $sql, %params) = @_;
 
 	my ($stdout, $stderr);
-	my $name = $self->name;
-	print("### Running SQL command on node \"$name\": $sql\n");
 
-	IPC::Run::run [ 'psql', '-XAtq', '-d', $self->connstr($dbname), '-f',
-		'-' ], '<', \$sql, '>', \$stdout, '2>', \$stderr
-	  or die;
+	my $ret = $self->psql(
+		$dbname, $sql,
+		%params,
+		stdout        => \$stdout,
+		stderr        => \$stderr,
+		on_error_die  => 1,
+		on_error_stop => 1);
 
+	# psql can emit stderr from NOTICEs etc
 	if ($stderr ne "")
 	{
 		print "#### Begin standard error\n";
 		print $stderr;
-		print "#### End standard error\n";
+		print "\n#### End standard error\n";
 	}
-	chomp $stdout;
-	$stdout =~ s/\r//g if $Config{osname} eq 'msys';
+
 	return $stdout;
 }
 
 =pod
 
+=item $node->psql($dbname, $sql, %params) => psql_retval
+
+Invoke B<psql> to execute B<$sql> on B<$dbname> and return the return value
+from B<psql>, which is run with on_error_stop by default so that it will
+stop running sql and return 3 if the passed SQL results in an error.
+
+As a convenience, if B<psql> is called in array context it returns an
+array containing ($retval, $stdout, $stderr).
+
+psql is invoked in tuples-only unaligned mode with reading of B<.psqlrc>
+disabled.  That may be overridden by passing extra psql parameters.
+
+stdout and stderr are transformed to UNIX line endings if on Windows. Any
+trailing newline is removed.
+
+Dies on failure to invoke psql but not if psql exits with a nonzero
+return code (unless on_error_die specified).
+
+If psql exits because of a signal, an exception is raised.
+
+=over
+
+=item stdout => \$stdout
+
+B<stdout>, if given, must be a scalar reference to which standard output is
+written.  If not given, standard output is not redirected and will be printed
+unless B<psql> is called in array context, in which case it's captured and
+returned.
+
+=item stderr => \$stderr
+
+Same as B<stdout> but gets standard error. If the same scalar is passed for
+both B<stdout> and B<stderr> the results may be interleaved unpredictably.
+
+=item on_error_stop => 1
+
+By default, the B<psql> method invokes the B<psql> program with ON_ERROR_STOP=1
+set, so SQL execution is stopped at the first error and exit code 2 is
+returned.  Set B<on_error_stop> to 0 to ignore errors instead.
+
+=item on_error_die => 0
+
+By default, this method returns psql's result code. Pass on_error_die to
+instead die with an informative message.
+
+=item timeout => 'interval'
+
+Set a timeout for the psql call as an interval accepted by B<IPC::Run::timer>
+(integer seconds is fine).  This method raises an exception on timeout, unless
+the B<timed_out> parameter is also given.
+
+=item timed_out => \$timed_out
+
+If B<timeout> is set and this parameter is given, the scalar it references
+is set to true if the psql call times out.
+
+=item extra_params => ['--single-transaction']
+
+If given, it must be an array reference containing additional parameters to B<psql>.
+
+=back
+
+e.g.
+
+	my ($stdout, $stderr, $timed_out);
+	my $cmdret = $node->psql('postgres', 'SELECT pg_sleep(60)',
+		stdout => \$stdout, stderr => \$stderr,
+		timeout => 30, timed_out => \$timed_out,
+		extra_params => ['--single-transaction'])
+
+will set $cmdret to undef and $timed_out to a true value.
+
+	$node->psql('postgres', $sql, on_error_die => 1);
+
+dies with an informative message if $sql fails.
+
+=cut
+
+sub psql
+{
+	my ($self, $dbname, $sql, %params) = @_;
+
+	my $stdout            = $params{stdout};
+	my $stderr            = $params{stderr};
+	my $timeout           = undef;
+	my $timeout_exception = 'psql timed out';
+	my @psql_params =
+	  ('psql', '-XAtq', '-d', $self->connstr($dbname), '-f', '-');
+
+	# If the caller wants an array and hasn't passed stdout/stderr
+	# references, allocate temporary ones to capture them so we
+	# can return them. Otherwise we won't redirect them at all.
+	if (wantarray)
+	{
+		if (!defined($stdout))
+		{
+			my $temp_stdout = "";
+			$stdout = \$temp_stdout;
+		}
+		if (!defined($stderr))
+		{
+			my $temp_stderr = "";
+			$stderr = \$temp_stderr;
+		}
+	}
+
+	$params{on_error_stop} = 1 unless defined $params{on_error_stop};
+	$params{on_error_die}  = 0 unless defined $params{on_error_die};
+
+	push @psql_params, '-v', 'ON_ERROR_STOP=1' if $params{on_error_stop};
+	push @psql_params, @{ $params{extra_params} }
+	  if defined $params{extra_params};
+
+	$timeout =
+	  IPC::Run::timeout($params{timeout}, exception => $timeout_exception)
+	  if (defined($params{timeout}));
+
+	# IPC::Run would otherwise append to existing contents:
+	$$stdout = "" if ref($stdout);
+	$$stderr = "" if ref($stderr);
+
+	my $ret;
+
+   # Run psql and capture any possible exceptions.  If the exception is
+   # because of a timeout and the caller requested to handle that, just return
+   # and set the flag.  Otherwise, and for any other exception, rethrow.
+   #
+   # For background, see
+   # http://search.cpan.org/~ether/Try-Tiny-0.24/lib/Try/Tiny.pm
+	do
+	{
+		local $@;
+		eval {
+			my @ipcrun_opts = (\@psql_params, '<', \$sql);
+			push @ipcrun_opts, '>',  $stdout if defined $stdout;
+			push @ipcrun_opts, '2>', $stderr if defined $stderr;
+			push @ipcrun_opts, $timeout if defined $timeout;
+
+			IPC::Run::run @ipcrun_opts;
+			$ret = $?;
+		};
+		my $exc_save = $@;
+		if ($exc_save)
+		{
+			# IPC::Run::run threw an exception. re-throw unless it's a
+			# timeout, which we'll handle by testing is_expired
+			die $exc_save
+			  if (blessed($exc_save) || $exc_save ne $timeout_exception);
+
+			$ret = undef;
+
+			die "Got timeout exception '$exc_save' but timer not expired?!"
+			  unless $timeout->is_expired;
+
+			if (defined($params{timed_out}))
+			{
+				${ $params{timed_out} } = 1;
+			}
+			else
+			{
+				die "psql timed out: stderr: '$$stderr'\n"
+				  . "while running '@psql_params'";
+			}
+		}
+	};
+
+	if (defined $$stdout)
+	{
+		chomp $$stdout;
+		$$stdout =~ s/\r//g if $TestLib::windows_os;
+	}
+
+	if (defined $$stderr)
+	{
+		chomp $$stderr;
+		$$stderr =~ s/\r//g if $TestLib::windows_os;
+	}
+
+	# See http://perldoc.perl.org/perlvar.html#%24CHILD_ERROR
+	# We don't use IPC::Run::Simple to limit dependencies.
+	#
+	# We always die on signal.
+	my $core = $ret & 128 ? " (core dumped)" : "";
+	die "psql exited with signal "
+	  . ($ret & 127)
+	  . "$core: '$$stderr' while running '@psql_params'"
+	  if $ret & 127;
+	$ret = $ret >> 8;
+
+	if ($ret && $params{on_error_die})
+	{
+		die "psql error: stderr: '$$stderr'\nwhile running '@psql_params'"
+		  if $ret == 1;
+		die "connection error: '$$stderr'\nwhile running '@psql_params'"
+		  if $ret == 2;
+		die "error running SQL: '$$stderr'\nwhile running '@psql_params'"
+		  if $ret == 3;
+		die "psql returns $ret: '$$stderr'\nwhile running '@psql_params'";
+	}
+
+	if (wantarray)
+	{
+		return ($ret, $$stdout, $$stderr);
+	}
+	else
+	{
+		return $ret;
+	}
+}
+
+=pod
+
 =item $node->poll_query_until(dbname, query)
 
 Run a query once a second, until it returns 't' (i.e. SQL boolean true).
@@ -837,7 +1067,7 @@ sub poll_query_until
 		my $result = IPC::Run::run $cmd, '>', \$stdout, '2>', \$stderr;
 
 		chomp($stdout);
-		$stdout =~ s/\r//g if $Config{osname} eq 'msys';
+		$stdout =~ s/\r//g if $TestLib::windows_os;
 		if ($stdout eq "t")
 		{
 			return 1;

src/test/recovery/t/001_stream_rep.pl

@@ -31,7 +31,7 @@ $node_standby_2->init_from_backup($node_standby_1, $backup_name,
 $node_standby_2->start;
 
 # Create some content on master and check its presence in standby 1
-$node_master->psql('postgres',
+$node_master->safe_psql('postgres',
 	"CREATE TABLE tab_int AS SELECT generate_series(1,1002) AS a");
 
 # Wait for standbys to catch up
@@ -47,24 +47,14 @@ $node_standby_1->poll_query_until('postgres', $caughtup_query)
   or die "Timed out while waiting for standby 2 to catch up";
 
 my $result =
-  $node_standby_1->psql('postgres', "SELECT count(*) FROM tab_int");
+  $node_standby_1->safe_psql('postgres', "SELECT count(*) FROM tab_int");
 print "standby 1: $result\n";
 is($result, qq(1002), 'check streamed content on standby 1');
 
-$result = $node_standby_2->psql('postgres', "SELECT count(*) FROM tab_int");
+$result = $node_standby_2->safe_psql('postgres', "SELECT count(*) FROM tab_int");
 print "standby 2: $result\n";
 is($result, qq(1002), 'check streamed content on standby 2');
 
 # Check that only READ-only queries can run on standbys
-$node_standby_1->command_fails(
-	[   'psql', '-A',
-		'-t',   '--no-psqlrc',
-		'-d',   $node_standby_1->connstr,
-		'-c',   "INSERT INTO tab_int VALUES (1)" ],
-	'Read-only queries on standby 1');
-$node_standby_2->command_fails(
-	[   'psql', '-A',
-		'-t',   '--no-psqlrc',
-		'-d',   $node_standby_2->connstr,
-		'-c',   "INSERT INTO tab_int VALUES (1)" ],
-	'Read-only queries on standby 2');
+is($node_standby_1->psql('postgres', 'INSERT INTO tab_int VALUES (1)'), 3, 'Read-only queries on standby 1');
+is($node_standby_2->psql('postgres', 'INSERT INTO tab_int VALUES (1)'), 3, 'Read-only queries on standby 2');

src/test/recovery/t/002_archiving.pl

@@ -30,16 +30,16 @@ wal_retrieve_retry_interval = '100ms'
 $node_standby->start;
 
 # Create some content on master
-$node_master->psql('postgres',
+$node_master->safe_psql('postgres',
 	"CREATE TABLE tab_int AS SELECT generate_series(1,1000) AS a");
 my $current_lsn =
-  $node_master->psql('postgres', "SELECT pg_current_xlog_location();");
+  $node_master->safe_psql('postgres', "SELECT pg_current_xlog_location();");
 
 # Force archiving of WAL file to make it present on master
-$node_master->psql('postgres', "SELECT pg_switch_xlog()");
+$node_master->safe_psql('postgres', "SELECT pg_switch_xlog()");
 
 # Add some more content, it should not be present on standby
-$node_master->psql('postgres',
+$node_master->safe_psql('postgres',
 	"INSERT INTO tab_int VALUES (generate_series(1001,2000))");
 
 # Wait until necessary replay has been done on standby
@@ -48,5 +48,5 @@ my $caughtup_query =
 $node_standby->poll_query_until('postgres', $caughtup_query)
   or die "Timed out while waiting for standby to catch up";
 
-my $result = $node_standby->psql('postgres', "SELECT count(*) FROM tab_int");
+my $result = $node_standby->safe_psql('postgres', "SELECT count(*) FROM tab_int");
 is($result, qq(1000), 'check content from archives');

src/test/recovery/t/003_recovery_targets.pl

@@ -38,7 +38,7 @@ sub test_recovery_standby
 
 	# Create some content on master and check its presence in standby
 	my $result =
-	  $node_standby->psql('postgres', "SELECT count(*) FROM tab_int");
+	  $node_standby->safe_psql('postgres', "SELECT count(*) FROM tab_int");
 	is($result, qq($num_rows), "check standby content for $test_name");
 
 	# Stop standby node
@@ -54,40 +54,40 @@ $node_master->start;
 
 # Create data before taking the backup, aimed at testing
 # recovery_target = 'immediate'
-$node_master->psql('postgres',
+$node_master->safe_psql('postgres',
 	"CREATE TABLE tab_int AS SELECT generate_series(1,1000) AS a");
 my $lsn1 =
-  $node_master->psql('postgres', "SELECT pg_current_xlog_location();");
+  $node_master->safe_psql('postgres', "SELECT pg_current_xlog_location();");
 
 # Take backup from which all operations will be run
 $node_master->backup('my_backup');
 
 # Insert some data with used as a replay reference, with a recovery
 # target TXID.
-$node_master->psql('postgres',
+$node_master->safe_psql('postgres',
 	"INSERT INTO tab_int VALUES (generate_series(1001,2000))");
-my $recovery_txid = $node_master->psql('postgres', "SELECT txid_current()");
+my $recovery_txid = $node_master->safe_psql('postgres', "SELECT txid_current()");
 my $lsn2 =
-  $node_master->psql('postgres', "SELECT pg_current_xlog_location();");
+  $node_master->safe_psql('postgres', "SELECT pg_current_xlog_location();");
 
 # More data, with recovery target timestamp
-$node_master->psql('postgres',
+$node_master->safe_psql('postgres',
 	"INSERT INTO tab_int VALUES (generate_series(2001,3000))");
-my $recovery_time = $node_master->psql('postgres', "SELECT now()");
+my $recovery_time = $node_master->safe_psql('postgres', "SELECT now()");
 my $lsn3 =
-  $node_master->psql('postgres', "SELECT pg_current_xlog_location();");
+  $node_master->safe_psql('postgres', "SELECT pg_current_xlog_location();");
 
 # Even more data, this time with a recovery target name
-$node_master->psql('postgres',
+$node_master->safe_psql('postgres',
 	"INSERT INTO tab_int VALUES (generate_series(3001,4000))");
 my $recovery_name = "my_target";
 my $lsn4 =
-  $node_master->psql('postgres', "SELECT pg_current_xlog_location();");
-$node_master->psql('postgres',
+  $node_master->safe_psql('postgres', "SELECT pg_current_xlog_location();");
+$node_master->safe_psql('postgres',
 	"SELECT pg_create_restore_point('$recovery_name');");
 
 # Force archiving of WAL file
-$node_master->psql('postgres', "SELECT pg_switch_xlog()");
+$node_master->safe_psql('postgres', "SELECT pg_switch_xlog()");
 
 # Test recovery targets
 my @recovery_params = ("recovery_target = 'immediate'");

src/test/recovery/t/004_timeline_switch.pl

@@ -30,10 +30,10 @@ $node_standby_2->init_from_backup($node_master, $backup_name,
 $node_standby_2->start;
 
 # Create some content on master
-$node_master->psql('postgres',
+$node_master->safe_psql('postgres',
 	"CREATE TABLE tab_int AS SELECT generate_series(1,1000) AS a");
 my $until_lsn =
-  $node_master->psql('postgres', "SELECT pg_current_xlog_location();");
+  $node_master->safe_psql('postgres', "SELECT pg_current_xlog_location();");
 
 # Wait until standby has replayed enough data on standby 1
 my $caughtup_query =
@@ -61,15 +61,15 @@ $node_standby_2->restart;
 # to exit recovery first before moving on with the test.
 $node_standby_1->poll_query_until('postgres',
 	"SELECT pg_is_in_recovery() <> true");
-$node_standby_1->psql('postgres',
+$node_standby_1->safe_psql('postgres',
 	"INSERT INTO tab_int VALUES (generate_series(1001,2000))");
 $until_lsn =
-  $node_standby_1->psql('postgres', "SELECT pg_current_xlog_location();");
+  $node_standby_1->safe_psql('postgres', "SELECT pg_current_xlog_location();");
 $caughtup_query =
   "SELECT '$until_lsn'::pg_lsn <= pg_last_xlog_replay_location()";
 $node_standby_2->poll_query_until('postgres', $caughtup_query)
   or die "Timed out while waiting for standby to catch up";
 
 my $result =
-  $node_standby_2->psql('postgres', "SELECT count(*) FROM tab_int");
+  $node_standby_2->safe_psql('postgres', "SELECT count(*) FROM tab_int");
 is($result, qq(2000), 'check content of standby 2');

src/test/recovery/t/005_replay_delay.pl

@@ -11,7 +11,7 @@ $node_master->init(allows_streaming => 1);
 $node_master->start;
 
 # And some content
-$node_master->psql('postgres',
+$node_master->safe_psql('postgres',
 	"CREATE TABLE tab_int AS SELECT generate_series(1,10) AS a");
 
 # Take backup
@@ -30,20 +30,20 @@ $node_standby->start;
 
 # Make new content on master and check its presence in standby
 # depending on the delay of 2s applied above.
-$node_master->psql('postgres',
+$node_master->safe_psql('postgres',
 	"INSERT INTO tab_int VALUES (generate_series(11,20))");
 sleep 1;
 
 # Here we should have only 10 rows
-my $result = $node_standby->psql('postgres', "SELECT count(*) FROM tab_int");
+my $result = $node_standby->safe_psql('postgres', "SELECT count(*) FROM tab_int");
 is($result, qq(10), 'check content with delay of 1s');
 
 # Now wait for replay to complete on standby
 my $until_lsn =
-  $node_master->psql('postgres', "SELECT pg_current_xlog_location();");
+  $node_master->safe_psql('postgres', "SELECT pg_current_xlog_location();");
 my $caughtup_query =
   "SELECT '$until_lsn'::pg_lsn <= pg_last_xlog_replay_location()";
 $node_standby->poll_query_until('postgres', $caughtup_query)
   or die "Timed out while waiting for standby to catch up";
-$result = $node_standby->psql('postgres', "SELECT count(*) FROM tab_int");
+$result = $node_standby->safe_psql('postgres', "SELECT count(*) FROM tab_int");
 is($result, qq(20), 'check content with delay of 2s');