Add regression test for recovery pause.

Previously there was no regression test for recovery pause feature.
This commit adds the test that checks

- recovery can be paused or resumed expectedly
- pg_get_wal_replay_pause_state() reports the correct pause state
- the paused state ends and promotion continues if a promotion
   is triggered while recovery is paused

Suggested-by: Michael Paquier
Author: Fujii Masao
Reviewed-by: Kyotaro Horiguchi, Dilip Kumar
Discussion: https://postgr.es/m/YKNirzqM1HYyk5h4@paquier.xyz

src/test/recovery/t/005_replay_delay.pl

 
 # Copyright (c) 2021, PostgreSQL Global Development Group
 
-# Checks for recovery_min_apply_delay
+# Checks for recovery_min_apply_delay and recovery pause
 use strict;
 use warnings;
 
 use PostgresNode;
 use TestLib;
-use Test::More tests => 1;
+use Test::More tests => 3;
 
 # Initialize primary node
 my $node_primary = get_new_node('primary');
@@ -55,3 +55,58 @@ $node_standby->poll_query_until('postgres',
 # the configured apply delay.
 ok(time() - $primary_insert_time >= $delay,
 	"standby applies WAL only after replication delay");
+
+
+# Check that recovery can be paused or resumed expectedly.
+my $node_standby2 = get_new_node('standby2');
+$node_standby2->init_from_backup($node_primary, $backup_name,
+	has_streaming => 1);
+$node_standby2->start;
+
+# Recovery is not yet paused.
+is($node_standby2->safe_psql('postgres',
+	"SELECT pg_get_wal_replay_pause_state()"),
+	'not paused', 'pg_get_wal_replay_pause_state() reports not paused');
+
+# Request to pause recovery and wait until it's actually paused.
+$node_standby2->safe_psql('postgres', "SELECT pg_wal_replay_pause()");
+$node_primary->safe_psql('postgres',
+	"INSERT INTO tab_int VALUES (generate_series(21,30))");
+$node_standby2->poll_query_until('postgres',
+	"SELECT pg_get_wal_replay_pause_state() = 'paused'")
+	or die "Timed out while waiting for recovery to be paused";
+
+# Even if new WAL records are streamed from the primary,
+# recovery in the paused state doesn't replay them.
+my $receive_lsn = $node_standby2->safe_psql('postgres',
+	"SELECT pg_last_wal_receive_lsn()");
+my $replay_lsn = $node_standby2->safe_psql('postgres',
+	"SELECT pg_last_wal_replay_lsn()");
+$node_primary->safe_psql('postgres',
+	"INSERT INTO tab_int VALUES (generate_series(31,40))");
+$node_standby2->poll_query_until('postgres',
+	"SELECT '$receive_lsn'::pg_lsn < pg_last_wal_receive_lsn()")
+	or die "Timed out while waiting for new WAL to be streamed";
+is($node_standby2->safe_psql('postgres',
+	"SELECT pg_last_wal_replay_lsn()"),
+	qq($replay_lsn), 'no WAL is replayed in the paused state');
+
+# Request to resume recovery and wait until it's actually resumed.
+$node_standby2->safe_psql('postgres', "SELECT pg_wal_replay_resume()");
+$node_standby2->poll_query_until('postgres',
+	"SELECT pg_get_wal_replay_pause_state() = 'not paused' AND pg_last_wal_replay_lsn() > '$replay_lsn'::pg_lsn")
+	or die "Timed out while waiting for recovery to be resumed";
+
+# Check that the paused state ends and promotion continues if a promotion
+# is triggered while recovery is paused.
+$node_standby2->safe_psql('postgres', "SELECT pg_wal_replay_pause()");
+$node_primary->safe_psql('postgres',
+	"INSERT INTO tab_int VALUES (generate_series(41,50))");
+$node_standby2->poll_query_until('postgres',
+	"SELECT pg_get_wal_replay_pause_state() = 'paused'")
+  or die "Timed out while waiting for recovery to be paused";
+
+$node_standby2->promote;
+$node_standby2->poll_query_until('postgres',
+	"SELECT NOT pg_is_in_recovery()")
+  or die "Timed out while waiting for promotion to finish";