Add support for --jobs in reindexdb

When doing a schema-level or a database-level operation, a list of
relations to build is created which gets processed in parallel using
multiple connections, based on the recent refactoring for parallel slots
in src/bin/scripts/.  System catalogs are processed first in a
serialized fashion to prevent deadlocks, followed by the rest done in
parallel.

This new option is not compatible with --system as reindexing system
catalogs in parallel can lead to deadlocks, and with --index as there is
no conflict handling for indexes rebuilt in parallel depending in the
same relation.

Author: Julien Rouhaud
Reviewed-by: Sergei Kornilov, Michael Paquier
Discussion: https://postgr.es/m/CAOBaU_YrnH_Jqo46NhaJ7uRBiWWEcS40VNRQxgFbqYo9kApUsg@mail.gmail.com

doc/src/sgml/ref/reindexdb.sgml

@@ -166,6 +166,29 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>-j <replaceable class="parameter">njobs</replaceable></option></term>
+      <term><option>--jobs=<replaceable class="parameter">njobs</replaceable></option></term>
+      <listitem>
+       <para>
+        Execute the reindex commands in parallel by running
+        <replaceable class="parameter">njobs</replaceable>
+        commands simultaneously.  This option reduces the time of the
+        processing but it also increases the load on the database server.
+       </para>
+       <para>
+        <application>reindexdb</application> will open
+        <replaceable class="parameter">njobs</replaceable> connections to the
+        database, so make sure your <xref linkend="guc-max-connections"/>
+        setting is high enough to accommodate all connections.
+       </para>
+       <para>
+        Note that this option is incompatible with the <option>--index</option>
+        and <option>--system</option> options.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><option>-q</option></term>
       <term><option>--quiet</option></term>

src/bin/scripts/Makefile

@@ -29,7 +29,7 @@ dropdb: dropdb.o common.o $(WIN32RES) | submake-libpq submake-libpgport submake-
 dropuser: dropuser.o common.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
 clusterdb: clusterdb.o common.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
 vacuumdb: vacuumdb.o common.o scripts_parallel.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
-reindexdb: reindexdb.o common.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
+reindexdb: reindexdb.o common.o scripts_parallel.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
 pg_isready: pg_isready.o common.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
 
 install: all installdirs

src/bin/scripts/t/090_reindexdb.pl

@@ -3,7 +3,7 @@ use warnings;
 
 use PostgresNode;
 use TestLib;
-use Test::More tests => 34;
+use Test::More tests => 44;
 
 program_help_ok('reindexdb');
 program_version_ok('reindexdb');
@@ -77,3 +77,45 @@ $node->command_ok(
 $node->command_ok(
 	[qw(reindexdb --echo --system dbname=template1)],
 	'reindexdb system with connection string');
+
+# parallel processing
+$node->safe_psql(
+	'postgres', q|
+	CREATE SCHEMA s1;
+	CREATE TABLE s1.t1(id integer);
+	CREATE INDEX ON s1.t1(id);
+	CREATE SCHEMA s2;
+	CREATE TABLE s2.t2(id integer);
+	CREATE INDEX ON s2.t2(id);
+	-- empty schema
+	CREATE SCHEMA s3;
+|);
+
+$node->command_fails(
+	[ 'reindexdb', '-j', '2', '-s', 'postgres' ],
+	'parallel reindexdb cannot process system catalogs');
+$node->command_fails(
+	[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
+	'parallel reindexdb cannot process indexes');
+$node->issues_sql_like(
+	[ 'reindexdb', '-j', '2', 'postgres' ],
+	qr/statement:\ REINDEX SYSTEM postgres;
+.*statement:\ REINDEX TABLE public\.test1/s,
+	'parallel reindexdb for database issues REINDEX SYSTEM first');
+# Note that the ordering of the commands is not stable, so the second
+# command for s2.t2 is not checked after.
+$node->issues_sql_like(
+	[ 'reindexdb', '-j', '2', '-S', 's1', '-S', 's2', 'postgres' ],
+	qr/statement:\ REINDEX TABLE s1.t1;/,
+	'parallel reindexdb for schemas does a per-table REINDEX');
+$node->command_ok(
+	['reindexdb', '-j', '2', '-S', 's3'],
+	'parallel reindexdb with empty schema');
+$node->command_checks_all(
+	[ 'reindexdb', '-j', '2', '--concurrently', '-d', 'postgres' ],
+	0,
+	[qr/^$/],
+	[
+		qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
+	],
+	'parallel reindexdb for system with --concurrently skips catalogs');