In pg_upgrade, use cached copy of directory listing, rather than calling

scandir() with a pattern for every table.

Optimization after report of pg_upgrade slowness with 150k tables.

contrib/pg_upgrade/info.c

@@ -30,8 +30,8 @@ static void map_rel_by_id(Oid oldid, Oid newid,
 			  const char *old_tablespace, const DbInfo *old_db,
 			  const DbInfo *new_db, const char *olddata,
 			  const char *newdata, FileNameMap *map);
-static RelInfo *relarr_lookup_reloid(
-					 RelInfoArr *rel_arr, Oid oid, Cluster whichCluster);
+static RelInfo *relarr_lookup_reloid(RelInfoArr *rel_arr,
+				Oid oid, Cluster whichCluster);
 static RelInfo *relarr_lookup_rel(RelInfoArr *rel_arr,
 				  const char *nspname, const char *relname,
 				  Cluster whichCluster);
@@ -172,8 +172,8 @@ map_rel_by_id(Oid oldid, Oid newid,
 		 * relation belongs to the default tablespace, hence relfiles would
 		 * exist in the data directories.
 		 */
-		snprintf(map->old_file, sizeof(map->old_file), "%s/base/%u", olddata, old_db->db_oid);
-		snprintf(map->new_file, sizeof(map->new_file), "%s/base/%u", newdata, new_db->db_oid);
+		snprintf(map->old_dir, sizeof(map->old_dir), "%s/base/%u", olddata, old_db->db_oid);
+		snprintf(map->new_dir, sizeof(map->new_dir), "%s/base/%u", newdata, new_db->db_oid);
 	}
 	else
 	{
@@ -181,9 +181,9 @@ map_rel_by_id(Oid oldid, Oid newid,
 		 * relation belongs to some tablespace, hence copy its physical
 		 * location
 		 */
-		snprintf(map->old_file, sizeof(map->old_file), "%s%s/%u", old_tablespace,
+		snprintf(map->old_dir, sizeof(map->old_dir), "%s%s/%u", old_tablespace,
 				 old_cluster.tablespace_suffix, old_db->db_oid);
-		snprintf(map->new_file, sizeof(map->new_file), "%s%s/%u", old_tablespace,
+		snprintf(map->new_dir, sizeof(map->new_dir), "%s%s/%u", old_tablespace,
 				 new_cluster.tablespace_suffix, new_db->db_oid);
 	}
 }
@@ -318,6 +318,7 @@ get_rel_infos(const DbInfo *dbinfo,
 	 * pg_largeobject_loid_pn_index's relfilenode can change if the table was
 	 * reindexed so we get the relfilenode for each database and migrate it as
 	 * a normal user table.
+	 * Order by tablespace so we can cache the directory contents efficiently.
 	 */
 
 	snprintf(query, sizeof(query),
@@ -338,7 +339,7 @@ get_rel_infos(const DbInfo *dbinfo,
 			 "GROUP BY  c.oid, n.nspname, c.relname, c.relfilenode,"
 			 "			c.reltoastrelid, t.spclocation, "
 			 "			n.nspname "
-			 "ORDER BY n.nspname, c.relname;",
+			 "ORDER BY t.spclocation, n.nspname, c.relname;",
 			 FirstNormalObjectId,
 	/* see the comment at the top of old_8_3_create_sequence_script() */
 			 (GET_MAJOR_VERSION(old_cluster.major_version) <= 803) ?

contrib/pg_upgrade/pg_upgrade.h

@@ -85,10 +85,10 @@ typedef struct
  */
 typedef struct
 {
+	char		old_dir[MAXPGPATH];
+	char		new_dir[MAXPGPATH];
 	Oid			old_relfilenode;	/* Relfilenode of the old relation */
 	Oid			new_relfilenode;	/* Relfilenode of the new relation */
-	char		old_file[MAXPGPATH];
-	char		new_file[MAXPGPATH];
 	char		old_nspname[NAMEDATALEN];		/* old name of the namespace */
 	char		old_relname[NAMEDATALEN];		/* old name of the relation */
 	char		new_nspname[NAMEDATALEN];		/* new name of the namespace */
@@ -255,10 +255,8 @@ void check_old_cluster(bool live_check,
 				  char **sequence_script_file_name);
 void		check_new_cluster(void);
 void		report_clusters_compatible(void);
-void issue_warnings(
-			   char *sequence_script_file_name);
-void output_completion_banner(
-						 char *deletion_script_file_name);
+void issue_warnings(char *sequence_script_file_name);
+void output_completion_banner(char *deletion_script_file_name);
 void		check_cluster_versions(void);
 void		check_cluster_compatibility(bool live_check);
 void		create_script_for_old_cluster_deletion(char **deletion_script_file_name);
@@ -319,14 +317,12 @@ typedef void *pageCnvCtx;
 #endif
 
 int			dir_matching_filenames(const struct dirent * scan_ent);
-int pg_scandir(const char *dirname,
-		   struct dirent *** namelist,
-		   int (*selector) (const struct dirent *));
-const char *copyAndUpdateFile(
-				  pageCnvCtx *pageConverter, const char *src,
+int pg_scandir(const char *dirname, struct dirent *** namelist,
+			   int (*selector) (const struct dirent *));
+const char *copyAndUpdateFile(pageCnvCtx *pageConverter, const char *src,
 				  const char *dst, bool force);
-const char *linkAndUpdateFile(
-				pageCnvCtx *pageConverter, const char *src, const char *dst);
+const char *linkAndUpdateFile(pageCnvCtx *pageConverter, const char *src,
+							  const char *dst);
 
 void		check_hard_link(void);
 
@@ -374,8 +370,7 @@ PGresult *executeQueryOrDie(PGconn *conn,
 
 void		start_postmaster(Cluster whichCluster, bool quiet);
 void		stop_postmaster(bool fast, bool quiet);
-uint32 get_major_server_version(char **verstr,
-						 Cluster whichCluster);
+uint32 get_major_server_version(char **verstr, Cluster whichCluster);
 void		check_for_libpq_envvars(void);
 
 

contrib/pg_upgrade/relfilenode.c

@@ -110,20 +110,38 @@ static void
 transfer_single_new_db(pageCnvCtx *pageConverter,
 					   FileNameMap *maps, int size)
 {
+	char		old_dir[MAXPGPATH];
+	struct dirent **namelist = NULL;
+	int			numFiles = 0;
 	int			mapnum;
+	int			fileno;
+
+	old_dir[0] = '\0';
 
 	for (mapnum = 0; mapnum < size; mapnum++)
 	{
 		char		old_file[MAXPGPATH];
 		char		new_file[MAXPGPATH];
-		struct dirent **namelist = NULL;
-		int			numFiles;
+
+		/* Changed tablespaces?  Need a new directory scan? */
+		if (strcmp(maps[mapnum].old_dir, old_dir) != 0)
+		{
+			if (numFiles > 0)
+			{
+				for (fileno = 0; fileno < numFiles; fileno++)
+					pg_free(namelist[fileno]);
+				pg_free(namelist);
+			}
+
+			snprintf(old_dir, sizeof(old_dir), "%s", maps[mapnum].old_dir);
+			numFiles = pg_scandir(old_dir, &namelist, NULL);
+		}
 
 		/* Copying files might take some time, so give feedback. */
 
-		snprintf(old_file, sizeof(old_file), "%s/%u", maps[mapnum].old_file,
+		snprintf(old_file, sizeof(old_file), "%s/%u", maps[mapnum].old_dir,
 				 maps[mapnum].old_relfilenode);
-		snprintf(new_file, sizeof(new_file), "%s/%u", maps[mapnum].new_file,
+		snprintf(new_file, sizeof(new_file), "%s/%u", maps[mapnum].new_dir,
 				 maps[mapnum].new_relfilenode);
 		pg_log(PG_REPORT, OVERWRITE_MESSAGE, old_file);
 
@@ -139,28 +157,27 @@ transfer_single_new_db(pageCnvCtx *pageConverter,
 		if (GET_MAJOR_VERSION(old_cluster.major_version) >= 804)
 		{
 			/*
-			 * Now copy/link any fsm and vm files, if they exist
+			 * Copy/link any fsm and vm files, if they exist
 			 */
 			snprintf(scandir_file_pattern, sizeof(scandir_file_pattern), "%u_",
 					 maps[mapnum].old_relfilenode);
-			numFiles = pg_scandir(maps[mapnum].old_file, &namelist, dir_matching_filenames);
 
-			while (numFiles--)
+			for (fileno = 0; fileno < numFiles; fileno++)
 			{
-				snprintf(old_file, sizeof(old_file), "%s/%s", maps[mapnum].old_file,
-						 namelist[numFiles]->d_name);
-				snprintf(new_file, sizeof(new_file), "%s/%u%s", maps[mapnum].new_file,
-						 maps[mapnum].new_relfilenode, strchr(namelist[numFiles]->d_name, '_'));
-
-				unlink(new_file);
-				transfer_relfile(pageConverter, old_file, new_file,
-						  maps[mapnum].old_nspname, maps[mapnum].old_relname,
-						 maps[mapnum].new_nspname, maps[mapnum].new_relname);
-
-				pg_free(namelist[numFiles]);
+				if (strncmp(namelist[fileno]->d_name, scandir_file_pattern,
+					strlen(scandir_file_pattern)) == 0)
+				{
+					snprintf(old_file, sizeof(old_file), "%s/%s", maps[mapnum].old_dir,
+							 namelist[fileno]->d_name);
+					snprintf(new_file, sizeof(new_file), "%s/%u%s", maps[mapnum].new_dir,
+							 maps[mapnum].new_relfilenode, strchr(namelist[fileno]->d_name, '_'));
+	
+					unlink(new_file);
+					transfer_relfile(pageConverter, old_file, new_file,
+							  maps[mapnum].old_nspname, maps[mapnum].old_relname,
+							 maps[mapnum].new_nspname, maps[mapnum].new_relname);
+				}
 			}
-
-			pg_free(namelist);
 		}
 
 		/*
@@ -172,23 +189,30 @@ transfer_single_new_db(pageCnvCtx *pageConverter,
 		 */
 		snprintf(scandir_file_pattern, sizeof(scandir_file_pattern), "%u.",
 				 maps[mapnum].old_relfilenode);
-		numFiles = pg_scandir(maps[mapnum].old_file, &namelist, dir_matching_filenames);
 
-		while (numFiles--)
+		for (fileno = 0; fileno < numFiles; fileno++)
 		{
-			snprintf(old_file, sizeof(old_file), "%s/%s", maps[mapnum].old_file,
-					 namelist[numFiles]->d_name);
-			snprintf(new_file, sizeof(new_file), "%s/%u%s", maps[mapnum].new_file,
-					 maps[mapnum].new_relfilenode, strchr(namelist[numFiles]->d_name, '.'));
-
-			unlink(new_file);
-			transfer_relfile(pageConverter, old_file, new_file,
-						  maps[mapnum].old_nspname, maps[mapnum].old_relname,
-						 maps[mapnum].new_nspname, maps[mapnum].new_relname);
+			if (strncmp(namelist[fileno]->d_name, scandir_file_pattern,
+				strlen(scandir_file_pattern)) == 0)
+			{
+				snprintf(old_file, sizeof(old_file), "%s/%s", maps[mapnum].old_dir,
+						 namelist[fileno]->d_name);
+				snprintf(new_file, sizeof(new_file), "%s/%u%s", maps[mapnum].new_dir,
+						 maps[mapnum].new_relfilenode, strchr(namelist[fileno]->d_name, '.'));
 
-			pg_free(namelist[numFiles]);
+				unlink(new_file);
+				transfer_relfile(pageConverter, old_file, new_file,
+							  maps[mapnum].old_nspname, maps[mapnum].old_relname,
+							 maps[mapnum].new_nspname, maps[mapnum].new_relname);
+			}
 		}
+	}
 
+
+	if (numFiles > 0)
+	{
+		for (fileno = 0; fileno < numFiles; fileno++)
+			pg_free(namelist[fileno]);
 		pg_free(namelist);
 	}
 }