Improve psql's tab completion for filenames.

The Readline library contains a fair amount of knowledge about how to
tab-complete filenames, but it turns out that that doesn't work too well
unless we follow its expectation that we use its filename quoting hooks
to quote and de-quote filenames.  We were trying to do such quote handling
within complete_from_files(), and that's still what we have to do if we're
using libedit, which lacks those hooks.  But for Readline, it works a lot
better if we tell Readline that single-quote is a quoting character and
then provide hooks that know the details of the quoting rules for SQL
and psql meta-commands.

Hence, resurrect the quoting hook functions that existed in the original
version of tab-complete.c (and were disabled by commit f6689a32 because
they "didn't work so well yet"), and whack on them until they do seem to
work well.

Notably, this fixes bug #16059 from Steven Winfield, who pointed out
that the previous coding would strip quote marks from filenames in SQL
COPY commands, even though they're syntactically necessary there.
Now, we not only don't do that, but we'll add a quote mark when you
tab-complete, even if you didn't type one.

Getting this to work across a range of libedit versions (and, to a
lesser extent, libreadline versions) was depressingly difficult.
It will be interesting to see whether the new regression test cases
pass everywhere in the buildfarm.

Some future patch might try to handle quoted SQL identifiers with
similar explicit quoting/dequoting logic, but that's for another day.

Patch by me, reviewed by Peter Eisentraut.

Discussion: https://postgr.es/m/16059-8836946734c02b84@postgresql.org

config/programs.m4

@@ -209,17 +209,20 @@ fi
 
 
 
-# PGAC_VAR_RL_COMPLETION_APPEND_CHARACTER
-# ---------------------------------------
+# PGAC_READLINE_VARIABLES
+# -----------------------
 # Readline versions < 2.1 don't have rl_completion_append_character
+# Libedit lacks rl_filename_quote_characters and rl_filename_quoting_function
 
-AC_DEFUN([PGAC_VAR_RL_COMPLETION_APPEND_CHARACTER],
+AC_DEFUN([PGAC_READLINE_VARIABLES],
 [AC_CACHE_CHECK([for rl_completion_append_character], pgac_cv_var_rl_completion_append_character,
 [AC_LINK_IFELSE([AC_LANG_PROGRAM([#include <stdio.h>
-#ifdef HAVE_READLINE_READLINE_H
-# include <readline/readline.h>
+#if defined(HAVE_READLINE_READLINE_H)
+#include <readline/readline.h>
+#elif defined(HAVE_EDITLINE_READLINE_H)
+#include <editline/readline.h>
 #elif defined(HAVE_READLINE_H)
-# include <readline.h>
+#include <readline.h>
 #endif
 ],
 [rl_completion_append_character = 'x';])],
@@ -228,7 +231,42 @@ AC_DEFUN([PGAC_VAR_RL_COMPLETION_APPEND_CHARACTER],
 if test x"$pgac_cv_var_rl_completion_append_character" = x"yes"; then
 AC_DEFINE(HAVE_RL_COMPLETION_APPEND_CHARACTER, 1,
           [Define to 1 if you have the global variable 'rl_completion_append_character'.])
-fi])# PGAC_VAR_RL_COMPLETION_APPEND_CHARACTER
+fi
+AC_CACHE_CHECK([for rl_filename_quote_characters], pgac_cv_var_rl_filename_quote_characters,
+[AC_LINK_IFELSE([AC_LANG_PROGRAM([#include <stdio.h>
+#if defined(HAVE_READLINE_READLINE_H)
+#include <readline/readline.h>
+#elif defined(HAVE_EDITLINE_READLINE_H)
+#include <editline/readline.h>
+#elif defined(HAVE_READLINE_H)
+#include <readline.h>
+#endif
+],
+[rl_filename_quote_characters = "x";])],
+[pgac_cv_var_rl_filename_quote_characters=yes],
+[pgac_cv_var_rl_filename_quote_characters=no])])
+if test x"$pgac_cv_var_rl_filename_quote_characters" = x"yes"; then
+AC_DEFINE(HAVE_RL_FILENAME_QUOTE_CHARACTERS, 1,
+          [Define to 1 if you have the global variable 'rl_filename_quote_characters'.])
+fi
+AC_CACHE_CHECK([for rl_filename_quoting_function], pgac_cv_var_rl_filename_quoting_function,
+[AC_LINK_IFELSE([AC_LANG_PROGRAM([#include <stdio.h>
+#if defined(HAVE_READLINE_READLINE_H)
+#include <readline/readline.h>
+#elif defined(HAVE_EDITLINE_READLINE_H)
+#include <editline/readline.h>
+#elif defined(HAVE_READLINE_H)
+#include <readline.h>
+#endif
+],
+[rl_filename_quoting_function = 0;])],
+[pgac_cv_var_rl_filename_quoting_function=yes],
+[pgac_cv_var_rl_filename_quoting_function=no])])
+if test x"$pgac_cv_var_rl_filename_quoting_function" = x"yes"; then
+AC_DEFINE(HAVE_RL_FILENAME_QUOTING_FUNCTION, 1,
+          [Define to 1 if you have the global variable 'rl_filename_quoting_function'.])
+fi
+])# PGAC_READLINE_VARIABLES
 
 
 

configure

@@ -16316,10 +16316,12 @@ else
   cat confdefs.h - <<_ACEOF >conftest.$ac_ext
 /* end confdefs.h.  */
 #include <stdio.h>
-#ifdef HAVE_READLINE_READLINE_H
-# include <readline/readline.h>
+#if defined(HAVE_READLINE_READLINE_H)
+#include <readline/readline.h>
+#elif defined(HAVE_EDITLINE_READLINE_H)
+#include <editline/readline.h>
 #elif defined(HAVE_READLINE_H)
-# include <readline.h>
+#include <readline.h>
 #endif
 
 int
@@ -16345,6 +16347,85 @@ if test x"$pgac_cv_var_rl_completion_append_character" = x"yes"; then
 $as_echo "#define HAVE_RL_COMPLETION_APPEND_CHARACTER 1" >>confdefs.h
 
 fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for rl_filename_quote_characters" >&5
+$as_echo_n "checking for rl_filename_quote_characters... " >&6; }
+if ${pgac_cv_var_rl_filename_quote_characters+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#include <stdio.h>
+#if defined(HAVE_READLINE_READLINE_H)
+#include <readline/readline.h>
+#elif defined(HAVE_EDITLINE_READLINE_H)
+#include <editline/readline.h>
+#elif defined(HAVE_READLINE_H)
+#include <readline.h>
+#endif
+
+int
+main ()
+{
+rl_filename_quote_characters = "x";
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+  pgac_cv_var_rl_filename_quote_characters=yes
+else
+  pgac_cv_var_rl_filename_quote_characters=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_var_rl_filename_quote_characters" >&5
+$as_echo "$pgac_cv_var_rl_filename_quote_characters" >&6; }
+if test x"$pgac_cv_var_rl_filename_quote_characters" = x"yes"; then
+
+$as_echo "#define HAVE_RL_FILENAME_QUOTE_CHARACTERS 1" >>confdefs.h
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for rl_filename_quoting_function" >&5
+$as_echo_n "checking for rl_filename_quoting_function... " >&6; }
+if ${pgac_cv_var_rl_filename_quoting_function+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#include <stdio.h>
+#if defined(HAVE_READLINE_READLINE_H)
+#include <readline/readline.h>
+#elif defined(HAVE_EDITLINE_READLINE_H)
+#include <editline/readline.h>
+#elif defined(HAVE_READLINE_H)
+#include <readline.h>
+#endif
+
+int
+main ()
+{
+rl_filename_quoting_function = 0;
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+  pgac_cv_var_rl_filename_quoting_function=yes
+else
+  pgac_cv_var_rl_filename_quoting_function=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_var_rl_filename_quoting_function" >&5
+$as_echo "$pgac_cv_var_rl_filename_quoting_function" >&6; }
+if test x"$pgac_cv_var_rl_filename_quoting_function" = x"yes"; then
+
+$as_echo "#define HAVE_RL_FILENAME_QUOTING_FUNCTION 1" >>confdefs.h
+
+fi
+
   for ac_func in rl_completion_matches rl_filename_completion_function rl_reset_screen_size
 do :
   as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh`

configure.in

@@ -1874,7 +1874,7 @@ fi
 LIBS="$LIBS_including_readline"
 
 if test "$with_readline" = yes; then
-  PGAC_VAR_RL_COMPLETION_APPEND_CHARACTER
+  PGAC_READLINE_VARIABLES
   AC_CHECK_FUNCS([rl_completion_matches rl_filename_completion_function rl_reset_screen_size])
   AC_CHECK_FUNCS([append_history history_truncate_file])
 fi

src/bin/psql/stringutils.c

@@ -282,6 +282,7 @@ strip_quotes(char *source, char quote, char escape, int encoding)
  * entails_quote -	any of these present?  need outer quotes
  * quote -			doubled within string, affixed to both ends
  * escape -			doubled within string
+ * force_quote -	if true, quote the output even if it doesn't "need" it
  * encoding -		the active character-set encoding
  *
  * Do not use this as a substitute for PQescapeStringConn().  Use it for
@@ -289,12 +290,13 @@ strip_quotes(char *source, char quote, char escape, int encoding)
  */
 char *
 quote_if_needed(const char *source, const char *entails_quote,
-				char quote, char escape, int encoding)
+				char quote, char escape, bool force_quote,
+				int encoding)
 {
 	const char *src;
 	char	   *ret;
 	char	   *dst;
-	bool		need_quotes = false;
+	bool		need_quotes = force_quote;
 
 	Assert(source != NULL);
 	Assert(quote != '\0');

src/bin/psql/stringutils.h

@@ -22,6 +22,7 @@ extern char *strtokx(const char *s,
 extern void strip_quotes(char *source, char quote, char escape, int encoding);
 
 extern char *quote_if_needed(const char *source, const char *entails_quote,
-							 char quote, char escape, int encoding);
+							 char quote, char escape, bool force_quote,
+							 int encoding);
 
 #endif							/* STRINGUTILS_H */

src/bin/psql/t/010_tab_completion.pl

@@ -60,6 +60,30 @@ delete $ENV{TERM};
 # Some versions of readline inspect LS_COLORS, so for luck unset that too.
 delete $ENV{LS_COLORS};
 
+# In a VPATH build, we'll be started in the source directory, but we want
+# to run in the build directory so that we can use relative paths to
+# access the tmp_check subdirectory; otherwise the output from filename
+# completion tests is too variable.
+if ($ENV{TESTDIR})
+{
+	chdir $ENV{TESTDIR} or die "could not chdir to \"$ENV{TESTDIR}\": $!";
+}
+
+# Create some junk files for filename completion testing.
+my $FH;
+open $FH, ">", "tmp_check/somefile"
+  or die("could not create file \"tmp_check/somefile\": $!");
+print $FH "some stuff\n";
+close $FH;
+open $FH, ">", "tmp_check/afile123"
+  or die("could not create file \"tmp_check/afile123\": $!");
+print $FH "more stuff\n";
+close $FH;
+open $FH, ">", "tmp_check/afile456"
+  or die("could not create file \"tmp_check/afile456\": $!");
+print $FH "other stuff\n";
+close $FH;
+
 # fire up an interactive psql session
 my $in  = '';
 my $out = '';
@@ -104,6 +128,15 @@ sub clear_query
 	return;
 }
 
+# Clear current line to start over
+# (this will work in an incomplete string literal, but it's less desirable
+# than clear_query because we lose evidence in the history file)
+sub clear_line
+{
+	check_completion("\025\n", qr/postgres=# /, "control-U works");
+	return;
+}
+
 # check basic command completion: SEL<tab> produces SELECT<space>
 check_completion("SEL\t", qr/SELECT /, "complete SEL<tab> to SELECT");
 
@@ -142,6 +175,47 @@ check_completion("\\DRD\t", qr/drds /, "complete \\DRD<tab> to \\drds");
 
 clear_query();
 
+# check filename completion
+check_completion(
+	"\\lo_import tmp_check/some\t",
+	qr|tmp_check/somefile |,
+	"filename completion with one possibility");
+
+clear_query();
+
+# note: readline might print a bell before the completion
+check_completion(
+	"\\lo_import tmp_check/af\t",
+	qr|tmp_check/af\a?ile|,
+	"filename completion with multiple possibilities");
+
+clear_query();
+
+# COPY requires quoting
+# note: broken versions of libedit want to backslash the closing quote;
+# not much we can do about that
+check_completion(
+	"COPY foo FROM tmp_check/some\t",
+	qr|'tmp_check/somefile\\?' |,
+	"quoted filename completion with one possibility");
+
+clear_line();
+
+check_completion(
+	"COPY foo FROM tmp_check/af\t",
+	qr|'tmp_check/afile|,
+	"quoted filename completion with multiple possibilities");
+
+# some versions of readline/libedit require two tabs here, some only need one
+# also, some will offer the whole path name and some just the file name
+# the quotes might appear, too
+check_completion(
+	"\t\t",
+	qr|afile123'? +'?(tmp_check/)?afile456|,
+	"offer multiple file choices");
+
+clear_line();
+
 # send psql an explicit \q to shut it down, else pty won't close properly
 $timer->start(5);
 $in .= "\\q\n";

src/include/pg_config.h.in

@@ -488,6 +488,14 @@
 /* Define to 1 if you have the `rl_filename_completion_function' function. */
 #undef HAVE_RL_FILENAME_COMPLETION_FUNCTION
 
+/* Define to 1 if you have the global variable 'rl_filename_quote_characters'.
+   */
+#undef HAVE_RL_FILENAME_QUOTE_CHARACTERS
+
+/* Define to 1 if you have the global variable 'rl_filename_quoting_function'.
+   */
+#undef HAVE_RL_FILENAME_QUOTING_FUNCTION
+
 /* Define to 1 if you have the `rl_reset_screen_size' function. */
 #undef HAVE_RL_RESET_SCREEN_SIZE
 

src/tools/msvc/Solution.pm

@@ -333,6 +333,8 @@ sub GenerateFiles
 		HAVE_RL_COMPLETION_APPEND_CHARACTER      => undef,
 		HAVE_RL_COMPLETION_MATCHES               => undef,
 		HAVE_RL_FILENAME_COMPLETION_FUNCTION     => undef,
+		HAVE_RL_FILENAME_QUOTE_CHARACTERS        => undef,
+		HAVE_RL_FILENAME_QUOTING_FUNCTION        => undef,
 		HAVE_RL_RESET_SCREEN_SIZE                => undef,
 		HAVE_SECURITY_PAM_APPL_H                 => undef,
 		HAVE_SETPROCTITLE                        => undef,