Fix our Windows stat() emulation to handle file sizes > 4GB.

Hack things so that our idea of "struct stat" is equivalent to Windows'
struct __stat64, allowing it to have a wide enough st_size field.

Instead of relying on native stat(), use GetFileInformationByHandle().
This avoids a number of issues with Microsoft's multiple and rather
slipshod emulations of stat().  We still need to jump through hoops
to deal with ERROR_DELETE_PENDING, though :-(

Pull the relevant support code out of dirmod.c and put it into
its own file, win32stat.c.

Still TODO: do we need to do something different with lstat(),
rather than treating it identically to stat()?

Juan José Santamaría Flecha, reviewed by Emil Iggland;
based on prior work by Michael Paquier, Sergey Zubkovsky, and others

Discussion: https://postgr.es/m/1803D792815FC24D871C00D17AE95905CF5099@g01jpexmbkw24
Discussion: https://postgr.es/m/15858-9572469fd3b73263@postgresql.org

configure

@@ -16137,6 +16137,12 @@ esac
  ;;
 esac
 
+  case " $LIBOBJS " in
+  *" win32stat.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS win32stat.$ac_objext"
+ ;;
+esac
+
 
 $as_echo "#define HAVE_SYMLINK 1" >>confdefs.h
 

configure.ac

@@ -1807,6 +1807,7 @@ if test "$PORTNAME" = "win32"; then
   AC_LIBOBJ(win32error)
   AC_LIBOBJ(win32security)
   AC_LIBOBJ(win32setlocale)
+  AC_LIBOBJ(win32stat)
   AC_DEFINE([HAVE_SYMLINK], 1,
             [Define to 1 if you have the `symlink' function.])
   AC_CHECK_TYPES(MINIDUMP_TYPE, [pgac_minidump_type=yes], [pgac_minidump_type=no], [

src/include/port/win32_port.h

@@ -51,7 +51,13 @@
 #include <signal.h>
 #include <direct.h>
 #undef near
-#include <sys/stat.h>			/* needed before sys/stat hacking below */
+
+/* needed before sys/stat hacking below: */
+#define fstat microsoft_native_fstat
+#define stat microsoft_native_stat
+#include <sys/stat.h>
+#undef fstat
+#undef stat
 
 /* Must be here to avoid conflicting with prototype in windows.h */
 #define mkdir(a,b)	mkdir(a)
@@ -240,20 +246,34 @@ typedef int pid_t;
  * Supplement to <sys/stat.h>.
  *
  * We must pull in sys/stat.h before this part, else our overrides lose.
- */
-#define lstat(path, sb) stat(path, sb)
-
-/*
+ *
  * stat() is not guaranteed to set the st_size field on win32, so we
- * redefine it to our own implementation that is.
+ * redefine it to our own implementation.  See src/port/win32stat.c.
  *
- * Some frontends don't need the size from stat, so if UNSAFE_STAT_OK
- * is defined we don't bother with this.
+ * The struct stat is 32 bit in MSVC, so we redefine it as a copy of
+ * struct __stat64.  This also fixes the struct size for MINGW builds.
  */
-#ifndef UNSAFE_STAT_OK
-extern int	pgwin32_safestat(const char *path, struct stat *buf);
-#define stat(a,b) pgwin32_safestat(a,b)
-#endif
+struct stat						/* This should match struct __stat64 */
+{
+	_dev_t		st_dev;
+	_ino_t		st_ino;
+	unsigned short st_mode;
+	short		st_nlink;
+	short		st_uid;
+	short		st_gid;
+	_dev_t		st_rdev;
+	__int64		st_size;
+	__time64_t	st_atime;
+	__time64_t	st_mtime;
+	__time64_t	st_ctime;
+};
+
+extern int	_pgfstat64(int fileno, struct stat *buf);
+extern int	_pgstat64(const char *name, struct stat *buf);
+
+#define fstat(fileno, sb)	_pgfstat64(fileno, sb)
+#define stat(path, sb)		_pgstat64(path, sb)
+#define lstat(path, sb)		_pgstat64(path, sb)
 
 /* These macros are not provided by older MinGW, nor by MSVC */
 #ifndef S_IRUSR

src/port/dirmod.c

@@ -353,55 +353,3 @@ pgwin32_is_junction(const char *path)
 	return ((attr & FILE_ATTRIBUTE_REPARSE_POINT) == FILE_ATTRIBUTE_REPARSE_POINT);
 }
 #endif							/* defined(WIN32) && !defined(__CYGWIN__) */
-
-
-#if defined(WIN32) && !defined(__CYGWIN__)
-
-#undef stat
-
-/*
- * The stat() function in win32 is not guaranteed to update the st_size
- * field when run. So we define our own version that uses the Win32 API
- * to update this field.
- */
-int
-pgwin32_safestat(const char *path, struct stat *buf)
-{
-	int			r;
-	WIN32_FILE_ATTRIBUTE_DATA attr;
-
-	r = stat(path, buf);
-	if (r < 0)
-	{
-		if (GetLastError() == ERROR_DELETE_PENDING)
-		{
-			/*
-			 * File has been deleted, but is not gone from the filesystem yet.
-			 * This can happen when some process with FILE_SHARE_DELETE has it
-			 * open and it will be fully removed once that handle is closed.
-			 * Meanwhile, we can't open it, so indicate that the file just
-			 * doesn't exist.
-			 */
-			errno = ENOENT;
-			return -1;
-		}
-
-		return r;
-	}
-
-	if (!GetFileAttributesEx(path, GetFileExInfoStandard, &attr))
-	{
-		_dosmaperr(GetLastError());
-		return -1;
-	}
-
-	/*
-	 * XXX no support for large files here, but we don't do that in general on
-	 * Win32 yet.
-	 */
-	buf->st_size = attr.nFileSizeLow;
-
-	return 0;
-}
-
-#endif

src/port/win32stat.c

+/*-------------------------------------------------------------------------
+ *
+ * win32stat.c
+ *	  Replacements for <sys/stat.h> functions using GetFileInformationByHandle
+ *
+ * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ *
+ * IDENTIFICATION
+ *	  src/port/win32stat.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifdef WIN32
+
+#include "c.h"
+#include <windows.h>
+
+/*
+ * In order to support MinGW and MSVC2013 we use NtQueryInformationFile as an
+ * alternative for GetFileInformationByHandleEx. It is loaded from the ntdll
+ * library.
+ */
+#if _WIN32_WINNT < 0x0600
+#include <winternl.h>
+
+#if !defined(__MINGW32__) && !defined(__MINGW64__)
+/* MinGW includes this in <winternl.h>, but it is missing in MSVC */
+typedef struct _FILE_STANDARD_INFORMATION
+{
+	LARGE_INTEGER AllocationSize;
+	LARGE_INTEGER EndOfFile;
+	ULONG		NumberOfLinks;
+	BOOLEAN		DeletePending;
+	BOOLEAN		Directory;
+} FILE_STANDARD_INFORMATION;
+#define FileStandardInformation 5
+#endif							/* !defined(__MINGW32__) &&
+								 * !defined(__MINGW64__) */
+
+typedef NTSTATUS(NTAPI * PFN_NTQUERYINFORMATIONFILE)
+(
+ IN HANDLE FileHandle,
+ OUT PIO_STATUS_BLOCK IoStatusBlock,
+ OUT PVOID FileInformation,
+ IN ULONG Length,
+ IN FILE_INFORMATION_CLASS FileInformationClass
+);
+
+static PFN_NTQUERYINFORMATIONFILE _NtQueryInformationFile = NULL;
+
+static HMODULE ntdll = NULL;
+
+/*
+ * Load DLL file just once regardless of how many functions we load/call in it.
+ */
+static void
+LoadNtdll(void)
+{
+	if (ntdll != NULL)
+		return;
+	ntdll = LoadLibraryEx("ntdll.dll", NULL, 0);
+}
+
+#endif							/* _WIN32_WINNT < 0x0600 */
+
+
+/*
+ * Convert a FILETIME struct into a 64 bit time_t.
+ */
+static __time64_t
+filetime_to_time(const FILETIME *ft)
+{
+	ULARGE_INTEGER unified_ft = {0};
+	static const uint64 EpochShift = UINT64CONST(116444736000000000);
+
+	unified_ft.LowPart = ft->dwLowDateTime;
+	unified_ft.HighPart = ft->dwHighDateTime;
+
+	if (unified_ft.QuadPart < EpochShift)
+		return -1;
+
+	unified_ft.QuadPart -= EpochShift;
+	unified_ft.QuadPart /= 10 * 1000 * 1000;
+
+	return unified_ft.QuadPart;
+}
+
+/*
+ * Convert WIN32 file attributes to a Unix-style mode.
+ *
+ * Only owner permissions are set.
+ */
+static unsigned short
+fileattr_to_unixmode(int attr)
+{
+	unsigned short uxmode = 0;
+
+	uxmode |= (unsigned short) ((attr & FILE_ATTRIBUTE_DIRECTORY) ?
+								(_S_IFDIR) : (_S_IFREG));
+
+	uxmode |= (unsigned short) (attr & FILE_ATTRIBUTE_READONLY) ?
+		(_S_IREAD) : (_S_IREAD | _S_IWRITE);
+
+	/* there is no need to simulate _S_IEXEC using CMD's PATHEXT extensions */
+	uxmode |= _S_IEXEC;
+
+	return uxmode;
+}
+
+/*
+ * Convert WIN32 file information (from a HANDLE) to a struct stat.
+ */
+static int
+fileinfo_to_stat(HANDLE hFile, struct stat *buf)
+{
+	BY_HANDLE_FILE_INFORMATION fiData;
+
+	memset(buf, 0, sizeof(*buf));
+
+	/*
+	 * GetFileInformationByHandle minimum supported version: Windows XP and
+	 * Windows Server 2003, so it exists everywhere we care about.
+	 */
+	if (!GetFileInformationByHandle(hFile, &fiData))
+	{
+		_dosmaperr(GetLastError());
+		return -1;
+	}
+
+	if (fiData.ftLastWriteTime.dwLowDateTime ||
+		fiData.ftLastWriteTime.dwHighDateTime)
+		buf->st_mtime = filetime_to_time(&fiData.ftLastWriteTime);
+
+	if (fiData.ftLastAccessTime.dwLowDateTime ||
+		fiData.ftLastAccessTime.dwHighDateTime)
+		buf->st_atime = filetime_to_time(&fiData.ftLastAccessTime);
+	else
+		buf->st_atime = buf->st_mtime;
+
+	if (fiData.ftCreationTime.dwLowDateTime ||
+		fiData.ftCreationTime.dwHighDateTime)
+		buf->st_ctime = filetime_to_time(&fiData.ftCreationTime);
+	else
+		buf->st_ctime = buf->st_mtime;
+
+	buf->st_mode = fileattr_to_unixmode(fiData.dwFileAttributes);
+	buf->st_nlink = fiData.nNumberOfLinks;
+
+	buf->st_size = (((uint64) fiData.nFileSizeHigh) << 32) |
+		(uint64) fiData.nFileSizeLow;
+
+	return 0;
+}
+
+/*
+ * Windows implementation of stat().
+ *
+ * This currently also implements lstat(), though perhaps that should change.
+ */
+int
+_pgstat64(const char *name, struct stat *buf)
+{
+	/*
+	 * We must use a handle so lstat() returns the information of the target
+	 * file.  To have a reliable test for ERROR_DELETE_PENDING, we use
+	 * NtQueryInformationFile from Windows 2000 or
+	 * GetFileInformationByHandleEx from Server 2008 / Vista.
+	 */
+	SECURITY_ATTRIBUTES sa;
+	HANDLE		hFile;
+	int			ret;
+#if _WIN32_WINNT < 0x0600
+	IO_STATUS_BLOCK ioStatus;
+	FILE_STANDARD_INFORMATION standardInfo;
+#else
+	FILE_STANDARD_INFO standardInfo;
+#endif
+
+	if (name == NULL || buf == NULL)
+	{
+		errno = EINVAL;
+		return -1;
+	}
+
+	/* fast not-exists check */
+	if (GetFileAttributes(name) == INVALID_FILE_ATTRIBUTES)
+	{
+		_dosmaperr(GetLastError());
+		return -1;
+	}
+
+	/* get a file handle as lightweight as we can */
+	sa.nLength = sizeof(SECURITY_ATTRIBUTES);
+	sa.bInheritHandle = TRUE;
+	sa.lpSecurityDescriptor = NULL;
+	hFile = CreateFile(name,
+					   GENERIC_READ,
+					   (FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE),
+					   &sa,
+					   OPEN_EXISTING,
+					   (FILE_FLAG_NO_BUFFERING | FILE_FLAG_BACKUP_SEMANTICS |
+						FILE_FLAG_OVERLAPPED),
+					   NULL);
+	if (hFile == INVALID_HANDLE_VALUE)
+	{
+		CloseHandle(hFile);
+		errno = ENOENT;
+		return -1;
+	}
+
+	memset(&standardInfo, 0, sizeof(standardInfo));
+
+#if _WIN32_WINNT < 0x0600
+	if (_NtQueryInformationFile == NULL)
+	{
+		/* First time through: load ntdll.dll and find NtQueryInformationFile */
+		LoadNtdll();
+		if (ntdll == NULL)
+		{
+			_dosmaperr(GetLastError());
+			CloseHandle(hFile);
+			return -1;
+		}
+
+		_NtQueryInformationFile = (PFN_NTQUERYINFORMATIONFILE)
+			GetProcAddress(ntdll, "NtQueryInformationFile");
+		if (_NtQueryInformationFile == NULL)
+		{
+			_dosmaperr(GetLastError());
+			CloseHandle(hFile);
+			return -1;
+		}
+	}
+
+	if (!NT_SUCCESS(_NtQueryInformationFile(hFile, &ioStatus, &standardInfo,
+											sizeof(standardInfo),
+											FileStandardInformation)))
+	{
+		_dosmaperr(GetLastError());
+		CloseHandle(hFile);
+		return -1;
+	}
+#else
+	if (!GetFileInformationByHandleEx(hFile, FileStandardInfo, &standardInfo,
+									  sizeof(standardInfo)))
+	{
+		_dosmaperr(GetLastError());
+		CloseHandle(hFile);
+		return -1;
+	}
+#endif							/* _WIN32_WINNT < 0x0600 */
+
+	if (standardInfo.DeletePending)
+	{
+		/*
+		 * File has been deleted, but is not gone from the filesystem yet.
+		 * This can happen when some process with FILE_SHARE_DELETE has it
+		 * open, and it will be fully removed once that handle is closed.
+		 * Meanwhile, we can't open it, so indicate that the file just doesn't
+		 * exist.
+		 */
+		CloseHandle(hFile);
+		errno = ENOENT;
+		return -1;
+	}
+
+	/* At last we can invoke fileinfo_to_stat */
+	ret = fileinfo_to_stat(hFile, buf);
+
+	CloseHandle(hFile);
+	return ret;
+}
+
+/*
+ * Windows implementation of fstat().
+ */
+int
+_pgfstat64(int fileno, struct stat *buf)
+{
+	HANDLE		hFile = (HANDLE) _get_osfhandle(fileno);
+
+	if (hFile == INVALID_HANDLE_VALUE || buf == NULL)
+	{
+		errno = EINVAL;
+		return -1;
+	}
+
+	/*
+	 * Since we already have a file handle there is no need to check for
+	 * ERROR_DELETE_PENDING.
+	 */
+
+	return fileinfo_to_stat(hFile, buf);
+}
+
+#endif							/* WIN32 */

src/tools/msvc/Mkvcbuild.pm

@@ -103,7 +103,7 @@ sub mkvcbuild
 	  pg_strong_random.c pgcheckdir.c pgmkdirp.c pgsleep.c pgstrcasecmp.c
 	  pqsignal.c mkdtemp.c qsort.c qsort_arg.c quotes.c system.c
 	  strerror.c tar.c thread.c
-	  win32env.c win32error.c win32security.c win32setlocale.c);
+	  win32env.c win32error.c win32security.c win32setlocale.c win32stat.c);
 
 	push(@pgportfiles, 'strtof.c') if ($vsVersion < '14.00');