xlog.c 213 KB
Newer Older
1
/*-------------------------------------------------------------------------
2 3
 *
 * xlog.c
4
 *		PostgreSQL transaction log manager
5 6
 *
 *
Bruce Momjian's avatar
Bruce Momjian committed
7
 * Portions Copyright (c) 1996-2009, PostgreSQL Global Development Group
Bruce Momjian's avatar
Add:  
Bruce Momjian committed
8
 * Portions Copyright (c) 1994, Regents of the University of California
9
 *
10
 * $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.327 2009/01/11 18:02:17 tgl Exp $
11 12 13
 *
 *-------------------------------------------------------------------------
 */
14

15 16
#include "postgres.h"

17
#include <ctype.h>
Tom Lane's avatar
Tom Lane committed
18
#include <signal.h>
19
#include <time.h>
20
#include <fcntl.h>
21
#include <sys/stat.h>
22
#include <sys/time.h>
23 24
#include <sys/wait.h>
#include <unistd.h>
25

26
#include "access/clog.h"
27
#include "access/multixact.h"
28
#include "access/subtrans.h"
29
#include "access/transam.h"
30
#include "access/tuptoaster.h"
31
#include "access/twophase.h"
32
#include "access/xact.h"
33
#include "access/xlog_internal.h"
34
#include "access/xlogutils.h"
35
#include "catalog/catversion.h"
Tom Lane's avatar
Tom Lane committed
36
#include "catalog/pg_control.h"
37 38
#include "catalog/pg_type.h"
#include "funcapi.h"
39
#include "miscadmin.h"
40
#include "pgstat.h"
41
#include "postmaster/bgwriter.h"
42
#include "storage/bufmgr.h"
43
#include "storage/fd.h"
44
#include "storage/ipc.h"
45
#include "storage/pmsignal.h"
46
#include "storage/procarray.h"
47
#include "storage/smgr.h"
48
#include "storage/spin.h"
49
#include "utils/builtins.h"
50
#include "utils/guc.h"
51
#include "utils/ps_status.h"
52
#include "pg_trace.h"
53

54

55 56
/* File path names (all relative to $PGDATA) */
#define BACKUP_LABEL_FILE		"backup_label"
57
#define BACKUP_LABEL_OLD		"backup_label.old"
58 59 60 61
#define RECOVERY_COMMAND_FILE	"recovery.conf"
#define RECOVERY_COMMAND_DONE	"recovery.done"


Tom Lane's avatar
Tom Lane committed
62 63
/* User-settable parameters */
int			CheckPointSegments = 3;
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
64
int			XLOGbuffers = 8;
65
int			XLogArchiveTimeout = 0;
66
bool		XLogArchiveMode = false;
67
char	   *XLogArchiveCommand = NULL;
68
bool		fullPageWrites = true;
69
bool		log_checkpoints = false;
70
int 		sync_method = DEFAULT_SYNC_METHOD;
Tom Lane's avatar
Tom Lane committed
71

72 73 74 75
#ifdef WAL_DEBUG
bool		XLOG_DEBUG = false;
#endif

76
/*
77 78 79 80 81
 * XLOGfileslop is the maximum number of preallocated future XLOG segments.
 * When we are done with an old XLOG segment file, we will recycle it as a
 * future XLOG segment as long as there aren't already XLOGfileslop future
 * segments; else we'll delete it.  This could be made a separate GUC
 * variable, but at present I think it's sufficient to hardwire it as
Bruce Momjian's avatar
Bruce Momjian committed
82
 * 2*CheckPointSegments+1.	Under normal conditions, a checkpoint will free
83 84 85
 * no more than 2*CheckPointSegments log segments, and we want to recycle all
 * of them; the +1 allows boundary cases to happen without wasting a
 * delete/create-segment cycle.
86 87 88
 */
#define XLOGfileslop	(2*CheckPointSegments + 1)

89 90 91 92
/*
 * GUC support
 */
const struct config_enum_entry sync_method_options[] = {
93
	{"fsync", SYNC_METHOD_FSYNC, false},
94
#ifdef HAVE_FSYNC_WRITETHROUGH
95
	{"fsync_writethrough", SYNC_METHOD_FSYNC_WRITETHROUGH, false},
96 97
#endif
#ifdef HAVE_FDATASYNC
98
	{"fdatasync", SYNC_METHOD_FDATASYNC, false},
99 100
#endif
#ifdef OPEN_SYNC_FLAG
101
	{"open_sync", SYNC_METHOD_OPEN, false},
102 103
#endif
#ifdef OPEN_DATASYNC_FLAG
104
	{"open_datasync", SYNC_METHOD_OPEN_DSYNC, false},
105
#endif
106
	{NULL, 0, false}
107
};
Tom Lane's avatar
Tom Lane committed
108

109 110 111 112 113 114 115
/*
 * Statistics for current checkpoint are collected in this global struct.
 * Because only the background writer or a stand-alone backend can perform
 * checkpoints, this will be unused in normal backends.
 */
CheckpointStatsData CheckpointStats;

Tom Lane's avatar
Tom Lane committed
116
/*
117 118
 * ThisTimeLineID will be same in all backends --- it identifies current
 * WAL timeline for the database system.
Tom Lane's avatar
Tom Lane committed
119
 */
120
TimeLineID	ThisTimeLineID = 0;
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
121

122
/* Are we doing recovery from XLOG? */
Tom Lane's avatar
Tom Lane committed
123
bool		InRecovery = false;
Bruce Momjian's avatar
Bruce Momjian committed
124

125
/* Are we recovering using offline XLOG archives? */
Bruce Momjian's avatar
Bruce Momjian committed
126 127
static bool InArchiveRecovery = false;

128
/* Was the last xlog file restored from archive, or local? */
Bruce Momjian's avatar
Bruce Momjian committed
129
static bool restoredFromArchive = false;
130

131 132
/* options taken from recovery.conf */
static char *recoveryRestoreCommand = NULL;
133 134 135
static bool recoveryTarget = false;
static bool recoveryTargetExact = false;
static bool recoveryTargetInclusive = true;
136
static bool recoveryLogRestartpoints = false;
Bruce Momjian's avatar
Bruce Momjian committed
137
static TransactionId recoveryTargetXid;
138
static TimestampTz recoveryTargetTime;
139
static TimestampTz recoveryLastXTime = 0;
140

141
/* if recoveryStopsHere returns true, it saves actual stop xid/time here */
Bruce Momjian's avatar
Bruce Momjian committed
142
static TransactionId recoveryStopXid;
143
static TimestampTz recoveryStopTime;
Bruce Momjian's avatar
Bruce Momjian committed
144
static bool recoveryStopAfter;
145 146 147 148 149 150 151 152 153 154 155 156 157

/*
 * During normal operation, the only timeline we care about is ThisTimeLineID.
 * During recovery, however, things are more complicated.  To simplify life
 * for rmgr code, we keep ThisTimeLineID set to the "current" timeline as we
 * scan through the WAL history (that is, it is the line that was active when
 * the currently-scanned WAL record was generated).  We also need these
 * timeline values:
 *
 * recoveryTargetTLI: the desired timeline that we want to end in.
 *
 * expectedTLIs: an integer list of recoveryTargetTLI and the TLIs of
 * its known parents, newest first (so recoveryTargetTLI is always the
Bruce Momjian's avatar
Bruce Momjian committed
158
 * first list member).	Only these TLIs are expected to be seen in the WAL
159 160 161 162 163 164 165 166 167
 * segments we read, and indeed only these TLIs will be considered as
 * candidate WAL files to open at all.
 *
 * curFileTLI: the TLI appearing in the name of the current input WAL file.
 * (This is not necessarily the same as ThisTimeLineID, because we could
 * be scanning data that was copied from an ancestor timeline when the current
 * file was created.)  During a sequential scan we do not allow this value
 * to decrease.
 */
Bruce Momjian's avatar
Bruce Momjian committed
168 169 170
static TimeLineID recoveryTargetTLI;
static List *expectedTLIs;
static TimeLineID curFileTLI;
171

Tom Lane's avatar
Tom Lane committed
172 173
/*
 * ProcLastRecPtr points to the start of the last XLOG record inserted by the
174 175 176 177
 * current backend.  It is updated for all inserts.  XactLastRecEnd points to
 * end+1 of the last record, and is reset when we end a top-level transaction,
 * or start a new one; so it can be used to tell if the current transaction has
 * created any XLOG records.
Tom Lane's avatar
Tom Lane committed
178 179
 */
static XLogRecPtr ProcLastRecPtr = {0, 0};
180

181
XLogRecPtr	XactLastRecEnd = {0, 0};
182

Tom Lane's avatar
Tom Lane committed
183 184 185
/*
 * RedoRecPtr is this backend's local copy of the REDO record pointer
 * (which is almost but not quite the same as a pointer to the most recent
186
 * CHECKPOINT record).	We update this from the shared-memory copy,
Tom Lane's avatar
Tom Lane committed
187
 * XLogCtl->Insert.RedoRecPtr, whenever we can safely do so (ie, when we
Bruce Momjian's avatar
Bruce Momjian committed
188
 * hold the Insert lock).  See XLogInsert for details.	We are also allowed
189
 * to update from XLogCtl->Insert.RedoRecPtr if we hold the info_lck;
190 191
 * see GetRedoRecPtr.  A freshly spawned backend obtains the value during
 * InitXLOGAccess.
Tom Lane's avatar
Tom Lane committed
192
 */
193
static XLogRecPtr RedoRecPtr;
194

Tom Lane's avatar
Tom Lane committed
195 196 197 198 199 200 201 202 203
/*----------
 * Shared-memory data structures for XLOG control
 *
 * LogwrtRqst indicates a byte position that we need to write and/or fsync
 * the log up to (all records before that point must be written or fsynced).
 * LogwrtResult indicates the byte positions we have already written/fsynced.
 * These structs are identical but are declared separately to indicate their
 * slightly different functions.
 *
204
 * We do a lot of pushups to minimize the amount of access to lockable
Tom Lane's avatar
Tom Lane committed
205 206 207
 * shared memory values.  There are actually three shared-memory copies of
 * LogwrtResult, plus one unshared copy in each backend.  Here's how it works:
 *		XLogCtl->LogwrtResult is protected by info_lck
208 209 210 211
 *		XLogCtl->Write.LogwrtResult is protected by WALWriteLock
 *		XLogCtl->Insert.LogwrtResult is protected by WALInsertLock
 * One must hold the associated lock to read or write any of these, but
 * of course no lock is needed to read/write the unshared LogwrtResult.
Tom Lane's avatar
Tom Lane committed
212 213 214
 *
 * XLogCtl->LogwrtResult and XLogCtl->Write.LogwrtResult are both "always
 * right", since both are updated by a write or flush operation before
215 216
 * it releases WALWriteLock.  The point of keeping XLogCtl->Write.LogwrtResult
 * is that it can be examined/modified by code that already holds WALWriteLock
Tom Lane's avatar
Tom Lane committed
217 218 219
 * without needing to grab info_lck as well.
 *
 * XLogCtl->Insert.LogwrtResult may lag behind the reality of the other two,
220
 * but is updated when convenient.	Again, it exists for the convenience of
221
 * code that is already holding WALInsertLock but not the other locks.
Tom Lane's avatar
Tom Lane committed
222 223 224 225 226 227 228 229 230 231
 *
 * The unshared LogwrtResult may lag behind any or all of these, and again
 * is updated when convenient.
 *
 * The request bookkeeping is simpler: there is a shared XLogCtl->LogwrtRqst
 * (protected by info_lck), but we don't need to cache any copies of it.
 *
 * Note that this all works because the request and result positions can only
 * advance forward, never back up, and so we can easily determine which of two
 * values is "more up to date".
232 233 234 235 236 237 238 239 240 241 242 243 244 245
 *
 * info_lck is only held long enough to read/update the protected variables,
 * so it's a plain spinlock.  The other locks are held longer (potentially
 * over I/O operations), so we use LWLocks for them.  These locks are:
 *
 * WALInsertLock: must be held to insert a record into the WAL buffers.
 *
 * WALWriteLock: must be held to write WAL buffers to disk (XLogWrite or
 * XLogFlush).
 *
 * ControlFileLock: must be held to read/update control file or create
 * new log file.
 *
 * CheckpointLock: must be held to do a checkpoint (ensures only one
246 247
 * checkpointer at a time; currently, with all checkpoints done by the
 * bgwriter, this is just pro forma).
248
 *
Tom Lane's avatar
Tom Lane committed
249 250
 *----------
 */
251

Tom Lane's avatar
Tom Lane committed
252
typedef struct XLogwrtRqst
253
{
Tom Lane's avatar
Tom Lane committed
254 255
	XLogRecPtr	Write;			/* last byte + 1 to write out */
	XLogRecPtr	Flush;			/* last byte + 1 to flush */
256
} XLogwrtRqst;
257

258 259 260 261 262 263
typedef struct XLogwrtResult
{
	XLogRecPtr	Write;			/* last byte + 1 written out */
	XLogRecPtr	Flush;			/* last byte + 1 flushed */
} XLogwrtResult;

Tom Lane's avatar
Tom Lane committed
264 265 266
/*
 * Shared state data for XLogInsert.
 */
267 268
typedef struct XLogCtlInsert
{
269 270
	XLogwrtResult LogwrtResult; /* a recent value of LogwrtResult */
	XLogRecPtr	PrevRecord;		/* start of previously-inserted record */
271
	int			curridx;		/* current block index in cache */
272 273 274
	XLogPageHeader currpage;	/* points to header of block in cache */
	char	   *currpos;		/* current insertion point in cache */
	XLogRecPtr	RedoRecPtr;		/* current redo point for insertions */
275
	bool		forcePageWrites;	/* forcing full-page writes for PITR? */
276 277
} XLogCtlInsert;

Tom Lane's avatar
Tom Lane committed
278 279 280
/*
 * Shared state data for XLogWrite/XLogFlush.
 */
281 282
typedef struct XLogCtlWrite
{
Bruce Momjian's avatar
Bruce Momjian committed
283 284
	XLogwrtResult LogwrtResult; /* current value of LogwrtResult */
	int			curridx;		/* cache index of next block to write */
285
	pg_time_t	lastSegSwitchTime;		/* time of last xlog segment switch */
286 287
} XLogCtlWrite;

Tom Lane's avatar
Tom Lane committed
288 289 290
/*
 * Total shared-memory state for XLOG.
 */
291 292
typedef struct XLogCtlData
{
293
	/* Protected by WALInsertLock: */
294
	XLogCtlInsert Insert;
295

Tom Lane's avatar
Tom Lane committed
296
	/* Protected by info_lck: */
297 298
	XLogwrtRqst LogwrtRqst;
	XLogwrtResult LogwrtResult;
299 300
	uint32		ckptXidEpoch;	/* nextXID & epoch of latest checkpoint */
	TransactionId ckptXid;
Bruce Momjian's avatar
Bruce Momjian committed
301
	XLogRecPtr	asyncCommitLSN; /* LSN of newest async commit */
302

303
	/* Protected by WALWriteLock: */
304 305
	XLogCtlWrite Write;

Tom Lane's avatar
Tom Lane committed
306
	/*
307 308 309
	 * These values do not change after startup, although the pointed-to pages
	 * and xlblocks values certainly do.  Permission to read/write the pages
	 * and xlblocks values depends on WALInsertLock and WALWriteLock.
Tom Lane's avatar
Tom Lane committed
310
	 */
311
	char	   *pages;			/* buffers for unwritten XLOG pages */
312
	XLogRecPtr *xlblocks;		/* 1st byte ptr-s + XLOG_BLCKSZ */
313
	int			XLogCacheBlck;	/* highest allocated xlog buffer index */
314
	TimeLineID	ThisTimeLineID;
Tom Lane's avatar
Tom Lane committed
315

316
	slock_t		info_lck;		/* locks shared variables shown above */
317 318
} XLogCtlData;

319
static XLogCtlData *XLogCtl = NULL;
320

321
/*
Tom Lane's avatar
Tom Lane committed
322
 * We maintain an image of pg_control in shared memory.
323
 */
324
static ControlFileData *ControlFile = NULL;
325

Tom Lane's avatar
Tom Lane committed
326 327 328 329 330
/*
 * Macros for managing XLogInsert state.  In most cases, the calling routine
 * has local copies of XLogCtl->Insert and/or XLogCtl->Insert->curridx,
 * so these are passed as parameters instead of being fetched via XLogCtl.
 */
331

Tom Lane's avatar
Tom Lane committed
332 333
/* Free space remaining in the current xlog page buffer */
#define INSERT_FREESPACE(Insert)  \
334
	(XLOG_BLCKSZ - ((Insert)->currpos - (char *) (Insert)->currpage))
Tom Lane's avatar
Tom Lane committed
335 336 337 338 339 340

/* Construct XLogRecPtr value for current insertion point */
#define INSERT_RECPTR(recptr,Insert,curridx)  \
	( \
	  (recptr).xlogid = XLogCtl->xlblocks[curridx].xlogid, \
	  (recptr).xrecoff = \
341
		XLogCtl->xlblocks[curridx].xrecoff - INSERT_FREESPACE(Insert) \
Tom Lane's avatar
Tom Lane committed
342 343 344 345 346 347 348
	)

#define PrevBufIdx(idx)		\
		(((idx) == 0) ? XLogCtl->XLogCacheBlck : ((idx) - 1))

#define NextBufIdx(idx)		\
		(((idx) == XLogCtl->XLogCacheBlck) ? 0 : ((idx) + 1))
349

Tom Lane's avatar
Tom Lane committed
350 351 352 353
/*
 * Private, possibly out-of-date copy of shared LogwrtResult.
 * See discussion above.
 */
354
static XLogwrtResult LogwrtResult = {{0, 0}, {0, 0}};
355

Tom Lane's avatar
Tom Lane committed
356 357 358 359 360 361 362 363 364 365
/*
 * openLogFile is -1 or a kernel FD for an open log file segment.
 * When it's open, openLogOff is the current seek offset in the file.
 * openLogId/openLogSeg identify the segment.  These variables are only
 * used to write the XLOG, and so will normally refer to the active segment.
 */
static int	openLogFile = -1;
static uint32 openLogId = 0;
static uint32 openLogSeg = 0;
static uint32 openLogOff = 0;
366

Tom Lane's avatar
Tom Lane committed
367 368 369 370 371 372
/*
 * These variables are used similarly to the ones above, but for reading
 * the XLOG.  Note, however, that readOff generally represents the offset
 * of the page just read, not the seek position of the FD itself, which
 * will be just past that page.
 */
373 374 375 376
static int	readFile = -1;
static uint32 readId = 0;
static uint32 readSeg = 0;
static uint32 readOff = 0;
377

378
/* Buffer for currently read page (XLOG_BLCKSZ bytes) */
Tom Lane's avatar
Tom Lane committed
379
static char *readBuf = NULL;
380

381 382 383 384
/* Buffer for current ReadRecord result (expandable) */
static char *readRecordBuf = NULL;
static uint32 readRecordBufSize = 0;

Tom Lane's avatar
Tom Lane committed
385
/* State information for XLOG reading */
386 387
static XLogRecPtr ReadRecPtr;	/* start of last record read */
static XLogRecPtr EndRecPtr;	/* end+1 of last record read */
388
static XLogRecord *nextRecord = NULL;
389
static TimeLineID lastPageTLI = 0;
390

Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
391 392
static bool InRedo = false;

393

394 395
static void XLogArchiveNotify(const char *xlog);
static void XLogArchiveNotifySeg(uint32 log, uint32 seg);
396 397
static bool XLogArchiveCheckDone(const char *xlog);
static bool XLogArchiveIsBusy(const char *xlog);
398 399
static void XLogArchiveCleanup(const char *xlog);
static void readRecoveryCommandFile(void);
400
static void exitArchiveRecovery(TimeLineID endTLI,
Bruce Momjian's avatar
Bruce Momjian committed
401
					uint32 endLogId, uint32 endLogSeg);
402
static bool recoveryStopsHere(XLogRecord *record, bool *includeThis);
403
static void CheckPointGuts(XLogRecPtr checkPointRedo, int flags);
Tom Lane's avatar
Tom Lane committed
404

405
static bool XLogCheckBuffer(XLogRecData *rdata, bool doPageWrites,
406
				XLogRecPtr *lsn, BkpBlock *bkpb);
407 408
static bool AdvanceXLInsertBuffer(bool new_segment);
static void XLogWrite(XLogwrtRqst WriteRqst, bool flexible, bool xlog_switch);
409 410
static int XLogFileInit(uint32 log, uint32 seg,
			 bool *use_existent, bool use_lock);
411 412
static bool InstallXLogFileSegment(uint32 *log, uint32 *seg, char *tmppath,
					   bool find_free, int *max_advance,
413
					   bool use_lock);
414 415
static int	XLogFileOpen(uint32 log, uint32 seg);
static int	XLogFileRead(uint32 log, uint32 seg, int emode);
Bruce Momjian's avatar
Bruce Momjian committed
416
static void XLogFileClose(void);
417
static bool RestoreArchivedFile(char *path, const char *xlogfname,
Bruce Momjian's avatar
Bruce Momjian committed
418
					const char *recovername, off_t expectedSize);
419 420
static void PreallocXlogFiles(XLogRecPtr endptr);
static void RemoveOldXlogFiles(uint32 log, uint32 seg, XLogRecPtr endptr);
421
static void ValidateXLOGDirectoryStructure(void);
422
static void CleanupBackupHistory(void);
423
static XLogRecord *ReadRecord(XLogRecPtr *RecPtr, int emode);
424
static bool ValidXLOGHeader(XLogPageHeader hdr, int emode);
425
static XLogRecord *ReadCheckpointRecord(XLogRecPtr RecPtr, int whichChkpt);
426 427 428 429
static List *readTimeLineHistory(TimeLineID targetTLI);
static bool existsTimeLineHistory(TimeLineID probeTLI);
static TimeLineID findNewestTimeLine(TimeLineID startTLI);
static void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI,
Bruce Momjian's avatar
Bruce Momjian committed
430 431
					 TimeLineID endTLI,
					 uint32 endLogId, uint32 endLogSeg);
Tom Lane's avatar
Tom Lane committed
432 433
static void WriteControlFile(void);
static void ReadControlFile(void);
434
static char *str_time(pg_time_t tnow);
435
#ifdef WAL_DEBUG
436
static void xlog_outrec(StringInfo buf, XLogRecord *record);
437
#endif
438 439
static void issue_xlog_fsync(void);
static void pg_start_backup_callback(int code, Datum arg);
440
static bool read_backup_label(XLogRecPtr *checkPointLoc,
Bruce Momjian's avatar
Bruce Momjian committed
441
				  XLogRecPtr *minRecoveryLoc);
442
static void rm_redo_error_callback(void *arg);
443
static int get_sync_bit(int method);
Tom Lane's avatar
Tom Lane committed
444 445 446 447 448


/*
 * Insert an XLOG record having the specified RMID and info bytes,
 * with the body of the record being the data chunk(s) described by
449
 * the rdata chain (see xlog.h for notes about rdata).
Tom Lane's avatar
Tom Lane committed
450 451 452 453 454 455 456 457 458 459 460
 *
 * Returns XLOG pointer to end of record (beginning of next record).
 * This can be used as LSN for data pages affected by the logged action.
 * (LSN is the XLOG point up to which the XLOG must be flushed to disk
 * before the data page can be written out.  This implements the basic
 * WAL rule "write the log before the data".)
 *
 * NB: this routine feels free to scribble on the XLogRecData structs,
 * though not on the data they reference.  This is OK since the XLogRecData
 * structs are always just temporaries in the calling code.
 */
461
XLogRecPtr
462
XLogInsert(RmgrId rmid, uint8 info, XLogRecData *rdata)
463
{
464 465
	XLogCtlInsert *Insert = &XLogCtl->Insert;
	XLogRecord *record;
Tom Lane's avatar
Tom Lane committed
466
	XLogContRecord *contrecord;
467 468 469
	XLogRecPtr	RecPtr;
	XLogRecPtr	WriteRqst;
	uint32		freespace;
470
	int			curridx;
471 472 473 474 475
	XLogRecData *rdt;
	Buffer		dtbuf[XLR_MAX_BKP_BLOCKS];
	bool		dtbuf_bkp[XLR_MAX_BKP_BLOCKS];
	BkpBlock	dtbuf_xlg[XLR_MAX_BKP_BLOCKS];
	XLogRecPtr	dtbuf_lsn[XLR_MAX_BKP_BLOCKS];
476 477 478 479
	XLogRecData dtbuf_rdt1[XLR_MAX_BKP_BLOCKS];
	XLogRecData dtbuf_rdt2[XLR_MAX_BKP_BLOCKS];
	XLogRecData dtbuf_rdt3[XLR_MAX_BKP_BLOCKS];
	pg_crc32	rdata_crc;
480 481 482 483
	uint32		len,
				write_len;
	unsigned	i;
	bool		updrqst;
484
	bool		doPageWrites;
485
	bool		isLogSwitch = (rmid == RM_XLOG_ID && info == XLOG_SWITCH);
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
486

487
	/* info's high bits are reserved for use by me */
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
488
	if (info & XLR_INFO_MASK)
489
		elog(PANIC, "invalid xlog info mask %02X", info);
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
490

491 492
	TRACE_POSTGRESQL_XLOG_INSERT(rmid, info);

Tom Lane's avatar
Tom Lane committed
493
	/*
494 495
	 * In bootstrap mode, we don't actually log anything but XLOG resources;
	 * return a phony record pointer.
Tom Lane's avatar
Tom Lane committed
496
	 */
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
497
	if (IsBootstrapProcessingMode() && rmid != RM_XLOG_ID)
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
498 499
	{
		RecPtr.xlogid = 0;
500
		RecPtr.xrecoff = SizeOfXLogLongPHD;		/* start of 1st chkpt record */
501
		return RecPtr;
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
502 503
	}

Tom Lane's avatar
Tom Lane committed
504
	/*
505
	 * Here we scan the rdata chain, determine which buffers must be backed
Tom Lane's avatar
Tom Lane committed
506
	 * up, and compute the CRC values for the data.  Note that the record
507 508 509 510
	 * header isn't added into the CRC initially since we don't know the final
	 * length or info bits quite yet.  Thus, the CRC will represent the CRC of
	 * the whole record in the order "rdata, then backup blocks, then record
	 * header".
Tom Lane's avatar
Tom Lane committed
511
	 *
512 513 514 515 516
	 * We may have to loop back to here if a race condition is detected below.
	 * We could prevent the race by doing all this work while holding the
	 * insert lock, but it seems better to avoid doing CRC calculations while
	 * holding the lock.  This means we have to be careful about modifying the
	 * rdata chain until we know we aren't going to loop back again.  The only
517 518 519 520 521
	 * change we allow ourselves to make earlier is to set rdt->data = NULL in
	 * chain items we have decided we will have to back up the whole buffer
	 * for.  This is OK because we will certainly decide the same thing again
	 * for those items if we do it over; doing it here saves an extra pass
	 * over the chain later.
Tom Lane's avatar
Tom Lane committed
522
	 */
523
begin:;
Tom Lane's avatar
Tom Lane committed
524 525 526 527 528 529
	for (i = 0; i < XLR_MAX_BKP_BLOCKS; i++)
	{
		dtbuf[i] = InvalidBuffer;
		dtbuf_bkp[i] = false;
	}

530 531 532 533 534 535 536 537
	/*
	 * Decide if we need to do full-page writes in this XLOG record: true if
	 * full_page_writes is on or we have a PITR request for it.  Since we
	 * don't yet have the insert lock, forcePageWrites could change under us,
	 * but we'll recheck it once we have the lock.
	 */
	doPageWrites = fullPageWrites || Insert->forcePageWrites;

538
	INIT_CRC32(rdata_crc);
Tom Lane's avatar
Tom Lane committed
539
	len = 0;
540
	for (rdt = rdata;;)
541 542 543
	{
		if (rdt->buffer == InvalidBuffer)
		{
Tom Lane's avatar
Tom Lane committed
544
			/* Simple data, just include it */
545
			len += rdt->len;
546
			COMP_CRC32(rdata_crc, rdt->data, rdt->len);
547
		}
Tom Lane's avatar
Tom Lane committed
548
		else
549
		{
Tom Lane's avatar
Tom Lane committed
550 551
			/* Find info for buffer */
			for (i = 0; i < XLR_MAX_BKP_BLOCKS; i++)
552
			{
Tom Lane's avatar
Tom Lane committed
553
				if (rdt->buffer == dtbuf[i])
554
				{
555
					/* Buffer already referenced by earlier chain item */
Tom Lane's avatar
Tom Lane committed
556 557 558 559 560
					if (dtbuf_bkp[i])
						rdt->data = NULL;
					else if (rdt->data)
					{
						len += rdt->len;
561
						COMP_CRC32(rdata_crc, rdt->data, rdt->len);
Tom Lane's avatar
Tom Lane committed
562 563
					}
					break;
564
				}
Tom Lane's avatar
Tom Lane committed
565
				if (dtbuf[i] == InvalidBuffer)
566
				{
Tom Lane's avatar
Tom Lane committed
567 568
					/* OK, put it in this slot */
					dtbuf[i] = rdt->buffer;
569 570
					if (XLogCheckBuffer(rdt, doPageWrites,
										&(dtbuf_lsn[i]), &(dtbuf_xlg[i])))
Tom Lane's avatar
Tom Lane committed
571 572 573 574 575 576 577
					{
						dtbuf_bkp[i] = true;
						rdt->data = NULL;
					}
					else if (rdt->data)
					{
						len += rdt->len;
578
						COMP_CRC32(rdata_crc, rdt->data, rdt->len);
Tom Lane's avatar
Tom Lane committed
579 580
					}
					break;
581 582
				}
			}
Tom Lane's avatar
Tom Lane committed
583
			if (i >= XLR_MAX_BKP_BLOCKS)
584
				elog(PANIC, "can backup at most %d blocks per xlog record",
Tom Lane's avatar
Tom Lane committed
585
					 XLR_MAX_BKP_BLOCKS);
586
		}
587
		/* Break out of loop when rdt points to last chain item */
588 589 590 591 592
		if (rdt->next == NULL)
			break;
		rdt = rdt->next;
	}

593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625
	/*
	 * Now add the backup block headers and data into the CRC
	 */
	for (i = 0; i < XLR_MAX_BKP_BLOCKS; i++)
	{
		if (dtbuf_bkp[i])
		{
			BkpBlock   *bkpb = &(dtbuf_xlg[i]);
			char	   *page;

			COMP_CRC32(rdata_crc,
					   (char *) bkpb,
					   sizeof(BkpBlock));
			page = (char *) BufferGetBlock(dtbuf[i]);
			if (bkpb->hole_length == 0)
			{
				COMP_CRC32(rdata_crc,
						   page,
						   BLCKSZ);
			}
			else
			{
				/* must skip the hole */
				COMP_CRC32(rdata_crc,
						   page,
						   bkpb->hole_offset);
				COMP_CRC32(rdata_crc,
						   page + (bkpb->hole_offset + bkpb->hole_length),
						   BLCKSZ - (bkpb->hole_offset + bkpb->hole_length));
			}
		}
	}

Tom Lane's avatar
Tom Lane committed
626
	/*
627 628
	 * NOTE: We disallow len == 0 because it provides a useful bit of extra
	 * error checking in ReadRecord.  This means that all callers of
Bruce Momjian's avatar
Bruce Momjian committed
629 630 631
	 * XLogInsert must supply at least some not-in-a-buffer data.  However, we
	 * make an exception for XLOG SWITCH records because we don't want them to
	 * ever cross a segment boundary.
Tom Lane's avatar
Tom Lane committed
632
	 */
633
	if (len == 0 && !isLogSwitch)
634
		elog(PANIC, "invalid xlog record length %u", len);
635

636
	START_CRIT_SECTION();
637

638 639 640
	/* Now wait to get insert lock */
	LWLockAcquire(WALInsertLock, LW_EXCLUSIVE);

Tom Lane's avatar
Tom Lane committed
641
	/*
642 643 644
	 * Check to see if my RedoRecPtr is out of date.  If so, may have to go
	 * back and recompute everything.  This can only happen just after a
	 * checkpoint, so it's better to be slow in this case and fast otherwise.
645 646
	 *
	 * If we aren't doing full-page writes then RedoRecPtr doesn't actually
Bruce Momjian's avatar
Bruce Momjian committed
647 648
	 * affect the contents of the XLOG record, so we'll update our local copy
	 * but not force a recomputation.
Tom Lane's avatar
Tom Lane committed
649 650
	 */
	if (!XLByteEQ(RedoRecPtr, Insert->RedoRecPtr))
651
	{
Tom Lane's avatar
Tom Lane committed
652 653 654
		Assert(XLByteLT(RedoRecPtr, Insert->RedoRecPtr));
		RedoRecPtr = Insert->RedoRecPtr;

655
		if (doPageWrites)
656
		{
657
			for (i = 0; i < XLR_MAX_BKP_BLOCKS; i++)
Tom Lane's avatar
Tom Lane committed
658
			{
659 660 661 662 663 664 665 666 667 668 669 670 671
				if (dtbuf[i] == InvalidBuffer)
					continue;
				if (dtbuf_bkp[i] == false &&
					XLByteLE(dtbuf_lsn[i], RedoRecPtr))
				{
					/*
					 * Oops, this buffer now needs to be backed up, but we
					 * didn't think so above.  Start over.
					 */
					LWLockRelease(WALInsertLock);
					END_CRIT_SECTION();
					goto begin;
				}
Tom Lane's avatar
Tom Lane committed
672
			}
673 674 675
		}
	}

676
	/*
Bruce Momjian's avatar
Bruce Momjian committed
677 678 679 680
	 * Also check to see if forcePageWrites was just turned on; if we weren't
	 * already doing full-page writes then go back and recompute. (If it was
	 * just turned off, we could recompute the record without full pages, but
	 * we choose not to bother.)
681 682 683 684 685 686 687 688 689
	 */
	if (Insert->forcePageWrites && !doPageWrites)
	{
		/* Oops, must redo it with full-page data */
		LWLockRelease(WALInsertLock);
		END_CRIT_SECTION();
		goto begin;
	}

Tom Lane's avatar
Tom Lane committed
690
	/*
691 692 693 694
	 * Make additional rdata chain entries for the backup blocks, so that we
	 * don't need to special-case them in the write loop.  Note that we have
	 * now irrevocably changed the input rdata chain.  At the exit of this
	 * loop, write_len includes the backup block data.
Tom Lane's avatar
Tom Lane committed
695
	 *
696 697 698
	 * Also set the appropriate info bits to show which buffers were backed
	 * up. The i'th XLR_SET_BKP_BLOCK bit corresponds to the i'th distinct
	 * buffer value (ignoring InvalidBuffer) appearing in the rdata chain.
Tom Lane's avatar
Tom Lane committed
699 700 701
	 */
	write_len = len;
	for (i = 0; i < XLR_MAX_BKP_BLOCKS; i++)
702
	{
703 704 705
		BkpBlock   *bkpb;
		char	   *page;

706
		if (!dtbuf_bkp[i])
707 708
			continue;

Tom Lane's avatar
Tom Lane committed
709
		info |= XLR_SET_BKP_BLOCK(i);
710

711 712 713 714 715
		bkpb = &(dtbuf_xlg[i]);
		page = (char *) BufferGetBlock(dtbuf[i]);

		rdt->next = &(dtbuf_rdt1[i]);
		rdt = rdt->next;
716

717 718
		rdt->data = (char *) bkpb;
		rdt->len = sizeof(BkpBlock);
Tom Lane's avatar
Tom Lane committed
719
		write_len += sizeof(BkpBlock);
720

721 722
		rdt->next = &(dtbuf_rdt2[i]);
		rdt = rdt->next;
723

724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745
		if (bkpb->hole_length == 0)
		{
			rdt->data = page;
			rdt->len = BLCKSZ;
			write_len += BLCKSZ;
			rdt->next = NULL;
		}
		else
		{
			/* must skip the hole */
			rdt->data = page;
			rdt->len = bkpb->hole_offset;
			write_len += bkpb->hole_offset;

			rdt->next = &(dtbuf_rdt3[i]);
			rdt = rdt->next;

			rdt->data = page + (bkpb->hole_offset + bkpb->hole_length);
			rdt->len = BLCKSZ - (bkpb->hole_offset + bkpb->hole_length);
			write_len += rdt->len;
			rdt->next = NULL;
		}
746 747
	}

748 749 750 751 752 753 754
	/*
	 * If we backed up any full blocks and online backup is not in progress,
	 * mark the backup blocks as removable.  This allows the WAL archiver to
	 * know whether it is safe to compress archived WAL data by transforming
	 * full-block records into the non-full-block format.
	 *
	 * Note: we could just set the flag whenever !forcePageWrites, but
Bruce Momjian's avatar
Bruce Momjian committed
755 756
	 * defining it like this leaves the info bit free for some potential other
	 * use in records without any backup blocks.
757 758 759 760
	 */
	if ((info & XLR_BKP_BLOCK_MASK) && !Insert->forcePageWrites)
		info |= XLR_BKP_REMOVABLE;

761
	/*
762
	 * If there isn't enough space on the current XLOG page for a record
763
	 * header, advance to the next page (leaving the unused space as zeroes).
764
	 */
Tom Lane's avatar
Tom Lane committed
765 766
	updrqst = false;
	freespace = INSERT_FREESPACE(Insert);
767 768
	if (freespace < SizeOfXLogRecord)
	{
769
		updrqst = AdvanceXLInsertBuffer(false);
770 771 772
		freespace = INSERT_FREESPACE(Insert);
	}

773
	/* Compute record's XLOG location */
Tom Lane's avatar
Tom Lane committed
774
	curridx = Insert->curridx;
775 776 777
	INSERT_RECPTR(RecPtr, Insert, curridx);

	/*
Bruce Momjian's avatar
Bruce Momjian committed
778 779 780 781 782
	 * If the record is an XLOG_SWITCH, and we are exactly at the start of a
	 * segment, we need not insert it (and don't want to because we'd like
	 * consecutive switch requests to be no-ops).  Instead, make sure
	 * everything is written and flushed through the end of the prior segment,
	 * and return the prior segment's end address.
783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813
	 */
	if (isLogSwitch &&
		(RecPtr.xrecoff % XLogSegSize) == SizeOfXLogLongPHD)
	{
		/* We can release insert lock immediately */
		LWLockRelease(WALInsertLock);

		RecPtr.xrecoff -= SizeOfXLogLongPHD;
		if (RecPtr.xrecoff == 0)
		{
			/* crossing a logid boundary */
			RecPtr.xlogid -= 1;
			RecPtr.xrecoff = XLogFileSize;
		}

		LWLockAcquire(WALWriteLock, LW_EXCLUSIVE);
		LogwrtResult = XLogCtl->Write.LogwrtResult;
		if (!XLByteLE(RecPtr, LogwrtResult.Flush))
		{
			XLogwrtRqst FlushRqst;

			FlushRqst.Write = RecPtr;
			FlushRqst.Flush = RecPtr;
			XLogWrite(FlushRqst, false, false);
		}
		LWLockRelease(WALWriteLock);

		END_CRIT_SECTION();

		return RecPtr;
	}
Tom Lane's avatar
Tom Lane committed
814

815 816
	/* Insert record header */

817
	record = (XLogRecord *) Insert->currpos;
818
	record->xl_prev = Insert->PrevRecord;
819
	record->xl_xid = GetCurrentTransactionIdIfAny();
820
	record->xl_tot_len = SizeOfXLogRecord + write_len;
Tom Lane's avatar
Tom Lane committed
821
	record->xl_len = len;		/* doesn't include backup blocks */
822
	record->xl_info = info;
823
	record->xl_rmid = rmid;
824

825 826 827 828
	/* Now we can finish computing the record's CRC */
	COMP_CRC32(rdata_crc, (char *) record + sizeof(pg_crc32),
			   SizeOfXLogRecord - sizeof(pg_crc32));
	FIN_CRC32(rdata_crc);
829 830
	record->xl_crc = rdata_crc;

831
#ifdef WAL_DEBUG
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
832 833
	if (XLOG_DEBUG)
	{
Bruce Momjian's avatar
Bruce Momjian committed
834
		StringInfoData buf;
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
835

836
		initStringInfo(&buf);
837 838
		appendStringInfo(&buf, "INSERT @ %X/%X: ",
						 RecPtr.xlogid, RecPtr.xrecoff);
839
		xlog_outrec(&buf, record);
840
		if (rdata->data != NULL)
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
841
		{
842 843
			appendStringInfo(&buf, " - ");
			RmgrTable[record->xl_rmid].rm_desc(&buf, record->xl_info, rdata->data);
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
844
		}
845 846
		elog(LOG, "%s", buf.data);
		pfree(buf.data);
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
847
	}
848
#endif
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
849

Tom Lane's avatar
Tom Lane committed
850 851 852 853
	/* Record begin of record in appropriate places */
	ProcLastRecPtr = RecPtr;
	Insert->PrevRecord = RecPtr;

854
	Insert->currpos += SizeOfXLogRecord;
Tom Lane's avatar
Tom Lane committed
855
	freespace -= SizeOfXLogRecord;
856

Tom Lane's avatar
Tom Lane committed
857 858 859 860
	/*
	 * Append the data, including backup blocks if any
	 */
	while (write_len)
861
	{
862 863 864 865
		while (rdata->data == NULL)
			rdata = rdata->next;

		if (freespace > 0)
866
		{
867 868 869 870 871
			if (rdata->len > freespace)
			{
				memcpy(Insert->currpos, rdata->data, freespace);
				rdata->data += freespace;
				rdata->len -= freespace;
Tom Lane's avatar
Tom Lane committed
872
				write_len -= freespace;
873 874 875 876 877
			}
			else
			{
				memcpy(Insert->currpos, rdata->data, rdata->len);
				freespace -= rdata->len;
Tom Lane's avatar
Tom Lane committed
878
				write_len -= rdata->len;
879 880 881 882
				Insert->currpos += rdata->len;
				rdata = rdata->next;
				continue;
			}
883 884
		}

885
		/* Use next buffer */
886
		updrqst = AdvanceXLInsertBuffer(false);
Tom Lane's avatar
Tom Lane committed
887 888 889 890 891 892
		curridx = Insert->curridx;
		/* Insert cont-record header */
		Insert->currpage->xlp_info |= XLP_FIRST_IS_CONTRECORD;
		contrecord = (XLogContRecord *) Insert->currpos;
		contrecord->xl_rem_len = write_len;
		Insert->currpos += SizeOfXLogContRecord;
893
		freespace = INSERT_FREESPACE(Insert);
894
	}
895

Tom Lane's avatar
Tom Lane committed
896 897
	/* Ensure next record will be properly aligned */
	Insert->currpos = (char *) Insert->currpage +
898
		MAXALIGN(Insert->currpos - (char *) Insert->currpage);
Tom Lane's avatar
Tom Lane committed
899
	freespace = INSERT_FREESPACE(Insert);
900

Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
901
	/*
902 903
	 * The recptr I return is the beginning of the *next* record. This will be
	 * stored as LSN for changed data pages...
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
904
	 */
Tom Lane's avatar
Tom Lane committed
905
	INSERT_RECPTR(RecPtr, Insert, curridx);
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
906

907 908 909 910 911 912 913 914 915 916 917 918 919 920
	/*
	 * If the record is an XLOG_SWITCH, we must now write and flush all the
	 * existing data, and then forcibly advance to the start of the next
	 * segment.  It's not good to do this I/O while holding the insert lock,
	 * but there seems too much risk of confusion if we try to release the
	 * lock sooner.  Fortunately xlog switch needn't be a high-performance
	 * operation anyway...
	 */
	if (isLogSwitch)
	{
		XLogCtlWrite *Write = &XLogCtl->Write;
		XLogwrtRqst FlushRqst;
		XLogRecPtr	OldSegEnd;

921 922
		TRACE_POSTGRESQL_XLOG_SWITCH();

923 924 925
		LWLockAcquire(WALWriteLock, LW_EXCLUSIVE);

		/*
Bruce Momjian's avatar
Bruce Momjian committed
926 927
		 * Flush through the end of the page containing XLOG_SWITCH, and
		 * perform end-of-segment actions (eg, notifying archiver).
928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977
		 */
		WriteRqst = XLogCtl->xlblocks[curridx];
		FlushRqst.Write = WriteRqst;
		FlushRqst.Flush = WriteRqst;
		XLogWrite(FlushRqst, false, true);

		/* Set up the next buffer as first page of next segment */
		/* Note: AdvanceXLInsertBuffer cannot need to do I/O here */
		(void) AdvanceXLInsertBuffer(true);

		/* There should be no unwritten data */
		curridx = Insert->curridx;
		Assert(curridx == Write->curridx);

		/* Compute end address of old segment */
		OldSegEnd = XLogCtl->xlblocks[curridx];
		OldSegEnd.xrecoff -= XLOG_BLCKSZ;
		if (OldSegEnd.xrecoff == 0)
		{
			/* crossing a logid boundary */
			OldSegEnd.xlogid -= 1;
			OldSegEnd.xrecoff = XLogFileSize;
		}

		/* Make it look like we've written and synced all of old segment */
		LogwrtResult.Write = OldSegEnd;
		LogwrtResult.Flush = OldSegEnd;

		/*
		 * Update shared-memory status --- this code should match XLogWrite
		 */
		{
			/* use volatile pointer to prevent code rearrangement */
			volatile XLogCtlData *xlogctl = XLogCtl;

			SpinLockAcquire(&xlogctl->info_lck);
			xlogctl->LogwrtResult = LogwrtResult;
			if (XLByteLT(xlogctl->LogwrtRqst.Write, LogwrtResult.Write))
				xlogctl->LogwrtRqst.Write = LogwrtResult.Write;
			if (XLByteLT(xlogctl->LogwrtRqst.Flush, LogwrtResult.Flush))
				xlogctl->LogwrtRqst.Flush = LogwrtResult.Flush;
			SpinLockRelease(&xlogctl->info_lck);
		}

		Write->LogwrtResult = LogwrtResult;

		LWLockRelease(WALWriteLock);

		updrqst = false;		/* done already */
	}
978
	else
979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994
	{
		/* normal case, ie not xlog switch */

		/* Need to update shared LogwrtRqst if some block was filled up */
		if (freespace < SizeOfXLogRecord)
		{
			/* curridx is filled and available for writing out */
			updrqst = true;
		}
		else
		{
			/* if updrqst already set, write through end of previous buf */
			curridx = PrevBufIdx(curridx);
		}
		WriteRqst = XLogCtl->xlblocks[curridx];
	}
995

996
	LWLockRelease(WALInsertLock);
997 998 999

	if (updrqst)
	{
1000 1001 1002
		/* use volatile pointer to prevent code rearrangement */
		volatile XLogCtlData *xlogctl = XLogCtl;

1003
		SpinLockAcquire(&xlogctl->info_lck);
Tom Lane's avatar
Tom Lane committed
1004
		/* advance global request to include new block(s) */
1005 1006
		if (XLByteLT(xlogctl->LogwrtRqst.Write, WriteRqst))
			xlogctl->LogwrtRqst.Write = WriteRqst;
Tom Lane's avatar
Tom Lane committed
1007
		/* update local result copy while I have the chance */
1008
		LogwrtResult = xlogctl->LogwrtResult;
1009
		SpinLockRelease(&xlogctl->info_lck);
1010 1011
	}

1012
	XactLastRecEnd = RecPtr;
1013

1014
	END_CRIT_SECTION();
1015

1016
	return RecPtr;
1017
}
1018

1019
/*
1020 1021 1022
 * Determine whether the buffer referenced by an XLogRecData item has to
 * be backed up, and if so fill a BkpBlock struct for it.  In any case
 * save the buffer's LSN at *lsn.
1023
 */
1024
static bool
1025
XLogCheckBuffer(XLogRecData *rdata, bool doPageWrites,
1026
				XLogRecPtr *lsn, BkpBlock *bkpb)
1027
{
1028
	Page		page;
1029

1030
	page = BufferGetPage(rdata->buffer);
1031 1032

	/*
1033 1034 1035
	 * XXX We assume page LSN is first data on *every* page that can be passed
	 * to XLogInsert, whether it otherwise has the standard page layout or
	 * not.
1036
	 */
1037
	*lsn = PageGetLSN(page);
1038

1039
	if (doPageWrites &&
1040
		XLByteLE(PageGetLSN(page), RedoRecPtr))
1041
	{
1042 1043 1044
		/*
		 * The page needs to be backed up, so set up *bkpb
		 */
1045
		BufferGetTag(rdata->buffer, &bkpb->node, &bkpb->fork, &bkpb->block);
1046

1047 1048 1049
		if (rdata->buffer_std)
		{
			/* Assume we can omit data between pd_lower and pd_upper */
1050 1051
			uint16		lower = ((PageHeader) page)->pd_lower;
			uint16		upper = ((PageHeader) page)->pd_upper;
1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072

			if (lower >= SizeOfPageHeaderData &&
				upper > lower &&
				upper <= BLCKSZ)
			{
				bkpb->hole_offset = lower;
				bkpb->hole_length = upper - lower;
			}
			else
			{
				/* No "hole" to compress out */
				bkpb->hole_offset = 0;
				bkpb->hole_length = 0;
			}
		}
		else
		{
			/* Not a standard page header, don't try to eliminate "hole" */
			bkpb->hole_offset = 0;
			bkpb->hole_length = 0;
		}
1073

1074
		return true;			/* buffer requires backup */
1075
	}
1076 1077

	return false;				/* buffer does not need to be backed up */
1078 1079
}

1080 1081 1082 1083 1084 1085
/*
 * XLogArchiveNotify
 *
 * Create an archive notification file
 *
 * The name of the notification file is the message that will be picked up
1086
 * by the archiver, e.g. we write 0000000100000001000000C6.ready
1087
 * and the archiver then knows to archive XLOGDIR/0000000100000001000000C6,
1088
 * then when complete, rename it to 0000000100000001000000C6.done
1089 1090 1091 1092 1093
 */
static void
XLogArchiveNotify(const char *xlog)
{
	char		archiveStatusPath[MAXPGPATH];
Bruce Momjian's avatar
Bruce Momjian committed
1094
	FILE	   *fd;
1095 1096 1097 1098

	/* insert an otherwise empty file called <XLOG>.ready */
	StatusFilePath(archiveStatusPath, xlog, ".ready");
	fd = AllocateFile(archiveStatusPath, "w");
Bruce Momjian's avatar
Bruce Momjian committed
1099 1100
	if (fd == NULL)
	{
1101 1102 1103 1104 1105 1106
		ereport(LOG,
				(errcode_for_file_access(),
				 errmsg("could not create archive status file \"%s\": %m",
						archiveStatusPath)));
		return;
	}
Bruce Momjian's avatar
Bruce Momjian committed
1107 1108
	if (FreeFile(fd))
	{
1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128
		ereport(LOG,
				(errcode_for_file_access(),
				 errmsg("could not write archive status file \"%s\": %m",
						archiveStatusPath)));
		return;
	}

	/* Notify archiver that it's got something to do */
	if (IsUnderPostmaster)
		SendPostmasterSignal(PMSIGNAL_WAKEN_ARCHIVER);
}

/*
 * Convenience routine to notify using log/seg representation of filename
 */
static void
XLogArchiveNotifySeg(uint32 log, uint32 seg)
{
	char		xlog[MAXFNAMELEN];

1129
	XLogFileName(xlog, ThisTimeLineID, log, seg);
1130 1131 1132 1133
	XLogArchiveNotify(xlog);
}

/*
1134
 * XLogArchiveCheckDone
1135
 *
1136 1137 1138 1139
 * This is called when we are ready to delete or recycle an old XLOG segment
 * file or backup history file.  If it is okay to delete it then return true.
 * If it is not time to delete it, make sure a .ready file exists, and return
 * false.
1140 1141
 *
 * If <XLOG>.done exists, then return true; else if <XLOG>.ready exists,
1142 1143 1144 1145
 * then return false; else create <XLOG>.ready and return false.
 *
 * The reason we do things this way is so that if the original attempt to
 * create <XLOG>.ready fails, we'll retry during subsequent checkpoints.
1146 1147
 */
static bool
1148
XLogArchiveCheckDone(const char *xlog)
1149 1150 1151 1152
{
	char		archiveStatusPath[MAXPGPATH];
	struct stat stat_buf;

1153 1154 1155 1156 1157
	/* Always deletable if archiving is off */
	if (!XLogArchivingActive())
		return true;

	/* First check for .done --- this means archiver is done with it */
1158 1159 1160 1161 1162 1163 1164
	StatusFilePath(archiveStatusPath, xlog, ".done");
	if (stat(archiveStatusPath, &stat_buf) == 0)
		return true;

	/* check for .ready --- this means archiver is still busy with it */
	StatusFilePath(archiveStatusPath, xlog, ".ready");
	if (stat(archiveStatusPath, &stat_buf) == 0)
Bruce Momjian's avatar
Bruce Momjian committed
1165
		return false;
1166 1167 1168 1169 1170 1171 1172

	/* Race condition --- maybe archiver just finished, so recheck */
	StatusFilePath(archiveStatusPath, xlog, ".done");
	if (stat(archiveStatusPath, &stat_buf) == 0)
		return true;

	/* Retry creation of the .ready file */
1173
	XLogArchiveNotify(xlog);
1174 1175 1176
	return false;
}

1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220
/*
 * XLogArchiveIsBusy
 *
 * Check to see if an XLOG segment file is still unarchived.
 * This is almost but not quite the inverse of XLogArchiveCheckDone: in
 * the first place we aren't chartered to recreate the .ready file, and
 * in the second place we should consider that if the file is already gone
 * then it's not busy.  (This check is needed to handle the race condition
 * that a checkpoint already deleted the no-longer-needed file.)
 */
static bool
XLogArchiveIsBusy(const char *xlog)
{
	char		archiveStatusPath[MAXPGPATH];
	struct stat stat_buf;

	/* First check for .done --- this means archiver is done with it */
	StatusFilePath(archiveStatusPath, xlog, ".done");
	if (stat(archiveStatusPath, &stat_buf) == 0)
		return false;

	/* check for .ready --- this means archiver is still busy with it */
	StatusFilePath(archiveStatusPath, xlog, ".ready");
	if (stat(archiveStatusPath, &stat_buf) == 0)
		return true;

	/* Race condition --- maybe archiver just finished, so recheck */
	StatusFilePath(archiveStatusPath, xlog, ".done");
	if (stat(archiveStatusPath, &stat_buf) == 0)
		return false;

	/*
	 * Check to see if the WAL file has been removed by checkpoint,
	 * which implies it has already been archived, and explains why we
	 * can't see a status file for it.
	 */
	snprintf(archiveStatusPath, MAXPGPATH, XLOGDIR "/%s", xlog);
	if (stat(archiveStatusPath, &stat_buf) != 0 &&
		errno == ENOENT)
		return false;

	return true;
}

1221 1222 1223
/*
 * XLogArchiveCleanup
 *
1224
 * Cleanup archive notification file(s) for a particular xlog segment
1225 1226 1227 1228
 */
static void
XLogArchiveCleanup(const char *xlog)
{
Bruce Momjian's avatar
Bruce Momjian committed
1229
	char		archiveStatusPath[MAXPGPATH];
1230

1231
	/* Remove the .done file */
1232 1233 1234
	StatusFilePath(archiveStatusPath, xlog, ".done");
	unlink(archiveStatusPath);
	/* should we complain about failure? */
1235 1236 1237 1238 1239

	/* Remove the .ready file if present --- normally it shouldn't be */
	StatusFilePath(archiveStatusPath, xlog, ".ready");
	unlink(archiveStatusPath);
	/* should we complain about failure? */
1240 1241
}

Tom Lane's avatar
Tom Lane committed
1242 1243 1244 1245
/*
 * Advance the Insert state to the next buffer page, writing out the next
 * buffer if it still contains unwritten data.
 *
1246 1247 1248 1249
 * If new_segment is TRUE then we set up the next buffer page as the first
 * page of the next xlog segment file, possibly but not usually the next
 * consecutive file page.
 *
Tom Lane's avatar
Tom Lane committed
1250
 * The global LogwrtRqst.Write pointer needs to be advanced to include the
1251
 * just-filled page.  If we can do this for free (without an extra lock),
Tom Lane's avatar
Tom Lane committed
1252 1253 1254
 * we do so here.  Otherwise the caller must do it.  We return TRUE if the
 * request update still needs to be done, FALSE if we did it internally.
 *
1255
 * Must be called with WALInsertLock held.
Tom Lane's avatar
Tom Lane committed
1256 1257
 */
static bool
1258
AdvanceXLInsertBuffer(bool new_segment)
1259
{
Tom Lane's avatar
Tom Lane committed
1260 1261
	XLogCtlInsert *Insert = &XLogCtl->Insert;
	XLogCtlWrite *Write = &XLogCtl->Write;
1262
	int			nextidx = NextBufIdx(Insert->curridx);
Tom Lane's avatar
Tom Lane committed
1263 1264 1265
	bool		update_needed = true;
	XLogRecPtr	OldPageRqstPtr;
	XLogwrtRqst WriteRqst;
1266 1267
	XLogRecPtr	NewPageEndPtr;
	XLogPageHeader NewPage;
1268

Tom Lane's avatar
Tom Lane committed
1269 1270 1271
	/* Use Insert->LogwrtResult copy if it's more fresh */
	if (XLByteLT(LogwrtResult.Write, Insert->LogwrtResult.Write))
		LogwrtResult = Insert->LogwrtResult;
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
1272

Tom Lane's avatar
Tom Lane committed
1273
	/*
1274 1275 1276
	 * Get ending-offset of the buffer page we need to replace (this may be
	 * zero if the buffer hasn't been used yet).  Fall through if it's already
	 * written out.
Tom Lane's avatar
Tom Lane committed
1277 1278 1279 1280 1281 1282
	 */
	OldPageRqstPtr = XLogCtl->xlblocks[nextidx];
	if (!XLByteLE(OldPageRqstPtr, LogwrtResult.Write))
	{
		/* nope, got work to do... */
		XLogRecPtr	FinishedPageRqstPtr;
1283

Tom Lane's avatar
Tom Lane committed
1284
		FinishedPageRqstPtr = XLogCtl->xlblocks[Insert->curridx];
1285

1286
		/* Before waiting, get info_lck and update LogwrtResult */
1287 1288 1289 1290
		{
			/* use volatile pointer to prevent code rearrangement */
			volatile XLogCtlData *xlogctl = XLogCtl;

1291
			SpinLockAcquire(&xlogctl->info_lck);
1292 1293 1294
			if (XLByteLT(xlogctl->LogwrtRqst.Write, FinishedPageRqstPtr))
				xlogctl->LogwrtRqst.Write = FinishedPageRqstPtr;
			LogwrtResult = xlogctl->LogwrtResult;
1295
			SpinLockRelease(&xlogctl->info_lck);
1296
		}
1297 1298 1299 1300 1301 1302 1303 1304 1305

		update_needed = false;	/* Did the shared-request update */

		if (XLByteLE(OldPageRqstPtr, LogwrtResult.Write))
		{
			/* OK, someone wrote it already */
			Insert->LogwrtResult = LogwrtResult;
		}
		else
1306
		{
1307 1308 1309 1310
			/* Must acquire write lock */
			LWLockAcquire(WALWriteLock, LW_EXCLUSIVE);
			LogwrtResult = Write->LogwrtResult;
			if (XLByteLE(OldPageRqstPtr, LogwrtResult.Write))
1311
			{
1312 1313 1314
				/* OK, someone wrote it already */
				LWLockRelease(WALWriteLock);
				Insert->LogwrtResult = LogwrtResult;
Tom Lane's avatar
Tom Lane committed
1315
			}
1316
			else
Tom Lane's avatar
Tom Lane committed
1317 1318
			{
				/*
1319 1320
				 * Have to write buffers while holding insert lock. This is
				 * not good, so only write as much as we absolutely must.
Tom Lane's avatar
Tom Lane committed
1321
				 */
1322
				TRACE_POSTGRESQL_WAL_BUFFER_WRITE_DIRTY_START();
Tom Lane's avatar
Tom Lane committed
1323 1324 1325
				WriteRqst.Write = OldPageRqstPtr;
				WriteRqst.Flush.xlogid = 0;
				WriteRqst.Flush.xrecoff = 0;
1326
				XLogWrite(WriteRqst, false, false);
1327
				LWLockRelease(WALWriteLock);
Tom Lane's avatar
Tom Lane committed
1328
				Insert->LogwrtResult = LogwrtResult;
1329
				TRACE_POSTGRESQL_WAL_BUFFER_WRITE_DIRTY_DONE();
1330 1331 1332 1333
			}
		}
	}

Tom Lane's avatar
Tom Lane committed
1334
	/*
1335 1336
	 * Now the next buffer slot is free and we can set it up to be the next
	 * output page.
Tom Lane's avatar
Tom Lane committed
1337
	 */
1338
	NewPageEndPtr = XLogCtl->xlblocks[Insert->curridx];
1339 1340 1341 1342 1343 1344 1345 1346

	if (new_segment)
	{
		/* force it to a segment start point */
		NewPageEndPtr.xrecoff += XLogSegSize - 1;
		NewPageEndPtr.xrecoff -= NewPageEndPtr.xrecoff % XLogSegSize;
	}

1347
	if (NewPageEndPtr.xrecoff >= XLogFileSize)
1348
	{
Tom Lane's avatar
Tom Lane committed
1349
		/* crossing a logid boundary */
1350
		NewPageEndPtr.xlogid += 1;
1351
		NewPageEndPtr.xrecoff = XLOG_BLCKSZ;
1352
	}
Tom Lane's avatar
Tom Lane committed
1353
	else
1354
		NewPageEndPtr.xrecoff += XLOG_BLCKSZ;
1355
	XLogCtl->xlblocks[nextidx] = NewPageEndPtr;
1356
	NewPage = (XLogPageHeader) (XLogCtl->pages + nextidx * (Size) XLOG_BLCKSZ);
1357

Tom Lane's avatar
Tom Lane committed
1358
	Insert->curridx = nextidx;
1359
	Insert->currpage = NewPage;
1360 1361

	Insert->currpos = ((char *) NewPage) +SizeOfXLogShortPHD;
1362

Tom Lane's avatar
Tom Lane committed
1363
	/*
1364 1365
	 * Be sure to re-zero the buffer so that bytes beyond what we've written
	 * will look like zeroes and not valid XLOG records...
Tom Lane's avatar
Tom Lane committed
1366
	 */
1367
	MemSet((char *) NewPage, 0, XLOG_BLCKSZ);
1368

1369 1370 1371
	/*
	 * Fill the new page's header
	 */
1372 1373
	NewPage   ->xlp_magic = XLOG_PAGE_MAGIC;

1374
	/* NewPage->xlp_info = 0; */	/* done by memset */
1375 1376
	NewPage   ->xlp_tli = ThisTimeLineID;
	NewPage   ->xlp_pageaddr.xlogid = NewPageEndPtr.xlogid;
1377
	NewPage   ->xlp_pageaddr.xrecoff = NewPageEndPtr.xrecoff - XLOG_BLCKSZ;
Tom Lane's avatar
Tom Lane committed
1378

1379
	/*
1380
	 * If first page of an XLOG segment file, make it a long header.
1381 1382 1383
	 */
	if ((NewPage->xlp_pageaddr.xrecoff % XLogSegSize) == 0)
	{
1384
		XLogLongPageHeader NewLongPage = (XLogLongPageHeader) NewPage;
1385

1386 1387
		NewLongPage->xlp_sysid = ControlFile->system_identifier;
		NewLongPage->xlp_seg_size = XLogSegSize;
1388
		NewLongPage->xlp_xlog_blcksz = XLOG_BLCKSZ;
1389 1390 1391
		NewPage   ->xlp_info |= XLP_LONG_HEADER;

		Insert->currpos = ((char *) NewPage) +SizeOfXLogLongPHD;
1392 1393
	}

Tom Lane's avatar
Tom Lane committed
1394
	return update_needed;
1395 1396
}

1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409
/*
 * Check whether we've consumed enough xlog space that a checkpoint is needed.
 *
 * Caller must have just finished filling the open log file (so that
 * openLogId/openLogSeg are valid).  We measure the distance from RedoRecPtr
 * to the open log file and see if that exceeds CheckPointSegments.
 *
 * Note: it is caller's responsibility that RedoRecPtr is up-to-date.
 */
static bool
XLogCheckpointNeeded(void)
{
	/*
1410 1411
	 * A straight computation of segment number could overflow 32 bits. Rather
	 * than assuming we have working 64-bit arithmetic, we compare the
Bruce Momjian's avatar
Bruce Momjian committed
1412 1413
	 * highest-order bits separately, and force a checkpoint immediately when
	 * they change.
1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425
	 */
	uint32		old_segno,
				new_segno;
	uint32		old_highbits,
				new_highbits;

	old_segno = (RedoRecPtr.xlogid % XLogSegSize) * XLogSegsPerFile +
		(RedoRecPtr.xrecoff / XLogSegSize);
	old_highbits = RedoRecPtr.xlogid / XLogSegSize;
	new_segno = (openLogId % XLogSegSize) * XLogSegsPerFile + openLogSeg;
	new_highbits = openLogId / XLogSegSize;
	if (new_highbits != old_highbits ||
Bruce Momjian's avatar
Bruce Momjian committed
1426
		new_segno >= old_segno + (uint32) (CheckPointSegments - 1))
1427 1428 1429 1430
		return true;
	return false;
}

Tom Lane's avatar
Tom Lane committed
1431 1432 1433
/*
 * Write and/or fsync the log at least as far as WriteRqst indicates.
 *
1434 1435 1436 1437 1438
 * If flexible == TRUE, we don't have to write as far as WriteRqst, but
 * may stop at any convenient boundary (such as a cache or logfile boundary).
 * This option allows us to avoid uselessly issuing multiple writes when a
 * single one would do.
 *
1439 1440 1441 1442 1443 1444
 * If xlog_switch == TRUE, we are intending an xlog segment switch, so
 * perform end-of-segment actions after writing the last page, even if
 * it's not physically the end of its segment.  (NB: this will work properly
 * only if caller specifies WriteRqst == page-end and flexible == false,
 * and there is some data to write.)
 *
1445
 * Must be called with WALWriteLock held.
Tom Lane's avatar
Tom Lane committed
1446
 */
1447
static void
1448
XLogWrite(XLogwrtRqst WriteRqst, bool flexible, bool xlog_switch)
1449
{
1450
	XLogCtlWrite *Write = &XLogCtl->Write;
Tom Lane's avatar
Tom Lane committed
1451
	bool		ispartialpage;
1452
	bool		last_iteration;
1453
	bool		finishing_seg;
1454
	bool		use_existent;
1455 1456 1457 1458
	int			curridx;
	int			npages;
	int			startidx;
	uint32		startoffset;
1459

1460 1461 1462
	/* We should always be inside a critical section here */
	Assert(CritSectionCount > 0);

1463
	/*
1464
	 * Update local LogwrtResult (caller probably did this already, but...)
1465
	 */
Tom Lane's avatar
Tom Lane committed
1466 1467
	LogwrtResult = Write->LogwrtResult;

1468 1469 1470
	/*
	 * Since successive pages in the xlog cache are consecutively allocated,
	 * we can usually gather multiple pages together and issue just one
1471 1472 1473 1474 1475
	 * write() call.  npages is the number of pages we have determined can be
	 * written together; startidx is the cache block index of the first one,
	 * and startoffset is the file offset at which it should go. The latter
	 * two variables are only valid when npages > 0, but we must initialize
	 * all of them to keep the compiler quiet.
1476 1477 1478 1479 1480 1481 1482 1483 1484
	 */
	npages = 0;
	startidx = 0;
	startoffset = 0;

	/*
	 * Within the loop, curridx is the cache block index of the page to
	 * consider writing.  We advance Write->curridx only after successfully
	 * writing pages.  (Right now, this refinement is useless since we are
1485 1486
	 * going to PANIC if any error occurs anyway; but someday it may come in
	 * useful.)
1487 1488
	 */
	curridx = Write->curridx;
1489

Tom Lane's avatar
Tom Lane committed
1490
	while (XLByteLT(LogwrtResult.Write, WriteRqst.Write))
1491
	{
1492
		/*
1493 1494 1495
		 * Make sure we're not ahead of the insert process.  This could happen
		 * if we're passed a bogus WriteRqst.Write that is past the end of the
		 * last page that's been initialized by AdvanceXLInsertBuffer.
1496
		 */
1497
		if (!XLByteLT(LogwrtResult.Write, XLogCtl->xlblocks[curridx]))
1498
			elog(PANIC, "xlog write request %X/%X is past end of log %X/%X",
1499
				 LogwrtResult.Write.xlogid, LogwrtResult.Write.xrecoff,
1500 1501
				 XLogCtl->xlblocks[curridx].xlogid,
				 XLogCtl->xlblocks[curridx].xrecoff);
1502

Tom Lane's avatar
Tom Lane committed
1503
		/* Advance LogwrtResult.Write to end of current buffer page */
1504
		LogwrtResult.Write = XLogCtl->xlblocks[curridx];
Tom Lane's avatar
Tom Lane committed
1505 1506 1507
		ispartialpage = XLByteLT(WriteRqst.Write, LogwrtResult.Write);

		if (!XLByteInPrevSeg(LogwrtResult.Write, openLogId, openLogSeg))
1508
		{
Tom Lane's avatar
Tom Lane committed
1509
			/*
1510 1511
			 * Switch to new logfile segment.  We cannot have any pending
			 * pages here (since we dump what we have at segment end).
Tom Lane's avatar
Tom Lane committed
1512
			 */
1513
			Assert(npages == 0);
Tom Lane's avatar
Tom Lane committed
1514
			if (openLogFile >= 0)
1515
				XLogFileClose();
Tom Lane's avatar
Tom Lane committed
1516 1517
			XLByteToPrevSeg(LogwrtResult.Write, openLogId, openLogSeg);

1518 1519 1520 1521
			/* create/use new log file */
			use_existent = true;
			openLogFile = XLogFileInit(openLogId, openLogSeg,
									   &use_existent, true);
Tom Lane's avatar
Tom Lane committed
1522
			openLogOff = 0;
1523 1524
		}

1525
		/* Make sure we have the current logfile open */
Tom Lane's avatar
Tom Lane committed
1526
		if (openLogFile < 0)
1527
		{
Tom Lane's avatar
Tom Lane committed
1528
			XLByteToPrevSeg(LogwrtResult.Write, openLogId, openLogSeg);
1529
			openLogFile = XLogFileOpen(openLogId, openLogSeg);
Tom Lane's avatar
Tom Lane committed
1530
			openLogOff = 0;
1531 1532
		}

1533 1534 1535 1536 1537
		/* Add current page to the set of pending pages-to-dump */
		if (npages == 0)
		{
			/* first of group */
			startidx = curridx;
1538
			startoffset = (LogwrtResult.Write.xrecoff - XLOG_BLCKSZ) % XLogSegSize;
1539 1540
		}
		npages++;
1541

Tom Lane's avatar
Tom Lane committed
1542
		/*
1543 1544 1545 1546
		 * Dump the set if this will be the last loop iteration, or if we are
		 * at the last page of the cache area (since the next page won't be
		 * contiguous in memory), or if we are at the end of the logfile
		 * segment.
Tom Lane's avatar
Tom Lane committed
1547
		 */
1548 1549
		last_iteration = !XLByteLT(LogwrtResult.Write, WriteRqst.Write);

1550
		finishing_seg = !ispartialpage &&
1551
			(startoffset + npages * XLOG_BLCKSZ) >= XLogSegSize;
1552

1553
		if (last_iteration ||
1554 1555
			curridx == XLogCtl->XLogCacheBlck ||
			finishing_seg)
Tom Lane's avatar
Tom Lane committed
1556
		{
1557 1558
			char	   *from;
			Size		nbytes;
1559

1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572
			/* Need to seek in the file? */
			if (openLogOff != startoffset)
			{
				if (lseek(openLogFile, (off_t) startoffset, SEEK_SET) < 0)
					ereport(PANIC,
							(errcode_for_file_access(),
							 errmsg("could not seek in log file %u, "
									"segment %u to offset %u: %m",
									openLogId, openLogSeg, startoffset)));
				openLogOff = startoffset;
			}

			/* OK to write the page(s) */
1573 1574
			from = XLogCtl->pages + startidx * (Size) XLOG_BLCKSZ;
			nbytes = npages * (Size) XLOG_BLCKSZ;
1575 1576 1577 1578 1579 1580 1581 1582 1583
			errno = 0;
			if (write(openLogFile, from, nbytes) != nbytes)
			{
				/* if write didn't set errno, assume no disk space */
				if (errno == 0)
					errno = ENOSPC;
				ereport(PANIC,
						(errcode_for_file_access(),
						 errmsg("could not write to log file %u, segment %u "
Peter Eisentraut's avatar
Peter Eisentraut committed
1584
								"at offset %u, length %lu: %m",
1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600
								openLogId, openLogSeg,
								openLogOff, (unsigned long) nbytes)));
			}

			/* Update state for write */
			openLogOff += nbytes;
			Write->curridx = ispartialpage ? curridx : NextBufIdx(curridx);
			npages = 0;

			/*
			 * If we just wrote the whole last page of a logfile segment,
			 * fsync the segment immediately.  This avoids having to go back
			 * and re-open prior segments when an fsync request comes along
			 * later. Doing it here ensures that one and only one backend will
			 * perform this fsync.
			 *
1601 1602 1603
			 * We also do this if this is the last page written for an xlog
			 * switch.
			 *
1604
			 * This is also the right place to notify the Archiver that the
Bruce Momjian's avatar
Bruce Momjian committed
1605
			 * segment is ready to copy to archival storage, and to update the
1606 1607 1608
			 * timer for archive_timeout, and to signal for a checkpoint if
			 * too many logfile segments have been used since the last
			 * checkpoint.
1609
			 */
1610
			if (finishing_seg || (xlog_switch && last_iteration))
1611 1612
			{
				issue_xlog_fsync();
1613
				LogwrtResult.Flush = LogwrtResult.Write;		/* end of page */
1614 1615 1616

				if (XLogArchivingActive())
					XLogArchiveNotifySeg(openLogId, openLogSeg);
1617

1618
				Write->lastSegSwitchTime = (pg_time_t) time(NULL);
1619 1620

				/*
1621
				 * Signal bgwriter to start a checkpoint if we've consumed too
1622
				 * much xlog since the last one.  For speed, we first check
Bruce Momjian's avatar
Bruce Momjian committed
1623 1624 1625
				 * using the local copy of RedoRecPtr, which might be out of
				 * date; if it looks like a checkpoint is needed, forcibly
				 * update RedoRecPtr and recheck.
1626
				 */
1627 1628
				if (IsUnderPostmaster &&
					XLogCheckpointNeeded())
1629
				{
1630 1631
					(void) GetRedoRecPtr();
					if (XLogCheckpointNeeded())
1632
						RequestCheckpoint(CHECKPOINT_CAUSE_XLOG);
1633
				}
1634
			}
Tom Lane's avatar
Tom Lane committed
1635
		}
1636

Tom Lane's avatar
Tom Lane committed
1637 1638 1639 1640 1641 1642
		if (ispartialpage)
		{
			/* Only asked to write a partial page */
			LogwrtResult.Write = WriteRqst.Write;
			break;
		}
1643 1644 1645 1646 1647
		curridx = NextBufIdx(curridx);

		/* If flexible, break out of loop as soon as we wrote something */
		if (flexible && npages == 0)
			break;
1648
	}
1649 1650 1651

	Assert(npages == 0);
	Assert(curridx == Write->curridx);
1652

Tom Lane's avatar
Tom Lane committed
1653 1654 1655 1656 1657
	/*
	 * If asked to flush, do so
	 */
	if (XLByteLT(LogwrtResult.Flush, WriteRqst.Flush) &&
		XLByteLT(LogwrtResult.Flush, LogwrtResult.Write))
1658
	{
Tom Lane's avatar
Tom Lane committed
1659
		/*
1660 1661 1662
		 * Could get here without iterating above loop, in which case we might
		 * have no open file or the wrong one.	However, we do not need to
		 * fsync more than one file.
Tom Lane's avatar
Tom Lane committed
1663
		 */
1664 1665
		if (sync_method != SYNC_METHOD_OPEN &&
			sync_method != SYNC_METHOD_OPEN_DSYNC)
Tom Lane's avatar
Tom Lane committed
1666
		{
1667
			if (openLogFile >= 0 &&
1668
				!XLByteInPrevSeg(LogwrtResult.Write, openLogId, openLogSeg))
1669
				XLogFileClose();
1670 1671 1672
			if (openLogFile < 0)
			{
				XLByteToPrevSeg(LogwrtResult.Write, openLogId, openLogSeg);
1673
				openLogFile = XLogFileOpen(openLogId, openLogSeg);
1674 1675 1676
				openLogOff = 0;
			}
			issue_xlog_fsync();
Tom Lane's avatar
Tom Lane committed
1677 1678
		}
		LogwrtResult.Flush = LogwrtResult.Write;
1679 1680
	}

Tom Lane's avatar
Tom Lane committed
1681 1682 1683
	/*
	 * Update shared-memory status
	 *
1684
	 * We make sure that the shared 'request' values do not fall behind the
1685 1686
	 * 'result' values.  This is not absolutely essential, but it saves some
	 * code in a couple of places.
Tom Lane's avatar
Tom Lane committed
1687
	 */
1688 1689 1690 1691
	{
		/* use volatile pointer to prevent code rearrangement */
		volatile XLogCtlData *xlogctl = XLogCtl;

1692
		SpinLockAcquire(&xlogctl->info_lck);
1693 1694 1695 1696 1697
		xlogctl->LogwrtResult = LogwrtResult;
		if (XLByteLT(xlogctl->LogwrtRqst.Write, LogwrtResult.Write))
			xlogctl->LogwrtRqst.Write = LogwrtResult.Write;
		if (XLByteLT(xlogctl->LogwrtRqst.Flush, LogwrtResult.Flush))
			xlogctl->LogwrtRqst.Flush = LogwrtResult.Flush;
1698
		SpinLockRelease(&xlogctl->info_lck);
1699
	}
1700

Tom Lane's avatar
Tom Lane committed
1701 1702 1703
	Write->LogwrtResult = LogwrtResult;
}

1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719
/*
 * Record the LSN for an asynchronous transaction commit.
 * (This should not be called for aborts, nor for synchronous commits.)
 */
void
XLogSetAsyncCommitLSN(XLogRecPtr asyncCommitLSN)
{
	/* use volatile pointer to prevent code rearrangement */
	volatile XLogCtlData *xlogctl = XLogCtl;

	SpinLockAcquire(&xlogctl->info_lck);
	if (XLByteLT(xlogctl->asyncCommitLSN, asyncCommitLSN))
		xlogctl->asyncCommitLSN = asyncCommitLSN;
	SpinLockRelease(&xlogctl->info_lck);
}

Tom Lane's avatar
Tom Lane committed
1720 1721 1722
/*
 * Ensure that all XLOG data through the given position is flushed to disk.
 *
1723
 * NOTE: this differs from XLogWrite mainly in that the WALWriteLock is not
Tom Lane's avatar
Tom Lane committed
1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739
 * already held, and we try to avoid acquiring it if possible.
 */
void
XLogFlush(XLogRecPtr record)
{
	XLogRecPtr	WriteRqstPtr;
	XLogwrtRqst WriteRqst;

	/* Disabled during REDO */
	if (InRedo)
		return;

	/* Quick exit if already known flushed */
	if (XLByteLE(record, LogwrtResult.Flush))
		return;

1740
#ifdef WAL_DEBUG
1741
	if (XLOG_DEBUG)
1742
		elog(LOG, "xlog flush request %X/%X; write %X/%X; flush %X/%X",
1743 1744 1745
			 record.xlogid, record.xrecoff,
			 LogwrtResult.Write.xlogid, LogwrtResult.Write.xrecoff,
			 LogwrtResult.Flush.xlogid, LogwrtResult.Flush.xrecoff);
1746
#endif
1747

Tom Lane's avatar
Tom Lane committed
1748 1749 1750 1751
	START_CRIT_SECTION();

	/*
	 * Since fsync is usually a horribly expensive operation, we try to
1752 1753 1754 1755
	 * piggyback as much data as we can on each fsync: if we see any more data
	 * entered into the xlog buffer, we'll write and fsync that too, so that
	 * the final value of LogwrtResult.Flush is as large as possible. This
	 * gives us some chance of avoiding another fsync immediately after.
Tom Lane's avatar
Tom Lane committed
1756 1757 1758 1759 1760
	 */

	/* initialize to given target; may increase below */
	WriteRqstPtr = record;

1761
	/* read LogwrtResult and update local state */
1762 1763 1764 1765
	{
		/* use volatile pointer to prevent code rearrangement */
		volatile XLogCtlData *xlogctl = XLogCtl;

1766
		SpinLockAcquire(&xlogctl->info_lck);
1767 1768 1769
		if (XLByteLT(WriteRqstPtr, xlogctl->LogwrtRqst.Write))
			WriteRqstPtr = xlogctl->LogwrtRqst.Write;
		LogwrtResult = xlogctl->LogwrtResult;
1770
		SpinLockRelease(&xlogctl->info_lck);
1771
	}
1772 1773 1774

	/* done already? */
	if (!XLByteLE(record, LogwrtResult.Flush))
Tom Lane's avatar
Tom Lane committed
1775
	{
1776 1777 1778 1779
		/* now wait for the write lock */
		LWLockAcquire(WALWriteLock, LW_EXCLUSIVE);
		LogwrtResult = XLogCtl->Write.LogwrtResult;
		if (!XLByteLE(record, LogwrtResult.Flush))
Tom Lane's avatar
Tom Lane committed
1780
		{
1781 1782 1783 1784 1785 1786
			/* try to write/flush later additions to XLOG as well */
			if (LWLockConditionalAcquire(WALInsertLock, LW_EXCLUSIVE))
			{
				XLogCtlInsert *Insert = &XLogCtl->Insert;
				uint32		freespace = INSERT_FREESPACE(Insert);

Bruce Momjian's avatar
Bruce Momjian committed
1787
				if (freespace < SizeOfXLogRecord)		/* buffer is full */
1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802
					WriteRqstPtr = XLogCtl->xlblocks[Insert->curridx];
				else
				{
					WriteRqstPtr = XLogCtl->xlblocks[Insert->curridx];
					WriteRqstPtr.xrecoff -= freespace;
				}
				LWLockRelease(WALInsertLock);
				WriteRqst.Write = WriteRqstPtr;
				WriteRqst.Flush = WriteRqstPtr;
			}
			else
			{
				WriteRqst.Write = WriteRqstPtr;
				WriteRqst.Flush = record;
			}
1803
			XLogWrite(WriteRqst, false, false);
Tom Lane's avatar
Tom Lane committed
1804
		}
1805
		LWLockRelease(WALWriteLock);
Tom Lane's avatar
Tom Lane committed
1806 1807 1808
	}

	END_CRIT_SECTION();
1809 1810 1811

	/*
	 * If we still haven't flushed to the request point then we have a
1812 1813
	 * problem; most likely, the requested flush point is past end of XLOG.
	 * This has been seen to occur when a disk page has a corrupted LSN.
1814
	 *
1815 1816 1817 1818 1819 1820 1821 1822 1823
	 * Formerly we treated this as a PANIC condition, but that hurts the
	 * system's robustness rather than helping it: we do not want to take down
	 * the whole system due to corruption on one data page.  In particular, if
	 * the bad page is encountered again during recovery then we would be
	 * unable to restart the database at all!  (This scenario has actually
	 * happened in the field several times with 7.1 releases. Note that we
	 * cannot get here while InRedo is true, but if the bad page is brought in
	 * and marked dirty during recovery then CreateCheckPoint will try to
	 * flush it at the end of recovery.)
1824
	 *
1825 1826 1827 1828
	 * The current approach is to ERROR under normal conditions, but only
	 * WARNING during recovery, so that the system can be brought up even if
	 * there's a corrupt LSN.  Note that for calls from xact.c, the ERROR will
	 * be promoted to PANIC since xact.c calls this routine inside a critical
1829 1830
	 * section.  However, calls from bufmgr.c are not within critical sections
	 * and so we will not force a restart for a bad LSN on a data page.
1831 1832
	 */
	if (XLByteLT(LogwrtResult.Flush, record))
Bruce Momjian's avatar
Bruce Momjian committed
1833
		elog(InRecovery ? WARNING : ERROR,
1834
		"xlog flush request %X/%X is not satisfied --- flushed only to %X/%X",
1835 1836
			 record.xlogid, record.xrecoff,
			 LogwrtResult.Flush.xlogid, LogwrtResult.Flush.xrecoff);
1837 1838
}

1839 1840 1841 1842 1843 1844
/*
 * Flush xlog, but without specifying exactly where to flush to.
 *
 * We normally flush only completed blocks; but if there is nothing to do on
 * that basis, we check for unflushed async commits in the current incomplete
 * block, and flush through the latest one of those.  Thus, if async commits
Bruce Momjian's avatar
Bruce Momjian committed
1845
 * are not being used, we will flush complete blocks only.	We can guarantee
1846
 * that async commits reach disk after at most three cycles; normally only
Bruce Momjian's avatar
Bruce Momjian committed
1847
 * one or two.	(We allow XLogWrite to write "flexibly", meaning it can stop
1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879
 * at the end of the buffer ring; this makes a difference only with very high
 * load or long wal_writer_delay, but imposes one extra cycle for the worst
 * case for async commits.)
 *
 * This routine is invoked periodically by the background walwriter process.
 */
void
XLogBackgroundFlush(void)
{
	XLogRecPtr	WriteRqstPtr;
	bool		flexible = true;

	/* read LogwrtResult and update local state */
	{
		/* use volatile pointer to prevent code rearrangement */
		volatile XLogCtlData *xlogctl = XLogCtl;

		SpinLockAcquire(&xlogctl->info_lck);
		LogwrtResult = xlogctl->LogwrtResult;
		WriteRqstPtr = xlogctl->LogwrtRqst.Write;
		SpinLockRelease(&xlogctl->info_lck);
	}

	/* back off to last completed page boundary */
	WriteRqstPtr.xrecoff -= WriteRqstPtr.xrecoff % XLOG_BLCKSZ;

	/* if we have already flushed that far, consider async commit records */
	if (XLByteLE(WriteRqstPtr, LogwrtResult.Flush))
	{
		/* use volatile pointer to prevent code rearrangement */
		volatile XLogCtlData *xlogctl = XLogCtl;

1880
		SpinLockAcquire(&xlogctl->info_lck);
1881
		WriteRqstPtr = xlogctl->asyncCommitLSN;
1882
		SpinLockRelease(&xlogctl->info_lck);
1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915
		flexible = false;		/* ensure it all gets written */
	}

	/* Done if already known flushed */
	if (XLByteLE(WriteRqstPtr, LogwrtResult.Flush))
		return;

#ifdef WAL_DEBUG
	if (XLOG_DEBUG)
		elog(LOG, "xlog bg flush request %X/%X; write %X/%X; flush %X/%X",
			 WriteRqstPtr.xlogid, WriteRqstPtr.xrecoff,
			 LogwrtResult.Write.xlogid, LogwrtResult.Write.xrecoff,
			 LogwrtResult.Flush.xlogid, LogwrtResult.Flush.xrecoff);
#endif

	START_CRIT_SECTION();

	/* now wait for the write lock */
	LWLockAcquire(WALWriteLock, LW_EXCLUSIVE);
	LogwrtResult = XLogCtl->Write.LogwrtResult;
	if (!XLByteLE(WriteRqstPtr, LogwrtResult.Flush))
	{
		XLogwrtRqst WriteRqst;

		WriteRqst.Write = WriteRqstPtr;
		WriteRqst.Flush = WriteRqstPtr;
		XLogWrite(WriteRqst, flexible, false);
	}
	LWLockRelease(WALWriteLock);

	END_CRIT_SECTION();
}

1916 1917
/*
 * Flush any previous asynchronously-committed transactions' commit records.
1918 1919 1920 1921
 *
 * NOTE: it is unwise to assume that this provides any strong guarantees.
 * In particular, because of the inexact LSN bookkeeping used by clog.c,
 * we cannot assume that hint bits will be settable for these transactions.
1922 1923 1924 1925 1926
 */
void
XLogAsyncCommitFlush(void)
{
	XLogRecPtr	WriteRqstPtr;
Bruce Momjian's avatar
Bruce Momjian committed
1927

1928 1929 1930 1931 1932 1933 1934 1935 1936 1937
	/* use volatile pointer to prevent code rearrangement */
	volatile XLogCtlData *xlogctl = XLogCtl;

	SpinLockAcquire(&xlogctl->info_lck);
	WriteRqstPtr = xlogctl->asyncCommitLSN;
	SpinLockRelease(&xlogctl->info_lck);

	XLogFlush(WriteRqstPtr);
}

1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967
/*
 * Test whether XLOG data has been flushed up to (at least) the given position.
 *
 * Returns true if a flush is still needed.  (It may be that someone else
 * is already in process of flushing that far, however.)
 */
bool
XLogNeedsFlush(XLogRecPtr record)
{
	/* Quick exit if already known flushed */
	if (XLByteLE(record, LogwrtResult.Flush))
		return false;

	/* read LogwrtResult and update local state */
	{
		/* use volatile pointer to prevent code rearrangement */
		volatile XLogCtlData *xlogctl = XLogCtl;

		SpinLockAcquire(&xlogctl->info_lck);
		LogwrtResult = xlogctl->LogwrtResult;
		SpinLockRelease(&xlogctl->info_lck);
	}

	/* check again */
	if (XLByteLE(record, LogwrtResult.Flush))
		return false;

	return true;
}

Tom Lane's avatar
Tom Lane committed
1968 1969 1970
/*
 * Create a new XLOG file segment, or open a pre-existing one.
 *
1971 1972 1973
 * log, seg: identify segment to be created/opened.
 *
 * *use_existent: if TRUE, OK to use a pre-existing file (else, any
1974
 * pre-existing file will be deleted).	On return, TRUE if a pre-existing
1975 1976
 * file was used.
 *
1977
 * use_lock: if TRUE, acquire ControlFileLock while moving file into
1978
 * place.  This should be TRUE except during bootstrap log creation.  The
1979
 * caller must *not* hold the lock at call.
1980
 *
Tom Lane's avatar
Tom Lane committed
1981
 * Returns FD of opened file.
1982 1983 1984 1985 1986
 *
 * Note: errors here are ERROR not PANIC because we might or might not be
 * inside a critical section (eg, during checkpoint there is no reason to
 * take down the system on failure).  They will promote to PANIC if we are
 * in a critical section.
Tom Lane's avatar
Tom Lane committed
1987
 */
1988
static int
1989 1990
XLogFileInit(uint32 log, uint32 seg,
			 bool *use_existent, bool use_lock)
1991
{
1992
	char		path[MAXPGPATH];
1993
	char		tmppath[MAXPGPATH];
1994
	char	   *zbuffer;
1995 1996 1997
	uint32		installed_log;
	uint32		installed_seg;
	int			max_advance;
1998
	int			fd;
1999
	int			nbytes;
2000

2001
	XLogFilePath(path, ThisTimeLineID, log, seg);
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
2002 2003

	/*
2004
	 * Try to use existent file (checkpoint maker may have created it already)
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
2005
	 */
2006
	if (*use_existent)
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
2007
	{
2008
		fd = BasicOpenFile(path, O_RDWR | PG_BINARY | get_sync_bit(sync_method),
2009
						   S_IRUSR | S_IWUSR);
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
2010 2011 2012
		if (fd < 0)
		{
			if (errno != ENOENT)
2013
				ereport(ERROR,
2014
						(errcode_for_file_access(),
2015
						 errmsg("could not open file \"%s\" (log file %u, segment %u): %m",
2016
								path, log, seg)));
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
2017 2018
		}
		else
2019
			return fd;
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
2020 2021
	}

2022
	/*
2023 2024 2025 2026
	 * Initialize an empty (all zeroes) segment.  NOTE: it is possible that
	 * another process is doing the same thing.  If so, we will end up
	 * pre-creating an extra log segment.  That seems OK, and better than
	 * holding the lock throughout this lengthy process.
2027
	 */
2028 2029
	elog(DEBUG2, "creating and filling new WAL file");

2030
	snprintf(tmppath, MAXPGPATH, XLOGDIR "/xlogtemp.%d", (int) getpid());
2031 2032

	unlink(tmppath);
2033

2034
	/* do not use get_sync_bit() here --- want to fsync only at end of fill */
2035
	fd = BasicOpenFile(tmppath, O_RDWR | O_CREAT | O_EXCL | PG_BINARY,
Tom Lane's avatar
Tom Lane committed
2036
					   S_IRUSR | S_IWUSR);
2037
	if (fd < 0)
2038
		ereport(ERROR,
2039
				(errcode_for_file_access(),
2040
				 errmsg("could not create file \"%s\": %m", tmppath)));
2041

2042
	/*
2043 2044 2045 2046 2047 2048 2049
	 * Zero-fill the file.	We have to do this the hard way to ensure that all
	 * the file space has really been allocated --- on platforms that allow
	 * "holes" in files, just seeking to the end doesn't allocate intermediate
	 * space.  This way, we know that we have all the space and (after the
	 * fsync below) that all the indirect blocks are down on disk.	Therefore,
	 * fdatasync(2) or O_DSYNC will be sufficient to sync future writes to the
	 * log file.
2050 2051 2052 2053
	 *
	 * Note: palloc zbuffer, instead of just using a local char array, to
	 * ensure it is reasonably well-aligned; this may save a few cycles
	 * transferring data to the kernel.
2054
	 */
2055 2056
	zbuffer = (char *) palloc0(XLOG_BLCKSZ);
	for (nbytes = 0; nbytes < XLogSegSize; nbytes += XLOG_BLCKSZ)
2057
	{
2058
		errno = 0;
2059
		if ((int) write(fd, zbuffer, XLOG_BLCKSZ) != (int) XLOG_BLCKSZ)
Tom Lane's avatar
Tom Lane committed
2060
		{
2061
			int			save_errno = errno;
Tom Lane's avatar
Tom Lane committed
2062

2063
			/*
2064
			 * If we fail to make the file, delete it to release disk space
2065
			 */
2066
			unlink(tmppath);
2067 2068
			/* if write didn't set errno, assume problem is no disk space */
			errno = save_errno ? save_errno : ENOSPC;
Tom Lane's avatar
Tom Lane committed
2069

2070
			ereport(ERROR,
2071
					(errcode_for_file_access(),
2072
					 errmsg("could not write to file \"%s\": %m", tmppath)));
Tom Lane's avatar
Tom Lane committed
2073
		}
2074
	}
2075
	pfree(zbuffer);
2076

2077
	if (pg_fsync(fd) != 0)
2078
		ereport(ERROR,
2079
				(errcode_for_file_access(),
2080
				 errmsg("could not fsync file \"%s\": %m", tmppath)));
2081

2082
	if (close(fd))
2083
		ereport(ERROR,
2084 2085
				(errcode_for_file_access(),
				 errmsg("could not close file \"%s\": %m", tmppath)));
Tom Lane's avatar
Tom Lane committed
2086

2087
	/*
2088 2089
	 * Now move the segment into place with its final name.
	 *
2090
	 * If caller didn't want to use a pre-existing file, get rid of any
2091 2092 2093
	 * pre-existing file.  Otherwise, cope with possibility that someone else
	 * has created the file while we were filling ours: if so, use ours to
	 * pre-create a future log segment.
2094
	 */
2095 2096 2097 2098 2099
	installed_log = log;
	installed_seg = seg;
	max_advance = XLOGfileslop;
	if (!InstallXLogFileSegment(&installed_log, &installed_seg, tmppath,
								*use_existent, &max_advance,
2100 2101 2102 2103 2104 2105
								use_lock))
	{
		/* No need for any more future segments... */
		unlink(tmppath);
	}

2106 2107
	elog(DEBUG2, "done creating and filling new WAL file");

2108 2109 2110 2111
	/* Set flag to tell caller there was no existent file */
	*use_existent = false;

	/* Now open original target segment (might not be file I just made) */
2112
	fd = BasicOpenFile(path, O_RDWR | PG_BINARY | get_sync_bit(sync_method),
2113 2114
					   S_IRUSR | S_IWUSR);
	if (fd < 0)
2115
		ereport(ERROR,
2116
				(errcode_for_file_access(),
2117 2118
		   errmsg("could not open file \"%s\" (log file %u, segment %u): %m",
				  path, log, seg)));
2119

2120
	return fd;
2121 2122
}

2123 2124 2125 2126 2127 2128 2129 2130 2131
/*
 * Create a new XLOG file segment by copying a pre-existing one.
 *
 * log, seg: identify segment to be created.
 *
 * srcTLI, srclog, srcseg: identify segment to be copied (could be from
 *		a different timeline)
 *
 * Currently this is only used during recovery, and so there are no locking
Bruce Momjian's avatar
Bruce Momjian committed
2132
 * considerations.	But we should be just as tense as XLogFileInit to avoid
2133 2134 2135 2136 2137 2138 2139 2140
 * emplacing a bogus file.
 */
static void
XLogFileCopy(uint32 log, uint32 seg,
			 TimeLineID srcTLI, uint32 srclog, uint32 srcseg)
{
	char		path[MAXPGPATH];
	char		tmppath[MAXPGPATH];
2141
	char		buffer[XLOG_BLCKSZ];
2142 2143 2144 2145 2146 2147 2148 2149 2150 2151
	int			srcfd;
	int			fd;
	int			nbytes;

	/*
	 * Open the source file
	 */
	XLogFilePath(path, srcTLI, srclog, srcseg);
	srcfd = BasicOpenFile(path, O_RDONLY | PG_BINARY, 0);
	if (srcfd < 0)
2152
		ereport(ERROR,
2153 2154 2155 2156 2157 2158
				(errcode_for_file_access(),
				 errmsg("could not open file \"%s\": %m", path)));

	/*
	 * Copy into a temp file name.
	 */
2159
	snprintf(tmppath, MAXPGPATH, XLOGDIR "/xlogtemp.%d", (int) getpid());
2160 2161 2162

	unlink(tmppath);

2163
	/* do not use get_sync_bit() here --- want to fsync only at end of fill */
2164 2165 2166
	fd = BasicOpenFile(tmppath, O_RDWR | O_CREAT | O_EXCL | PG_BINARY,
					   S_IRUSR | S_IWUSR);
	if (fd < 0)
2167
		ereport(ERROR,
2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179
				(errcode_for_file_access(),
				 errmsg("could not create file \"%s\": %m", tmppath)));

	/*
	 * Do the data copying.
	 */
	for (nbytes = 0; nbytes < XLogSegSize; nbytes += sizeof(buffer))
	{
		errno = 0;
		if ((int) read(srcfd, buffer, sizeof(buffer)) != (int) sizeof(buffer))
		{
			if (errno != 0)
2180
				ereport(ERROR,
2181 2182 2183
						(errcode_for_file_access(),
						 errmsg("could not read file \"%s\": %m", path)));
			else
2184
				ereport(ERROR,
2185
						(errmsg("not enough data in file \"%s\"", path)));
2186 2187 2188 2189 2190 2191 2192
		}
		errno = 0;
		if ((int) write(fd, buffer, sizeof(buffer)) != (int) sizeof(buffer))
		{
			int			save_errno = errno;

			/*
2193
			 * If we fail to make the file, delete it to release disk space
2194 2195 2196 2197 2198
			 */
			unlink(tmppath);
			/* if write didn't set errno, assume problem is no disk space */
			errno = save_errno ? save_errno : ENOSPC;

2199
			ereport(ERROR,
2200
					(errcode_for_file_access(),
2201
					 errmsg("could not write to file \"%s\": %m", tmppath)));
2202 2203 2204 2205
		}
	}

	if (pg_fsync(fd) != 0)
2206
		ereport(ERROR,
2207 2208 2209 2210
				(errcode_for_file_access(),
				 errmsg("could not fsync file \"%s\": %m", tmppath)));

	if (close(fd))
2211
		ereport(ERROR,
2212 2213 2214 2215 2216 2217 2218 2219
				(errcode_for_file_access(),
				 errmsg("could not close file \"%s\": %m", tmppath)));

	close(srcfd);

	/*
	 * Now move the segment into place with its final name.
	 */
2220
	if (!InstallXLogFileSegment(&log, &seg, tmppath, false, NULL, false))
2221
		elog(ERROR, "InstallXLogFileSegment should not have failed");
2222 2223
}

2224 2225 2226 2227 2228 2229
/*
 * Install a new XLOG segment file as a current or future log segment.
 *
 * This is used both to install a newly-created segment (which has a temp
 * filename while it's being created) and to recycle an old segment.
 *
2230 2231 2232
 * *log, *seg: identify segment to install as (or first possible target).
 * When find_free is TRUE, these are modified on return to indicate the
 * actual installation location or last segment searched.
2233 2234 2235 2236 2237 2238 2239
 *
 * tmppath: initial name of file to install.  It will be renamed into place.
 *
 * find_free: if TRUE, install the new segment at the first empty log/seg
 * number at or after the passed numbers.  If FALSE, install the new segment
 * exactly where specified, deleting any existing segment file there.
 *
2240 2241 2242 2243
 * *max_advance: maximum number of log/seg slots to advance past the starting
 * point.  Fail if no free slot is found in this range.  On return, reduced
 * by the number of slots skipped over.  (Irrelevant, and may be NULL,
 * when find_free is FALSE.)
2244
 *
2245
 * use_lock: if TRUE, acquire ControlFileLock while moving file into
2246
 * place.  This should be TRUE except during bootstrap log creation.  The
2247
 * caller must *not* hold the lock at call.
2248 2249
 *
 * Returns TRUE if file installed, FALSE if not installed because of
2250 2251 2252
 * exceeding max_advance limit.  On Windows, we also return FALSE if we
 * can't rename the file into place because someone's got it open.
 * (Any other kind of failure causes ereport().)
2253 2254
 */
static bool
2255 2256
InstallXLogFileSegment(uint32 *log, uint32 *seg, char *tmppath,
					   bool find_free, int *max_advance,
2257 2258 2259
					   bool use_lock)
{
	char		path[MAXPGPATH];
2260
	struct stat stat_buf;
2261

2262
	XLogFilePath(path, ThisTimeLineID, *log, *seg);
2263 2264 2265 2266 2267

	/*
	 * We want to be sure that only one process does this at a time.
	 */
	if (use_lock)
2268
		LWLockAcquire(ControlFileLock, LW_EXCLUSIVE);
2269

2270 2271 2272 2273 2274
	if (!find_free)
	{
		/* Force installation: get rid of any pre-existing segment file */
		unlink(path);
	}
2275 2276
	else
	{
2277
		/* Find a free slot to put it in */
2278
		while (stat(path, &stat_buf) == 0)
2279
		{
2280
			if (*max_advance <= 0)
2281 2282 2283
			{
				/* Failed to find a free slot within specified range */
				if (use_lock)
2284
					LWLockRelease(ControlFileLock);
2285 2286
				return false;
			}
2287 2288 2289
			NextLogSeg(*log, *seg);
			(*max_advance)--;
			XLogFilePath(path, ThisTimeLineID, *log, *seg);
2290 2291 2292 2293 2294 2295 2296
		}
	}

	/*
	 * Prefer link() to rename() here just to be really sure that we don't
	 * overwrite an existing logfile.  However, there shouldn't be one, so
	 * rename() is an acceptable substitute except for the truly paranoid.
2297
	 */
2298
#if HAVE_WORKING_LINK
2299
	if (link(tmppath, path) < 0)
2300
		ereport(ERROR,
2301
				(errcode_for_file_access(),
2302
				 errmsg("could not link file \"%s\" to \"%s\" (initialization of log file %u, segment %u): %m",
2303
						tmppath, path, *log, *seg)));
2304
	unlink(tmppath);
2305
#else
2306
	if (rename(tmppath, path) < 0)
2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318
	{
#ifdef WIN32
#if !defined(__CYGWIN__)
		if (GetLastError() == ERROR_ACCESS_DENIED)
#else
		if (errno == EACCES)
#endif
		{
			if (use_lock)
				LWLockRelease(ControlFileLock);
			return false;
		}
Bruce Momjian's avatar
Bruce Momjian committed
2319
#endif   /* WIN32 */
2320

2321
		ereport(ERROR,
2322
				(errcode_for_file_access(),
2323
				 errmsg("could not rename file \"%s\" to \"%s\" (initialization of log file %u, segment %u): %m",
2324
						tmppath, path, *log, *seg)));
2325
	}
2326
#endif
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
2327

2328
	if (use_lock)
2329
		LWLockRelease(ControlFileLock);
2330

2331
	return true;
2332 2333
}

Tom Lane's avatar
Tom Lane committed
2334
/*
2335
 * Open a pre-existing logfile segment for writing.
Tom Lane's avatar
Tom Lane committed
2336
 */
2337
static int
2338
XLogFileOpen(uint32 log, uint32 seg)
2339
{
2340 2341
	char		path[MAXPGPATH];
	int			fd;
2342

2343
	XLogFilePath(path, ThisTimeLineID, log, seg);
2344

2345
	fd = BasicOpenFile(path, O_RDWR | PG_BINARY | get_sync_bit(sync_method),
2346
					   S_IRUSR | S_IWUSR);
2347
	if (fd < 0)
2348 2349
		ereport(PANIC,
				(errcode_for_file_access(),
2350 2351
		   errmsg("could not open file \"%s\" (log file %u, segment %u): %m",
				  path, log, seg)));
2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363

	return fd;
}

/*
 * Open a logfile segment for reading (during recovery).
 */
static int
XLogFileRead(uint32 log, uint32 seg, int emode)
{
	char		path[MAXPGPATH];
	char		xlogfname[MAXFNAMELEN];
2364
	char		activitymsg[MAXFNAMELEN + 16];
2365 2366
	ListCell   *cell;
	int			fd;
2367

2368
	/*
2369 2370
	 * Loop looking for a suitable timeline ID: we might need to read any of
	 * the timelines listed in expectedTLIs.
2371
	 *
Bruce Momjian's avatar
Bruce Momjian committed
2372
	 * We expect curFileTLI on entry to be the TLI of the preceding file in
2373 2374 2375 2376
	 * sequence, or 0 if there was no predecessor.	We do not allow curFileTLI
	 * to go backwards; this prevents us from picking up the wrong file when a
	 * parent timeline extends to higher segment numbers than the child we
	 * want to read.
2377
	 */
2378 2379 2380
	foreach(cell, expectedTLIs)
	{
		TimeLineID	tli = (TimeLineID) lfirst_int(cell);
2381

2382 2383 2384
		if (tli < curFileTLI)
			break;				/* don't bother looking at too-old TLIs */

2385 2386
		XLogFileName(xlogfname, tli, log, seg);

2387 2388
		if (InArchiveRecovery)
		{
2389 2390 2391 2392 2393
			/* Report recovery progress in PS display */
			snprintf(activitymsg, sizeof(activitymsg), "waiting for %s",
					 xlogfname);
			set_ps_display(activitymsg, false);

2394
			restoredFromArchive = RestoreArchivedFile(path, xlogfname,
2395 2396
													  "RECOVERYXLOG",
													  XLogSegSize);
2397 2398 2399 2400 2401 2402 2403 2404 2405
		}
		else
			XLogFilePath(path, tli, log, seg);

		fd = BasicOpenFile(path, O_RDONLY | PG_BINARY, 0);
		if (fd >= 0)
		{
			/* Success! */
			curFileTLI = tli;
2406 2407

			/* Report recovery progress in PS display */
2408 2409
			snprintf(activitymsg, sizeof(activitymsg), "recovering %s",
					 xlogfname);
2410 2411
			set_ps_display(activitymsg, false);

2412 2413 2414 2415 2416
			return fd;
		}
		if (errno != ENOENT)	/* unexpected failure? */
			ereport(PANIC,
					(errcode_for_file_access(),
2417 2418
			errmsg("could not open file \"%s\" (log file %u, segment %u): %m",
				   path, log, seg)));
2419 2420 2421 2422 2423 2424 2425
	}

	/* Couldn't find it.  For simplicity, complain about front timeline */
	XLogFilePath(path, recoveryTargetTLI, log, seg);
	errno = ENOENT;
	ereport(emode,
			(errcode_for_file_access(),
2426 2427
		   errmsg("could not open file \"%s\" (log file %u, segment %u): %m",
				  path, log, seg)));
2428
	return -1;
2429 2430
}

2431 2432 2433 2434 2435 2436 2437 2438 2439
/*
 * Close the current logfile segment for writing.
 */
static void
XLogFileClose(void)
{
	Assert(openLogFile >= 0);

	/*
2440
	 * WAL segment files will not be re-read in normal operation, so we advise
2441 2442 2443 2444
	 * the OS to release any cached pages.  But do not do so if WAL archiving
	 * is active, because archiver process could use the cache to read the WAL
	 * segment.  Also, don't bother with it if we are using O_DIRECT, since
	 * the kernel is presumably not caching in that case.
2445
	 */
2446 2447 2448 2449
#if defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED)
	if (!XLogArchivingActive() &&
		(get_sync_bit(sync_method) & PG_O_DIRECT) == 0)
		(void) posix_fadvise(openLogFile, 0, 0, POSIX_FADV_DONTNEED);
2450
#endif
2451

2452 2453
	if (close(openLogFile))
		ereport(PANIC,
Bruce Momjian's avatar
Bruce Momjian committed
2454 2455 2456
				(errcode_for_file_access(),
				 errmsg("could not close log file %u, segment %u: %m",
						openLogId, openLogSeg)));
2457 2458 2459
	openLogFile = -1;
}

2460
/*
2461
 * Attempt to retrieve the specified file from off-line archival storage.
2462
 * If successful, fill "path" with its complete path (note that this will be
2463 2464
 * a temp file name that doesn't follow the normal naming convention), and
 * return TRUE.
2465
 *
2466 2467 2468
 * If not successful, fill "path" with the name of the normal on-line file
 * (which may or may not actually exist, but we'll try to use it), and return
 * FALSE.
2469 2470 2471 2472
 *
 * For fixed-size files, the caller may pass the expected size as an
 * additional crosscheck on successful recovery.  If the file size is not
 * known, set expectedSize = 0.
2473
 */
2474 2475
static bool
RestoreArchivedFile(char *path, const char *xlogfname,
2476
					const char *recovername, off_t expectedSize)
2477
{
Bruce Momjian's avatar
Bruce Momjian committed
2478 2479
	char		xlogpath[MAXPGPATH];
	char		xlogRestoreCmd[MAXPGPATH];
2480
	char		lastRestartPointFname[MAXPGPATH];
Bruce Momjian's avatar
Bruce Momjian committed
2481 2482
	char	   *dp;
	char	   *endp;
2483
	const char *sp;
Bruce Momjian's avatar
Bruce Momjian committed
2484
	int			rc;
2485
	bool		signaled;
2486
	struct stat stat_buf;
Bruce Momjian's avatar
Bruce Momjian committed
2487 2488
	uint32		restartLog;
	uint32		restartSeg;
2489 2490

	/*
2491 2492 2493 2494
	 * When doing archive recovery, we always prefer an archived log file even
	 * if a file of the same name exists in XLOGDIR.  The reason is that the
	 * file in XLOGDIR could be an old, un-filled or partly-filled version
	 * that was copied and restored as part of backing up $PGDATA.
2495
	 *
Bruce Momjian's avatar
Bruce Momjian committed
2496
	 * We could try to optimize this slightly by checking the local copy
2497 2498 2499 2500
	 * lastchange timestamp against the archived copy, but we have no API to
	 * do this, nor can we guarantee that the lastchange timestamp was
	 * preserved correctly when we copied to archive. Our aim is robustness,
	 * so we elect not to do this.
2501
	 *
2502 2503 2504
	 * If we cannot obtain the log file from the archive, however, we will try
	 * to use the XLOGDIR file if it exists.  This is so that we can make use
	 * of log segments that weren't yet transferred to the archive.
2505
	 *
2506 2507 2508 2509
	 * Notice that we don't actually overwrite any files when we copy back
	 * from archive because the recoveryRestoreCommand may inadvertently
	 * restore inappropriate xlogs, or they may be corrupt, so we may wish to
	 * fallback to the segments remaining in current XLOGDIR later. The
2510 2511
	 * copy-from-archive filename is always the same, ensuring that we don't
	 * run out of disk space on long recoveries.
2512
	 */
2513
	snprintf(xlogpath, MAXPGPATH, XLOGDIR "/%s", recovername);
2514 2515

	/*
2516
	 * Make sure there is no existing file named recovername.
2517 2518 2519 2520 2521 2522
	 */
	if (stat(xlogpath, &stat_buf) != 0)
	{
		if (errno != ENOENT)
			ereport(FATAL,
					(errcode_for_file_access(),
Peter Eisentraut's avatar
Peter Eisentraut committed
2523
					 errmsg("could not stat file \"%s\": %m",
2524 2525 2526 2527 2528 2529 2530
							xlogpath)));
	}
	else
	{
		if (unlink(xlogpath) != 0)
			ereport(FATAL,
					(errcode_for_file_access(),
2531
					 errmsg("could not remove file \"%s\": %m",
2532 2533 2534
							xlogpath)));
	}

2535 2536 2537 2538 2539 2540 2541
	/*
	 * Calculate the archive file cutoff point for use during log shipping
	 * replication. All files earlier than this point can be deleted
	 * from the archive, though there is no requirement to do so.
	 *
	 * We initialise this with the filename of an InvalidXLogRecPtr, which
	 * will prevent the deletion of any WAL files from the archive
2542
	 * because of the alphabetic sorting property of WAL filenames.
2543 2544 2545 2546 2547 2548
	 *
	 * Once we have successfully located the redo pointer of the checkpoint
	 * from which we start recovery we never request a file prior to the redo
	 * pointer of the last restartpoint. When redo begins we know that we
	 * have successfully located it, so there is no need for additional
	 * status flags to signify the point when we can begin deleting WAL files
2549
	 * from the archive.
2550 2551 2552 2553 2554 2555 2556 2557 2558
	 */
	if (InRedo)
	{
		XLByteToSeg(ControlFile->checkPointCopy.redo,
					restartLog, restartSeg);
		XLogFileName(lastRestartPointFname,
					 ControlFile->checkPointCopy.ThisTimeLineID,
					 restartLog, restartSeg);
		/* we shouldn't need anything earlier than last restart point */
2559
		Assert(strcmp(lastRestartPointFname, xlogfname) <= 0);
2560 2561 2562 2563
	}
	else
		XLogFileName(lastRestartPointFname, 0, 0, 0);

2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577
	/*
	 * construct the command to be executed
	 */
	dp = xlogRestoreCmd;
	endp = xlogRestoreCmd + MAXPGPATH - 1;
	*endp = '\0';

	for (sp = recoveryRestoreCommand; *sp; sp++)
	{
		if (*sp == '%')
		{
			switch (sp[1])
			{
				case 'p':
2578
					/* %p: relative path of target file */
2579
					sp++;
Bruce Momjian's avatar
Bruce Momjian committed
2580
					StrNCpy(dp, xlogpath, endp - dp);
2581
					make_native_path(dp);
2582 2583 2584 2585 2586
					dp += strlen(dp);
					break;
				case 'f':
					/* %f: filename of desired file */
					sp++;
Bruce Momjian's avatar
Bruce Momjian committed
2587
					StrNCpy(dp, xlogfname, endp - dp);
2588 2589
					dp += strlen(dp);
					break;
2590 2591 2592 2593 2594 2595
				case 'r':
					/* %r: filename of last restartpoint */
					sp++;
					StrNCpy(dp, lastRestartPointFname, endp - dp);
					dp += strlen(dp);
					break;
2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617
				case '%':
					/* convert %% to a single % */
					sp++;
					if (dp < endp)
						*dp++ = *sp;
					break;
				default:
					/* otherwise treat the % as not special */
					if (dp < endp)
						*dp++ = *sp;
					break;
			}
		}
		else
		{
			if (dp < endp)
				*dp++ = *sp;
		}
	}
	*dp = '\0';

	ereport(DEBUG3,
Bruce Momjian's avatar
Bruce Momjian committed
2618
			(errmsg_internal("executing restore command \"%s\"",
2619 2620 2621
							 xlogRestoreCmd)));

	/*
2622
	 * Copy xlog from archival storage to XLOGDIR
2623 2624 2625 2626
	 */
	rc = system(xlogRestoreCmd);
	if (rc == 0)
	{
2627 2628 2629 2630
		/*
		 * command apparently succeeded, but let's make sure the file is
		 * really there now and has the correct size.
		 *
2631 2632 2633 2634 2635
		 * XXX I made wrong-size a fatal error to ensure the DBA would notice
		 * it, but is that too strong?	We could try to plow ahead with a
		 * local copy of the file ... but the problem is that there probably
		 * isn't one, and we'd incorrectly conclude we've reached the end of
		 * WAL and we're done recovering ...
2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659
		 */
		if (stat(xlogpath, &stat_buf) == 0)
		{
			if (expectedSize > 0 && stat_buf.st_size != expectedSize)
				ereport(FATAL,
						(errmsg("archive file \"%s\" has wrong size: %lu instead of %lu",
								xlogfname,
								(unsigned long) stat_buf.st_size,
								(unsigned long) expectedSize)));
			else
			{
				ereport(LOG,
						(errmsg("restored log file \"%s\" from archive",
								xlogfname)));
				strcpy(path, xlogpath);
				return true;
			}
		}
		else
		{
			/* stat failed */
			if (errno != ENOENT)
				ereport(FATAL,
						(errcode_for_file_access(),
Peter Eisentraut's avatar
Peter Eisentraut committed
2660
						 errmsg("could not stat file \"%s\": %m",
2661
								xlogpath)));
2662 2663 2664 2665
		}
	}

	/*
2666
	 * Remember, we rollforward UNTIL the restore fails so failure here is
Bruce Momjian's avatar
Bruce Momjian committed
2667
	 * just part of the process... that makes it difficult to determine
2668 2669 2670
	 * whether the restore failed because there isn't an archive to restore,
	 * or because the administrator has specified the restore program
	 * incorrectly.  We have to assume the former.
2671 2672
	 *
	 * However, if the failure was due to any sort of signal, it's best to
Bruce Momjian's avatar
Bruce Momjian committed
2673 2674 2675
	 * punt and abort recovery.  (If we "return false" here, upper levels will
	 * assume that recovery is complete and start up the database!) It's
	 * essential to abort on child SIGINT and SIGQUIT, because per spec
2676 2677 2678 2679
	 * system() ignores SIGINT and SIGQUIT while waiting; if we see one of
	 * those it's a good bet we should have gotten it too.  Aborting on other
	 * signals such as SIGTERM seems a good idea as well.
	 *
Bruce Momjian's avatar
Bruce Momjian committed
2680 2681 2682 2683
	 * Per the Single Unix Spec, shells report exit status > 128 when a called
	 * command died on a signal.  Also, 126 and 127 are used to report
	 * problems such as an unfindable command; treat those as fatal errors
	 * too.
2684
	 */
2685 2686 2687
	signaled = WIFSIGNALED(rc) || WEXITSTATUS(rc) > 125;

	ereport(signaled ? FATAL : DEBUG2,
2688 2689
		(errmsg("could not restore file \"%s\" from archive: return code %d",
				xlogfname, rc)));
2690 2691

	/*
2692 2693
	 * if an archived file is not available, there might still be a version of
	 * this file in XLOGDIR, so return that as the filename to open.
2694
	 *
Bruce Momjian's avatar
Bruce Momjian committed
2695 2696
	 * In many recovery scenarios we expect this to fail also, but if so that
	 * just means we've reached the end of WAL.
2697
	 */
2698
	snprintf(path, MAXPGPATH, XLOGDIR "/%s", xlogfname);
2699
	return false;
2700 2701
}

Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
2702
/*
2703 2704 2705 2706 2707 2708 2709 2710
 * Preallocate log files beyond the specified log endpoint.
 *
 * XXX this is currently extremely conservative, since it forces only one
 * future log segment to exist, and even that only if we are 75% done with
 * the current one.  This is only appropriate for very low-WAL-volume systems.
 * High-volume systems will be OK once they've built up a sufficient set of
 * recycled log segments, but the startup transient is likely to include
 * a lot of segment creations by foreground processes, which is not so good.
Tom Lane's avatar
Tom Lane committed
2711
 */
2712
static void
Tom Lane's avatar
Tom Lane committed
2713 2714 2715 2716 2717
PreallocXlogFiles(XLogRecPtr endptr)
{
	uint32		_logId;
	uint32		_logSeg;
	int			lf;
2718
	bool		use_existent;
Tom Lane's avatar
Tom Lane committed
2719 2720

	XLByteToPrevSeg(endptr, _logId, _logSeg);
Bruce Momjian's avatar
Bruce Momjian committed
2721
	if ((endptr.xrecoff - 1) % XLogSegSize >=
Bruce Momjian's avatar
Bruce Momjian committed
2722
		(uint32) (0.75 * XLogSegSize))
Tom Lane's avatar
Tom Lane committed
2723 2724
	{
		NextLogSeg(_logId, _logSeg);
2725 2726
		use_existent = true;
		lf = XLogFileInit(_logId, _logSeg, &use_existent, true);
Tom Lane's avatar
Tom Lane committed
2727
		close(lf);
2728
		if (!use_existent)
2729
			CheckpointStats.ckpt_segs_added++;
Tom Lane's avatar
Tom Lane committed
2730 2731 2732 2733
	}
}

/*
2734
 * Recycle or remove all log files older or equal to passed log/seg#
2735 2736 2737
 *
 * endptr is current (or recent) end of xlog; this is used to determine
 * whether we want to recycle rather than delete no-longer-wanted log files.
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
2738 2739
 */
static void
2740
RemoveOldXlogFiles(uint32 log, uint32 seg, XLogRecPtr endptr)
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
2741
{
2742 2743
	uint32		endlogId;
	uint32		endlogSeg;
2744
	int			max_advance;
2745 2746
	DIR		   *xldir;
	struct dirent *xlde;
2747
	char		lastoff[MAXFNAMELEN];
2748
	char		path[MAXPGPATH];
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
2749

2750 2751 2752 2753
	/*
	 * Initialize info about where to try to recycle to.  We allow recycling
	 * segments up to XLOGfileslop segments beyond the current XLOG location.
	 */
2754
	XLByteToPrevSeg(endptr, endlogId, endlogSeg);
2755
	max_advance = XLOGfileslop;
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
2756

2757
	xldir = AllocateDir(XLOGDIR);
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
2758
	if (xldir == NULL)
2759
		ereport(ERROR,
2760
				(errcode_for_file_access(),
2761 2762
				 errmsg("could not open transaction log directory \"%s\": %m",
						XLOGDIR)));
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
2763

2764
	XLogFileName(lastoff, ThisTimeLineID, log, seg);
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
2765

2766
	while ((xlde = ReadDir(xldir, XLOGDIR)) != NULL)
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
2767
	{
2768
		/*
2769
		 * We ignore the timeline part of the XLOG segment identifiers in
2770 2771 2772 2773 2774
		 * deciding whether a segment is still needed.	This ensures that we
		 * won't prematurely remove a segment from a parent timeline. We could
		 * probably be a little more proactive about removing segments of
		 * non-parent timelines, but that would be a whole lot more
		 * complicated.
2775
		 *
2776 2777
		 * We use the alphanumeric sorting property of the filenames to decide
		 * which ones are earlier than the lastoff segment.
2778
		 */
2779 2780 2781
		if (strlen(xlde->d_name) == 24 &&
			strspn(xlde->d_name, "0123456789ABCDEF") == 24 &&
			strcmp(xlde->d_name + 8, lastoff + 8) <= 0)
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
2782
		{
2783
			if (XLogArchiveCheckDone(xlde->d_name))
2784
			{
2785
				snprintf(path, MAXPGPATH, XLOGDIR "/%s", xlde->d_name);
2786

2787
				/*
2788 2789
				 * Before deleting the file, see if it can be recycled as a
				 * future log segment.
2790
				 */
2791 2792
				if (InstallXLogFileSegment(&endlogId, &endlogSeg, path,
										   true, &max_advance,
2793 2794
										   true))
				{
2795
					ereport(DEBUG2,
2796 2797
							(errmsg("recycled transaction log file \"%s\"",
									xlde->d_name)));
2798
					CheckpointStats.ckpt_segs_recycled++;
2799 2800 2801 2802 2803 2804
					/* Needn't recheck that slot on future iterations */
					if (max_advance > 0)
					{
						NextLogSeg(endlogId, endlogSeg);
						max_advance--;
					}
2805 2806 2807 2808
				}
				else
				{
					/* No need for any more future segments... */
2809
					ereport(DEBUG2,
2810 2811
							(errmsg("removing transaction log file \"%s\"",
									xlde->d_name)));
2812
					unlink(path);
2813
					CheckpointStats.ckpt_segs_removed++;
2814
				}
2815 2816

				XLogArchiveCleanup(xlde->d_name);
2817
			}
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
2818 2819
		}
	}
Bruce Momjian's avatar
Bruce Momjian committed
2820

2821
	FreeDir(xldir);
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
2822 2823
}

2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870
/*
 * Verify whether pg_xlog and pg_xlog/archive_status exist.
 * If the latter does not exist, recreate it.
 *
 * It is not the goal of this function to verify the contents of these
 * directories, but to help in cases where someone has performed a cluster
 * copy for PITR purposes but omitted pg_xlog from the copy.
 *
 * We could also recreate pg_xlog if it doesn't exist, but a deliberate
 * policy decision was made not to.  It is fairly common for pg_xlog to be
 * a symlink, and if that was the DBA's intent then automatically making a
 * plain directory would result in degraded performance with no notice.
 */
static void
ValidateXLOGDirectoryStructure(void)
{
	char		path[MAXPGPATH];
	struct stat	stat_buf;

	/* Check for pg_xlog; if it doesn't exist, error out */
	if (stat(XLOGDIR, &stat_buf) != 0 ||
		!S_ISDIR(stat_buf.st_mode))
		ereport(FATAL, 
				(errmsg("required WAL directory \"%s\" does not exist",
						XLOGDIR)));

	/* Check for archive_status */
	snprintf(path, MAXPGPATH, XLOGDIR "/archive_status");
	if (stat(path, &stat_buf) == 0)
	{
		/* Check for weird cases where it exists but isn't a directory */
		if (!S_ISDIR(stat_buf.st_mode))
			ereport(FATAL, 
					(errmsg("required WAL directory \"%s\" does not exist",
							path)));
	}
	else
	{
		ereport(LOG,
				(errmsg("creating missing WAL directory \"%s\"", path)));
		if (mkdir(path, 0700) < 0)
			ereport(FATAL, 
					(errmsg("could not create missing directory \"%s\": %m",
							path)));
	}
}

2871
/*
2872 2873 2874
 * Remove previous backup history files.  This also retries creation of
 * .ready files for any backup history files for which XLogArchiveNotify
 * failed earlier.
2875 2876
 */
static void
2877
CleanupBackupHistory(void)
2878 2879 2880 2881 2882
{
	DIR		   *xldir;
	struct dirent *xlde;
	char		path[MAXPGPATH];

2883
	xldir = AllocateDir(XLOGDIR);
2884 2885 2886
	if (xldir == NULL)
		ereport(ERROR,
				(errcode_for_file_access(),
2887 2888
				 errmsg("could not open transaction log directory \"%s\": %m",
						XLOGDIR)));
2889

2890
	while ((xlde = ReadDir(xldir, XLOGDIR)) != NULL)
2891 2892 2893 2894 2895 2896
	{
		if (strlen(xlde->d_name) > 24 &&
			strspn(xlde->d_name, "0123456789ABCDEF") == 24 &&
			strcmp(xlde->d_name + strlen(xlde->d_name) - strlen(".backup"),
				   ".backup") == 0)
		{
2897
			if (XLogArchiveCheckDone(xlde->d_name))
2898 2899
			{
				ereport(DEBUG2,
2900 2901
				(errmsg("removing transaction log backup history file \"%s\"",
						xlde->d_name)));
2902
				snprintf(path, MAXPGPATH, XLOGDIR "/%s", xlde->d_name);
2903 2904 2905 2906 2907 2908 2909 2910 2911
				unlink(path);
				XLogArchiveCleanup(xlde->d_name);
			}
		}
	}

	FreeDir(xldir);
}

Tom Lane's avatar
Tom Lane committed
2912 2913 2914 2915
/*
 * Restore the backup blocks present in an XLOG record, if any.
 *
 * We assume all of the record has been read into memory at *record.
2916 2917 2918 2919 2920 2921 2922 2923 2924
 *
 * Note: when a backup block is available in XLOG, we restore it
 * unconditionally, even if the page in the database appears newer.
 * This is to protect ourselves against database pages that were partially
 * or incorrectly written during a crash.  We assume that the XLOG data
 * must be good because it has passed a CRC check, while the database
 * page might not be.  This will force us to replay all subsequent
 * modifications of the page that appear in XLOG, rather than possibly
 * ignoring them as already applied, but that's not a huge drawback.
Tom Lane's avatar
Tom Lane committed
2925
 */
2926 2927 2928 2929 2930 2931 2932 2933 2934
static void
RestoreBkpBlocks(XLogRecord *record, XLogRecPtr lsn)
{
	Buffer		buffer;
	Page		page;
	BkpBlock	bkpb;
	char	   *blk;
	int			i;

2935
	blk = (char *) XLogRecGetData(record) + record->xl_len;
Tom Lane's avatar
Tom Lane committed
2936
	for (i = 0; i < XLR_MAX_BKP_BLOCKS; i++)
2937
	{
Tom Lane's avatar
Tom Lane committed
2938
		if (!(record->xl_info & XLR_SET_BKP_BLOCK(i)))
2939 2940
			continue;

2941
		memcpy(&bkpb, blk, sizeof(BkpBlock));
2942 2943
		blk += sizeof(BkpBlock);

2944 2945
		buffer = XLogReadBufferExtended(bkpb.node, bkpb.fork, bkpb.block,
										RBM_ZERO);
2946 2947
		Assert(BufferIsValid(buffer));
		page = (Page) BufferGetPage(buffer);
2948

2949
		if (bkpb.hole_length == 0)
2950
		{
2951 2952 2953 2954 2955 2956 2957 2958 2959 2960
			memcpy((char *) page, blk, BLCKSZ);
		}
		else
		{
			/* must zero-fill the hole */
			MemSet((char *) page, 0, BLCKSZ);
			memcpy((char *) page, blk, bkpb.hole_offset);
			memcpy((char *) page + (bkpb.hole_offset + bkpb.hole_length),
				   blk + bkpb.hole_offset,
				   BLCKSZ - (bkpb.hole_offset + bkpb.hole_length));
2961 2962
		}

2963 2964
		PageSetLSN(page, lsn);
		PageSetTLI(page, ThisTimeLineID);
2965 2966
		MarkBufferDirty(buffer);
		UnlockReleaseBuffer(buffer);
2967

2968
		blk += BLCKSZ - bkpb.hole_length;
2969 2970 2971
	}
}

Tom Lane's avatar
Tom Lane committed
2972 2973 2974 2975 2976 2977 2978
/*
 * CRC-check an XLOG record.  We do not believe the contents of an XLOG
 * record (other than to the minimal extent of computing the amount of
 * data to read in) until we've checked the CRCs.
 *
 * We assume all of the record has been read into memory at *record.
 */
2979 2980 2981
static bool
RecordIsValid(XLogRecord *record, XLogRecPtr recptr, int emode)
{
2982
	pg_crc32	crc;
2983 2984
	int			i;
	uint32		len = record->xl_len;
2985
	BkpBlock	bkpb;
2986 2987
	char	   *blk;

2988 2989 2990
	/* First the rmgr data */
	INIT_CRC32(crc);
	COMP_CRC32(crc, XLogRecGetData(record), len);
2991

2992
	/* Add in the backup blocks, if any */
2993
	blk = (char *) XLogRecGetData(record) + len;
Tom Lane's avatar
Tom Lane committed
2994
	for (i = 0; i < XLR_MAX_BKP_BLOCKS; i++)
2995
	{
2996
		uint32		blen;
2997

Tom Lane's avatar
Tom Lane committed
2998
		if (!(record->xl_info & XLR_SET_BKP_BLOCK(i)))
2999 3000
			continue;

3001 3002
		memcpy(&bkpb, blk, sizeof(BkpBlock));
		if (bkpb.hole_offset + bkpb.hole_length > BLCKSZ)
3003
		{
3004
			ereport(emode,
3005 3006 3007
					(errmsg("incorrect hole size in record at %X/%X",
							recptr.xlogid, recptr.xrecoff)));
			return false;
3008
		}
3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030
		blen = sizeof(BkpBlock) + BLCKSZ - bkpb.hole_length;
		COMP_CRC32(crc, blk, blen);
		blk += blen;
	}

	/* Check that xl_tot_len agrees with our calculation */
	if (blk != (char *) record + record->xl_tot_len)
	{
		ereport(emode,
				(errmsg("incorrect total length in record at %X/%X",
						recptr.xlogid, recptr.xrecoff)));
		return false;
	}

	/* Finally include the record header */
	COMP_CRC32(crc, (char *) record + sizeof(pg_crc32),
			   SizeOfXLogRecord - sizeof(pg_crc32));
	FIN_CRC32(crc);

	if (!EQ_CRC32(record->xl_crc, crc))
	{
		ereport(emode,
3031 3032
		(errmsg("incorrect resource manager data checksum in record at %X/%X",
				recptr.xlogid, recptr.xrecoff)));
3033
		return false;
3034 3035
	}

3036
	return true;
3037 3038
}

Tom Lane's avatar
Tom Lane committed
3039 3040 3041 3042 3043 3044
/*
 * Attempt to read an XLOG record.
 *
 * If RecPtr is not NULL, try to read a record at that position.  Otherwise
 * try to read a record just after the last one previously read.
 *
3045 3046
 * If no valid record is available, returns NULL, or fails if emode is PANIC.
 * (emode must be either PANIC or LOG.)
Tom Lane's avatar
Tom Lane committed
3047
 *
3048 3049
 * The record is copied into readRecordBuf, so that on successful return,
 * the returned record pointer always points there.
Tom Lane's avatar
Tom Lane committed
3050
 */
3051
static XLogRecord *
3052
ReadRecord(XLogRecPtr *RecPtr, int emode)
3053
{
3054
	XLogRecord *record;
3055
	char	   *buffer;
3056
	XLogRecPtr	tmpRecPtr = EndRecPtr;
3057
	bool		randAccess = false;
Tom Lane's avatar
Tom Lane committed
3058 3059 3060
	uint32		len,
				total_len;
	uint32		targetPageOff;
3061 3062
	uint32		targetRecOff;
	uint32		pageHeaderSize;
Tom Lane's avatar
Tom Lane committed
3063 3064 3065 3066

	if (readBuf == NULL)
	{
		/*
3067 3068 3069 3070 3071
		 * First time through, permanently allocate readBuf.  We do it this
		 * way, rather than just making a static array, for two reasons: (1)
		 * no need to waste the storage in most instantiations of the backend;
		 * (2) a static char array isn't guaranteed to have any particular
		 * alignment, whereas malloc() will provide MAXALIGN'd storage.
Tom Lane's avatar
Tom Lane committed
3072
		 */
3073
		readBuf = (char *) malloc(XLOG_BLCKSZ);
Tom Lane's avatar
Tom Lane committed
3074 3075
		Assert(readBuf != NULL);
	}
3076

Tom Lane's avatar
Tom Lane committed
3077
	if (RecPtr == NULL)
3078
	{
3079
		RecPtr = &tmpRecPtr;
Tom Lane's avatar
Tom Lane committed
3080
		/* fast case if next record is on same page */
3081 3082 3083 3084 3085
		if (nextRecord != NULL)
		{
			record = nextRecord;
			goto got_record;
		}
Tom Lane's avatar
Tom Lane committed
3086
		/* align old recptr to next page */
3087 3088
		if (tmpRecPtr.xrecoff % XLOG_BLCKSZ != 0)
			tmpRecPtr.xrecoff += (XLOG_BLCKSZ - tmpRecPtr.xrecoff % XLOG_BLCKSZ);
3089 3090 3091 3092 3093
		if (tmpRecPtr.xrecoff >= XLogFileSize)
		{
			(tmpRecPtr.xlogid)++;
			tmpRecPtr.xrecoff = 0;
		}
3094 3095 3096 3097 3098 3099 3100 3101
		/* We will account for page header size below */
	}
	else
	{
		if (!XRecOffIsValid(RecPtr->xrecoff))
			ereport(PANIC,
					(errmsg("invalid record offset at %X/%X",
							RecPtr->xlogid, RecPtr->xrecoff)));
Bruce Momjian's avatar
Bruce Momjian committed
3102

3103
		/*
3104 3105 3106 3107 3108
		 * Since we are going to a random position in WAL, forget any prior
		 * state about what timeline we were in, and allow it to be any
		 * timeline in expectedTLIs.  We also set a flag to allow curFileTLI
		 * to go backwards (but we can't reset that variable right here, since
		 * we might not change files at all).
3109 3110 3111
		 */
		lastPageTLI = 0;		/* see comment in ValidXLOGHeader */
		randAccess = true;		/* allow curFileTLI to go backwards too */
3112 3113
	}

Tom Lane's avatar
Tom Lane committed
3114
	if (readFile >= 0 && !XLByteInSeg(*RecPtr, readId, readSeg))
3115
	{
3116 3117
		close(readFile);
		readFile = -1;
3118
	}
Tom Lane's avatar
Tom Lane committed
3119
	XLByteToSeg(*RecPtr, readId, readSeg);
3120
	if (readFile < 0)
3121
	{
3122 3123 3124 3125 3126
		/* Now it's okay to reset curFileTLI if random fetch */
		if (randAccess)
			curFileTLI = 0;

		readFile = XLogFileRead(readId, readSeg, emode);
3127 3128
		if (readFile < 0)
			goto next_record_is_invalid;
3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147

		/*
		 * Whenever switching to a new WAL segment, we read the first page of
		 * the file and validate its header, even if that's not where the
		 * target record is.  This is so that we can check the additional
		 * identification info that is present in the first page's "long"
		 * header.
		 */
		readOff = 0;
		if (read(readFile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ)
		{
			ereport(emode,
					(errcode_for_file_access(),
					 errmsg("could not read from log file %u, segment %u, offset %u: %m",
							readId, readSeg, readOff)));
			goto next_record_is_invalid;
		}
		if (!ValidXLOGHeader((XLogPageHeader) readBuf, emode))
			goto next_record_is_invalid;
3148 3149
	}

3150
	targetPageOff = ((RecPtr->xrecoff % XLogSegSize) / XLOG_BLCKSZ) * XLOG_BLCKSZ;
Tom Lane's avatar
Tom Lane committed
3151
	if (readOff != targetPageOff)
3152
	{
Tom Lane's avatar
Tom Lane committed
3153 3154 3155
		readOff = targetPageOff;
		if (lseek(readFile, (off_t) readOff, SEEK_SET) < 0)
		{
3156 3157
			ereport(emode,
					(errcode_for_file_access(),
3158
					 errmsg("could not seek in log file %u, segment %u to offset %u: %m",
3159
							readId, readSeg, readOff)));
Tom Lane's avatar
Tom Lane committed
3160 3161
			goto next_record_is_invalid;
		}
3162
		if (read(readFile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ)
Tom Lane's avatar
Tom Lane committed
3163
		{
3164 3165
			ereport(emode,
					(errcode_for_file_access(),
3166
					 errmsg("could not read from log file %u, segment %u, offset %u: %m",
3167
							readId, readSeg, readOff)));
Tom Lane's avatar
Tom Lane committed
3168 3169
			goto next_record_is_invalid;
		}
3170
		if (!ValidXLOGHeader((XLogPageHeader) readBuf, emode))
3171 3172
			goto next_record_is_invalid;
	}
3173
	pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) readBuf);
3174
	targetRecOff = RecPtr->xrecoff % XLOG_BLCKSZ;
3175 3176 3177
	if (targetRecOff == 0)
	{
		/*
3178 3179 3180
		 * Can only get here in the continuing-from-prev-page case, because
		 * XRecOffIsValid eliminated the zero-page-offset case otherwise. Need
		 * to skip over the new page's header.
3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191
		 */
		tmpRecPtr.xrecoff += pageHeaderSize;
		targetRecOff = pageHeaderSize;
	}
	else if (targetRecOff < pageHeaderSize)
	{
		ereport(emode,
				(errmsg("invalid record offset at %X/%X",
						RecPtr->xlogid, RecPtr->xrecoff)));
		goto next_record_is_invalid;
	}
Tom Lane's avatar
Tom Lane committed
3192
	if ((((XLogPageHeader) readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
3193
		targetRecOff == pageHeaderSize)
3194
	{
3195 3196 3197
		ereport(emode,
				(errmsg("contrecord is requested by %X/%X",
						RecPtr->xlogid, RecPtr->xrecoff)));
3198 3199
		goto next_record_is_invalid;
	}
3200
	record = (XLogRecord *) ((char *) readBuf + RecPtr->xrecoff % XLOG_BLCKSZ);
3201 3202

got_record:;
3203

Tom Lane's avatar
Tom Lane committed
3204
	/*
Bruce Momjian's avatar
Bruce Momjian committed
3205 3206
	 * xl_len == 0 is bad data for everything except XLOG SWITCH, where it is
	 * required.
Tom Lane's avatar
Tom Lane committed
3207
	 */
3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218
	if (record->xl_rmid == RM_XLOG_ID && record->xl_info == XLOG_SWITCH)
	{
		if (record->xl_len != 0)
		{
			ereport(emode,
					(errmsg("invalid xlog switch record at %X/%X",
							RecPtr->xlogid, RecPtr->xrecoff)));
			goto next_record_is_invalid;
		}
	}
	else if (record->xl_len == 0)
3219
	{
3220 3221 3222
		ereport(emode,
				(errmsg("record with zero length at %X/%X",
						RecPtr->xlogid, RecPtr->xrecoff)));
3223 3224
		goto next_record_is_invalid;
	}
3225 3226 3227 3228 3229 3230 3231 3232 3233
	if (record->xl_tot_len < SizeOfXLogRecord + record->xl_len ||
		record->xl_tot_len > SizeOfXLogRecord + record->xl_len +
		XLR_MAX_BKP_BLOCKS * (sizeof(BkpBlock) + BLCKSZ))
	{
		ereport(emode,
				(errmsg("invalid record length at %X/%X",
						RecPtr->xlogid, RecPtr->xrecoff)));
		goto next_record_is_invalid;
	}
3234 3235 3236 3237
	if (record->xl_rmid > RM_MAX_ID)
	{
		ereport(emode,
				(errmsg("invalid resource manager ID %u at %X/%X",
3238
						record->xl_rmid, RecPtr->xlogid, RecPtr->xrecoff)));
3239 3240
		goto next_record_is_invalid;
	}
3241 3242 3243
	if (randAccess)
	{
		/*
3244 3245
		 * We can't exactly verify the prev-link, but surely it should be less
		 * than the record's own address.
3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258
		 */
		if (!XLByteLT(record->xl_prev, *RecPtr))
		{
			ereport(emode,
					(errmsg("record with incorrect prev-link %X/%X at %X/%X",
							record->xl_prev.xlogid, record->xl_prev.xrecoff,
							RecPtr->xlogid, RecPtr->xrecoff)));
			goto next_record_is_invalid;
		}
	}
	else
	{
		/*
3259 3260 3261
		 * Record's prev-link should exactly match our previous location. This
		 * check guards against torn WAL pages where a stale but valid-looking
		 * WAL record starts on a sector boundary.
3262 3263 3264 3265 3266 3267 3268 3269 3270 3271
		 */
		if (!XLByteEQ(record->xl_prev, ReadRecPtr))
		{
			ereport(emode,
					(errmsg("record with incorrect prev-link %X/%X at %X/%X",
							record->xl_prev.xlogid, record->xl_prev.xrecoff,
							RecPtr->xlogid, RecPtr->xrecoff)));
			goto next_record_is_invalid;
		}
	}
3272

Tom Lane's avatar
Tom Lane committed
3273
	/*
3274
	 * Allocate or enlarge readRecordBuf as needed.  To avoid useless small
3275 3276 3277 3278
	 * increases, round its size to a multiple of XLOG_BLCKSZ, and make sure
	 * it's at least 4*Max(BLCKSZ, XLOG_BLCKSZ) to start with.  (That is
	 * enough for all "normal" records, but very large commit or abort records
	 * might need more space.)
Tom Lane's avatar
Tom Lane committed
3279
	 */
3280
	total_len = record->xl_tot_len;
3281
	if (total_len > readRecordBufSize)
3282
	{
3283 3284
		uint32		newSize = total_len;

3285 3286
		newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
		newSize = Max(newSize, 4 * Max(BLCKSZ, XLOG_BLCKSZ));
3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299
		if (readRecordBuf)
			free(readRecordBuf);
		readRecordBuf = (char *) malloc(newSize);
		if (!readRecordBuf)
		{
			readRecordBufSize = 0;
			/* We treat this as a "bogus data" condition */
			ereport(emode,
					(errmsg("record length %u at %X/%X too long",
							total_len, RecPtr->xlogid, RecPtr->xrecoff)));
			goto next_record_is_invalid;
		}
		readRecordBufSize = newSize;
3300
	}
3301 3302

	buffer = readRecordBuf;
3303
	nextRecord = NULL;
3304
	len = XLOG_BLCKSZ - RecPtr->xrecoff % XLOG_BLCKSZ;
Tom Lane's avatar
Tom Lane committed
3305
	if (total_len > len)
3306
	{
Tom Lane's avatar
Tom Lane committed
3307 3308
		/* Need to reassemble record */
		XLogContRecord *contrecord;
3309
		uint32		gotlen = len;
3310

Tom Lane's avatar
Tom Lane committed
3311
		memcpy(buffer, record, len);
3312
		record = (XLogRecord *) buffer;
Tom Lane's avatar
Tom Lane committed
3313
		buffer += len;
3314
		for (;;)
3315
		{
3316
			readOff += XLOG_BLCKSZ;
Tom Lane's avatar
Tom Lane committed
3317
			if (readOff >= XLogSegSize)
3318 3319
			{
				close(readFile);
Tom Lane's avatar
Tom Lane committed
3320 3321
				readFile = -1;
				NextLogSeg(readId, readSeg);
3322
				readFile = XLogFileRead(readId, readSeg, emode);
3323 3324
				if (readFile < 0)
					goto next_record_is_invalid;
Tom Lane's avatar
Tom Lane committed
3325
				readOff = 0;
3326
			}
3327
			if (read(readFile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ)
Tom Lane's avatar
Tom Lane committed
3328
			{
3329 3330
				ereport(emode,
						(errcode_for_file_access(),
Tom Lane's avatar
Tom Lane committed
3331
						 errmsg("could not read from log file %u, segment %u, offset %u: %m",
3332
								readId, readSeg, readOff)));
Tom Lane's avatar
Tom Lane committed
3333 3334
				goto next_record_is_invalid;
			}
3335
			if (!ValidXLOGHeader((XLogPageHeader) readBuf, emode))
3336
				goto next_record_is_invalid;
Tom Lane's avatar
Tom Lane committed
3337
			if (!(((XLogPageHeader) readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD))
3338
			{
3339 3340 3341
				ereport(emode,
						(errmsg("there is no contrecord flag in log file %u, segment %u, offset %u",
								readId, readSeg, readOff)));
3342 3343
				goto next_record_is_invalid;
			}
3344 3345
			pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) readBuf);
			contrecord = (XLogContRecord *) ((char *) readBuf + pageHeaderSize);
3346
			if (contrecord->xl_rem_len == 0 ||
Tom Lane's avatar
Tom Lane committed
3347
				total_len != (contrecord->xl_rem_len + gotlen))
3348
			{
3349 3350 3351 3352
				ereport(emode,
						(errmsg("invalid contrecord length %u in log file %u, segment %u, offset %u",
								contrecord->xl_rem_len,
								readId, readSeg, readOff)));
3353 3354
				goto next_record_is_invalid;
			}
3355
			len = XLOG_BLCKSZ - pageHeaderSize - SizeOfXLogContRecord;
Tom Lane's avatar
Tom Lane committed
3356
			if (contrecord->xl_rem_len > len)
3357
			{
3358
				memcpy(buffer, (char *) contrecord + SizeOfXLogContRecord, len);
Tom Lane's avatar
Tom Lane committed
3359 3360 3361 3362 3363 3364 3365 3366 3367 3368
				gotlen += len;
				buffer += len;
				continue;
			}
			memcpy(buffer, (char *) contrecord + SizeOfXLogContRecord,
				   contrecord->xl_rem_len);
			break;
		}
		if (!RecordIsValid(record, *RecPtr, emode))
			goto next_record_is_invalid;
3369
		pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) readBuf);
3370
		if (XLOG_BLCKSZ - SizeOfXLogRecord >= pageHeaderSize +
3371
			MAXALIGN(SizeOfXLogContRecord + contrecord->xl_rem_len))
Tom Lane's avatar
Tom Lane committed
3372
		{
3373
			nextRecord = (XLogRecord *) ((char *) contrecord +
3374
					MAXALIGN(SizeOfXLogContRecord + contrecord->xl_rem_len));
Tom Lane's avatar
Tom Lane committed
3375 3376 3377
		}
		EndRecPtr.xlogid = readId;
		EndRecPtr.xrecoff = readSeg * XLogSegSize + readOff +
3378 3379
			pageHeaderSize +
			MAXALIGN(SizeOfXLogContRecord + contrecord->xl_rem_len);
Tom Lane's avatar
Tom Lane committed
3380
		ReadRecPtr = *RecPtr;
3381
		/* needn't worry about XLOG SWITCH, it can't cross page boundaries */
Tom Lane's avatar
Tom Lane committed
3382
		return record;
3383 3384
	}

Tom Lane's avatar
Tom Lane committed
3385 3386 3387
	/* Record does not cross a page boundary */
	if (!RecordIsValid(record, *RecPtr, emode))
		goto next_record_is_invalid;
3388
	if (XLOG_BLCKSZ - SizeOfXLogRecord >= RecPtr->xrecoff % XLOG_BLCKSZ +
Tom Lane's avatar
Tom Lane committed
3389 3390 3391 3392 3393 3394
		MAXALIGN(total_len))
		nextRecord = (XLogRecord *) ((char *) record + MAXALIGN(total_len));
	EndRecPtr.xlogid = RecPtr->xlogid;
	EndRecPtr.xrecoff = RecPtr->xrecoff + MAXALIGN(total_len);
	ReadRecPtr = *RecPtr;
	memcpy(buffer, record, total_len);
Bruce Momjian's avatar
Bruce Momjian committed
3395

3396 3397 3398 3399 3400 3401 3402 3403 3404
	/*
	 * Special processing if it's an XLOG SWITCH record
	 */
	if (record->xl_rmid == RM_XLOG_ID && record->xl_info == XLOG_SWITCH)
	{
		/* Pretend it extends to end of segment */
		EndRecPtr.xrecoff += XLogSegSize - 1;
		EndRecPtr.xrecoff -= EndRecPtr.xrecoff % XLogSegSize;
		nextRecord = NULL;		/* definitely not on same page */
Bruce Momjian's avatar
Bruce Momjian committed
3405

3406
		/*
Bruce Momjian's avatar
Bruce Momjian committed
3407 3408 3409
		 * Pretend that readBuf contains the last page of the segment. This is
		 * just to avoid Assert failure in StartupXLOG if XLOG ends with this
		 * segment.
3410 3411 3412
		 */
		readOff = XLogSegSize - XLOG_BLCKSZ;
	}
Tom Lane's avatar
Tom Lane committed
3413
	return (XLogRecord *) buffer;
3414

Tom Lane's avatar
Tom Lane committed
3415
next_record_is_invalid:;
3416 3417 3418 3419 3420
	if (readFile >= 0)
	{
		close(readFile);
		readFile = -1;
	}
Tom Lane's avatar
Tom Lane committed
3421 3422
	nextRecord = NULL;
	return NULL;
3423 3424
}

3425 3426 3427 3428
/*
 * Check whether the xlog header of a page just read in looks valid.
 *
 * This is just a convenience subroutine to avoid duplicated code in
3429
 * ReadRecord.	It's not intended for use from anywhere else.
3430 3431
 */
static bool
3432
ValidXLOGHeader(XLogPageHeader hdr, int emode)
3433
{
3434 3435
	XLogRecPtr	recaddr;

3436 3437
	if (hdr->xlp_magic != XLOG_PAGE_MAGIC)
	{
3438 3439 3440
		ereport(emode,
				(errmsg("invalid magic number %04X in log file %u, segment %u, offset %u",
						hdr->xlp_magic, readId, readSeg, readOff)));
3441 3442 3443 3444
		return false;
	}
	if ((hdr->xlp_info & ~XLP_ALL_FLAGS) != 0)
	{
3445 3446 3447
		ereport(emode,
				(errmsg("invalid info bits %04X in log file %u, segment %u, offset %u",
						hdr->xlp_info, readId, readSeg, readOff)));
3448 3449
		return false;
	}
3450
	if (hdr->xlp_info & XLP_LONG_HEADER)
3451
	{
3452
		XLogLongPageHeader longhdr = (XLogLongPageHeader) hdr;
3453

3454
		if (longhdr->xlp_sysid != ControlFile->system_identifier)
3455
		{
3456 3457
			char		fhdrident_str[32];
			char		sysident_str[32];
3458

3459
			/*
3460 3461
			 * Format sysids separately to keep platform-dependent format code
			 * out of the translatable message string.
3462 3463 3464 3465 3466 3467 3468
			 */
			snprintf(fhdrident_str, sizeof(fhdrident_str), UINT64_FORMAT,
					 longhdr->xlp_sysid);
			snprintf(sysident_str, sizeof(sysident_str), UINT64_FORMAT,
					 ControlFile->system_identifier);
			ereport(emode,
					(errmsg("WAL file is from different system"),
3469 3470
					 errdetail("WAL file SYSID is %s, pg_control SYSID is %s",
							   fhdrident_str, sysident_str)));
3471 3472 3473 3474 3475 3476
			return false;
		}
		if (longhdr->xlp_seg_size != XLogSegSize)
		{
			ereport(emode,
					(errmsg("WAL file is from different system"),
3477
					 errdetail("Incorrect XLOG_SEG_SIZE in page header.")));
3478 3479
			return false;
		}
3480 3481 3482 3483 3484 3485 3486
		if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
		{
			ereport(emode,
					(errmsg("WAL file is from different system"),
					 errdetail("Incorrect XLOG_BLCKSZ in page header.")));
			return false;
		}
3487
	}
3488 3489 3490 3491 3492 3493 3494 3495 3496
	else if (readOff == 0)
	{
		/* hmm, first page of file doesn't have a long header? */
		ereport(emode,
				(errmsg("invalid info bits %04X in log file %u, segment %u, offset %u",
						hdr->xlp_info, readId, readSeg, readOff)));
		return false;
	}

3497 3498 3499 3500 3501 3502
	recaddr.xlogid = readId;
	recaddr.xrecoff = readSeg * XLogSegSize + readOff;
	if (!XLByteEQ(hdr->xlp_pageaddr, recaddr))
	{
		ereport(emode,
				(errmsg("unexpected pageaddr %X/%X in log file %u, segment %u, offset %u",
3503
						hdr->xlp_pageaddr.xlogid, hdr->xlp_pageaddr.xrecoff,
3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524
						readId, readSeg, readOff)));
		return false;
	}

	/*
	 * Check page TLI is one of the expected values.
	 */
	if (!list_member_int(expectedTLIs, (int) hdr->xlp_tli))
	{
		ereport(emode,
				(errmsg("unexpected timeline ID %u in log file %u, segment %u, offset %u",
						hdr->xlp_tli,
						readId, readSeg, readOff)));
		return false;
	}

	/*
	 * Since child timelines are always assigned a TLI greater than their
	 * immediate parent's TLI, we should never see TLI go backwards across
	 * successive pages of a consistent WAL sequence.
	 *
3525 3526 3527
	 * Of course this check should only be applied when advancing sequentially
	 * across pages; therefore ReadRecord resets lastPageTLI to zero when
	 * going to a random page.
3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544
	 */
	if (hdr->xlp_tli < lastPageTLI)
	{
		ereport(emode,
				(errmsg("out-of-sequence timeline ID %u (after %u) in log file %u, segment %u, offset %u",
						hdr->xlp_tli, lastPageTLI,
						readId, readSeg, readOff)));
		return false;
	}
	lastPageTLI = hdr->xlp_tli;
	return true;
}

/*
 * Try to read a timeline's history file.
 *
 * If successful, return the list of component TLIs (the given TLI followed by
Bruce Momjian's avatar
Bruce Momjian committed
3545
 * its ancestor TLIs).	If we can't find the history file, assume that the
3546 3547 3548 3549 3550 3551 3552 3553 3554 3555
 * timeline has no parents, and return a list of just the specified timeline
 * ID.
 */
static List *
readTimeLineHistory(TimeLineID targetTLI)
{
	List	   *result;
	char		path[MAXPGPATH];
	char		histfname[MAXFNAMELEN];
	char		fline[MAXPGPATH];
Bruce Momjian's avatar
Bruce Momjian committed
3556
	FILE	   *fd;
3557 3558 3559 3560

	if (InArchiveRecovery)
	{
		TLHistoryFileName(histfname, targetTLI);
3561
		RestoreArchivedFile(path, histfname, "RECOVERYHISTORY", 0);
3562 3563 3564 3565
	}
	else
		TLHistoryFilePath(path, targetTLI);

Bruce Momjian's avatar
Bruce Momjian committed
3566
	fd = AllocateFile(path, "r");
3567 3568 3569 3570 3571
	if (fd == NULL)
	{
		if (errno != ENOENT)
			ereport(FATAL,
					(errcode_for_file_access(),
Peter Eisentraut's avatar
Peter Eisentraut committed
3572
					 errmsg("could not open file \"%s\": %m", path)));
3573 3574 3575 3576 3577 3578
		/* Not there, so assume no parents */
		return list_make1_int((int) targetTLI);
	}

	result = NIL;

Bruce Momjian's avatar
Bruce Momjian committed
3579 3580 3581
	/*
	 * Parse the file...
	 */
3582
	while (fgets(fline, sizeof(fline), fd) != NULL)
3583 3584
	{
		/* skip leading whitespace and check for # comment */
Bruce Momjian's avatar
Bruce Momjian committed
3585 3586 3587
		char	   *ptr;
		char	   *endptr;
		TimeLineID	tli;
3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607

		for (ptr = fline; *ptr; ptr++)
		{
			if (!isspace((unsigned char) *ptr))
				break;
		}
		if (*ptr == '\0' || *ptr == '#')
			continue;

		/* expect a numeric timeline ID as first field of line */
		tli = (TimeLineID) strtoul(ptr, &endptr, 0);
		if (endptr == ptr)
			ereport(FATAL,
					(errmsg("syntax error in history file: %s", fline),
					 errhint("Expected a numeric timeline ID.")));

		if (result &&
			tli <= (TimeLineID) linitial_int(result))
			ereport(FATAL,
					(errmsg("invalid data in history file: %s", fline),
3608
				   errhint("Timeline IDs must be in increasing sequence.")));
3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621

		/* Build list with newest item first */
		result = lcons_int((int) tli, result);

		/* we ignore the remainder of each line */
	}

	FreeFile(fd);

	if (result &&
		targetTLI <= (TimeLineID) linitial_int(result))
		ereport(FATAL,
				(errmsg("invalid data in history file \"%s\"", path),
3622
			errhint("Timeline IDs must be less than child timeline's ID.")));
3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640

	result = lcons_int((int) targetTLI, result);

	ereport(DEBUG3,
			(errmsg_internal("history of timeline %u is %s",
							 targetTLI, nodeToString(result))));

	return result;
}

/*
 * Probe whether a timeline history file exists for the given timeline ID
 */
static bool
existsTimeLineHistory(TimeLineID probeTLI)
{
	char		path[MAXPGPATH];
	char		histfname[MAXFNAMELEN];
Bruce Momjian's avatar
Bruce Momjian committed
3641
	FILE	   *fd;
3642 3643 3644 3645

	if (InArchiveRecovery)
	{
		TLHistoryFileName(histfname, probeTLI);
3646
		RestoreArchivedFile(path, histfname, "RECOVERYHISTORY", 0);
3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661
	}
	else
		TLHistoryFilePath(path, probeTLI);

	fd = AllocateFile(path, "r");
	if (fd != NULL)
	{
		FreeFile(fd);
		return true;
	}
	else
	{
		if (errno != ENOENT)
			ereport(FATAL,
					(errcode_for_file_access(),
Peter Eisentraut's avatar
Peter Eisentraut committed
3662
					 errmsg("could not open file \"%s\": %m", path)));
3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680
		return false;
	}
}

/*
 * Find the newest existing timeline, assuming that startTLI exists.
 *
 * Note: while this is somewhat heuristic, it does positively guarantee
 * that (result + 1) is not a known timeline, and therefore it should
 * be safe to assign that ID to a new timeline.
 */
static TimeLineID
findNewestTimeLine(TimeLineID startTLI)
{
	TimeLineID	newestTLI;
	TimeLineID	probeTLI;

	/*
3681 3682
	 * The algorithm is just to probe for the existence of timeline history
	 * files.  XXX is it useful to allow gaps in the sequence?
3683 3684 3685
	 */
	newestTLI = startTLI;

Bruce Momjian's avatar
Bruce Momjian committed
3686
	for (probeTLI = startTLI + 1;; probeTLI++)
3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709
	{
		if (existsTimeLineHistory(probeTLI))
		{
			newestTLI = probeTLI;		/* probeTLI exists */
		}
		else
		{
			/* doesn't exist, assume we're done */
			break;
		}
	}

	return newestTLI;
}

/*
 * Create a new timeline history file.
 *
 *	newTLI: ID of the new timeline
 *	parentTLI: ID of its immediate parent
 *	endTLI et al: ID of the last used WAL file, for annotation purposes
 *
 * Currently this is only used during recovery, and so there are no locking
Bruce Momjian's avatar
Bruce Momjian committed
3710
 * considerations.	But we should be just as tense as XLogFileInit to avoid
3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725
 * emplacing a bogus file.
 */
static void
writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI,
					 TimeLineID endTLI, uint32 endLogId, uint32 endLogSeg)
{
	char		path[MAXPGPATH];
	char		tmppath[MAXPGPATH];
	char		histfname[MAXFNAMELEN];
	char		xlogfname[MAXFNAMELEN];
	char		buffer[BLCKSZ];
	int			srcfd;
	int			fd;
	int			nbytes;

Bruce Momjian's avatar
Bruce Momjian committed
3726
	Assert(newTLI > parentTLI); /* else bad selection of newTLI */
3727 3728 3729 3730

	/*
	 * Write into a temp file name.
	 */
3731
	snprintf(tmppath, MAXPGPATH, XLOGDIR "/xlogtemp.%d", (int) getpid());
3732 3733 3734

	unlink(tmppath);

3735
	/* do not use get_sync_bit() here --- want to fsync only at end of fill */
3736 3737 3738
	fd = BasicOpenFile(tmppath, O_RDWR | O_CREAT | O_EXCL,
					   S_IRUSR | S_IWUSR);
	if (fd < 0)
3739
		ereport(ERROR,
3740 3741 3742 3743 3744 3745 3746 3747 3748
				(errcode_for_file_access(),
				 errmsg("could not create file \"%s\": %m", tmppath)));

	/*
	 * If a history file exists for the parent, copy it verbatim
	 */
	if (InArchiveRecovery)
	{
		TLHistoryFileName(histfname, parentTLI);
3749
		RestoreArchivedFile(path, histfname, "RECOVERYHISTORY", 0);
3750 3751 3752 3753 3754 3755 3756 3757
	}
	else
		TLHistoryFilePath(path, parentTLI);

	srcfd = BasicOpenFile(path, O_RDONLY, 0);
	if (srcfd < 0)
	{
		if (errno != ENOENT)
3758
			ereport(ERROR,
3759
					(errcode_for_file_access(),
Peter Eisentraut's avatar
Peter Eisentraut committed
3760
					 errmsg("could not open file \"%s\": %m", path)));
3761 3762 3763 3764 3765 3766 3767 3768 3769
		/* Not there, so assume parent has no parents */
	}
	else
	{
		for (;;)
		{
			errno = 0;
			nbytes = (int) read(srcfd, buffer, sizeof(buffer));
			if (nbytes < 0 || errno != 0)
3770
				ereport(ERROR,
3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784
						(errcode_for_file_access(),
						 errmsg("could not read file \"%s\": %m", path)));
			if (nbytes == 0)
				break;
			errno = 0;
			if ((int) write(fd, buffer, nbytes) != nbytes)
			{
				int			save_errno = errno;

				/*
				 * If we fail to make the file, delete it to release disk
				 * space
				 */
				unlink(tmppath);
Bruce Momjian's avatar
Bruce Momjian committed
3785 3786

				/*
3787
				 * if write didn't set errno, assume problem is no disk space
Bruce Momjian's avatar
Bruce Momjian committed
3788
				 */
3789 3790
				errno = save_errno ? save_errno : ENOSPC;

3791
				ereport(ERROR,
3792
						(errcode_for_file_access(),
3793
					 errmsg("could not write to file \"%s\": %m", tmppath)));
3794 3795 3796 3797 3798 3799 3800 3801
			}
		}
		close(srcfd);
	}

	/*
	 * Append one line with the details of this timeline split.
	 *
Bruce Momjian's avatar
Bruce Momjian committed
3802 3803
	 * If we did have a parent file, insert an extra newline just in case the
	 * parent file failed to end with one.
3804 3805 3806 3807 3808 3809 3810 3811 3812 3813
	 */
	XLogFileName(xlogfname, endTLI, endLogId, endLogSeg);

	snprintf(buffer, sizeof(buffer),
			 "%s%u\t%s\t%s transaction %u at %s\n",
			 (srcfd < 0) ? "" : "\n",
			 parentTLI,
			 xlogfname,
			 recoveryStopAfter ? "after" : "before",
			 recoveryStopXid,
3814
			 timestamptz_to_str(recoveryStopTime));
3815 3816 3817 3818 3819 3820 3821 3822

	nbytes = strlen(buffer);
	errno = 0;
	if ((int) write(fd, buffer, nbytes) != nbytes)
	{
		int			save_errno = errno;

		/*
Bruce Momjian's avatar
Bruce Momjian committed
3823
		 * If we fail to make the file, delete it to release disk space
3824 3825 3826 3827 3828
		 */
		unlink(tmppath);
		/* if write didn't set errno, assume problem is no disk space */
		errno = save_errno ? save_errno : ENOSPC;

3829
		ereport(ERROR,
3830 3831 3832 3833 3834
				(errcode_for_file_access(),
				 errmsg("could not write to file \"%s\": %m", tmppath)));
	}

	if (pg_fsync(fd) != 0)
3835
		ereport(ERROR,
3836 3837 3838 3839
				(errcode_for_file_access(),
				 errmsg("could not fsync file \"%s\": %m", tmppath)));

	if (close(fd))
3840
		ereport(ERROR,
3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856
				(errcode_for_file_access(),
				 errmsg("could not close file \"%s\": %m", tmppath)));


	/*
	 * Now move the completed history file into place with its final name.
	 */
	TLHistoryFilePath(path, newTLI);

	/*
	 * Prefer link() to rename() here just to be really sure that we don't
	 * overwrite an existing logfile.  However, there shouldn't be one, so
	 * rename() is an acceptable substitute except for the truly paranoid.
	 */
#if HAVE_WORKING_LINK
	if (link(tmppath, path) < 0)
3857
		ereport(ERROR,
3858 3859 3860 3861 3862 3863
				(errcode_for_file_access(),
				 errmsg("could not link file \"%s\" to \"%s\": %m",
						tmppath, path)));
	unlink(tmppath);
#else
	if (rename(tmppath, path) < 0)
3864
		ereport(ERROR,
3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876
				(errcode_for_file_access(),
				 errmsg("could not rename file \"%s\" to \"%s\": %m",
						tmppath, path)));
#endif

	/* The history file can be archived immediately. */
	TLHistoryFileName(histfname, newTLI);
	XLogArchiveNotify(histfname);
}

/*
 * I/O routines for pg_control
3877 3878
 *
 * *ControlFile is a buffer in shared memory that holds an image of the
3879
 * contents of pg_control.	WriteControlFile() initializes pg_control
3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892
 * given a preloaded buffer, ReadControlFile() loads the buffer from
 * the pg_control file (during postmaster or standalone-backend startup),
 * and UpdateControlFile() rewrites pg_control after we modify xlog state.
 *
 * For simplicity, WriteControlFile() initializes the fields of pg_control
 * that are related to checking backend/database compatibility, and
 * ReadControlFile() verifies they are correct.  We could split out the
 * I/O and compatibility-check functions, but there seems no need currently.
 */
static void
WriteControlFile(void)
{
	int			fd;
Bruce Momjian's avatar
Bruce Momjian committed
3893
	char		buffer[PG_CONTROL_SIZE];		/* need not be aligned */
3894 3895

	/*
Tom Lane's avatar
Tom Lane committed
3896
	 * Initialize version and compatibility-check fields
3897
	 */
Tom Lane's avatar
Tom Lane committed
3898 3899
	ControlFile->pg_control_version = PG_CONTROL_VERSION;
	ControlFile->catalog_version_no = CATALOG_VERSION_NO;
3900 3901 3902 3903

	ControlFile->maxAlign = MAXIMUM_ALIGNOF;
	ControlFile->floatFormat = FLOATFORMAT_VALUE;

3904 3905
	ControlFile->blcksz = BLCKSZ;
	ControlFile->relseg_size = RELSEG_SIZE;
3906
	ControlFile->xlog_blcksz = XLOG_BLCKSZ;
3907
	ControlFile->xlog_seg_size = XLOG_SEG_SIZE;
3908 3909

	ControlFile->nameDataLen = NAMEDATALEN;
3910
	ControlFile->indexMaxKeys = INDEX_MAX_KEYS;
3911

3912 3913
	ControlFile->toast_max_chunk_size = TOAST_MAX_CHUNK_SIZE;

3914
#ifdef HAVE_INT64_TIMESTAMP
3915
	ControlFile->enableIntTimes = true;
3916
#else
3917
	ControlFile->enableIntTimes = false;
3918
#endif
3919 3920
	ControlFile->float4ByVal = FLOAT4PASSBYVAL;
	ControlFile->float8ByVal = FLOAT8PASSBYVAL;
3921

Tom Lane's avatar
Tom Lane committed
3922
	/* Contents are protected with a CRC */
3923 3924 3925 3926 3927
	INIT_CRC32(ControlFile->crc);
	COMP_CRC32(ControlFile->crc,
			   (char *) ControlFile,
			   offsetof(ControlFileData, crc));
	FIN_CRC32(ControlFile->crc);
Tom Lane's avatar
Tom Lane committed
3928

3929
	/*
3930 3931 3932 3933 3934
	 * We write out PG_CONTROL_SIZE bytes into pg_control, zero-padding the
	 * excess over sizeof(ControlFileData).  This reduces the odds of
	 * premature-EOF errors when reading pg_control.  We'll still fail when we
	 * check the contents of the file, but hopefully with a more specific
	 * error than "couldn't read pg_control".
3935
	 */
3936 3937
	if (sizeof(ControlFileData) > PG_CONTROL_SIZE)
		elog(PANIC, "sizeof(ControlFileData) is larger than PG_CONTROL_SIZE; fix either one");
3938

3939
	memset(buffer, 0, PG_CONTROL_SIZE);
3940 3941
	memcpy(buffer, ControlFile, sizeof(ControlFileData));

3942 3943
	fd = BasicOpenFile(XLOG_CONTROL_FILE,
					   O_RDWR | O_CREAT | O_EXCL | PG_BINARY,
3944
					   S_IRUSR | S_IWUSR);
3945
	if (fd < 0)
3946 3947 3948
		ereport(PANIC,
				(errcode_for_file_access(),
				 errmsg("could not create control file \"%s\": %m",
3949
						XLOG_CONTROL_FILE)));
3950

3951
	errno = 0;
3952
	if (write(fd, buffer, PG_CONTROL_SIZE) != PG_CONTROL_SIZE)
3953 3954 3955 3956
	{
		/* if write didn't set errno, assume problem is no disk space */
		if (errno == 0)
			errno = ENOSPC;
3957 3958
		ereport(PANIC,
				(errcode_for_file_access(),
3959
				 errmsg("could not write to control file: %m")));
3960
	}
3961

3962
	if (pg_fsync(fd) != 0)
3963 3964
		ereport(PANIC,
				(errcode_for_file_access(),
3965
				 errmsg("could not fsync control file: %m")));
3966

3967 3968 3969 3970
	if (close(fd))
		ereport(PANIC,
				(errcode_for_file_access(),
				 errmsg("could not close control file: %m")));
3971 3972 3973 3974 3975
}

static void
ReadControlFile(void)
{
3976
	pg_crc32	crc;
3977 3978 3979 3980 3981
	int			fd;

	/*
	 * Read data...
	 */
3982 3983 3984
	fd = BasicOpenFile(XLOG_CONTROL_FILE,
					   O_RDWR | PG_BINARY,
					   S_IRUSR | S_IWUSR);
3985
	if (fd < 0)
3986 3987 3988
		ereport(PANIC,
				(errcode_for_file_access(),
				 errmsg("could not open control file \"%s\": %m",
3989
						XLOG_CONTROL_FILE)));
3990 3991

	if (read(fd, ControlFile, sizeof(ControlFileData)) != sizeof(ControlFileData))
3992 3993
		ereport(PANIC,
				(errcode_for_file_access(),
3994
				 errmsg("could not read from control file: %m")));
3995 3996 3997

	close(fd);

Tom Lane's avatar
Tom Lane committed
3998
	/*
3999 4000 4001 4002
	 * Check for expected pg_control format version.  If this is wrong, the
	 * CRC check will likely fail because we'll be checking the wrong number
	 * of bytes.  Complaining about wrong version will probably be more
	 * enlightening than complaining about wrong CRC.
Tom Lane's avatar
Tom Lane committed
4003
	 */
4004 4005 4006 4007 4008 4009 4010 4011 4012 4013

	if (ControlFile->pg_control_version != PG_CONTROL_VERSION && ControlFile->pg_control_version % 65536 == 0 && ControlFile->pg_control_version / 65536 != 0)
		ereport(FATAL,
				(errmsg("database files are incompatible with server"),
				 errdetail("The database cluster was initialized with PG_CONTROL_VERSION %d (0x%08x),"
						   " but the server was compiled with PG_CONTROL_VERSION %d (0x%08x).",
						   ControlFile->pg_control_version, ControlFile->pg_control_version,
						   PG_CONTROL_VERSION, PG_CONTROL_VERSION),
				 errhint("This could be a problem of mismatched byte ordering.  It looks like you need to initdb.")));

Tom Lane's avatar
Tom Lane committed
4014
	if (ControlFile->pg_control_version != PG_CONTROL_VERSION)
4015 4016 4017
		ereport(FATAL,
				(errmsg("database files are incompatible with server"),
				 errdetail("The database cluster was initialized with PG_CONTROL_VERSION %d,"
4018 4019
				  " but the server was compiled with PG_CONTROL_VERSION %d.",
						ControlFile->pg_control_version, PG_CONTROL_VERSION),
4020
				 errhint("It looks like you need to initdb.")));
4021

Tom Lane's avatar
Tom Lane committed
4022
	/* Now check the CRC. */
4023 4024 4025 4026 4027
	INIT_CRC32(crc);
	COMP_CRC32(crc,
			   (char *) ControlFile,
			   offsetof(ControlFileData, crc));
	FIN_CRC32(crc);
4028

4029
	if (!EQ_CRC32(crc, ControlFile->crc))
4030
		ereport(FATAL,
4031
				(errmsg("incorrect checksum in control file")));
4032

4033
	/*
4034
	 * Do compatibility checking immediately.  We do this here for 2 reasons:
4035
	 *
4036 4037
	 * (1) if the database isn't compatible with the backend executable, we
	 * want to abort before we can possibly do any damage;
4038 4039
	 *
	 * (2) this code is executed in the postmaster, so the setlocale() will
4040 4041
	 * propagate to forked backends, which aren't going to read this file for
	 * themselves.	(These locale settings are considered critical
4042 4043
	 * compatibility items because they can affect sort order of indexes.)
	 */
Tom Lane's avatar
Tom Lane committed
4044
	if (ControlFile->catalog_version_no != CATALOG_VERSION_NO)
4045 4046 4047
		ereport(FATAL,
				(errmsg("database files are incompatible with server"),
				 errdetail("The database cluster was initialized with CATALOG_VERSION_NO %d,"
4048 4049
				  " but the server was compiled with CATALOG_VERSION_NO %d.",
						ControlFile->catalog_version_no, CATALOG_VERSION_NO),
4050
				 errhint("It looks like you need to initdb.")));
4051 4052 4053
	if (ControlFile->maxAlign != MAXIMUM_ALIGNOF)
		ereport(FATAL,
				(errmsg("database files are incompatible with server"),
4054 4055 4056 4057
		   errdetail("The database cluster was initialized with MAXALIGN %d,"
					 " but the server was compiled with MAXALIGN %d.",
					 ControlFile->maxAlign, MAXIMUM_ALIGNOF),
				 errhint("It looks like you need to initdb.")));
4058 4059 4060
	if (ControlFile->floatFormat != FLOATFORMAT_VALUE)
		ereport(FATAL,
				(errmsg("database files are incompatible with server"),
Peter Eisentraut's avatar
Peter Eisentraut committed
4061
				 errdetail("The database cluster appears to use a different floating-point number format than the server executable."),
4062
				 errhint("It looks like you need to initdb.")));
4063
	if (ControlFile->blcksz != BLCKSZ)
4064 4065
		ereport(FATAL,
				(errmsg("database files are incompatible with server"),
4066 4067 4068 4069
			 errdetail("The database cluster was initialized with BLCKSZ %d,"
					   " but the server was compiled with BLCKSZ %d.",
					   ControlFile->blcksz, BLCKSZ),
				 errhint("It looks like you need to recompile or initdb.")));
4070
	if (ControlFile->relseg_size != RELSEG_SIZE)
4071 4072
		ereport(FATAL,
				(errmsg("database files are incompatible with server"),
4073 4074 4075 4076
		errdetail("The database cluster was initialized with RELSEG_SIZE %d,"
				  " but the server was compiled with RELSEG_SIZE %d.",
				  ControlFile->relseg_size, RELSEG_SIZE),
				 errhint("It looks like you need to recompile or initdb.")));
4077 4078 4079
	if (ControlFile->xlog_blcksz != XLOG_BLCKSZ)
		ereport(FATAL,
				(errmsg("database files are incompatible with server"),
Bruce Momjian's avatar
Bruce Momjian committed
4080 4081 4082
		errdetail("The database cluster was initialized with XLOG_BLCKSZ %d,"
				  " but the server was compiled with XLOG_BLCKSZ %d.",
				  ControlFile->xlog_blcksz, XLOG_BLCKSZ),
4083
				 errhint("It looks like you need to recompile or initdb.")));
4084 4085 4086 4087
	if (ControlFile->xlog_seg_size != XLOG_SEG_SIZE)
		ereport(FATAL,
				(errmsg("database files are incompatible with server"),
				 errdetail("The database cluster was initialized with XLOG_SEG_SIZE %d,"
4088
					   " but the server was compiled with XLOG_SEG_SIZE %d.",
4089
						   ControlFile->xlog_seg_size, XLOG_SEG_SIZE),
4090
				 errhint("It looks like you need to recompile or initdb.")));
4091
	if (ControlFile->nameDataLen != NAMEDATALEN)
4092 4093
		ereport(FATAL,
				(errmsg("database files are incompatible with server"),
4094 4095 4096 4097
		errdetail("The database cluster was initialized with NAMEDATALEN %d,"
				  " but the server was compiled with NAMEDATALEN %d.",
				  ControlFile->nameDataLen, NAMEDATALEN),
				 errhint("It looks like you need to recompile or initdb.")));
4098
	if (ControlFile->indexMaxKeys != INDEX_MAX_KEYS)
4099 4100
		ereport(FATAL,
				(errmsg("database files are incompatible with server"),
4101
				 errdetail("The database cluster was initialized with INDEX_MAX_KEYS %d,"
4102
					  " but the server was compiled with INDEX_MAX_KEYS %d.",
4103
						   ControlFile->indexMaxKeys, INDEX_MAX_KEYS),
4104
				 errhint("It looks like you need to recompile or initdb.")));
4105 4106 4107 4108
	if (ControlFile->toast_max_chunk_size != TOAST_MAX_CHUNK_SIZE)
		ereport(FATAL,
				(errmsg("database files are incompatible with server"),
				 errdetail("The database cluster was initialized with TOAST_MAX_CHUNK_SIZE %d,"
Bruce Momjian's avatar
Bruce Momjian committed
4109 4110
				" but the server was compiled with TOAST_MAX_CHUNK_SIZE %d.",
			  ControlFile->toast_max_chunk_size, (int) TOAST_MAX_CHUNK_SIZE),
4111
				 errhint("It looks like you need to recompile or initdb.")));
4112 4113

#ifdef HAVE_INT64_TIMESTAMP
4114
	if (ControlFile->enableIntTimes != true)
4115 4116 4117
		ereport(FATAL,
				(errmsg("database files are incompatible with server"),
				 errdetail("The database cluster was initialized without HAVE_INT64_TIMESTAMP"
4118 4119
				  " but the server was compiled with HAVE_INT64_TIMESTAMP."),
				 errhint("It looks like you need to recompile or initdb.")));
4120
#else
4121
	if (ControlFile->enableIntTimes != false)
4122 4123 4124
		ereport(FATAL,
				(errmsg("database files are incompatible with server"),
				 errdetail("The database cluster was initialized with HAVE_INT64_TIMESTAMP"
4125 4126
			   " but the server was compiled without HAVE_INT64_TIMESTAMP."),
				 errhint("It looks like you need to recompile or initdb.")));
4127 4128
#endif

4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159
#ifdef USE_FLOAT4_BYVAL
	if (ControlFile->float4ByVal != true)
		ereport(FATAL,
				(errmsg("database files are incompatible with server"),
				 errdetail("The database cluster was initialized without USE_FLOAT4_BYVAL"
						   " but the server was compiled with USE_FLOAT4_BYVAL."),
				 errhint("It looks like you need to recompile or initdb.")));
#else
	if (ControlFile->float4ByVal != false)
		ereport(FATAL,
				(errmsg("database files are incompatible with server"),
				 errdetail("The database cluster was initialized with USE_FLOAT4_BYVAL"
						   " but the server was compiled without USE_FLOAT4_BYVAL."),
				 errhint("It looks like you need to recompile or initdb.")));
#endif

#ifdef USE_FLOAT8_BYVAL
	if (ControlFile->float8ByVal != true)
		ereport(FATAL,
				(errmsg("database files are incompatible with server"),
				 errdetail("The database cluster was initialized without USE_FLOAT8_BYVAL"
						   " but the server was compiled with USE_FLOAT8_BYVAL."),
				 errhint("It looks like you need to recompile or initdb.")));
#else
	if (ControlFile->float8ByVal != false)
		ereport(FATAL,
				(errmsg("database files are incompatible with server"),
				 errdetail("The database cluster was initialized with USE_FLOAT8_BYVAL"
						   " but the server was compiled without USE_FLOAT8_BYVAL."),
				 errhint("It looks like you need to recompile or initdb.")));
#endif
4160 4161
}

4162
void
4163
UpdateControlFile(void)
4164
{
4165
	int			fd;
4166

4167 4168 4169 4170 4171
	INIT_CRC32(ControlFile->crc);
	COMP_CRC32(ControlFile->crc,
			   (char *) ControlFile,
			   offsetof(ControlFileData, crc));
	FIN_CRC32(ControlFile->crc);
4172

4173 4174 4175
	fd = BasicOpenFile(XLOG_CONTROL_FILE,
					   O_RDWR | PG_BINARY,
					   S_IRUSR | S_IWUSR);
4176
	if (fd < 0)
4177 4178 4179
		ereport(PANIC,
				(errcode_for_file_access(),
				 errmsg("could not open control file \"%s\": %m",
4180
						XLOG_CONTROL_FILE)));
4181

4182
	errno = 0;
4183
	if (write(fd, ControlFile, sizeof(ControlFileData)) != sizeof(ControlFileData))
4184 4185 4186 4187
	{
		/* if write didn't set errno, assume problem is no disk space */
		if (errno == 0)
			errno = ENOSPC;
4188 4189
		ereport(PANIC,
				(errcode_for_file_access(),
4190
				 errmsg("could not write to control file: %m")));
4191
	}
4192

4193
	if (pg_fsync(fd) != 0)
4194 4195
		ereport(PANIC,
				(errcode_for_file_access(),
4196
				 errmsg("could not fsync control file: %m")));
4197

4198 4199 4200 4201
	if (close(fd))
		ereport(PANIC,
				(errcode_for_file_access(),
				 errmsg("could not close control file: %m")));
4202 4203
}

4204
/*
Tom Lane's avatar
Tom Lane committed
4205
 * Initialization of shared memory for XLOG
4206
 */
4207
Size
4208
XLOGShmemSize(void)
4209
{
4210
	Size		size;
4211

4212 4213 4214 4215 4216 4217 4218
	/* XLogCtl */
	size = sizeof(XLogCtlData);
	/* xlblocks array */
	size = add_size(size, mul_size(sizeof(XLogRecPtr), XLOGbuffers));
	/* extra alignment padding for XLOG I/O buffers */
	size = add_size(size, ALIGNOF_XLOG_BUFFER);
	/* and the buffers themselves */
4219
	size = add_size(size, mul_size(XLOG_BLCKSZ, XLOGbuffers));
4220 4221

	/*
4222 4223 4224
	 * Note: we don't count ControlFileData, it comes out of the "slop factor"
	 * added by CreateSharedMemoryAndSemaphores.  This lets us use this
	 * routine again below to compute the actual allocation size.
4225 4226 4227
	 */

	return size;
4228 4229 4230 4231 4232
}

void
XLOGShmemInit(void)
{
4233 4234
	bool		foundCFile,
				foundXLog;
4235
	char	   *allocptr;
4236

4237
	ControlFile = (ControlFileData *)
4238
		ShmemInitStruct("Control File", sizeof(ControlFileData), &foundCFile);
4239 4240
	XLogCtl = (XLogCtlData *)
		ShmemInitStruct("XLOG Ctl", XLOGShmemSize(), &foundXLog);
4241

4242
	if (foundCFile || foundXLog)
4243 4244
	{
		/* both should be present or neither */
4245
		Assert(foundCFile && foundXLog);
4246 4247
		return;
	}
4248

Tom Lane's avatar
Tom Lane committed
4249
	memset(XLogCtl, 0, sizeof(XLogCtlData));
4250

Tom Lane's avatar
Tom Lane committed
4251
	/*
4252 4253 4254
	 * Since XLogCtlData contains XLogRecPtr fields, its sizeof should be a
	 * multiple of the alignment for same, so no extra alignment padding is
	 * needed here.
Tom Lane's avatar
Tom Lane committed
4255
	 */
4256 4257
	allocptr = ((char *) XLogCtl) + sizeof(XLogCtlData);
	XLogCtl->xlblocks = (XLogRecPtr *) allocptr;
Tom Lane's avatar
Tom Lane committed
4258
	memset(XLogCtl->xlblocks, 0, sizeof(XLogRecPtr) * XLOGbuffers);
4259
	allocptr += sizeof(XLogRecPtr) * XLOGbuffers;
4260

Tom Lane's avatar
Tom Lane committed
4261
	/*
4262
	 * Align the start of the page buffers to an ALIGNOF_XLOG_BUFFER boundary.
Tom Lane's avatar
Tom Lane committed
4263
	 */
4264 4265
	allocptr = (char *) TYPEALIGN(ALIGNOF_XLOG_BUFFER, allocptr);
	XLogCtl->pages = allocptr;
4266
	memset(XLogCtl->pages, 0, (Size) XLOG_BLCKSZ * XLOGbuffers);
Tom Lane's avatar
Tom Lane committed
4267 4268

	/*
4269 4270
	 * Do basic initialization of XLogCtl shared data. (StartupXLOG will fill
	 * in additional info.)
Tom Lane's avatar
Tom Lane committed
4271 4272 4273
	 */
	XLogCtl->XLogCacheBlck = XLOGbuffers - 1;
	XLogCtl->Insert.currpage = (XLogPageHeader) (XLogCtl->pages);
4274
	SpinLockInit(&XLogCtl->info_lck);
Tom Lane's avatar
Tom Lane committed
4275

4276
	/*
4277 4278 4279
	 * If we are not in bootstrap mode, pg_control should already exist. Read
	 * and validate it immediately (see comments in ReadControlFile() for the
	 * reasons why).
4280 4281 4282
	 */
	if (!IsBootstrapProcessingMode())
		ReadControlFile();
4283 4284 4285
}

/*
Tom Lane's avatar
Tom Lane committed
4286 4287
 * This func must be called ONCE on system install.  It creates pg_control
 * and the initial XLOG segment.
4288 4289
 */
void
Tom Lane's avatar
Tom Lane committed
4290
BootStrapXLOG(void)
4291
{
4292
	CheckPoint	checkPoint;
Tom Lane's avatar
Tom Lane committed
4293 4294
	char	   *buffer;
	XLogPageHeader page;
4295
	XLogLongPageHeader longpage;
4296
	XLogRecord *record;
4297
	bool		use_existent;
4298 4299
	uint64		sysidentifier;
	struct timeval tv;
4300
	pg_crc32	crc;
4301

4302
	/*
4303 4304 4305 4306 4307 4308 4309 4310 4311 4312
	 * Select a hopefully-unique system identifier code for this installation.
	 * We use the result of gettimeofday(), including the fractional seconds
	 * field, as being about as unique as we can easily get.  (Think not to
	 * use random(), since it hasn't been seeded and there's no portable way
	 * to seed it other than the system clock value...)  The upper half of the
	 * uint64 value is just the tv_sec part, while the lower half is the XOR
	 * of tv_sec and tv_usec.  This is to ensure that we don't lose uniqueness
	 * unnecessarily if "uint64" is really only 32 bits wide.  A person
	 * knowing this encoding can determine the initialization time of the
	 * installation, which could perhaps be useful sometimes.
4313 4314 4315 4316 4317
	 */
	gettimeofday(&tv, NULL);
	sysidentifier = ((uint64) tv.tv_sec) << 32;
	sysidentifier |= (uint32) (tv.tv_sec | tv.tv_usec);

4318 4319 4320
	/* First timeline ID is always 1 */
	ThisTimeLineID = 1;

4321
	/* page buffer must be aligned suitably for O_DIRECT */
4322
	buffer = (char *) palloc(XLOG_BLCKSZ + ALIGNOF_XLOG_BUFFER);
4323
	page = (XLogPageHeader) TYPEALIGN(ALIGNOF_XLOG_BUFFER, buffer);
4324
	memset(page, 0, XLOG_BLCKSZ);
Tom Lane's avatar
Tom Lane committed
4325

4326
	/* Set up information for the initial checkpoint record */
4327
	checkPoint.redo.xlogid = 0;
4328 4329
	checkPoint.redo.xrecoff = SizeOfXLogLongPHD;
	checkPoint.ThisTimeLineID = ThisTimeLineID;
4330
	checkPoint.nextXidEpoch = 0;
4331
	checkPoint.nextXid = FirstNormalTransactionId;
4332
	checkPoint.nextOid = FirstBootstrapObjectId;
4333
	checkPoint.nextMulti = FirstMultiXactId;
4334
	checkPoint.nextMultiOffset = 0;
4335
	checkPoint.time = (pg_time_t) time(NULL);
4336

4337 4338 4339
	ShmemVariableCache->nextXid = checkPoint.nextXid;
	ShmemVariableCache->nextOid = checkPoint.nextOid;
	ShmemVariableCache->oidCount = 0;
4340
	MultiXactSetNextMXact(checkPoint.nextMulti, checkPoint.nextMultiOffset);
4341

4342
	/* Set up the XLOG page header */
4343
	page->xlp_magic = XLOG_PAGE_MAGIC;
4344 4345
	page->xlp_info = XLP_LONG_HEADER;
	page->xlp_tli = ThisTimeLineID;
4346 4347
	page->xlp_pageaddr.xlogid = 0;
	page->xlp_pageaddr.xrecoff = 0;
4348 4349 4350
	longpage = (XLogLongPageHeader) page;
	longpage->xlp_sysid = sysidentifier;
	longpage->xlp_seg_size = XLogSegSize;
4351
	longpage->xlp_xlog_blcksz = XLOG_BLCKSZ;
4352 4353

	/* Insert the initial checkpoint record */
4354
	record = (XLogRecord *) ((char *) page + SizeOfXLogLongPHD);
4355
	record->xl_prev.xlogid = 0;
4356
	record->xl_prev.xrecoff = 0;
4357
	record->xl_xid = InvalidTransactionId;
4358
	record->xl_tot_len = SizeOfXLogRecord + sizeof(checkPoint);
4359
	record->xl_len = sizeof(checkPoint);
Tom Lane's avatar
Tom Lane committed
4360
	record->xl_info = XLOG_CHECKPOINT_SHUTDOWN;
4361
	record->xl_rmid = RM_XLOG_ID;
Tom Lane's avatar
Tom Lane committed
4362
	memcpy(XLogRecGetData(record), &checkPoint, sizeof(checkPoint));
4363

4364 4365 4366 4367 4368
	INIT_CRC32(crc);
	COMP_CRC32(crc, &checkPoint, sizeof(checkPoint));
	COMP_CRC32(crc, (char *) record + sizeof(pg_crc32),
			   SizeOfXLogRecord - sizeof(pg_crc32));
	FIN_CRC32(crc);
4369 4370
	record->xl_crc = crc;

4371
	/* Create first XLOG segment file */
4372 4373
	use_existent = false;
	openLogFile = XLogFileInit(0, 0, &use_existent, false);
4374

4375
	/* Write the first page with the initial record */
4376
	errno = 0;
4377
	if (write(openLogFile, page, XLOG_BLCKSZ) != XLOG_BLCKSZ)
4378 4379 4380 4381
	{
		/* if write didn't set errno, assume problem is no disk space */
		if (errno == 0)
			errno = ENOSPC;
4382 4383
		ereport(PANIC,
				(errcode_for_file_access(),
4384
			  errmsg("could not write bootstrap transaction log file: %m")));
4385
	}
4386

Tom Lane's avatar
Tom Lane committed
4387
	if (pg_fsync(openLogFile) != 0)
4388 4389
		ereport(PANIC,
				(errcode_for_file_access(),
4390
			  errmsg("could not fsync bootstrap transaction log file: %m")));
4391

4392 4393 4394
	if (close(openLogFile))
		ereport(PANIC,
				(errcode_for_file_access(),
4395
			  errmsg("could not close bootstrap transaction log file: %m")));
4396

Tom Lane's avatar
Tom Lane committed
4397
	openLogFile = -1;
4398

4399 4400
	/* Now create pg_control */

4401
	memset(ControlFile, 0, sizeof(ControlFileData));
Tom Lane's avatar
Tom Lane committed
4402
	/* Initialize pg_control status fields */
4403
	ControlFile->system_identifier = sysidentifier;
Tom Lane's avatar
Tom Lane committed
4404 4405
	ControlFile->state = DB_SHUTDOWNED;
	ControlFile->time = checkPoint.time;
4406
	ControlFile->checkPoint = checkPoint.redo;
Tom Lane's avatar
Tom Lane committed
4407
	ControlFile->checkPointCopy = checkPoint;
4408
	/* some additional ControlFile fields are set in WriteControlFile() */
4409

4410
	WriteControlFile();
4411 4412 4413

	/* Bootstrap the commit log, too */
	BootStrapCLOG();
4414
	BootStrapSUBTRANS();
4415
	BootStrapMultiXact();
4416

4417
	pfree(buffer);
4418 4419
}

4420
static char *
4421
str_time(pg_time_t tnow)
4422
{
4423
	static char buf[128];
4424

4425 4426 4427
	pg_strftime(buf, sizeof(buf),
				"%Y-%m-%d %H:%M:%S %Z",
				pg_localtime(&tnow, log_timezone));
4428

4429
	return buf;
4430 4431
}

4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442
/*
 * See if there is a recovery command file (recovery.conf), and if so
 * read in parameters for archive recovery.
 *
 * XXX longer term intention is to expand this to
 * cater for additional parameters and controls
 * possibly use a flex lexer similar to the GUC one
 */
static void
readRecoveryCommandFile(void)
{
Bruce Momjian's avatar
Bruce Momjian committed
4443 4444 4445 4446 4447 4448
	FILE	   *fd;
	char		cmdline[MAXPGPATH];
	TimeLineID	rtli = 0;
	bool		rtliGiven = false;
	bool		syntaxError = false;

4449
	fd = AllocateFile(RECOVERY_COMMAND_FILE, "r");
4450 4451 4452 4453 4454
	if (fd == NULL)
	{
		if (errno == ENOENT)
			return;				/* not there, so no archive recovery */
		ereport(FATAL,
Bruce Momjian's avatar
Bruce Momjian committed
4455
				(errcode_for_file_access(),
4456
				 errmsg("could not open recovery command file \"%s\": %m",
4457
						RECOVERY_COMMAND_FILE)));
4458 4459 4460
	}

	ereport(LOG,
Bruce Momjian's avatar
Bruce Momjian committed
4461
			(errmsg("starting archive recovery")));
4462

Bruce Momjian's avatar
Bruce Momjian committed
4463 4464 4465
	/*
	 * Parse the file...
	 */
4466
	while (fgets(cmdline, sizeof(cmdline), fd) != NULL)
4467 4468
	{
		/* skip leading whitespace and check for # comment */
Bruce Momjian's avatar
Bruce Momjian committed
4469 4470 4471
		char	   *ptr;
		char	   *tok1;
		char	   *tok2;
4472 4473 4474 4475 4476 4477 4478 4479 4480 4481

		for (ptr = cmdline; *ptr; ptr++)
		{
			if (!isspace((unsigned char) *ptr))
				break;
		}
		if (*ptr == '\0' || *ptr == '#')
			continue;

		/* identify the quoted parameter value */
Bruce Momjian's avatar
Bruce Momjian committed
4482
		tok1 = strtok(ptr, "'");
4483 4484 4485 4486 4487
		if (!tok1)
		{
			syntaxError = true;
			break;
		}
Bruce Momjian's avatar
Bruce Momjian committed
4488
		tok2 = strtok(NULL, "'");
4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501
		if (!tok2)
		{
			syntaxError = true;
			break;
		}
		/* reparse to get just the parameter name */
		tok1 = strtok(ptr, " \t=");
		if (!tok1)
		{
			syntaxError = true;
			break;
		}

Bruce Momjian's avatar
Bruce Momjian committed
4502 4503
		if (strcmp(tok1, "restore_command") == 0)
		{
4504
			recoveryRestoreCommand = pstrdup(tok2);
4505
			ereport(LOG,
4506
					(errmsg("restore_command = '%s'",
4507 4508
							recoveryRestoreCommand)));
		}
Bruce Momjian's avatar
Bruce Momjian committed
4509 4510
		else if (strcmp(tok1, "recovery_target_timeline") == 0)
		{
4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529
			rtliGiven = true;
			if (strcmp(tok2, "latest") == 0)
				rtli = 0;
			else
			{
				errno = 0;
				rtli = (TimeLineID) strtoul(tok2, NULL, 0);
				if (errno == EINVAL || errno == ERANGE)
					ereport(FATAL,
							(errmsg("recovery_target_timeline is not a valid number: \"%s\"",
									tok2)));
			}
			if (rtli)
				ereport(LOG,
						(errmsg("recovery_target_timeline = %u", rtli)));
			else
				ereport(LOG,
						(errmsg("recovery_target_timeline = latest")));
		}
Bruce Momjian's avatar
Bruce Momjian committed
4530 4531
		else if (strcmp(tok1, "recovery_target_xid") == 0)
		{
4532 4533 4534 4535
			errno = 0;
			recoveryTargetXid = (TransactionId) strtoul(tok2, NULL, 0);
			if (errno == EINVAL || errno == ERANGE)
				ereport(FATAL,
4536 4537
				 (errmsg("recovery_target_xid is not a valid number: \"%s\"",
						 tok2)));
4538 4539 4540 4541 4542 4543
			ereport(LOG,
					(errmsg("recovery_target_xid = %u",
							recoveryTargetXid)));
			recoveryTarget = true;
			recoveryTargetExact = true;
		}
Bruce Momjian's avatar
Bruce Momjian committed
4544 4545
		else if (strcmp(tok1, "recovery_target_time") == 0)
		{
4546 4547 4548 4549 4550 4551 4552 4553
			/*
			 * if recovery_target_xid specified, then this overrides
			 * recovery_target_time
			 */
			if (recoveryTargetExact)
				continue;
			recoveryTarget = true;
			recoveryTargetExact = false;
Bruce Momjian's avatar
Bruce Momjian committed
4554

4555
			/*
4556
			 * Convert the time string given by the user to TimestampTz form.
4557
			 */
4558 4559
			recoveryTargetTime =
				DatumGetTimestampTz(DirectFunctionCall3(timestamptz_in,
Bruce Momjian's avatar
Bruce Momjian committed
4560
														CStringGetDatum(tok2),
4561 4562
												ObjectIdGetDatum(InvalidOid),
														Int32GetDatum(-1)));
4563
			ereport(LOG,
4564
					(errmsg("recovery_target_time = '%s'",
4565
							timestamptz_to_str(recoveryTargetTime))));
4566
		}
Bruce Momjian's avatar
Bruce Momjian committed
4567 4568
		else if (strcmp(tok1, "recovery_target_inclusive") == 0)
		{
4569 4570 4571
			/*
			 * does nothing if a recovery_target is not also set
			 */
4572 4573 4574 4575
			if (!parse_bool(tok2, &recoveryTargetInclusive))
				  ereport(ERROR,
							(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
					  errmsg("parameter \"recovery_target_inclusive\" requires a Boolean value")));
4576 4577 4578
			ereport(LOG,
					(errmsg("recovery_target_inclusive = %s", tok2)));
		}
4579 4580 4581 4582 4583
		else if (strcmp(tok1, "log_restartpoints") == 0)
		{
			/*
			 * does nothing if a recovery_target is not also set
			 */
4584 4585 4586 4587
			if (!parse_bool(tok2, &recoveryLogRestartpoints))
				  ereport(ERROR,
							(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
					  errmsg("parameter \"log_restartpoints\" requires a Boolean value")));
4588 4589 4590
			ereport(LOG,
					(errmsg("log_restartpoints = %s", tok2)));
		}
4591 4592 4593 4594 4595 4596 4597 4598
		else
			ereport(FATAL,
					(errmsg("unrecognized recovery parameter \"%s\"",
							tok1)));
	}

	FreeFile(fd);

Bruce Momjian's avatar
Bruce Momjian committed
4599 4600
	if (syntaxError)
		ereport(FATAL,
4601 4602
				(errmsg("syntax error in recovery command file: %s",
						cmdline),
4603
			  errhint("Lines should have the format parameter = 'value'.")));
4604 4605

	/* Check that required parameters were supplied */
4606
	if (recoveryRestoreCommand == NULL)
4607 4608
		ereport(FATAL,
				(errmsg("recovery command file \"%s\" did not specify restore_command",
4609
						RECOVERY_COMMAND_FILE)));
4610

4611 4612 4613
	/* Enable fetching from archive recovery area */
	InArchiveRecovery = true;

4614
	/*
4615 4616 4617 4618
	 * If user specified recovery_target_timeline, validate it or compute the
	 * "latest" value.	We can't do this until after we've gotten the restore
	 * command and set InArchiveRecovery, because we need to fetch timeline
	 * history files from the archive.
4619
	 */
4620 4621 4622 4623 4624 4625 4626
	if (rtliGiven)
	{
		if (rtli)
		{
			/* Timeline 1 does not have a history file, all else should */
			if (rtli != 1 && !existsTimeLineHistory(rtli))
				ereport(FATAL,
4627
						(errmsg("recovery target timeline %u does not exist",
4628
								rtli)));
4629 4630 4631 4632 4633 4634 4635 4636
			recoveryTargetTLI = rtli;
		}
		else
		{
			/* We start the "latest" search from pg_control's timeline */
			recoveryTargetTLI = findNewestTimeLine(recoveryTargetTLI);
		}
	}
4637 4638 4639 4640 4641 4642
}

/*
 * Exit archive-recovery state
 */
static void
4643
exitArchiveRecovery(TimeLineID endTLI, uint32 endLogId, uint32 endLogSeg)
4644
{
Bruce Momjian's avatar
Bruce Momjian committed
4645 4646
	char		recoveryPath[MAXPGPATH];
	char		xlogpath[MAXPGPATH];
4647 4648

	/*
4649
	 * We are no longer in archive recovery state.
4650 4651 4652 4653
	 */
	InArchiveRecovery = false;

	/*
4654 4655 4656
	 * We should have the ending log segment currently open.  Verify, and then
	 * close it (to avoid problems on Windows with trying to rename or delete
	 * an open file).
4657 4658 4659 4660 4661 4662 4663 4664 4665
	 */
	Assert(readFile >= 0);
	Assert(readId == endLogId);
	Assert(readSeg == endLogSeg);

	close(readFile);
	readFile = -1;

	/*
4666 4667 4668 4669 4670 4671 4672
	 * If the segment was fetched from archival storage, we want to replace
	 * the existing xlog segment (if any) with the archival version.  This is
	 * because whatever is in XLOGDIR is very possibly older than what we have
	 * from the archives, since it could have come from restoring a PGDATA
	 * backup.	In any case, the archival version certainly is more
	 * descriptive of what our current database state is, because that is what
	 * we replayed from.
4673
	 *
4674 4675
	 * Note that if we are establishing a new timeline, ThisTimeLineID is
	 * already set to the new value, and so we will create a new file instead
4676 4677
	 * of overwriting any existing file.  (This is, in fact, always the case
	 * at present.)
4678
	 */
4679
	snprintf(recoveryPath, MAXPGPATH, XLOGDIR "/RECOVERYXLOG");
4680
	XLogFilePath(xlogpath, ThisTimeLineID, endLogId, endLogSeg);
4681 4682 4683 4684 4685 4686 4687 4688 4689 4690

	if (restoredFromArchive)
	{
		ereport(DEBUG3,
				(errmsg_internal("moving last restored xlog to \"%s\"",
								 xlogpath)));
		unlink(xlogpath);		/* might or might not exist */
		if (rename(recoveryPath, xlogpath) != 0)
			ereport(FATAL,
					(errcode_for_file_access(),
4691
					 errmsg("could not rename file \"%s\" to \"%s\": %m",
4692 4693 4694 4695 4696 4697 4698 4699 4700 4701
							recoveryPath, xlogpath)));
		/* XXX might we need to fix permissions on the file? */
	}
	else
	{
		/*
		 * If the latest segment is not archival, but there's still a
		 * RECOVERYXLOG laying about, get rid of it.
		 */
		unlink(recoveryPath);	/* ignore any error */
Bruce Momjian's avatar
Bruce Momjian committed
4702

4703
		/*
4704 4705 4706
		 * If we are establishing a new timeline, we have to copy data from
		 * the last WAL segment of the old timeline to create a starting WAL
		 * segment for the new timeline.
4707 4708 4709 4710
		 */
		if (endTLI != ThisTimeLineID)
			XLogFileCopy(endLogId, endLogSeg,
						 endTLI, endLogId, endLogSeg);
4711 4712 4713
	}

	/*
4714 4715
	 * Let's just make real sure there are not .ready or .done flags posted
	 * for the new segment.
4716
	 */
4717 4718
	XLogFileName(xlogpath, ThisTimeLineID, endLogId, endLogSeg);
	XLogArchiveCleanup(xlogpath);
4719

4720
	/* Get rid of any remaining recovered timeline-history file, too */
4721
	snprintf(recoveryPath, MAXPGPATH, XLOGDIR "/RECOVERYHISTORY");
Bruce Momjian's avatar
Bruce Momjian committed
4722
	unlink(recoveryPath);		/* ignore any error */
4723 4724

	/*
4725 4726
	 * Rename the config file out of the way, so that we don't accidentally
	 * re-enter archive recovery mode in a subsequent crash.
4727
	 */
4728 4729
	unlink(RECOVERY_COMMAND_DONE);
	if (rename(RECOVERY_COMMAND_FILE, RECOVERY_COMMAND_DONE) != 0)
4730 4731
		ereport(FATAL,
				(errcode_for_file_access(),
4732
				 errmsg("could not rename file \"%s\" to \"%s\": %m",
4733
						RECOVERY_COMMAND_FILE, RECOVERY_COMMAND_DONE)));
4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744

	ereport(LOG,
			(errmsg("archive recovery complete")));
}

/*
 * For point-in-time recovery, this function decides whether we want to
 * stop applying the XLOG at or after the current record.
 *
 * Returns TRUE if we are stopping, FALSE otherwise.  On TRUE return,
 * *includeThis is set TRUE if we should apply this record before stopping.
4745 4746 4747
 *
 * We also track the timestamp of the latest applied COMMIT/ABORT record
 * in recoveryLastXTime, for logging purposes.
4748 4749
 * Also, some information is saved in recoveryStopXid et al for use in
 * annotating the new timeline's history file.
4750 4751 4752 4753 4754
 */
static bool
recoveryStopsHere(XLogRecord *record, bool *includeThis)
{
	bool		stopsHere;
Bruce Momjian's avatar
Bruce Momjian committed
4755
	uint8		record_info;
Bruce Momjian's avatar
Bruce Momjian committed
4756
	TimestampTz recordXtime;
4757 4758 4759 4760 4761 4762 4763

	/* We only consider stopping at COMMIT or ABORT records */
	if (record->xl_rmid != RM_XACT_ID)
		return false;
	record_info = record->xl_info & ~XLR_INFO_MASK;
	if (record_info == XLOG_XACT_COMMIT)
	{
Bruce Momjian's avatar
Bruce Momjian committed
4764
		xl_xact_commit *recordXactCommitData;
4765 4766

		recordXactCommitData = (xl_xact_commit *) XLogRecGetData(record);
4767
		recordXtime = recordXactCommitData->xact_time;
4768 4769 4770
	}
	else if (record_info == XLOG_XACT_ABORT)
	{
Bruce Momjian's avatar
Bruce Momjian committed
4771
		xl_xact_abort *recordXactAbortData;
4772 4773

		recordXactAbortData = (xl_xact_abort *) XLogRecGetData(record);
4774
		recordXtime = recordXactAbortData->xact_time;
4775 4776 4777 4778
	}
	else
		return false;

4779 4780
	/* Do we have a PITR target at all? */
	if (!recoveryTarget)
4781 4782
	{
		recoveryLastXTime = recordXtime;
4783
		return false;
4784
	}
4785

4786 4787 4788
	if (recoveryTargetExact)
	{
		/*
Bruce Momjian's avatar
Bruce Momjian committed
4789 4790
		 * there can be only one transaction end record with this exact
		 * transactionid
4791
		 *
Bruce Momjian's avatar
Bruce Momjian committed
4792
		 * when testing for an xid, we MUST test for equality only, since
4793 4794 4795
		 * transactions are numbered in the order they start, not the order
		 * they complete. A higher numbered xid will complete before you about
		 * 50% of the time...
4796 4797 4798 4799 4800 4801 4802 4803
		 */
		stopsHere = (record->xl_xid == recoveryTargetXid);
		if (stopsHere)
			*includeThis = recoveryTargetInclusive;
	}
	else
	{
		/*
4804 4805 4806
		 * there can be many transactions that share the same commit time, so
		 * we stop after the last one, if we are inclusive, or stop at the
		 * first one if we are exclusive
4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817
		 */
		if (recoveryTargetInclusive)
			stopsHere = (recordXtime > recoveryTargetTime);
		else
			stopsHere = (recordXtime >= recoveryTargetTime);
		if (stopsHere)
			*includeThis = false;
	}

	if (stopsHere)
	{
4818 4819 4820 4821
		recoveryStopXid = record->xl_xid;
		recoveryStopTime = recordXtime;
		recoveryStopAfter = *includeThis;

4822 4823
		if (record_info == XLOG_XACT_COMMIT)
		{
4824
			if (recoveryStopAfter)
4825 4826
				ereport(LOG,
						(errmsg("recovery stopping after commit of transaction %u, time %s",
4827 4828
								recoveryStopXid,
								timestamptz_to_str(recoveryStopTime))));
4829 4830 4831
			else
				ereport(LOG,
						(errmsg("recovery stopping before commit of transaction %u, time %s",
4832 4833
								recoveryStopXid,
								timestamptz_to_str(recoveryStopTime))));
4834 4835 4836
		}
		else
		{
4837
			if (recoveryStopAfter)
4838 4839
				ereport(LOG,
						(errmsg("recovery stopping after abort of transaction %u, time %s",
4840 4841
								recoveryStopXid,
								timestamptz_to_str(recoveryStopTime))));
4842 4843 4844
			else
				ereport(LOG,
						(errmsg("recovery stopping before abort of transaction %u, time %s",
4845 4846
								recoveryStopXid,
								timestamptz_to_str(recoveryStopTime))));
4847
		}
4848 4849 4850

		if (recoveryStopAfter)
			recoveryLastXTime = recordXtime;
4851
	}
4852 4853
	else
		recoveryLastXTime = recordXtime;
4854 4855 4856 4857

	return stopsHere;
}

4858
/*
Tom Lane's avatar
Tom Lane committed
4859
 * This must be called ONCE during postmaster or standalone-backend startup
4860 4861
 */
void
Tom Lane's avatar
Tom Lane committed
4862
StartupXLOG(void)
4863
{
4864 4865
	XLogCtlInsert *Insert;
	CheckPoint	checkPoint;
Tom Lane's avatar
Tom Lane committed
4866
	bool		wasShutdown;
4867
	bool		reachedStopPoint = false;
4868
	bool		haveBackupLabel = false;
4869
	XLogRecPtr	RecPtr,
Tom Lane's avatar
Tom Lane committed
4870 4871
				LastRec,
				checkPointLoc,
4872
				minRecoveryLoc,
Tom Lane's avatar
Tom Lane committed
4873
				EndOfLog;
4874 4875
	uint32		endLogId;
	uint32		endLogSeg;
4876
	XLogRecord *record;
4877
	uint32		freespace;
4878
	TransactionId oldestActiveXID;
4879

4880
	/*
4881 4882
	 * Read control file and check XLOG status looks valid.
	 *
4883 4884
	 * Note: in most control paths, *ControlFile is already valid and we need
	 * not do ReadControlFile() here, but might as well do it to be sure.
4885
	 */
4886
	ReadControlFile();
4887

4888
	if (ControlFile->state < DB_SHUTDOWNED ||
4889
		ControlFile->state > DB_IN_PRODUCTION ||
4890
		!XRecOffIsValid(ControlFile->checkPoint.xrecoff))
4891 4892
		ereport(FATAL,
				(errmsg("control file contains invalid data")));
4893 4894

	if (ControlFile->state == DB_SHUTDOWNED)
4895 4896 4897
		ereport(LOG,
				(errmsg("database system was shut down at %s",
						str_time(ControlFile->time))));
4898
	else if (ControlFile->state == DB_SHUTDOWNING)
4899
		ereport(LOG,
4900
				(errmsg("database system shutdown was interrupted; last known up at %s",
4901
						str_time(ControlFile->time))));
4902
	else if (ControlFile->state == DB_IN_CRASH_RECOVERY)
4903
		ereport(LOG,
4904 4905 4906 4907
		   (errmsg("database system was interrupted while in recovery at %s",
				   str_time(ControlFile->time)),
			errhint("This probably means that some data is corrupted and"
					" you will have to use the last backup for recovery.")));
4908 4909
	else if (ControlFile->state == DB_IN_ARCHIVE_RECOVERY)
		ereport(LOG,
Bruce Momjian's avatar
Bruce Momjian committed
4910 4911
				(errmsg("database system was interrupted while in recovery at log time %s",
						str_time(ControlFile->checkPointCopy.time)),
4912
				 errhint("If this has occurred more than once some data might be corrupted"
Bruce Momjian's avatar
Bruce Momjian committed
4913
			  " and you might need to choose an earlier recovery target.")));
4914
	else if (ControlFile->state == DB_IN_PRODUCTION)
4915
		ereport(LOG,
Bruce Momjian's avatar
Bruce Momjian committed
4916 4917
			  (errmsg("database system was interrupted; last known up at %s",
					  str_time(ControlFile->time))));
4918

4919 4920
	/* This is just to allow attaching to startup process with a debugger */
#ifdef XLOG_REPLAY_DELAY
4921
	if (ControlFile->state != DB_SHUTDOWNED)
4922
		pg_usleep(60000000L);
4923 4924
#endif

4925 4926 4927 4928 4929 4930 4931
	/*
	 * Verify that pg_xlog and pg_xlog/archive_status exist.  In cases where
	 * someone has performed a copy for PITR, these directories may have
	 * been excluded and need to be re-created.
	 */
	ValidateXLOGDirectoryStructure();

4932
	/*
4933 4934
	 * Initialize on the assumption we want to recover to the same timeline
	 * that's active according to pg_control.
4935 4936 4937
	 */
	recoveryTargetTLI = ControlFile->checkPointCopy.ThisTimeLineID;

4938
	/*
Bruce Momjian's avatar
Bruce Momjian committed
4939 4940
	 * Check for recovery control file, and if so set up state for offline
	 * recovery
4941 4942 4943
	 */
	readRecoveryCommandFile();

4944 4945 4946
	/* Now we can determine the list of expected TLIs */
	expectedTLIs = readTimeLineHistory(recoveryTargetTLI);

4947 4948 4949 4950 4951 4952
	/*
	 * If pg_control's timeline is not in expectedTLIs, then we cannot
	 * proceed: the backup is not part of the history of the requested
	 * timeline.
	 */
	if (!list_member_int(expectedTLIs,
4953
						 (int) ControlFile->checkPointCopy.ThisTimeLineID))
4954 4955 4956 4957 4958
		ereport(FATAL,
				(errmsg("requested timeline %u is not a child of database system timeline %u",
						recoveryTargetTLI,
						ControlFile->checkPointCopy.ThisTimeLineID)));

4959
	if (read_backup_label(&checkPointLoc, &minRecoveryLoc))
Tom Lane's avatar
Tom Lane committed
4960
	{
4961
		/*
4962 4963
		 * When a backup_label file is present, we want to roll forward from
		 * the checkpoint it identifies, rather than using pg_control.
4964
		 */
4965
		record = ReadCheckpointRecord(checkPointLoc, 0);
4966 4967
		if (record != NULL)
		{
4968
			ereport(DEBUG1,
4969
					(errmsg("checkpoint record is at %X/%X",
4970
							checkPointLoc.xlogid, checkPointLoc.xrecoff)));
4971 4972 4973 4974 4975
			InRecovery = true;	/* force recovery even if SHUTDOWNED */
		}
		else
		{
			ereport(PANIC,
4976 4977
					(errmsg("could not locate required checkpoint record"),
					 errhint("If you are not restoring from a backup, try removing the file \"%s/backup_label\".", DataDir)));
4978
		}
4979 4980
		/* set flag to delete it later */
		haveBackupLabel = true;
Tom Lane's avatar
Tom Lane committed
4981 4982 4983
	}
	else
	{
4984
		/*
4985 4986
		 * Get the last valid checkpoint record.  If the latest one according
		 * to pg_control is broken, try the next-to-last one.
4987 4988
		 */
		checkPointLoc = ControlFile->checkPoint;
4989
		record = ReadCheckpointRecord(checkPointLoc, 1);
Tom Lane's avatar
Tom Lane committed
4990 4991
		if (record != NULL)
		{
4992
			ereport(DEBUG1,
4993
					(errmsg("checkpoint record is at %X/%X",
4994
							checkPointLoc.xlogid, checkPointLoc.xrecoff)));
Tom Lane's avatar
Tom Lane committed
4995 4996
		}
		else
4997 4998
		{
			checkPointLoc = ControlFile->prevCheckPoint;
4999
			record = ReadCheckpointRecord(checkPointLoc, 2);
5000 5001 5002
			if (record != NULL)
			{
				ereport(LOG,
5003 5004 5005
						(errmsg("using previous checkpoint record at %X/%X",
							  checkPointLoc.xlogid, checkPointLoc.xrecoff)));
				InRecovery = true;		/* force recovery even if SHUTDOWNED */
5006 5007 5008
			}
			else
				ereport(PANIC,
5009
					 (errmsg("could not locate a valid checkpoint record")));
5010
		}
Tom Lane's avatar
Tom Lane committed
5011
	}
5012

Tom Lane's avatar
Tom Lane committed
5013 5014 5015
	LastRec = RecPtr = checkPointLoc;
	memcpy(&checkPoint, XLogRecGetData(record), sizeof(CheckPoint));
	wasShutdown = (record->xl_info == XLOG_CHECKPOINT_SHUTDOWN);
5016

5017
	ereport(DEBUG1,
Bruce Momjian's avatar
Bruce Momjian committed
5018 5019 5020
			(errmsg("redo record is at %X/%X; shutdown %s",
					checkPoint.redo.xlogid, checkPoint.redo.xrecoff,
					wasShutdown ? "TRUE" : "FALSE")));
5021
	ereport(DEBUG1,
5022 5023 5024
			(errmsg("next transaction ID: %u/%u; next OID: %u",
					checkPoint.nextXidEpoch, checkPoint.nextXid,
					checkPoint.nextOid)));
5025
	ereport(DEBUG1,
5026 5027
			(errmsg("next MultiXactId: %u; next MultiXactOffset: %u",
					checkPoint.nextMulti, checkPoint.nextMultiOffset)));
5028
	if (!TransactionIdIsNormal(checkPoint.nextXid))
5029
		ereport(PANIC,
5030
				(errmsg("invalid next transaction ID")));
5031 5032 5033

	ShmemVariableCache->nextXid = checkPoint.nextXid;
	ShmemVariableCache->nextOid = checkPoint.nextOid;
5034
	ShmemVariableCache->oidCount = 0;
5035
	MultiXactSetNextMXact(checkPoint.nextMulti, checkPoint.nextMultiOffset);
5036

5037
	/*
5038 5039 5040
	 * We must replay WAL entries using the same TimeLineID they were created
	 * under, so temporarily adopt the TLI indicated by the checkpoint (see
	 * also xlog_redo()).
5041
	 */
5042
	ThisTimeLineID = checkPoint.ThisTimeLineID;
5043

5044
	RedoRecPtr = XLogCtl->Insert.RedoRecPtr = checkPoint.redo;
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
5045

5046
	if (XLByteLT(RecPtr, checkPoint.redo))
5047 5048
		ereport(PANIC,
				(errmsg("invalid redo in checkpoint record")));
5049

5050
	/*
Bruce Momjian's avatar
Bruce Momjian committed
5051
	 * Check whether we need to force recovery from WAL.  If it appears to
5052 5053
	 * have been a clean shutdown and we did not have a recovery.conf file,
	 * then assume no recovery needed.
5054
	 */
5055
	if (XLByteLT(checkPoint.redo, RecPtr))
5056
	{
Tom Lane's avatar
Tom Lane committed
5057
		if (wasShutdown)
5058
			ereport(PANIC,
Bruce Momjian's avatar
Bruce Momjian committed
5059
					(errmsg("invalid redo record in shutdown checkpoint")));
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
5060
		InRecovery = true;
5061 5062
	}
	else if (ControlFile->state != DB_SHUTDOWNED)
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
5063
		InRecovery = true;
5064 5065 5066 5067 5068
	else if (InArchiveRecovery)
	{
		/* force recovery due to presence of recovery.conf */
		InRecovery = true;
	}
5069

Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
5070
	/* REDO */
5071
	if (InRecovery)
5072
	{
Bruce Momjian's avatar
Bruce Momjian committed
5073
		int			rmid;
5074

5075
		/*
Bruce Momjian's avatar
Bruce Momjian committed
5076 5077 5078 5079
		 * Update pg_control to show that we are recovering and to show the
		 * selected checkpoint as the place we are starting from. We also mark
		 * pg_control with any minimum recovery stop point obtained from a
		 * backup history file.
5080
		 */
5081
		if (InArchiveRecovery)
5082
		{
5083
			ereport(LOG,
5084
					(errmsg("automatic recovery in progress")));
5085 5086
			ControlFile->state = DB_IN_ARCHIVE_RECOVERY;
		}
5087
		else
5088
		{
5089
			ereport(LOG,
5090 5091
					(errmsg("database system was not properly shut down; "
							"automatic recovery in progress")));
5092 5093 5094 5095 5096 5097 5098
			ControlFile->state = DB_IN_CRASH_RECOVERY;
		}
		ControlFile->prevCheckPoint = ControlFile->checkPoint;
		ControlFile->checkPoint = checkPointLoc;
		ControlFile->checkPointCopy = checkPoint;
		if (minRecoveryLoc.xlogid != 0 || minRecoveryLoc.xrecoff != 0)
			ControlFile->minRecoveryPoint = minRecoveryLoc;
5099
		ControlFile->time = (pg_time_t) time(NULL);
5100 5101
		UpdateControlFile();

5102
		/*
Bruce Momjian's avatar
Bruce Momjian committed
5103 5104 5105 5106 5107 5108
		 * If there was a backup label file, it's done its job and the info
		 * has now been propagated into pg_control.  We must get rid of the
		 * label file so that if we crash during recovery, we'll pick up at
		 * the latest recovery restartpoint instead of going all the way back
		 * to the backup start point.  It seems prudent though to just rename
		 * the file out of the way rather than delete it completely.
5109 5110 5111 5112 5113 5114 5115 5116 5117 5118 5119
		 */
		if (haveBackupLabel)
		{
			unlink(BACKUP_LABEL_OLD);
			if (rename(BACKUP_LABEL_FILE, BACKUP_LABEL_OLD) != 0)
				ereport(FATAL,
						(errcode_for_file_access(),
						 errmsg("could not rename file \"%s\" to \"%s\": %m",
								BACKUP_LABEL_FILE, BACKUP_LABEL_OLD)));
		}

5120
		/* Initialize resource managers */
5121 5122 5123 5124 5125 5126
		for (rmid = 0; rmid <= RM_MAX_ID; rmid++)
		{
			if (RmgrTable[rmid].rm_startup != NULL)
				RmgrTable[rmid].rm_startup();
		}

5127
		/*
5128 5129
		 * Find the first record that logically follows the checkpoint --- it
		 * might physically precede it, though.
5130
		 */
5131
		if (XLByteLT(checkPoint.redo, RecPtr))
5132 5133
		{
			/* back up to find the record */
5134
			record = ReadRecord(&(checkPoint.redo), PANIC);
5135
		}
5136
		else
5137
		{
5138
			/* just have to read next record after CheckPoint */
5139
			record = ReadRecord(NULL, LOG);
5140
		}
5141

Tom Lane's avatar
Tom Lane committed
5142
		if (record != NULL)
5143
		{
5144 5145
			bool		recoveryContinue = true;
			bool		recoveryApply = true;
Bruce Momjian's avatar
Bruce Momjian committed
5146
			ErrorContextCallback errcontext;
5147

Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
5148
			InRedo = true;
5149 5150 5151
			ereport(LOG,
					(errmsg("redo starts at %X/%X",
							ReadRecPtr.xlogid, ReadRecPtr.xrecoff)));
5152 5153 5154 5155

			/*
			 * main redo apply loop
			 */
5156 5157
			do
			{
5158
#ifdef WAL_DEBUG
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
5159 5160
				if (XLOG_DEBUG)
				{
Bruce Momjian's avatar
Bruce Momjian committed
5161
					StringInfoData buf;
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
5162

5163 5164
					initStringInfo(&buf);
					appendStringInfo(&buf, "REDO @ %X/%X; LSN %X/%X: ",
Bruce Momjian's avatar
Bruce Momjian committed
5165 5166
									 ReadRecPtr.xlogid, ReadRecPtr.xrecoff,
									 EndRecPtr.xlogid, EndRecPtr.xrecoff);
5167 5168 5169 5170
					xlog_outrec(&buf, record);
					appendStringInfo(&buf, " - ");
					RmgrTable[record->xl_rmid].rm_desc(&buf,
													   record->xl_info,
Bruce Momjian's avatar
Bruce Momjian committed
5171
													 XLogRecGetData(record));
5172 5173
					elog(LOG, "%s", buf.data);
					pfree(buf.data);
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
5174
				}
5175
#endif
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
5176

5177 5178 5179 5180 5181
				/*
				 * Have we reached our recovery target?
				 */
				if (recoveryStopsHere(record, &recoveryApply))
				{
Bruce Momjian's avatar
Bruce Momjian committed
5182
					reachedStopPoint = true;	/* see below */
5183 5184 5185 5186 5187
					recoveryContinue = false;
					if (!recoveryApply)
						break;
				}

5188 5189 5190 5191 5192 5193
				/* Setup error traceback support for ereport() */
				errcontext.callback = rm_redo_error_callback;
				errcontext.arg = (void *) record;
				errcontext.previous = error_context_stack;
				error_context_stack = &errcontext;

5194 5195
				/* nextXid must be beyond record's xid */
				if (TransactionIdFollowsOrEquals(record->xl_xid,
5196
												 ShmemVariableCache->nextXid))
5197 5198 5199 5200 5201
				{
					ShmemVariableCache->nextXid = record->xl_xid;
					TransactionIdAdvance(ShmemVariableCache->nextXid);
				}

Tom Lane's avatar
Tom Lane committed
5202
				if (record->xl_info & XLR_BKP_BLOCK_MASK)
5203 5204
					RestoreBkpBlocks(record, EndRecPtr);

5205
				RmgrTable[record->xl_rmid].rm_redo(EndRecPtr, record);
5206

5207 5208 5209
				/* Pop the error context stack */
				error_context_stack = errcontext.previous;

5210 5211
				LastRec = ReadRecPtr;

5212
				record = ReadRecord(NULL, LOG);
5213
			} while (record != NULL && recoveryContinue);
Bruce Momjian's avatar
Bruce Momjian committed
5214

5215 5216 5217 5218
			/*
			 * end of main redo apply loop
			 */

5219 5220 5221
			ereport(LOG,
					(errmsg("redo done at %X/%X",
							ReadRecPtr.xlogid, ReadRecPtr.xrecoff)));
5222 5223
			if (recoveryLastXTime)
				ereport(LOG,
Bruce Momjian's avatar
Bruce Momjian committed
5224 5225
					 (errmsg("last completed transaction was at log time %s",
							 timestamptz_to_str(recoveryLastXTime))));
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
5226
			InRedo = false;
5227 5228
		}
		else
5229 5230
		{
			/* there are no WAL records following the checkpoint */
5231 5232
			ereport(LOG,
					(errmsg("redo is not required")));
5233
		}
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
5234 5235
	}

Tom Lane's avatar
Tom Lane committed
5236
	/*
5237 5238
	 * Re-fetch the last valid or last applied record, so we can identify the
	 * exact endpoint of what we consider the valid portion of WAL.
Tom Lane's avatar
Tom Lane committed
5239
	 */
5240
	record = ReadRecord(&LastRec, PANIC);
Tom Lane's avatar
Tom Lane committed
5241
	EndOfLog = EndRecPtr;
5242 5243
	XLByteToPrevSeg(EndOfLog, endLogId, endLogSeg);

5244 5245 5246 5247
	/*
	 * Complain if we did not roll forward far enough to render the backup
	 * dump consistent.
	 */
5248
	if (XLByteLT(EndOfLog, ControlFile->minRecoveryPoint))
5249
	{
5250
		if (reachedStopPoint)	/* stopped because of stop request */
5251 5252
			ereport(FATAL,
					(errmsg("requested recovery stop point is before end time of backup dump")));
Bruce Momjian's avatar
Bruce Momjian committed
5253
		else	/* ran off end of WAL */
5254 5255 5256 5257
			ereport(FATAL,
					(errmsg("WAL ends before end time of backup dump")));
	}

5258 5259 5260
	/*
	 * Consider whether we need to assign a new timeline ID.
	 *
Bruce Momjian's avatar
Bruce Momjian committed
5261 5262
	 * If we are doing an archive recovery, we always assign a new ID.	This
	 * handles a couple of issues.	If we stopped short of the end of WAL
5263 5264
	 * during recovery, then we are clearly generating a new timeline and must
	 * assign it a unique new ID.  Even if we ran to the end, modifying the
Bruce Momjian's avatar
Bruce Momjian committed
5265 5266
	 * current last segment is problematic because it may result in trying to
	 * overwrite an already-archived copy of that segment, and we encourage
5267 5268 5269 5270
	 * DBAs to make their archive_commands reject that.  We can dodge the
	 * problem by making the new active segment have a new timeline ID.
	 *
	 * In a normal crash recovery, we can just extend the timeline we were in.
5271
	 */
5272
	if (InArchiveRecovery)
5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283
	{
		ThisTimeLineID = findNewestTimeLine(recoveryTargetTLI) + 1;
		ereport(LOG,
				(errmsg("selected new timeline ID: %u", ThisTimeLineID)));
		writeTimeLineHistory(ThisTimeLineID, recoveryTargetTLI,
							 curFileTLI, endLogId, endLogSeg);
	}

	/* Save the selected TimeLineID in shared memory, too */
	XLogCtl->ThisTimeLineID = ThisTimeLineID;

5284
	/*
5285 5286 5287 5288
	 * We are now done reading the old WAL.  Turn off archive fetching if it
	 * was active, and make a writable copy of the last WAL segment. (Note
	 * that we also have a copy of the last block of the old WAL in readBuf;
	 * we will use that below.)
5289 5290
	 */
	if (InArchiveRecovery)
5291
		exitArchiveRecovery(curFileTLI, endLogId, endLogSeg);
5292 5293 5294 5295 5296 5297 5298 5299

	/*
	 * Prepare to write WAL starting at EndOfLog position, and init xlog
	 * buffer cache using the block containing the last record from the
	 * previous incarnation.
	 */
	openLogId = endLogId;
	openLogSeg = endLogSeg;
5300
	openLogFile = XLogFileOpen(openLogId, openLogSeg);
Tom Lane's avatar
Tom Lane committed
5301
	openLogOff = 0;
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
5302
	Insert = &XLogCtl->Insert;
5303
	Insert->PrevRecord = LastRec;
5304 5305
	XLogCtl->xlblocks[0].xlogid = openLogId;
	XLogCtl->xlblocks[0].xrecoff =
5306
		((EndOfLog.xrecoff - 1) / XLOG_BLCKSZ + 1) * XLOG_BLCKSZ;
5307 5308

	/*
5309 5310 5311
	 * Tricky point here: readBuf contains the *last* block that the LastRec
	 * record spans, not the one it starts in.	The last block is indeed the
	 * one we want to use.
Tom Lane's avatar
Tom Lane committed
5312
	 */
5313 5314
	Assert(readOff == (XLogCtl->xlblocks[0].xrecoff - XLOG_BLCKSZ) % XLogSegSize);
	memcpy((char *) Insert->currpage, readBuf, XLOG_BLCKSZ);
5315
	Insert->currpos = (char *) Insert->currpage +
5316
		(EndOfLog.xrecoff + XLOG_BLCKSZ - XLogCtl->xlblocks[0].xrecoff);
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
5317

Tom Lane's avatar
Tom Lane committed
5318
	LogwrtResult.Write = LogwrtResult.Flush = EndOfLog;
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
5319

Tom Lane's avatar
Tom Lane committed
5320 5321 5322
	XLogCtl->Write.LogwrtResult = LogwrtResult;
	Insert->LogwrtResult = LogwrtResult;
	XLogCtl->LogwrtResult = LogwrtResult;
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
5323

Tom Lane's avatar
Tom Lane committed
5324 5325
	XLogCtl->LogwrtRqst.Write = EndOfLog;
	XLogCtl->LogwrtRqst.Flush = EndOfLog;
5326

5327 5328 5329 5330 5331 5332 5333 5334 5335 5336
	freespace = INSERT_FREESPACE(Insert);
	if (freespace > 0)
	{
		/* Make sure rest of page is zero */
		MemSet(Insert->currpos, 0, freespace);
		XLogCtl->Write.curridx = 0;
	}
	else
	{
		/*
5337 5338
		 * Whenever Write.LogwrtResult points to exactly the end of a page,
		 * Write.curridx must point to the *next* page (see XLogWrite()).
5339
		 *
Bruce Momjian's avatar
Bruce Momjian committed
5340
		 * Note: it might seem we should do AdvanceXLInsertBuffer() here, but
Bruce Momjian's avatar
Bruce Momjian committed
5341
		 * this is sufficient.	The first actual attempt to insert a log
5342
		 * record will advance the insert state.
5343 5344 5345 5346
		 */
		XLogCtl->Write.curridx = NextBufIdx(0);
	}

5347 5348
	/* Pre-scan prepared transactions to find out the range of XIDs present */
	oldestActiveXID = PrescanPreparedTransactions();
5349

Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
5350
	if (InRecovery)
5351
	{
Bruce Momjian's avatar
Bruce Momjian committed
5352
		int			rmid;
5353 5354 5355 5356 5357 5358 5359 5360 5361 5362

		/*
		 * Allow resource managers to do any required cleanup.
		 */
		for (rmid = 0; rmid <= RM_MAX_ID; rmid++)
		{
			if (RmgrTable[rmid].rm_cleanup != NULL)
				RmgrTable[rmid].rm_cleanup();
		}

5363 5364 5365 5366 5367 5368
		/*
		 * Check to see if the XLOG sequence contained any unresolved
		 * references to uninitialized pages.
		 */
		XLogCheckInvalidPages();

5369 5370 5371 5372 5373
		/*
		 * Reset pgstat data, because it may be invalid after recovery.
		 */
		pgstat_reset_all();

Tom Lane's avatar
Tom Lane committed
5374
		/*
5375
		 * Perform a checkpoint to update all our recovery activity to disk.
5376
		 *
5377 5378 5379 5380 5381
		 * Note that we write a shutdown checkpoint rather than an on-line
		 * one. This is not particularly critical, but since we may be
		 * assigning a new TLI, using a shutdown checkpoint allows us to have
		 * the rule that TLI only changes in shutdown checkpoints, which
		 * allows some extra error checking in xlog_redo.
Tom Lane's avatar
Tom Lane committed
5382
		 */
5383
		CreateCheckPoint(CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_IMMEDIATE);
5384
	}
5385

Tom Lane's avatar
Tom Lane committed
5386 5387 5388
	/*
	 * Preallocate additional log files, if wanted.
	 */
5389
	PreallocXlogFiles(EndOfLog);
5390

5391 5392 5393
	/*
	 * Okay, we're officially UP.
	 */
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
5394
	InRecovery = false;
5395 5396

	ControlFile->state = DB_IN_PRODUCTION;
5397
	ControlFile->time = (pg_time_t) time(NULL);
5398 5399
	UpdateControlFile();

5400 5401 5402
	/* start the archive_timeout timer running */
	XLogCtl->Write.lastSegSwitchTime = ControlFile->time;

5403 5404 5405 5406
	/* initialize shared-memory copy of latest checkpoint XID/epoch */
	XLogCtl->ckptXidEpoch = ControlFile->checkPointCopy.nextXidEpoch;
	XLogCtl->ckptXid = ControlFile->checkPointCopy.nextXid;

5407 5408 5409 5410
	/* also initialize latestCompletedXid, to nextXid - 1 */
	ShmemVariableCache->latestCompletedXid = ShmemVariableCache->nextXid;
	TransactionIdRetreat(ShmemVariableCache->latestCompletedXid);

5411
	/* Start up the commit log and related stuff, too */
5412
	StartupCLOG();
5413
	StartupSUBTRANS(oldestActiveXID);
5414
	StartupMultiXact();
5415

5416 5417 5418
	/* Reload shared-memory state for prepared transactions */
	RecoverPreparedTransactions();

Tom Lane's avatar
Tom Lane committed
5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429
	/* Shut down readFile facility, free space */
	if (readFile >= 0)
	{
		close(readFile);
		readFile = -1;
	}
	if (readBuf)
	{
		free(readBuf);
		readBuf = NULL;
	}
5430 5431 5432 5433 5434 5435
	if (readRecordBuf)
	{
		free(readRecordBuf);
		readRecordBuf = NULL;
		readRecordBufSize = 0;
	}
Tom Lane's avatar
Tom Lane committed
5436 5437
}

5438 5439
/*
 * Subroutine to try to fetch and validate a prior checkpoint record.
5440 5441 5442
 *
 * whichChkpt identifies the checkpoint (merely for reporting purposes).
 * 1 for "primary", 2 for "secondary", 0 for "other" (backup_label)
5443
 */
Tom Lane's avatar
Tom Lane committed
5444
static XLogRecord *
5445
ReadCheckpointRecord(XLogRecPtr RecPtr, int whichChkpt)
Tom Lane's avatar
Tom Lane committed
5446 5447 5448 5449 5450
{
	XLogRecord *record;

	if (!XRecOffIsValid(RecPtr.xrecoff))
	{
5451 5452 5453 5454
		switch (whichChkpt)
		{
			case 1:
				ereport(LOG,
5455
				(errmsg("invalid primary checkpoint link in control file")));
5456 5457 5458 5459 5460 5461 5462
				break;
			case 2:
				ereport(LOG,
						(errmsg("invalid secondary checkpoint link in control file")));
				break;
			default:
				ereport(LOG,
5463
				   (errmsg("invalid checkpoint link in backup_label file")));
5464 5465
				break;
		}
Tom Lane's avatar
Tom Lane committed
5466 5467 5468
		return NULL;
	}

5469
	record = ReadRecord(&RecPtr, LOG);
Tom Lane's avatar
Tom Lane committed
5470 5471 5472

	if (record == NULL)
	{
5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485 5486 5487
		switch (whichChkpt)
		{
			case 1:
				ereport(LOG,
						(errmsg("invalid primary checkpoint record")));
				break;
			case 2:
				ereport(LOG,
						(errmsg("invalid secondary checkpoint record")));
				break;
			default:
				ereport(LOG,
						(errmsg("invalid checkpoint record")));
				break;
		}
Tom Lane's avatar
Tom Lane committed
5488 5489 5490 5491
		return NULL;
	}
	if (record->xl_rmid != RM_XLOG_ID)
	{
5492 5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503
		switch (whichChkpt)
		{
			case 1:
				ereport(LOG,
						(errmsg("invalid resource manager ID in primary checkpoint record")));
				break;
			case 2:
				ereport(LOG,
						(errmsg("invalid resource manager ID in secondary checkpoint record")));
				break;
			default:
				ereport(LOG,
5504
				(errmsg("invalid resource manager ID in checkpoint record")));
5505 5506
				break;
		}
Tom Lane's avatar
Tom Lane committed
5507 5508 5509 5510 5511
		return NULL;
	}
	if (record->xl_info != XLOG_CHECKPOINT_SHUTDOWN &&
		record->xl_info != XLOG_CHECKPOINT_ONLINE)
	{
5512 5513 5514 5515
		switch (whichChkpt)
		{
			case 1:
				ereport(LOG,
5516
				   (errmsg("invalid xl_info in primary checkpoint record")));
5517 5518 5519
				break;
			case 2:
				ereport(LOG,
5520
				 (errmsg("invalid xl_info in secondary checkpoint record")));
5521 5522 5523 5524 5525 5526
				break;
			default:
				ereport(LOG,
						(errmsg("invalid xl_info in checkpoint record")));
				break;
		}
Tom Lane's avatar
Tom Lane committed
5527 5528
		return NULL;
	}
5529 5530
	if (record->xl_len != sizeof(CheckPoint) ||
		record->xl_tot_len != SizeOfXLogRecord + sizeof(CheckPoint))
Tom Lane's avatar
Tom Lane committed
5531
	{
5532 5533 5534 5535
		switch (whichChkpt)
		{
			case 1:
				ereport(LOG,
5536
					(errmsg("invalid length of primary checkpoint record")));
5537 5538 5539
				break;
			case 2:
				ereport(LOG,
5540
				  (errmsg("invalid length of secondary checkpoint record")));
5541 5542 5543 5544 5545 5546
				break;
			default:
				ereport(LOG,
						(errmsg("invalid length of checkpoint record")));
				break;
		}
Tom Lane's avatar
Tom Lane committed
5547 5548 5549
		return NULL;
	}
	return record;
5550 5551
}

Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
5552
/*
5553 5554
 * This must be called during startup of a backend process, except that
 * it need not be called in a standalone backend (which does StartupXLOG
5555
 * instead).  We need to initialize the local copies of ThisTimeLineID and
5556 5557
 * RedoRecPtr.
 *
5558
 * Note: before Postgres 8.0, we went to some effort to keep the postmaster
5559
 * process's copies of ThisTimeLineID and RedoRecPtr valid too.  This was
5560
 * unnecessary however, since the postmaster itself never touches XLOG anyway.
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
5561 5562
 */
void
5563
InitXLOGAccess(void)
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
5564
{
5565 5566
	/* ThisTimeLineID doesn't change so we need no lock to copy it */
	ThisTimeLineID = XLogCtl->ThisTimeLineID;
5567 5568
	/* Use GetRedoRecPtr to copy the RedoRecPtr safely */
	(void) GetRedoRecPtr();
5569 5570 5571 5572 5573 5574 5575 5576
}

/*
 * Once spawned, a backend may update its local RedoRecPtr from
 * XLogCtl->Insert.RedoRecPtr; it must hold the insert lock or info_lck
 * to do so.  This is done in XLogInsert() or GetRedoRecPtr().
 */
XLogRecPtr
5577 5578
GetRedoRecPtr(void)
{
5579 5580 5581
	/* use volatile pointer to prevent code rearrangement */
	volatile XLogCtlData *xlogctl = XLogCtl;

5582
	SpinLockAcquire(&xlogctl->info_lck);
5583 5584
	Assert(XLByteLE(RedoRecPtr, xlogctl->Insert.RedoRecPtr));
	RedoRecPtr = xlogctl->Insert.RedoRecPtr;
5585
	SpinLockRelease(&xlogctl->info_lck);
5586 5587

	return RedoRecPtr;
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
5588 5589
}

5590 5591 5592 5593 5594 5595 5596 5597 5598 5599 5600 5601 5602 5603
/*
 * GetInsertRecPtr -- Returns the current insert position.
 *
 * NOTE: The value *actually* returned is the position of the last full
 * xlog page. It lags behind the real insert position by at most 1 page.
 * For that, we don't need to acquire WALInsertLock which can be quite
 * heavily contended, and an approximation is enough for the current
 * usage of this function.
 */
XLogRecPtr
GetInsertRecPtr(void)
{
	/* use volatile pointer to prevent code rearrangement */
	volatile XLogCtlData *xlogctl = XLogCtl;
Bruce Momjian's avatar
Bruce Momjian committed
5604
	XLogRecPtr	recptr;
5605 5606 5607 5608 5609 5610 5611 5612

	SpinLockAcquire(&xlogctl->info_lck);
	recptr = xlogctl->LogwrtRqst.Write;
	SpinLockRelease(&xlogctl->info_lck);

	return recptr;
}

5613 5614 5615
/*
 * Get the time of the last xlog segment switch
 */
5616
pg_time_t
5617 5618
GetLastSegSwitchTime(void)
{
5619
	pg_time_t	result;
5620 5621 5622 5623 5624 5625 5626 5627 5628

	/* Need WALWriteLock, but shared lock is sufficient */
	LWLockAcquire(WALWriteLock, LW_SHARED);
	result = XLogCtl->Write.lastSegSwitchTime;
	LWLockRelease(WALWriteLock);

	return result;
}

5629 5630 5631 5632 5633 5634 5635 5636 5637 5638 5639
/*
 * GetNextXidAndEpoch - get the current nextXid value and associated epoch
 *
 * This is exported for use by code that would like to have 64-bit XIDs.
 * We don't really support such things, but all XIDs within the system
 * can be presumed "close to" the result, and thus the epoch associated
 * with them can be determined.
 */
void
GetNextXidAndEpoch(TransactionId *xid, uint32 *epoch)
{
Bruce Momjian's avatar
Bruce Momjian committed
5640 5641 5642
	uint32		ckptXidEpoch;
	TransactionId ckptXid;
	TransactionId nextXid;
5643 5644 5645 5646 5647 5648 5649 5650 5651 5652 5653 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664 5665 5666 5667 5668

	/* Must read checkpoint info first, else have race condition */
	{
		/* use volatile pointer to prevent code rearrangement */
		volatile XLogCtlData *xlogctl = XLogCtl;

		SpinLockAcquire(&xlogctl->info_lck);
		ckptXidEpoch = xlogctl->ckptXidEpoch;
		ckptXid = xlogctl->ckptXid;
		SpinLockRelease(&xlogctl->info_lck);
	}

	/* Now fetch current nextXid */
	nextXid = ReadNewTransactionId();

	/*
	 * nextXid is certainly logically later than ckptXid.  So if it's
	 * numerically less, it must have wrapped into the next epoch.
	 */
	if (nextXid < ckptXid)
		ckptXidEpoch++;

	*xid = nextXid;
	*epoch = ckptXidEpoch;
}

5669
/*
Tom Lane's avatar
Tom Lane committed
5670
 * This must be called ONCE during postmaster or standalone-backend shutdown
5671 5672
 */
void
5673
ShutdownXLOG(int code, Datum arg)
5674
{
5675 5676
	ereport(LOG,
			(errmsg("shutting down")));
5677

5678
	CreateCheckPoint(CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_IMMEDIATE);
5679
	ShutdownCLOG();
5680
	ShutdownSUBTRANS();
5681
	ShutdownMultiXact();
5682

5683 5684
	ereport(LOG,
			(errmsg("database system is shut down")));
5685 5686
}

5687
/*
5688 5689 5690 5691 5692 5693 5694 5695 5696 5697 5698 5699 5700 5701
 * Log start of a checkpoint.
 */
static void
LogCheckpointStart(int flags)
{
	elog(LOG, "checkpoint starting:%s%s%s%s%s%s",
		 (flags & CHECKPOINT_IS_SHUTDOWN) ? " shutdown" : "",
		 (flags & CHECKPOINT_IMMEDIATE) ? " immediate" : "",
		 (flags & CHECKPOINT_FORCE) ? " force" : "",
		 (flags & CHECKPOINT_WAIT) ? " wait" : "",
		 (flags & CHECKPOINT_CAUSE_XLOG) ? " xlog" : "",
		 (flags & CHECKPOINT_CAUSE_TIME) ? " time" : "");
}

5702
/*
5703 5704 5705 5706 5707
 * Log end of a checkpoint.
 */
static void
LogCheckpointEnd(void)
{
Bruce Momjian's avatar
Bruce Momjian committed
5708 5709 5710 5711 5712 5713
	long		write_secs,
				sync_secs,
				total_secs;
	int			write_usecs,
				sync_usecs,
				total_usecs;
5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736

	CheckpointStats.ckpt_end_t = GetCurrentTimestamp();

	TimestampDifference(CheckpointStats.ckpt_start_t,
						CheckpointStats.ckpt_end_t,
						&total_secs, &total_usecs);

	TimestampDifference(CheckpointStats.ckpt_write_t,
						CheckpointStats.ckpt_sync_t,
						&write_secs, &write_usecs);

	TimestampDifference(CheckpointStats.ckpt_sync_t,
						CheckpointStats.ckpt_sync_end_t,
						&sync_secs, &sync_usecs);

	elog(LOG, "checkpoint complete: wrote %d buffers (%.1f%%); "
		 "%d transaction log file(s) added, %d removed, %d recycled; "
		 "write=%ld.%03d s, sync=%ld.%03d s, total=%ld.%03d s",
		 CheckpointStats.ckpt_bufs_written,
		 (double) CheckpointStats.ckpt_bufs_written * 100 / NBuffers,
		 CheckpointStats.ckpt_segs_added,
		 CheckpointStats.ckpt_segs_removed,
		 CheckpointStats.ckpt_segs_recycled,
Bruce Momjian's avatar
Bruce Momjian committed
5737 5738 5739
		 write_secs, write_usecs / 1000,
		 sync_secs, sync_usecs / 1000,
		 total_secs, total_usecs / 1000);
5740 5741
}

Tom Lane's avatar
Tom Lane committed
5742 5743
/*
 * Perform a checkpoint --- either during shutdown, or on-the-fly
5744
 *
5745 5746 5747
 * flags is a bitwise OR of the following:
 *	CHECKPOINT_IS_SHUTDOWN: checkpoint is for database shutdown.
 *	CHECKPOINT_IMMEDIATE: finish the checkpoint ASAP,
5748
 *		ignoring checkpoint_completion_target parameter.
5749 5750 5751
 *	CHECKPOINT_FORCE: force a checkpoint even if no XLOG activity has occured
 *		since the last one (implied by CHECKPOINT_IS_SHUTDOWN).
 *
5752
 * Note: flags contains other bits, of interest here only for logging purposes.
5753 5754
 * In particular note that this routine is synchronous and does not pay
 * attention to CHECKPOINT_WAIT.
Tom Lane's avatar
Tom Lane committed
5755
 */
5756
void
5757
CreateCheckPoint(int flags)
5758
{
5759
	bool		shutdown = (flags & CHECKPOINT_IS_SHUTDOWN) != 0;
5760 5761 5762
	CheckPoint	checkPoint;
	XLogRecPtr	recptr;
	XLogCtlInsert *Insert = &XLogCtl->Insert;
5763
	XLogRecData rdata;
5764
	uint32		freespace;
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
5765 5766
	uint32		_logId;
	uint32		_logSeg;
5767 5768
	TransactionId *inCommitXids;
	int			nInCommit;
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
5769

5770
	/*
5771 5772 5773 5774
	 * Acquire CheckpointLock to ensure only one checkpoint happens at a time.
	 * (This is just pro forma, since in the present system structure there is
	 * only one process that is allowed to issue checkpoints at any given
	 * time.)
5775
	 */
5776
	LWLockAcquire(CheckpointLock, LW_EXCLUSIVE);
5777

5778 5779 5780 5781 5782 5783 5784 5785 5786 5787
	/*
	 * Prepare to accumulate statistics.
	 *
	 * Note: because it is possible for log_checkpoints to change while a
	 * checkpoint proceeds, we always accumulate stats, even if
	 * log_checkpoints is currently off.
	 */
	MemSet(&CheckpointStats, 0, sizeof(CheckpointStats));
	CheckpointStats.ckpt_start_t = GetCurrentTimestamp();

5788 5789 5790
	/*
	 * Use a critical section to force system panic if we have trouble.
	 */
5791 5792
	START_CRIT_SECTION();

5793 5794 5795
	if (shutdown)
	{
		ControlFile->state = DB_SHUTDOWNING;
5796
		ControlFile->time = (pg_time_t) time(NULL);
5797 5798
		UpdateControlFile();
	}
Tom Lane's avatar
Tom Lane committed
5799

5800
	/*
Bruce Momjian's avatar
Bruce Momjian committed
5801 5802 5803
	 * Let smgr prepare for checkpoint; this has to happen before we determine
	 * the REDO pointer.  Note that smgr must not do anything that'd have to
	 * be undone if we decide no checkpoint is needed.
5804 5805 5806 5807
	 */
	smgrpreckpt();

	/* Begin filling in the checkpoint WAL record */
5808
	MemSet(&checkPoint, 0, sizeof(checkPoint));
5809
	checkPoint.ThisTimeLineID = ThisTimeLineID;
5810
	checkPoint.time = (pg_time_t) time(NULL);
5811

5812
	/*
5813 5814
	 * We must hold WALInsertLock while examining insert state to determine
	 * the checkpoint REDO pointer.
5815
	 */
5816
	LWLockAcquire(WALInsertLock, LW_EXCLUSIVE);
Tom Lane's avatar
Tom Lane committed
5817 5818

	/*
5819 5820 5821 5822 5823 5824 5825 5826
	 * If this isn't a shutdown or forced checkpoint, and we have not inserted
	 * any XLOG records since the start of the last checkpoint, skip the
	 * checkpoint.	The idea here is to avoid inserting duplicate checkpoints
	 * when the system is idle. That wastes log space, and more importantly it
	 * exposes us to possible loss of both current and previous checkpoint
	 * records if the machine crashes just as we're writing the update.
	 * (Perhaps it'd make even more sense to checkpoint only when the previous
	 * checkpoint record is in a different xlog page?)
Tom Lane's avatar
Tom Lane committed
5827
	 *
5828 5829 5830 5831
	 * We have to make two tests to determine that nothing has happened since
	 * the start of the last checkpoint: current insertion point must match
	 * the end of the last checkpoint record, and its redo pointer must point
	 * to itself.
Tom Lane's avatar
Tom Lane committed
5832
	 */
5833
	if ((flags & (CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_FORCE)) == 0)
Tom Lane's avatar
Tom Lane committed
5834 5835 5836 5837 5838 5839 5840 5841 5842 5843 5844 5845
	{
		XLogRecPtr	curInsert;

		INSERT_RECPTR(curInsert, Insert, Insert->curridx);
		if (curInsert.xlogid == ControlFile->checkPoint.xlogid &&
			curInsert.xrecoff == ControlFile->checkPoint.xrecoff +
			MAXALIGN(SizeOfXLogRecord + sizeof(CheckPoint)) &&
			ControlFile->checkPoint.xlogid ==
			ControlFile->checkPointCopy.redo.xlogid &&
			ControlFile->checkPoint.xrecoff ==
			ControlFile->checkPointCopy.redo.xrecoff)
		{
5846 5847
			LWLockRelease(WALInsertLock);
			LWLockRelease(CheckpointLock);
Tom Lane's avatar
Tom Lane committed
5848 5849 5850 5851 5852 5853 5854 5855
			END_CRIT_SECTION();
			return;
		}
	}

	/*
	 * Compute new REDO record ptr = location of next XLOG record.
	 *
5856 5857 5858 5859
	 * NB: this is NOT necessarily where the checkpoint record itself will be,
	 * since other backends may insert more XLOG records while we're off doing
	 * the buffer flush work.  Those XLOG records are logically after the
	 * checkpoint, even though physically before it.  Got that?
Tom Lane's avatar
Tom Lane committed
5860 5861
	 */
	freespace = INSERT_FREESPACE(Insert);
5862 5863
	if (freespace < SizeOfXLogRecord)
	{
5864
		(void) AdvanceXLInsertBuffer(false);
Tom Lane's avatar
Tom Lane committed
5865
		/* OK to ignore update return flag, since we will do flush anyway */
5866
		freespace = INSERT_FREESPACE(Insert);
5867
	}
Tom Lane's avatar
Tom Lane committed
5868
	INSERT_RECPTR(checkPoint.redo, Insert, Insert->curridx);
5869

Tom Lane's avatar
Tom Lane committed
5870
	/*
5871 5872
	 * Here we update the shared RedoRecPtr for future XLogInsert calls; this
	 * must be done while holding the insert lock AND the info_lck.
5873
	 *
Bruce Momjian's avatar
Bruce Momjian committed
5874
	 * Note: if we fail to complete the checkpoint, RedoRecPtr will be left
5875 5876 5877 5878 5879
	 * pointing past where it really needs to point.  This is okay; the only
	 * consequence is that XLogInsert might back up whole buffers that it
	 * didn't really need to.  We can't postpone advancing RedoRecPtr because
	 * XLogInserts that happen while we are dumping buffers must assume that
	 * their buffer changes are not included in the checkpoint.
Tom Lane's avatar
Tom Lane committed
5880
	 */
5881 5882 5883 5884
	{
		/* use volatile pointer to prevent code rearrangement */
		volatile XLogCtlData *xlogctl = XLogCtl;

5885
		SpinLockAcquire(&xlogctl->info_lck);
5886
		RedoRecPtr = xlogctl->Insert.RedoRecPtr = checkPoint.redo;
5887
		SpinLockRelease(&xlogctl->info_lck);
5888
	}
5889

Tom Lane's avatar
Tom Lane committed
5890
	/*
5891 5892
	 * Now we can release WAL insert lock, allowing other xacts to proceed
	 * while we are flushing disk buffers.
Tom Lane's avatar
Tom Lane committed
5893
	 */
5894
	LWLockRelease(WALInsertLock);
5895

5896
	/*
Bruce Momjian's avatar
Bruce Momjian committed
5897 5898
	 * If enabled, log checkpoint start.  We postpone this until now so as not
	 * to log anything if we decided to skip the checkpoint.
5899 5900 5901
	 */
	if (log_checkpoints)
		LogCheckpointStart(flags);
5902

5903 5904
	TRACE_POSTGRESQL_CHECKPOINT_START(flags);

5905 5906 5907 5908 5909 5910 5911 5912 5913 5914 5915 5916 5917 5918 5919
	/*
	 * Before flushing data, we must wait for any transactions that are
	 * currently in their commit critical sections.  If an xact inserted its
	 * commit record into XLOG just before the REDO point, then a crash
	 * restart from the REDO point would not replay that record, which means
	 * that our flushing had better include the xact's update of pg_clog.  So
	 * we wait till he's out of his commit critical section before proceeding.
	 * See notes in RecordTransactionCommit().
	 *
	 * Because we've already released WALInsertLock, this test is a bit fuzzy:
	 * it is possible that we will wait for xacts we didn't really need to
	 * wait for.  But the delay should be short and it seems better to make
	 * checkpoint take a bit longer than to hold locks longer than necessary.
	 * (In fact, the whole reason we have this issue is that xact.c does
	 * commit record XLOG insertion and clog update as two separate steps
Bruce Momjian's avatar
Bruce Momjian committed
5920 5921
	 * protected by different locks, but again that seems best on grounds of
	 * minimizing lock contention.)
5922
	 *
Bruce Momjian's avatar
Bruce Momjian committed
5923 5924
	 * A transaction that has not yet set inCommit when we look cannot be at
	 * risk, since he's not inserted his commit record yet; and one that's
5925 5926 5927 5928 5929 5930 5931
	 * already cleared it is not at risk either, since he's done fixing clog
	 * and we will correctly flush the update below.  So we cannot miss any
	 * xacts we need to wait for.
	 */
	nInCommit = GetTransactionsInCommit(&inCommitXids);
	if (nInCommit > 0)
	{
Bruce Momjian's avatar
Bruce Momjian committed
5932 5933 5934
		do
		{
			pg_usleep(10000L);	/* wait for 10 msec */
5935 5936 5937 5938
		} while (HaveTransactionsInCommit(inCommitXids, nInCommit));
	}
	pfree(inCommitXids);

5939 5940 5941
	/*
	 * Get the other info we need for the checkpoint record.
	 */
5942
	LWLockAcquire(XidGenLock, LW_SHARED);
5943
	checkPoint.nextXid = ShmemVariableCache->nextXid;
5944
	LWLockRelease(XidGenLock);
Tom Lane's avatar
Tom Lane committed
5945

5946 5947 5948 5949 5950
	/* Increase XID epoch if we've wrapped around since last checkpoint */
	checkPoint.nextXidEpoch = ControlFile->checkPointCopy.nextXidEpoch;
	if (checkPoint.nextXid < ControlFile->checkPointCopy.nextXid)
		checkPoint.nextXidEpoch++;

5951
	LWLockAcquire(OidGenLock, LW_SHARED);
5952
	checkPoint.nextOid = ShmemVariableCache->nextOid;
5953 5954
	if (!shutdown)
		checkPoint.nextOid += ShmemVariableCache->oidCount;
5955
	LWLockRelease(OidGenLock);
5956

5957 5958 5959
	MultiXactGetCheckptMulti(shutdown,
							 &checkPoint.nextMulti,
							 &checkPoint.nextMultiOffset);
5960

Tom Lane's avatar
Tom Lane committed
5961
	/*
5962 5963
	 * Having constructed the checkpoint record, ensure all shmem disk buffers
	 * and commit-log buffers are flushed to disk.
5964
	 *
5965 5966
	 * This I/O could fail for various reasons.  If so, we will fail to
	 * complete the checkpoint, but there is no reason to force a system
5967
	 * panic. Accordingly, exit critical section while doing it.
Tom Lane's avatar
Tom Lane committed
5968
	 */
5969 5970
	END_CRIT_SECTION();

5971
	CheckPointGuts(checkPoint.redo, flags);
5972

5973 5974
	START_CRIT_SECTION();

Tom Lane's avatar
Tom Lane committed
5975 5976 5977
	/*
	 * Now insert the checkpoint record into XLOG.
	 */
5978
	rdata.data = (char *) (&checkPoint);
5979
	rdata.len = sizeof(checkPoint);
5980
	rdata.buffer = InvalidBuffer;
5981 5982
	rdata.next = NULL;

Tom Lane's avatar
Tom Lane committed
5983 5984 5985 5986 5987 5988
	recptr = XLogInsert(RM_XLOG_ID,
						shutdown ? XLOG_CHECKPOINT_SHUTDOWN :
						XLOG_CHECKPOINT_ONLINE,
						&rdata);

	XLogFlush(recptr);
5989

Tom Lane's avatar
Tom Lane committed
5990
	/*
5991 5992
	 * We now have ProcLastRecPtr = start of actual checkpoint record, recptr
	 * = end of actual checkpoint record.
Tom Lane's avatar
Tom Lane committed
5993 5994
	 */
	if (shutdown && !XLByteEQ(checkPoint.redo, ProcLastRecPtr))
5995 5996
		ereport(PANIC,
				(errmsg("concurrent transaction log activity while database system is shutting down")));
5997

Tom Lane's avatar
Tom Lane committed
5998
	/*
5999 6000
	 * Select point at which we can truncate the log, which we base on the
	 * prior checkpoint's earliest info.
Tom Lane's avatar
Tom Lane committed
6001
	 */
6002
	XLByteToSeg(ControlFile->checkPointCopy.redo, _logId, _logSeg);
6003

Tom Lane's avatar
Tom Lane committed
6004 6005 6006
	/*
	 * Update the control file.
	 */
6007
	LWLockAcquire(ControlFileLock, LW_EXCLUSIVE);
6008 6009
	if (shutdown)
		ControlFile->state = DB_SHUTDOWNED;
Tom Lane's avatar
Tom Lane committed
6010 6011 6012
	ControlFile->prevCheckPoint = ControlFile->checkPoint;
	ControlFile->checkPoint = ProcLastRecPtr;
	ControlFile->checkPointCopy = checkPoint;
6013
	ControlFile->time = (pg_time_t) time(NULL);
6014
	UpdateControlFile();
6015
	LWLockRelease(ControlFileLock);
6016

6017 6018 6019 6020 6021 6022 6023 6024 6025 6026 6027
	/* Update shared-memory copy of checkpoint XID/epoch */
	{
		/* use volatile pointer to prevent code rearrangement */
		volatile XLogCtlData *xlogctl = XLogCtl;

		SpinLockAcquire(&xlogctl->info_lck);
		xlogctl->ckptXidEpoch = checkPoint.nextXidEpoch;
		xlogctl->ckptXid = checkPoint.nextXid;
		SpinLockRelease(&xlogctl->info_lck);
	}

6028
	/*
6029
	 * We are now done with critical updates; no need for system panic if we
6030
	 * have trouble while fooling with old log segments.
6031 6032 6033
	 */
	END_CRIT_SECTION();

6034 6035 6036 6037 6038
	/*
	 * Let smgr do post-checkpoint cleanup (eg, deleting old files).
	 */
	smgrpostckpt();

Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
6039
	/*
6040
	 * Delete old log files (those no longer needed even for previous
Tom Lane's avatar
Tom Lane committed
6041
	 * checkpoint).
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
6042 6043 6044
	 */
	if (_logId || _logSeg)
	{
Tom Lane's avatar
Tom Lane committed
6045
		PrevLogSeg(_logId, _logSeg);
6046
		RemoveOldXlogFiles(_logId, _logSeg, recptr);
Vadim B. Mikheev's avatar
Vadim B. Mikheev committed
6047 6048
	}

Tom Lane's avatar
Tom Lane committed
6049
	/*
6050 6051
	 * Make more log segments if needed.  (Do this after recycling old log
	 * segments, since that may supply some of the needed files.)
Tom Lane's avatar
Tom Lane committed
6052 6053
	 */
	if (!shutdown)
6054
		PreallocXlogFiles(recptr);
Tom Lane's avatar
Tom Lane committed
6055

6056
	/*
6057 6058 6059 6060 6061
	 * Truncate pg_subtrans if possible.  We can throw away all data before
	 * the oldest XMIN of any running transaction.	No future transaction will
	 * attempt to reference any pg_subtrans entry older than that (see Asserts
	 * in subtrans.c).	During recovery, though, we mustn't do this because
	 * StartupSUBTRANS hasn't been called yet.
6062
	 */
6063
	if (!InRecovery)
6064
		TruncateSUBTRANS(GetOldestXmin(true, false));
6065

6066 6067 6068
	/* All real work is done, but log before releasing lock. */
	if (log_checkpoints)
		LogCheckpointEnd();
6069

6070 6071 6072 6073 6074
        TRACE_POSTGRESQL_CHECKPOINT_DONE(CheckpointStats.ckpt_bufs_written,
                                NBuffers, CheckpointStats.ckpt_segs_added,
                                CheckpointStats.ckpt_segs_removed,
                                CheckpointStats.ckpt_segs_recycled);

6075
	LWLockRelease(CheckpointLock);
6076
}
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
6077

6078 6079 6080 6081 6082 6083 6084
/*
 * Flush all data in shared memory to disk, and fsync
 *
 * This is the common code shared between regular checkpoints and
 * recovery restartpoints.
 */
static void
6085
CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
6086 6087 6088 6089
{
	CheckPointCLOG();
	CheckPointSUBTRANS();
	CheckPointMultiXact();
Bruce Momjian's avatar
Bruce Momjian committed
6090
	CheckPointBuffers(flags);	/* performs all required fsyncs */
6091 6092 6093 6094 6095 6096 6097
	/* We deliberately delay 2PC checkpointing as long as possible */
	CheckPointTwoPhase(checkPointRedo);
}

/*
 * Set a recovery restart point if appropriate
 *
6098
 * This is similar to CreateCheckPoint, but is used during WAL recovery
6099 6100 6101 6102 6103 6104 6105 6106
 * to establish a point from which recovery can roll forward without
 * replaying the entire recovery log.  This function is called each time
 * a checkpoint record is read from XLOG; it must determine whether a
 * restartpoint is needed or not.
 */
static void
RecoveryRestartPoint(const CheckPoint *checkPoint)
{
Bruce Momjian's avatar
Bruce Momjian committed
6107 6108
	int			elapsed_secs;
	int			rmid;
6109 6110

	/*
Bruce Momjian's avatar
Bruce Momjian committed
6111 6112
	 * Do nothing if the elapsed time since the last restartpoint is less than
	 * half of checkpoint_timeout.	(We use a value less than
6113 6114 6115 6116 6117 6118
	 * checkpoint_timeout so that variations in the timing of checkpoints on
	 * the master, or speed of transmission of WAL segments to a slave, won't
	 * make the slave skip a restartpoint once it's synced with the master.)
	 * Checking true elapsed time keeps us from doing restartpoints too often
	 * while rapidly scanning large amounts of WAL.
	 */
6119
	elapsed_secs = (pg_time_t) time(NULL) - ControlFile->time;
6120 6121 6122 6123 6124 6125 6126 6127 6128 6129 6130 6131 6132
	if (elapsed_secs < CheckPointTimeout / 2)
		return;

	/*
	 * Is it safe to checkpoint?  We must ask each of the resource managers
	 * whether they have any partial state information that might prevent a
	 * correct restart from this point.  If so, we skip this opportunity, but
	 * return at the next checkpoint record for another try.
	 */
	for (rmid = 0; rmid <= RM_MAX_ID; rmid++)
	{
		if (RmgrTable[rmid].rm_safe_restartpoint != NULL)
			if (!(RmgrTable[rmid].rm_safe_restartpoint()))
6133 6134 6135 6136 6137
			{
				elog(DEBUG2, "RM %d not safe to record restart point at %X/%X",
					 rmid,
					 checkPoint->redo.xlogid,
					 checkPoint->redo.xrecoff);
6138
				return;
6139
			}
6140 6141 6142 6143 6144
	}

	/*
	 * OK, force data out to disk
	 */
6145
	CheckPointGuts(checkPoint->redo, CHECKPOINT_IMMEDIATE);
6146 6147

	/*
Bruce Momjian's avatar
Bruce Momjian committed
6148 6149 6150
	 * Update pg_control so that any subsequent crash will restart from this
	 * checkpoint.	Note: ReadRecPtr gives the XLOG address of the checkpoint
	 * record itself.
6151 6152 6153 6154
	 */
	ControlFile->prevCheckPoint = ControlFile->checkPoint;
	ControlFile->checkPoint = ReadRecPtr;
	ControlFile->checkPointCopy = *checkPoint;
6155
	ControlFile->time = (pg_time_t) time(NULL);
6156 6157
	UpdateControlFile();

6158
	ereport((recoveryLogRestartpoints ? LOG : DEBUG2),
6159 6160
			(errmsg("recovery restart point at %X/%X",
					checkPoint->redo.xlogid, checkPoint->redo.xrecoff)));
6161 6162 6163 6164
	if (recoveryLastXTime)
		ereport((recoveryLogRestartpoints ? LOG : DEBUG2),
				(errmsg("last completed transaction was at log time %s",
						timestamptz_to_str(recoveryLastXTime))));
6165 6166
}

Tom Lane's avatar
Tom Lane committed
6167 6168 6169
/*
 * Write a NEXTOID log record
 */
6170 6171 6172
void
XLogPutNextOid(Oid nextOid)
{
6173
	XLogRecData rdata;
6174

6175
	rdata.data = (char *) (&nextOid);
6176
	rdata.len = sizeof(Oid);
6177
	rdata.buffer = InvalidBuffer;
6178 6179
	rdata.next = NULL;
	(void) XLogInsert(RM_XLOG_ID, XLOG_NEXTOID, &rdata);
6180

6181 6182
	/*
	 * We need not flush the NEXTOID record immediately, because any of the
6183 6184 6185 6186 6187
	 * just-allocated OIDs could only reach disk as part of a tuple insert or
	 * update that would have its own XLOG record that must follow the NEXTOID
	 * record.	Therefore, the standard buffer LSN interlock applied to those
	 * records will ensure no such OID reaches disk before the NEXTOID record
	 * does.
6188 6189
	 *
	 * Note, however, that the above statement only covers state "within" the
Bruce Momjian's avatar
Bruce Momjian committed
6190 6191
	 * database.  When we use a generated OID as a file or directory name, we
	 * are in a sense violating the basic WAL rule, because that filesystem
6192
	 * change may reach disk before the NEXTOID WAL record does.  The impact
Bruce Momjian's avatar
Bruce Momjian committed
6193 6194 6195 6196 6197
	 * of this is that if a database crash occurs immediately afterward, we
	 * might after restart re-generate the same OID and find that it conflicts
	 * with the leftover file or directory.  But since for safety's sake we
	 * always loop until finding a nonconflicting filename, this poses no real
	 * problem in practice. See pgsql-hackers discussion 27-Sep-2006.
6198 6199 6200
	 */
}

6201 6202 6203 6204 6205 6206 6207 6208 6209 6210
/*
 * Write an XLOG SWITCH record.
 *
 * Here we just blindly issue an XLogInsert request for the record.
 * All the magic happens inside XLogInsert.
 *
 * The return value is either the end+1 address of the switch record,
 * or the end+1 address of the prior segment if we did not need to
 * write a switch record because we are already at segment start.
 */
6211
XLogRecPtr
6212 6213 6214 6215 6216 6217 6218 6219 6220 6221 6222 6223 6224 6225 6226 6227
RequestXLogSwitch(void)
{
	XLogRecPtr	RecPtr;
	XLogRecData rdata;

	/* XLOG SWITCH, alone among xlog record types, has no data */
	rdata.buffer = InvalidBuffer;
	rdata.data = NULL;
	rdata.len = 0;
	rdata.next = NULL;

	RecPtr = XLogInsert(RM_XLOG_ID, XLOG_SWITCH, &rdata);

	return RecPtr;
}

Tom Lane's avatar
Tom Lane committed
6228 6229 6230
/*
 * XLOG resource manager's routines
 */
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
6231 6232 6233
void
xlog_redo(XLogRecPtr lsn, XLogRecord *record)
{
6234
	uint8		info = record->xl_info & ~XLR_INFO_MASK;
6235

6236
	if (info == XLOG_NEXTOID)
6237
	{
6238
		Oid			nextOid;
6239 6240 6241

		memcpy(&nextOid, XLogRecGetData(record), sizeof(Oid));
		if (ShmemVariableCache->nextOid < nextOid)
Tom Lane's avatar
Tom Lane committed
6242
		{
6243
			ShmemVariableCache->nextOid = nextOid;
Tom Lane's avatar
Tom Lane committed
6244 6245 6246 6247 6248 6249 6250 6251 6252 6253 6254 6255
			ShmemVariableCache->oidCount = 0;
		}
	}
	else if (info == XLOG_CHECKPOINT_SHUTDOWN)
	{
		CheckPoint	checkPoint;

		memcpy(&checkPoint, XLogRecGetData(record), sizeof(CheckPoint));
		/* In a SHUTDOWN checkpoint, believe the counters exactly */
		ShmemVariableCache->nextXid = checkPoint.nextXid;
		ShmemVariableCache->nextOid = checkPoint.nextOid;
		ShmemVariableCache->oidCount = 0;
6256 6257
		MultiXactSetNextMXact(checkPoint.nextMulti,
							  checkPoint.nextMultiOffset);
Bruce Momjian's avatar
Bruce Momjian committed
6258

6259 6260 6261 6262
		/* ControlFile->checkPointCopy always tracks the latest ckpt XID */
		ControlFile->checkPointCopy.nextXidEpoch = checkPoint.nextXidEpoch;
		ControlFile->checkPointCopy.nextXid = checkPoint.nextXid;

6263
		/*
6264
		 * TLI may change in a shutdown checkpoint, but it shouldn't decrease
6265 6266 6267 6268 6269 6270 6271 6272
		 */
		if (checkPoint.ThisTimeLineID != ThisTimeLineID)
		{
			if (checkPoint.ThisTimeLineID < ThisTimeLineID ||
				!list_member_int(expectedTLIs,
								 (int) checkPoint.ThisTimeLineID))
				ereport(PANIC,
						(errmsg("unexpected timeline ID %u (after %u) in checkpoint record",
6273
								checkPoint.ThisTimeLineID, ThisTimeLineID)));
6274 6275 6276
			/* Following WAL records should be run with new TLI */
			ThisTimeLineID = checkPoint.ThisTimeLineID;
		}
6277 6278

		RecoveryRestartPoint(&checkPoint);
Tom Lane's avatar
Tom Lane committed
6279 6280 6281 6282 6283 6284
	}
	else if (info == XLOG_CHECKPOINT_ONLINE)
	{
		CheckPoint	checkPoint;

		memcpy(&checkPoint, XLogRecGetData(record), sizeof(CheckPoint));
6285
		/* In an ONLINE checkpoint, treat the counters like NEXTOID */
6286 6287
		if (TransactionIdPrecedes(ShmemVariableCache->nextXid,
								  checkPoint.nextXid))
Tom Lane's avatar
Tom Lane committed
6288 6289 6290 6291 6292 6293
			ShmemVariableCache->nextXid = checkPoint.nextXid;
		if (ShmemVariableCache->nextOid < checkPoint.nextOid)
		{
			ShmemVariableCache->nextOid = checkPoint.nextOid;
			ShmemVariableCache->oidCount = 0;
		}
6294 6295
		MultiXactAdvanceNextMXact(checkPoint.nextMulti,
								  checkPoint.nextMultiOffset);
6296 6297 6298 6299 6300

		/* ControlFile->checkPointCopy always tracks the latest ckpt XID */
		ControlFile->checkPointCopy.nextXidEpoch = checkPoint.nextXidEpoch;
		ControlFile->checkPointCopy.nextXid = checkPoint.nextXid;

6301 6302
		/* TLI should not change in an on-line checkpoint */
		if (checkPoint.ThisTimeLineID != ThisTimeLineID)
6303
			ereport(PANIC,
6304 6305
					(errmsg("unexpected timeline ID %u (should be %u) in checkpoint record",
							checkPoint.ThisTimeLineID, ThisTimeLineID)));
6306 6307

		RecoveryRestartPoint(&checkPoint);
6308
	}
6309 6310 6311 6312
	else if (info == XLOG_NOOP)
	{
		/* nothing to do here */
	}
6313 6314 6315 6316
	else if (info == XLOG_SWITCH)
	{
		/* nothing to do here */
	}
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
6317
}
6318

Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
6319
void
6320
xlog_desc(StringInfo buf, uint8 xl_info, char *rec)
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
6321
{
Bruce Momjian's avatar
Bruce Momjian committed
6322
	uint8		info = xl_info & ~XLR_INFO_MASK;
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
6323

Tom Lane's avatar
Tom Lane committed
6324 6325
	if (info == XLOG_CHECKPOINT_SHUTDOWN ||
		info == XLOG_CHECKPOINT_ONLINE)
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
6326
	{
6327 6328
		CheckPoint *checkpoint = (CheckPoint *) rec;

6329
		appendStringInfo(buf, "checkpoint: redo %X/%X; "
Bruce Momjian's avatar
Bruce Momjian committed
6330 6331 6332 6333 6334 6335 6336 6337
						 "tli %u; xid %u/%u; oid %u; multi %u; offset %u; %s",
						 checkpoint->redo.xlogid, checkpoint->redo.xrecoff,
						 checkpoint->ThisTimeLineID,
						 checkpoint->nextXidEpoch, checkpoint->nextXid,
						 checkpoint->nextOid,
						 checkpoint->nextMulti,
						 checkpoint->nextMultiOffset,
				 (info == XLOG_CHECKPOINT_SHUTDOWN) ? "shutdown" : "online");
Tom Lane's avatar
Tom Lane committed
6338
	}
6339 6340 6341 6342
	else if (info == XLOG_NOOP)
	{
		appendStringInfo(buf, "xlog no-op");
	}
6343 6344
	else if (info == XLOG_NEXTOID)
	{
6345
		Oid			nextOid;
6346 6347

		memcpy(&nextOid, rec, sizeof(Oid));
6348
		appendStringInfo(buf, "nextOid: %u", nextOid);
6349
	}
6350 6351 6352 6353
	else if (info == XLOG_SWITCH)
	{
		appendStringInfo(buf, "xlog switch");
	}
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
6354
	else
6355
		appendStringInfo(buf, "UNKNOWN");
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
6356 6357
}

6358
#ifdef WAL_DEBUG
6359

Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
6360
static void
6361
xlog_outrec(StringInfo buf, XLogRecord *record)
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
6362
{
6363
	int			i;
6364

6365
	appendStringInfo(buf, "prev %X/%X; xid %u",
6366 6367
					 record->xl_prev.xlogid, record->xl_prev.xrecoff,
					 record->xl_xid);
6368

6369
	for (i = 0; i < XLR_MAX_BKP_BLOCKS; i++)
6370
	{
6371
		if (record->xl_info & XLR_SET_BKP_BLOCK(i))
Bruce Momjian's avatar
Bruce Momjian committed
6372
			appendStringInfo(buf, "; bkpb%d", i + 1);
6373 6374
	}

6375
	appendStringInfo(buf, ": %s", RmgrTable[record->xl_rmid].rm_name);
Vadim B. Mikheev's avatar
WAL  
Vadim B. Mikheev committed
6376
}
Bruce Momjian's avatar
Bruce Momjian committed
6377
#endif   /* WAL_DEBUG */
6378 6379 6380


/*
6381 6382
 * Return the (possible) sync flag used for opening a file, depending on the
 * value of the GUC wal_sync_method.
6383
 */
6384 6385
static int
get_sync_bit(int method)
6386
{
6387 6388 6389
	/* If fsync is disabled, never open in sync mode */
	if (!enableFsync)
		return 0;
6390

6391
	switch (method)
6392
	{
6393
		/*
6394 6395 6396 6397
		 * enum values for all sync options are defined even if they are not
		 * supported on the current platform.  But if not, they are not
		 * included in the enum option array, and therefore will never be seen
		 * here.
6398 6399 6400 6401
		 */
		case SYNC_METHOD_FSYNC:
		case SYNC_METHOD_FSYNC_WRITETHROUGH:
		case SYNC_METHOD_FDATASYNC:
6402
			return 0;
6403
#ifdef OPEN_SYNC_FLAG
6404
		case SYNC_METHOD_OPEN:
6405
			return OPEN_SYNC_FLAG;
6406 6407
#endif
#ifdef OPEN_DATASYNC_FLAG
6408
		case SYNC_METHOD_OPEN_DSYNC:
6409
			return OPEN_DATASYNC_FLAG;
6410
#endif
6411
		default:
6412 6413
			/* can't happen (unless we are out of sync with option array) */
			elog(ERROR, "unrecognized wal_sync_method: %d", method);
6414
			return 0; /* silence warning */
6415
	}
6416
}
6417

6418 6419 6420 6421 6422 6423
/*
 * GUC support
 */
bool
assign_xlog_sync_method(int new_sync_method, bool doit, GucSource source)
{
6424
	if (!doit)
6425
		return true;
6426

6427
	if (sync_method != new_sync_method)
6428 6429
	{
		/*
6430 6431
		 * To ensure that no blocks escape unsynced, force an fsync on the
		 * currently open log segment (if any).  Also, if the open flag is
6432 6433
		 * changing, close the log file so it will be reopened (with new flag
		 * bit) at next use.
6434 6435 6436 6437
		 */
		if (openLogFile >= 0)
		{
			if (pg_fsync(openLogFile) != 0)
6438 6439
				ereport(PANIC,
						(errcode_for_file_access(),
6440 6441
						 errmsg("could not fsync log file %u, segment %u: %m",
								openLogId, openLogSeg)));
6442
			if (get_sync_bit(sync_method) != get_sync_bit(new_sync_method))
6443
				XLogFileClose();
6444 6445
		}
	}
6446

6447
	return true;
6448 6449 6450 6451 6452 6453 6454 6455 6456 6457 6458
}


/*
 * Issue appropriate kind of fsync (if any) on the current XLOG output file
 */
static void
issue_xlog_fsync(void)
{
	switch (sync_method)
	{
6459
		case SYNC_METHOD_FSYNC:
6460
			if (pg_fsync_no_writethrough(openLogFile) != 0)
6461 6462
				ereport(PANIC,
						(errcode_for_file_access(),
6463 6464
						 errmsg("could not fsync log file %u, segment %u: %m",
								openLogId, openLogSeg)));
6465
			break;
6466 6467 6468 6469 6470
#ifdef HAVE_FSYNC_WRITETHROUGH
		case SYNC_METHOD_FSYNC_WRITETHROUGH:
			if (pg_fsync_writethrough(openLogFile) != 0)
				ereport(PANIC,
						(errcode_for_file_access(),
6471 6472
						 errmsg("could not fsync write-through log file %u, segment %u: %m",
								openLogId, openLogSeg)));
6473 6474
			break;
#endif
6475 6476 6477
#ifdef HAVE_FDATASYNC
		case SYNC_METHOD_FDATASYNC:
			if (pg_fdatasync(openLogFile) != 0)
6478 6479
				ereport(PANIC,
						(errcode_for_file_access(),
6480 6481
					errmsg("could not fdatasync log file %u, segment %u: %m",
						   openLogId, openLogSeg)));
6482 6483 6484
			break;
#endif
		case SYNC_METHOD_OPEN:
6485
		case SYNC_METHOD_OPEN_DSYNC:
6486 6487 6488
			/* write synced it already */
			break;
		default:
6489
			elog(PANIC, "unrecognized wal_sync_method: %d", sync_method);
6490 6491 6492
			break;
	}
}
6493 6494 6495 6496 6497 6498 6499 6500 6501


/*
 * pg_start_backup: set up for taking an on-line backup dump
 *
 * Essentially what this does is to create a backup label file in $PGDATA,
 * where it will be archived as part of the backup dump.  The label file
 * contains the user-supplied label string (typically this would be used
 * to tell where the backup dump will be stored) and the starting time and
6502
 * starting WAL location for the dump.
6503 6504 6505 6506 6507 6508
 */
Datum
pg_start_backup(PG_FUNCTION_ARGS)
{
	text	   *backupid = PG_GETARG_TEXT_P(0);
	char	   *backupidstr;
6509
	XLogRecPtr	checkpointloc;
6510
	XLogRecPtr	startpoint;
6511
	pg_time_t	stamp_time;
6512 6513 6514 6515 6516 6517 6518
	char		strfbuf[128];
	char		xlogfilename[MAXFNAMELEN];
	uint32		_logId;
	uint32		_logSeg;
	struct stat stat_buf;
	FILE	   *fp;

Bruce Momjian's avatar
Bruce Momjian committed
6519
	if (!superuser())
6520 6521
		ereport(ERROR,
				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
6522
				 errmsg("must be superuser to run a backup")));
6523 6524 6525 6526

	if (!XLogArchivingActive())
		ereport(ERROR,
				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
6527 6528 6529 6530 6531 6532 6533 6534 6535
				 errmsg("WAL archiving is not active"),
				 errhint("archive_mode must be enabled at server start.")));

	if (!XLogArchiveCommandSet())
		ereport(ERROR,
				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
				 errmsg("WAL archiving is not active"),
				 errhint("archive_command must be defined before "
						 "online backups can be made safely.")));
6536

6537
	backupidstr = text_to_cstring(backupid);
Bruce Momjian's avatar
Bruce Momjian committed
6538

6539
	/*
6540 6541 6542
	 * Mark backup active in shared memory.  We must do full-page WAL writes
	 * during an on-line backup even if not doing so at other times, because
	 * it's quite possible for the backup dump to obtain a "torn" (partially
Bruce Momjian's avatar
Bruce Momjian committed
6543 6544 6545 6546 6547 6548 6549 6550 6551
	 * written) copy of a database page if it reads the page concurrently with
	 * our write to the same page.	This can be fixed as long as the first
	 * write to the page in the WAL sequence is a full-page write. Hence, we
	 * turn on forcePageWrites and then force a CHECKPOINT, to ensure there
	 * are no dirty pages in shared memory that might get dumped while the
	 * backup is in progress without having a corresponding WAL record.  (Once
	 * the backup is complete, we need not force full-page writes anymore,
	 * since we expect that any pages not modified during the backup interval
	 * must have been correctly captured by the backup.)
6552
	 *
Bruce Momjian's avatar
Bruce Momjian committed
6553 6554
	 * We must hold WALInsertLock to change the value of forcePageWrites, to
	 * ensure adequate interlocking against XLogInsert().
6555
	 */
6556 6557 6558 6559 6560 6561 6562 6563 6564 6565 6566
	LWLockAcquire(WALInsertLock, LW_EXCLUSIVE);
	if (XLogCtl->Insert.forcePageWrites)
	{
		LWLockRelease(WALInsertLock);
		ereport(ERROR,
				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
				 errmsg("a backup is already in progress"),
				 errhint("Run pg_stop_backup() and try again.")));
	}
	XLogCtl->Insert.forcePageWrites = true;
	LWLockRelease(WALInsertLock);
Bruce Momjian's avatar
Bruce Momjian committed
6567

6568 6569
	/* Ensure we release forcePageWrites if fail below */
	PG_ENSURE_ERROR_CLEANUP(pg_start_backup_callback, (Datum) 0);
6570 6571
	{
		/*
Bruce Momjian's avatar
Bruce Momjian committed
6572
		 * Force a CHECKPOINT.	Aside from being necessary to prevent torn
6573 6574 6575
		 * page problems, this guarantees that two successive backup runs will
		 * have different checkpoint positions and hence different history
		 * file names, even if nothing happened in between.
6576 6577
		 *
		 * We don't use CHECKPOINT_IMMEDIATE, hence this can take awhile.
6578
		 */
6579
		RequestCheckpoint(CHECKPOINT_FORCE | CHECKPOINT_WAIT);
6580

6581 6582 6583 6584 6585 6586 6587 6588 6589
		/*
		 * Now we need to fetch the checkpoint record location, and also its
		 * REDO pointer.  The oldest point in WAL that would be needed to
		 * restore starting from the checkpoint is precisely the REDO pointer.
		 */
		LWLockAcquire(ControlFileLock, LW_EXCLUSIVE);
		checkpointloc = ControlFile->checkPoint;
		startpoint = ControlFile->checkPointCopy.redo;
		LWLockRelease(ControlFileLock);
Bruce Momjian's avatar
Bruce Momjian committed
6590

6591 6592
		XLByteToSeg(startpoint, _logId, _logSeg);
		XLogFileName(xlogfilename, ThisTimeLineID, _logId, _logSeg);
Bruce Momjian's avatar
Bruce Momjian committed
6593

6594 6595 6596 6597 6598
		/* Use the log timezone here, not the session timezone */
		stamp_time = (pg_time_t) time(NULL);
		pg_strftime(strfbuf, sizeof(strfbuf),
					"%Y-%m-%d %H:%M:%S %Z",
					pg_localtime(&stamp_time, log_timezone));
6599 6600 6601 6602 6603 6604 6605 6606 6607 6608 6609 6610 6611 6612 6613 6614 6615 6616 6617 6618 6619 6620 6621 6622 6623 6624

		/*
		 * Check for existing backup label --- implies a backup is already
		 * running.  (XXX given that we checked forcePageWrites above, maybe
		 * it would be OK to just unlink any such label file?)
		 */
		if (stat(BACKUP_LABEL_FILE, &stat_buf) != 0)
		{
			if (errno != ENOENT)
				ereport(ERROR,
						(errcode_for_file_access(),
						 errmsg("could not stat file \"%s\": %m",
								BACKUP_LABEL_FILE)));
		}
		else
			ereport(ERROR,
					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
					 errmsg("a backup is already in progress"),
					 errhint("If you're sure there is no backup in progress, remove file \"%s\" and try again.",
							 BACKUP_LABEL_FILE)));

		/*
		 * Okay, write the file
		 */
		fp = AllocateFile(BACKUP_LABEL_FILE, "w");
		if (!fp)
6625 6626
			ereport(ERROR,
					(errcode_for_file_access(),
6627 6628 6629 6630 6631 6632 6633 6634 6635 6636 6637 6638
					 errmsg("could not create file \"%s\": %m",
							BACKUP_LABEL_FILE)));
		fprintf(fp, "START WAL LOCATION: %X/%X (file %s)\n",
				startpoint.xlogid, startpoint.xrecoff, xlogfilename);
		fprintf(fp, "CHECKPOINT LOCATION: %X/%X\n",
				checkpointloc.xlogid, checkpointloc.xrecoff);
		fprintf(fp, "START TIME: %s\n", strfbuf);
		fprintf(fp, "LABEL: %s\n", backupidstr);
		if (fflush(fp) || ferror(fp) || FreeFile(fp))
			ereport(ERROR,
					(errcode_for_file_access(),
					 errmsg("could not write file \"%s\": %m",
6639
							BACKUP_LABEL_FILE)));
6640
	}
6641
	PG_END_ENSURE_ERROR_CLEANUP(pg_start_backup_callback, (Datum) 0);
Bruce Momjian's avatar
Bruce Momjian committed
6642

6643
	/*
6644
	 * We're done.  As a convenience, return the starting WAL location.
6645 6646 6647
	 */
	snprintf(xlogfilename, sizeof(xlogfilename), "%X/%X",
			 startpoint.xlogid, startpoint.xrecoff);
6648
	PG_RETURN_TEXT_P(cstring_to_text(xlogfilename));
6649 6650
}

6651 6652 6653 6654 6655 6656 6657 6658 6659 6660
/* Error cleanup callback for pg_start_backup */
static void
pg_start_backup_callback(int code, Datum arg)
{
	/* Turn off forcePageWrites on failure */
	LWLockAcquire(WALInsertLock, LW_EXCLUSIVE);
	XLogCtl->Insert.forcePageWrites = false;
	LWLockRelease(WALInsertLock);
}

6661 6662 6663 6664 6665 6666
/*
 * pg_stop_backup: finish taking an on-line backup dump
 *
 * We remove the backup label file created by pg_start_backup, and instead
 * create a backup history file in pg_xlog (whence it will immediately be
 * archived).  The backup history file contains the same info found in
6667
 * the label file, plus the backup-end time and WAL location.
6668
 * Note: different from CancelBackup which just cancels online backup mode.
6669 6670 6671 6672 6673 6674
 */
Datum
pg_stop_backup(PG_FUNCTION_ARGS)
{
	XLogRecPtr	startpoint;
	XLogRecPtr	stoppoint;
6675
	pg_time_t	stamp_time;
6676
	char		strfbuf[128];
6677
	char		histfilepath[MAXPGPATH];
6678 6679
	char		startxlogfilename[MAXFNAMELEN];
	char		stopxlogfilename[MAXFNAMELEN];
6680 6681
	char		lastxlogfilename[MAXFNAMELEN];
	char		histfilename[MAXFNAMELEN];
6682 6683 6684 6685 6686 6687
	uint32		_logId;
	uint32		_logSeg;
	FILE	   *lfp;
	FILE	   *fp;
	char		ch;
	int			ich;
6688 6689
	int			seconds_before_warning;
	int			waits = 0;
6690

Bruce Momjian's avatar
Bruce Momjian committed
6691
	if (!superuser())
6692 6693 6694
		ereport(ERROR,
				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
				 (errmsg("must be superuser to run a backup"))));
Bruce Momjian's avatar
Bruce Momjian committed
6695

6696 6697 6698 6699 6700 6701
	if (!XLogArchivingActive())
		ereport(ERROR,
				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
				 errmsg("WAL archiving is not active"),
				 errhint("archive_mode must be enabled at server start.")));

6702
	/*
6703
	 * OK to clear forcePageWrites
6704 6705
	 */
	LWLockAcquire(WALInsertLock, LW_EXCLUSIVE);
6706
	XLogCtl->Insert.forcePageWrites = false;
6707 6708
	LWLockRelease(WALInsertLock);

6709
	/*
Bruce Momjian's avatar
Bruce Momjian committed
6710 6711 6712
	 * Force a switch to a new xlog segment file, so that the backup is valid
	 * as soon as archiver moves out the current segment file. We'll report
	 * the end address of the XLOG SWITCH record as the backup stopping point.
6713 6714 6715
	 */
	stoppoint = RequestXLogSwitch();

6716 6717
	XLByteToSeg(stoppoint, _logId, _logSeg);
	XLogFileName(stopxlogfilename, ThisTimeLineID, _logId, _logSeg);
Bruce Momjian's avatar
Bruce Momjian committed
6718

6719 6720 6721 6722 6723
	/* Use the log timezone here, not the session timezone */
	stamp_time = (pg_time_t) time(NULL);
	pg_strftime(strfbuf, sizeof(strfbuf),
				"%Y-%m-%d %H:%M:%S %Z",
				pg_localtime(&stamp_time, log_timezone));
Bruce Momjian's avatar
Bruce Momjian committed
6724

6725 6726 6727
	/*
	 * Open the existing label file
	 */
6728
	lfp = AllocateFile(BACKUP_LABEL_FILE, "r");
6729 6730 6731 6732 6733 6734
	if (!lfp)
	{
		if (errno != ENOENT)
			ereport(ERROR,
					(errcode_for_file_access(),
					 errmsg("could not read file \"%s\": %m",
6735
							BACKUP_LABEL_FILE)));
6736 6737 6738 6739
		ereport(ERROR,
				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
				 errmsg("a backup is not in progress")));
	}
Bruce Momjian's avatar
Bruce Momjian committed
6740

6741
	/*
6742 6743
	 * Read and parse the START WAL LOCATION line (this code is pretty crude,
	 * but we are not expecting any variability in the file format).
6744 6745 6746 6747 6748 6749
	 */
	if (fscanf(lfp, "START WAL LOCATION: %X/%X (file %24s)%c",
			   &startpoint.xlogid, &startpoint.xrecoff, startxlogfilename,
			   &ch) != 4 || ch != '\n')
		ereport(ERROR,
				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
6750
				 errmsg("invalid data in file \"%s\"", BACKUP_LABEL_FILE)));
Bruce Momjian's avatar
Bruce Momjian committed
6751

6752 6753 6754 6755
	/*
	 * Write the backup history file
	 */
	XLByteToSeg(startpoint, _logId, _logSeg);
6756
	BackupHistoryFilePath(histfilepath, ThisTimeLineID, _logId, _logSeg,
6757
						  startpoint.xrecoff % XLogSegSize);
6758
	fp = AllocateFile(histfilepath, "w");
6759 6760 6761 6762
	if (!fp)
		ereport(ERROR,
				(errcode_for_file_access(),
				 errmsg("could not create file \"%s\": %m",
6763
						histfilepath)));
6764 6765 6766 6767
	fprintf(fp, "START WAL LOCATION: %X/%X (file %s)\n",
			startpoint.xlogid, startpoint.xrecoff, startxlogfilename);
	fprintf(fp, "STOP WAL LOCATION: %X/%X (file %s)\n",
			stoppoint.xlogid, stoppoint.xrecoff, stopxlogfilename);
6768
	/* transfer remaining lines from label to history file */
6769 6770 6771 6772 6773 6774 6775
	while ((ich = fgetc(lfp)) != EOF)
		fputc(ich, fp);
	fprintf(fp, "STOP TIME: %s\n", strfbuf);
	if (fflush(fp) || ferror(fp) || FreeFile(fp))
		ereport(ERROR,
				(errcode_for_file_access(),
				 errmsg("could not write file \"%s\": %m",
6776
						histfilepath)));
Bruce Momjian's avatar
Bruce Momjian committed
6777

6778 6779 6780 6781 6782 6783 6784
	/*
	 * Close and remove the backup label file
	 */
	if (ferror(lfp) || FreeFile(lfp))
		ereport(ERROR,
				(errcode_for_file_access(),
				 errmsg("could not read file \"%s\": %m",
6785 6786
						BACKUP_LABEL_FILE)));
	if (unlink(BACKUP_LABEL_FILE) != 0)
6787 6788 6789
		ereport(ERROR,
				(errcode_for_file_access(),
				 errmsg("could not remove file \"%s\": %m",
6790
						BACKUP_LABEL_FILE)));
Bruce Momjian's avatar
Bruce Momjian committed
6791

6792
	/*
Bruce Momjian's avatar
Bruce Momjian committed
6793 6794 6795
	 * Clean out any no-longer-needed history files.  As a side effect, this
	 * will post a .ready file for the newly created history file, notifying
	 * the archiver that history file may be archived immediately.
6796
	 */
6797
	CleanupBackupHistory();
Bruce Momjian's avatar
Bruce Momjian committed
6798

6799
	/*
6800 6801 6802 6803
	 * Wait until both the last WAL file filled during backup and the history
	 * file have been archived.  We assume that the alphabetic sorting
	 * property of the WAL files ensures any earlier WAL files are safely
	 * archived as well.
6804 6805
	 *
	 * We wait forever, since archive_command is supposed to work and
6806 6807
	 * we assume the admin wanted his backup to work completely. If you
	 * don't wish to wait, you can set statement_timeout.
6808
	 */
6809 6810 6811 6812 6813
	XLByteToPrevSeg(stoppoint, _logId, _logSeg);
	XLogFileName(lastxlogfilename, ThisTimeLineID, _logId, _logSeg);

	XLByteToSeg(startpoint, _logId, _logSeg);
	BackupHistoryFileName(histfilename, ThisTimeLineID, _logId, _logSeg,
6814 6815 6816 6817 6818
						  startpoint.xrecoff % XLogSegSize);

	seconds_before_warning = 60;
	waits = 0;

6819 6820
	while (XLogArchiveIsBusy(lastxlogfilename) ||
		   XLogArchiveIsBusy(histfilename))
6821 6822 6823 6824 6825 6826 6827 6828
	{
		CHECK_FOR_INTERRUPTS();

		pg_usleep(1000000L);

		if (++waits >= seconds_before_warning)
		{
			seconds_before_warning *= 2;     /* This wraps in >10 years... */
6829 6830 6831
			ereport(WARNING,
					(errmsg("pg_stop_backup still waiting for archive to complete (%d seconds elapsed)",
							waits)));
6832 6833 6834
		}
	}

6835
	/*
6836
	 * We're done.  As a convenience, return the ending WAL location.
6837 6838 6839
	 */
	snprintf(stopxlogfilename, sizeof(stopxlogfilename), "%X/%X",
			 stoppoint.xlogid, stoppoint.xrecoff);
6840
	PG_RETURN_TEXT_P(cstring_to_text(stopxlogfilename));
6841
}
6842

6843 6844 6845 6846 6847 6848
/*
 * pg_switch_xlog: switch to next xlog file
 */
Datum
pg_switch_xlog(PG_FUNCTION_ARGS)
{
Bruce Momjian's avatar
Bruce Momjian committed
6849
	XLogRecPtr	switchpoint;
6850 6851 6852 6853 6854
	char		location[MAXFNAMELEN];

	if (!superuser())
		ereport(ERROR,
				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
Bruce Momjian's avatar
Bruce Momjian committed
6855
			 (errmsg("must be superuser to switch transaction log files"))));
6856 6857 6858 6859 6860 6861 6862 6863

	switchpoint = RequestXLogSwitch();

	/*
	 * As a convenience, return the WAL location of the switch record
	 */
	snprintf(location, sizeof(location), "%X/%X",
			 switchpoint.xlogid, switchpoint.xrecoff);
6864
	PG_RETURN_TEXT_P(cstring_to_text(location));
6865 6866 6867
}

/*
6868 6869 6870 6871 6872
 * Report the current WAL write location (same format as pg_start_backup etc)
 *
 * This is useful for determining how much of WAL is visible to an external
 * archiving process.  Note that the data before this point is written out
 * to the kernel, but is not necessarily synced to disk.
6873 6874 6875
 */
Datum
pg_current_xlog_location(PG_FUNCTION_ARGS)
6876 6877 6878 6879 6880 6881 6882 6883 6884 6885 6886 6887 6888 6889 6890
{
	char		location[MAXFNAMELEN];

	/* Make sure we have an up-to-date local LogwrtResult */
	{
		/* use volatile pointer to prevent code rearrangement */
		volatile XLogCtlData *xlogctl = XLogCtl;

		SpinLockAcquire(&xlogctl->info_lck);
		LogwrtResult = xlogctl->LogwrtResult;
		SpinLockRelease(&xlogctl->info_lck);
	}

	snprintf(location, sizeof(location), "%X/%X",
			 LogwrtResult.Write.xlogid, LogwrtResult.Write.xrecoff);
6891
	PG_RETURN_TEXT_P(cstring_to_text(location));
6892 6893 6894 6895 6896 6897 6898 6899 6900
}

/*
 * Report the current WAL insert location (same format as pg_start_backup etc)
 *
 * This function is mostly for debugging purposes.
 */
Datum
pg_current_xlog_insert_location(PG_FUNCTION_ARGS)
6901 6902 6903 6904 6905 6906 6907 6908 6909 6910 6911 6912 6913 6914
{
	XLogCtlInsert *Insert = &XLogCtl->Insert;
	XLogRecPtr	current_recptr;
	char		location[MAXFNAMELEN];

	/*
	 * Get the current end-of-WAL position ... shared lock is sufficient
	 */
	LWLockAcquire(WALInsertLock, LW_SHARED);
	INSERT_RECPTR(current_recptr, Insert, Insert->curridx);
	LWLockRelease(WALInsertLock);

	snprintf(location, sizeof(location), "%X/%X",
			 current_recptr.xlogid, current_recptr.xrecoff);
6915
	PG_RETURN_TEXT_P(cstring_to_text(location));
6916 6917 6918 6919 6920 6921 6922 6923 6924 6925 6926 6927 6928 6929 6930 6931 6932 6933 6934 6935 6936 6937
}

/*
 * Compute an xlog file name and decimal byte offset given a WAL location,
 * such as is returned by pg_stop_backup() or pg_xlog_switch().
 *
 * Note that a location exactly at a segment boundary is taken to be in
 * the previous segment.  This is usually the right thing, since the
 * expected usage is to determine which xlog file(s) are ready to archive.
 */
Datum
pg_xlogfile_name_offset(PG_FUNCTION_ARGS)
{
	text	   *location = PG_GETARG_TEXT_P(0);
	char	   *locationstr;
	unsigned int uxlogid;
	unsigned int uxrecoff;
	uint32		xlogid;
	uint32		xlogseg;
	uint32		xrecoff;
	XLogRecPtr	locationpoint;
	char		xlogfilename[MAXFNAMELEN];
Bruce Momjian's avatar
Bruce Momjian committed
6938 6939 6940 6941 6942
	Datum		values[2];
	bool		isnull[2];
	TupleDesc	resultTupleDesc;
	HeapTuple	resultHeapTuple;
	Datum		result;
6943

6944 6945 6946
	/*
	 * Read input and parse
	 */
6947
	locationstr = text_to_cstring(location);
6948 6949 6950 6951

	if (sscanf(locationstr, "%X/%X", &uxlogid, &uxrecoff) != 2)
		ereport(ERROR,
				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
6952
				 errmsg("could not parse transaction log location \"%s\"",
6953 6954 6955 6956 6957
						locationstr)));

	locationpoint.xlogid = uxlogid;
	locationpoint.xrecoff = uxrecoff;

6958
	/*
Bruce Momjian's avatar
Bruce Momjian committed
6959 6960
	 * Construct a tuple descriptor for the result row.  This must match this
	 * function's pg_proc entry!
6961 6962 6963 6964 6965 6966 6967 6968 6969 6970 6971 6972
	 */
	resultTupleDesc = CreateTemplateTupleDesc(2, false);
	TupleDescInitEntry(resultTupleDesc, (AttrNumber) 1, "file_name",
					   TEXTOID, -1, 0);
	TupleDescInitEntry(resultTupleDesc, (AttrNumber) 2, "file_offset",
					   INT4OID, -1, 0);

	resultTupleDesc = BlessTupleDesc(resultTupleDesc);

	/*
	 * xlogfilename
	 */
6973 6974 6975
	XLByteToPrevSeg(locationpoint, xlogid, xlogseg);
	XLogFileName(xlogfilename, ThisTimeLineID, xlogid, xlogseg);

6976
	values[0] = CStringGetTextDatum(xlogfilename);
6977 6978 6979 6980 6981
	isnull[0] = false;

	/*
	 * offset
	 */
6982 6983
	xrecoff = locationpoint.xrecoff - xlogseg * XLogSegSize;

6984 6985 6986 6987 6988 6989 6990 6991 6992 6993 6994
	values[1] = UInt32GetDatum(xrecoff);
	isnull[1] = false;

	/*
	 * Tuple jam: Having first prepared your Datums, then squash together
	 */
	resultHeapTuple = heap_form_tuple(resultTupleDesc, values, isnull);

	result = HeapTupleGetDatum(resultHeapTuple);

	PG_RETURN_DATUM(result);
6995 6996 6997 6998 6999 7000 7001 7002 7003 7004 7005 7006 7007 7008 7009 7010 7011 7012
}

/*
 * Compute an xlog file name given a WAL location,
 * such as is returned by pg_stop_backup() or pg_xlog_switch().
 */
Datum
pg_xlogfile_name(PG_FUNCTION_ARGS)
{
	text	   *location = PG_GETARG_TEXT_P(0);
	char	   *locationstr;
	unsigned int uxlogid;
	unsigned int uxrecoff;
	uint32		xlogid;
	uint32		xlogseg;
	XLogRecPtr	locationpoint;
	char		xlogfilename[MAXFNAMELEN];

7013
	locationstr = text_to_cstring(location);
7014 7015 7016 7017

	if (sscanf(locationstr, "%X/%X", &uxlogid, &uxrecoff) != 2)
		ereport(ERROR,
				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
Peter Eisentraut's avatar
Peter Eisentraut committed
7018
				 errmsg("could not parse transaction log location \"%s\"",
7019 7020 7021 7022 7023 7024 7025 7026
						locationstr)));

	locationpoint.xlogid = uxlogid;
	locationpoint.xrecoff = uxrecoff;

	XLByteToPrevSeg(locationpoint, xlogid, xlogseg);
	XLogFileName(xlogfilename, ThisTimeLineID, xlogid, xlogseg);

7027
	PG_RETURN_TEXT_P(cstring_to_text(xlogfilename));
7028 7029
}

7030 7031 7032 7033 7034
/*
 * read_backup_label: check to see if a backup_label file is present
 *
 * If we see a backup_label during recovery, we assume that we are recovering
 * from a backup dump file, and we therefore roll forward from the checkpoint
Bruce Momjian's avatar
Bruce Momjian committed
7035
 * identified by the label file, NOT what pg_control says.	This avoids the
7036 7037 7038 7039 7040
 * problem that pg_control might have been archived one or more checkpoints
 * later than the start of the dump, and so if we rely on it as the start
 * point, we will fail to restore a consistent database state.
 *
 * We also attempt to retrieve the corresponding backup history file.
7041
 * If successful, set *minRecoveryLoc to constrain valid PITR stopping
7042 7043 7044 7045 7046 7047
 * points.
 *
 * Returns TRUE if a backup_label was found (and fills the checkpoint
 * location into *checkPointLoc); returns FALSE if not.
 */
static bool
7048
read_backup_label(XLogRecPtr *checkPointLoc, XLogRecPtr *minRecoveryLoc)
7049 7050 7051 7052 7053 7054 7055 7056 7057 7058 7059 7060 7061 7062
{
	XLogRecPtr	startpoint;
	XLogRecPtr	stoppoint;
	char		histfilename[MAXFNAMELEN];
	char		histfilepath[MAXPGPATH];
	char		startxlogfilename[MAXFNAMELEN];
	char		stopxlogfilename[MAXFNAMELEN];
	TimeLineID	tli;
	uint32		_logId;
	uint32		_logSeg;
	FILE	   *lfp;
	FILE	   *fp;
	char		ch;

7063 7064 7065 7066
	/* Default is to not constrain recovery stop point */
	minRecoveryLoc->xlogid = 0;
	minRecoveryLoc->xrecoff = 0;

7067 7068 7069
	/*
	 * See if label file is present
	 */
7070
	lfp = AllocateFile(BACKUP_LABEL_FILE, "r");
7071 7072 7073 7074 7075 7076
	if (!lfp)
	{
		if (errno != ENOENT)
			ereport(FATAL,
					(errcode_for_file_access(),
					 errmsg("could not read file \"%s\": %m",
7077
							BACKUP_LABEL_FILE)));
7078 7079
		return false;			/* it's not there, all is fine */
	}
Bruce Momjian's avatar
Bruce Momjian committed
7080

7081
	/*
7082 7083 7084
	 * Read and parse the START WAL LOCATION and CHECKPOINT lines (this code
	 * is pretty crude, but we are not expecting any variability in the file
	 * format).
7085 7086 7087 7088 7089 7090
	 */
	if (fscanf(lfp, "START WAL LOCATION: %X/%X (file %08X%16s)%c",
			   &startpoint.xlogid, &startpoint.xrecoff, &tli,
			   startxlogfilename, &ch) != 5 || ch != '\n')
		ereport(FATAL,
				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
7091
				 errmsg("invalid data in file \"%s\"", BACKUP_LABEL_FILE)));
7092 7093 7094 7095 7096
	if (fscanf(lfp, "CHECKPOINT LOCATION: %X/%X%c",
			   &checkPointLoc->xlogid, &checkPointLoc->xrecoff,
			   &ch) != 3 || ch != '\n')
		ereport(FATAL,
				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
7097
				 errmsg("invalid data in file \"%s\"", BACKUP_LABEL_FILE)));
7098 7099 7100 7101
	if (ferror(lfp) || FreeFile(lfp))
		ereport(FATAL,
				(errcode_for_file_access(),
				 errmsg("could not read file \"%s\": %m",
7102
						BACKUP_LABEL_FILE)));
Bruce Momjian's avatar
Bruce Momjian committed
7103

7104 7105 7106 7107 7108 7109 7110 7111 7112 7113 7114 7115 7116
	/*
	 * Try to retrieve the backup history file (no error if we can't)
	 */
	XLByteToSeg(startpoint, _logId, _logSeg);
	BackupHistoryFileName(histfilename, tli, _logId, _logSeg,
						  startpoint.xrecoff % XLogSegSize);

	if (InArchiveRecovery)
		RestoreArchivedFile(histfilepath, histfilename, "RECOVERYHISTORY", 0);
	else
		BackupHistoryFilePath(histfilepath, tli, _logId, _logSeg,
							  startpoint.xrecoff % XLogSegSize);

Bruce Momjian's avatar
Bruce Momjian committed
7117
	fp = AllocateFile(histfilepath, "r");
7118 7119 7120 7121 7122 7123
	if (fp)
	{
		/*
		 * Parse history file to identify stop point.
		 */
		if (fscanf(fp, "START WAL LOCATION: %X/%X (file %24s)%c",
7124
				   &startpoint.xlogid, &startpoint.xrecoff, startxlogfilename,
7125 7126 7127
				   &ch) != 4 || ch != '\n')
			ereport(FATAL,
					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
7128
					 errmsg("invalid data in file \"%s\"", histfilename)));
7129
		if (fscanf(fp, "STOP WAL LOCATION: %X/%X (file %24s)%c",
7130
				   &stoppoint.xlogid, &stoppoint.xrecoff, stopxlogfilename,
7131 7132 7133
				   &ch) != 4 || ch != '\n')
			ereport(FATAL,
					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
7134
					 errmsg("invalid data in file \"%s\"", histfilename)));
7135
		*minRecoveryLoc = stoppoint;
7136 7137 7138 7139 7140 7141 7142 7143 7144 7145
		if (ferror(fp) || FreeFile(fp))
			ereport(FATAL,
					(errcode_for_file_access(),
					 errmsg("could not read file \"%s\": %m",
							histfilepath)));
	}

	return true;
}

7146 7147 7148 7149 7150 7151
/*
 * Error context callback for errors occurring during rm_redo().
 */
static void
rm_redo_error_callback(void *arg)
{
Bruce Momjian's avatar
Bruce Momjian committed
7152 7153
	XLogRecord *record = (XLogRecord *) arg;
	StringInfoData buf;
7154 7155

	initStringInfo(&buf);
7156 7157
	RmgrTable[record->xl_rmid].rm_desc(&buf,
									   record->xl_info,
7158 7159 7160 7161 7162 7163 7164 7165
									   XLogRecGetData(record));

	/* don't bother emitting empty description */
	if (buf.len > 0)
		errcontext("xlog redo %s", buf.data);

	pfree(buf.data);
}
7166 7167 7168 7169 7170 7171 7172 7173 7174 7175 7176 7177 7178 7179 7180 7181 7182 7183 7184 7185 7186 7187 7188 7189 7190 7191 7192 7193 7194 7195 7196 7197 7198 7199 7200 7201 7202

/*
 * BackupInProgress: check if online backup mode is active
 *
 * This is done by checking for existence of the "backup_label" file.
 */
bool
BackupInProgress(void)
{
	struct stat stat_buf;

	return (stat(BACKUP_LABEL_FILE, &stat_buf) == 0);
}

/*
 * CancelBackup: rename the "backup_label" file to cancel backup mode
 *
 * If the "backup_label" file exists, it will be renamed to "backup_label.old".
 * Note that this will render an online backup in progress useless.
 * To correctly finish an online backup, pg_stop_backup must be called.
 */
void
CancelBackup(void)
{
	struct stat stat_buf;

	/* if the file is not there, return */
	if (stat(BACKUP_LABEL_FILE, &stat_buf) < 0)
		return;

	/* remove leftover file from previously cancelled backup if it exists */
	unlink(BACKUP_LABEL_OLD);

	if (rename(BACKUP_LABEL_FILE, BACKUP_LABEL_OLD) == 0)
	{
		ereport(LOG,
				(errmsg("online backup mode cancelled"),
7203
				 errdetail("\"%s\" was renamed to \"%s\".",
7204 7205 7206 7207 7208 7209
						BACKUP_LABEL_FILE, BACKUP_LABEL_OLD)));
	}
	else
	{
		ereport(WARNING,
				(errcode_for_file_access(),
7210 7211
				 errmsg("online backup mode was not cancelled"),
				 errdetail("Could not rename \"%s\" to \"%s\": %m.",
7212 7213 7214 7215
						BACKUP_LABEL_FILE, BACKUP_LABEL_OLD)));
	}
}