Skip to content

P
Postgres FD Implementation
Commit fb03d08a authored by Tom Lane's avatar Tom Lane
Browse files
Options

Rationalize parallel dump/restore's handling of worker cmd/status messages. 

The existing APIs for creating and parsing command and status messages are
rather messy; for example, archive-format modules have to provide code
for constructing command messages, which is entirely pointless since
the code to read them is hard-wired in WaitForCommands() and hence
no format-specific variation is actually possible.  But there's little
foreseeable reason to need format-specific variation anyway.

The situation for status messages is no better; at least those are both
constructed and parsed by format-specific code, but said code is quite
redundant since there's no actual need for format-specific variation.

To add insult to injury, the first API involves returning pointers to
static buffers, which is bad, while the second involves returning pointers
to malloc'd strings, which is safer but randomly inconsistent.

Hence, get rid of the MasterStartParallelItem and MasterEndParallelItem
APIs, and instead write centralized functions that construct and parse
command and status messages.  If we ever do need more flexibility, these
functions can be the standard implementations of format-specific
callback methods, but that's a long way off if it ever happens.

Tom Lane, reviewed by Kevin Grittner

Discussion: <17340.1464465717@sss.pgh.pa.us>
parent b7b8cc0c
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 162 additions and 274 deletions
src/bin/pg_dump/parallel.c
View file @ fb03d08a
This diff is collapsed.
src/bin/pg_dump/pg_backup_archiver.h
View file @ fb03d08a
......@@ -161,12 +161,8 @@ typedef void (*PrintTocDataPtr) (ArchiveHandle *AH, TocEntry *te);
typedef void (*ClonePtr) (ArchiveHandle *AH);
typedef void (*DeClonePtr) (ArchiveHandle *AH);
typedef char *(*WorkerJobRestorePtr) (ArchiveHandle *AH, TocEntry *te);
typedef char *(*WorkerJobDumpPtr) (ArchiveHandle *AH, TocEntry *te);
typedef char *(*MasterStartParallelItemPtr) (ArchiveHandle *AH, TocEntry *te,
T_Action act);
typedef int (*MasterEndParallelItemPtr) (ArchiveHandle *AH, TocEntry *te,
const char *str, T_Action act);
typedef int (*WorkerJobDumpPtr) (ArchiveHandle *AH, TocEntry *te);
typedef int (*WorkerJobRestorePtr) (ArchiveHandle *AH, TocEntry *te);
typedef size_t (*CustomOutPtr) (ArchiveHandle *AH, const void *buf, size_t len);
......@@ -266,9 +262,6 @@ struct _archiveHandle
StartBlobPtr StartBlobPtr;
EndBlobPtr EndBlobPtr;
MasterStartParallelItemPtr MasterStartParallelItemPtr;
MasterEndParallelItemPtr MasterEndParallelItemPtr;
SetupWorkerPtr SetupWorkerPtr;
WorkerJobDumpPtr WorkerJobDumpPtr;
WorkerJobRestorePtr WorkerJobRestorePtr;
......
src/bin/pg_dump/pg_backup_custom.c
View file @ fb03d08a
......@@ -61,9 +61,7 @@ static void _LoadBlobs(ArchiveHandle *AH, bool drop);
static void _Clone(ArchiveHandle *AH);
static void _DeClone(ArchiveHandle *AH);
static char *_MasterStartParallelItem(ArchiveHandle *AH, TocEntry *te, T_Action act);
static int _MasterEndParallelItem(ArchiveHandle *AH, TocEntry *te, const char *str, T_Action act);
char *_WorkerJobRestoreCustom(ArchiveHandle *AH, TocEntry *te);
static int _WorkerJobRestoreCustom(ArchiveHandle *AH, TocEntry *te);
typedef struct
{
......@@ -133,9 +131,6 @@ InitArchiveFmt_Custom(ArchiveHandle *AH)
AH->ClonePtr = _Clone;
AH->DeClonePtr = _DeClone;
AH->MasterStartParallelItemPtr = _MasterStartParallelItem;
AH->MasterEndParallelItemPtr = _MasterEndParallelItem;
/* no parallel dump in the custom archive, only parallel restore */
AH->WorkerJobDumpPtr = NULL;
AH->WorkerJobRestorePtr = _WorkerJobRestoreCustom;
......@@ -808,73 +803,13 @@ _DeClone(ArchiveHandle *AH)
}
/*
* This function is executed in the child of a parallel backup for the
* custom format archive and dumps the actual data.
*/
char *
_WorkerJobRestoreCustom(ArchiveHandle *AH, TocEntry *te)
{
/*
* short fixed-size string + some ID so far, this needs to be malloc'ed
* instead of static because we work with threads on windows
*/
const int buflen = 64;
char *buf = (char *) pg_malloc(buflen);
int status;
status = parallel_restore(AH, te);
snprintf(buf, buflen, "OK RESTORE %d %d %d", te->dumpId, status,
status == WORKER_IGNORED_ERRORS ? AH->public.n_errors : 0);
return buf;
}
/*
* This function is executed in the parent process. Depending on the desired
* action (dump or restore) it creates a string that is understood by the
* _WorkerJobDump /_WorkerJobRestore functions of the dump format.
*/
static char *
_MasterStartParallelItem(ArchiveHandle *AH, TocEntry *te, T_Action act)
{
/*
* A static char is okay here, even on Windows because we call this
* function only from one process (the master).
*/
static char buf[64]; /* short fixed-size string + number */
/* no parallel dump in the custom archive format */
Assert(act == ACT_RESTORE);
snprintf(buf, sizeof(buf), "RESTORE %d", te->dumpId);
return buf;
}
/*
* This function is executed in the parent process. It analyzes the response of
* the _WorkerJobDump / _WorkerJobRestore functions of the dump format.
* This function is executed in the child of a parallel restore from a
* custom-format archive and restores the actual data for one TOC entry.
*/
static int
_MasterEndParallelItem(ArchiveHandle *AH, TocEntry *te, const char *str, T_Action act)
_WorkerJobRestoreCustom(ArchiveHandle *AH, TocEntry *te)
{
DumpId dumpId;
int nBytes,
status,
n_errors;
/* no parallel dump in the custom archive */
Assert(act == ACT_RESTORE);
sscanf(str, "%d %d %d%n", &dumpId, &status, &n_errors, &nBytes);
Assert(nBytes == strlen(str));
Assert(dumpId == te->dumpId);
AH->public.n_errors += n_errors;
return status;
return parallel_restore(AH, te);
}
/*--------------------------------------------------
......
src/bin/pg_dump/pg_backup_directory.c
View file @ fb03d08a
......@@ -89,11 +89,8 @@ static void _LoadBlobs(ArchiveHandle *AH);
static void _Clone(ArchiveHandle *AH);
static void _DeClone(ArchiveHandle *AH);
static char *_MasterStartParallelItem(ArchiveHandle *AH, TocEntry *te, T_Action act);
static int _MasterEndParallelItem(ArchiveHandle *AH, TocEntry *te,
const char *str, T_Action act);
static char *_WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te);
static char *_WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te);
static int _WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te);
static int _WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te);
static void setFilePath(ArchiveHandle *AH, char *buf,
const char *relativeFilename);
......@@ -140,9 +137,6 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
AH->WorkerJobRestorePtr = _WorkerJobRestoreDirectory;
AH->WorkerJobDumpPtr = _WorkerJobDumpDirectory;
AH->MasterStartParallelItemPtr = _MasterStartParallelItem;
AH->MasterEndParallelItemPtr = _MasterEndParallelItem;
/* Set up our private context */
ctx = (lclContext *) pg_malloc0(sizeof(lclContext));
AH->formatData = (void *) ctx;
......@@ -754,53 +748,12 @@ _DeClone(ArchiveHandle *AH)
}
/*
* This function is executed in the parent process. Depending on the desired
* action (dump or restore) it creates a string that is understood by the
* _WorkerJobDump /_WorkerJobRestore functions of the dump format.
*/
static char *
_MasterStartParallelItem(ArchiveHandle *AH, TocEntry *te, T_Action act)
{
/*
* A static char is okay here, even on Windows because we call this
* function only from one process (the master).
*/
static char buf[64];
if (act == ACT_DUMP)
snprintf(buf, sizeof(buf), "DUMP %d", te->dumpId);
else if (act == ACT_RESTORE)
snprintf(buf, sizeof(buf), "RESTORE %d", te->dumpId);
return buf;
}
/*
* This function is executed in the child of a parallel backup for the
* directory archive and dumps the actual data.
*
* We are currently returning only the DumpId so theoretically we could
* make this function returning an int (or a DumpId). However, to
* facilitate further enhancements and because sooner or later we need to
* convert this to a string and send it via a message anyway, we stick with
* char *. It is parsed on the other side by the _EndMasterParallel()
* function of the respective dump format.
* This function is executed in the child of a parallel backup for a
* directory-format archive and dumps the actual data for one TOC entry.
*/
static char *
static int
_WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te)
{
/*
* short fixed-size string + some ID so far, this needs to be malloc'ed
* instead of static because we work with threads on windows
*/
const int buflen = 64;
char *buf = (char *) pg_malloc(buflen);
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
/* This should never happen */
if (!tctx)
exit_horribly(modulename, "error during backup\n");
/*
* This function returns void. We either fail and die horribly or
* succeed... A failure will be detected by the parent when the child dies
......@@ -808,63 +761,15 @@ _WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te)
*/
WriteDataChunksForTocEntry(AH, te);
snprintf(buf, buflen, "OK DUMP %d", te->dumpId);
return buf;
return 0;
}
/*
* This function is executed in the child of a parallel backup for the
* directory archive and dumps the actual data.
*/
static char *
_WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te)
{
/*
* short fixed-size string + some ID so far, this needs to be malloc'ed
* instead of static because we work with threads on windows
*/
const int buflen = 64;
char *buf = (char *) pg_malloc(buflen);
int status;
status = parallel_restore(AH, te);
snprintf(buf, buflen, "OK RESTORE %d %d %d", te->dumpId, status,
status == WORKER_IGNORED_ERRORS ? AH->public.n_errors : 0);
return buf;
}
/*
* This function is executed in the parent process. It analyzes the response of
* the _WorkerJobDumpDirectory/_WorkerJobRestoreDirectory functions of the
* respective dump format.
* This function is executed in the child of a parallel restore from a
* directory-format archive and restores the actual data for one TOC entry.
*/
static int
_MasterEndParallelItem(ArchiveHandle *AH, TocEntry *te, const char *str, T_Action act)
_WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te)
{
DumpId dumpId;
int nBytes,
n_errors;
int status = 0;
if (act == ACT_DUMP)
{
sscanf(str, "%d%n", &dumpId, &nBytes);
Assert(dumpId == te->dumpId);
Assert(nBytes == strlen(str));
}
else if (act == ACT_RESTORE)
{
sscanf(str, "%d %d %d%n", &dumpId, &status, &n_errors, &nBytes);
Assert(dumpId == te->dumpId);
Assert(nBytes == strlen(str));
AH->public.n_errors += n_errors;
}
return status;
return parallel_restore(AH, te);
}
src/bin/pg_dump/pg_backup_tar.c
View file @ fb03d08a
......@@ -152,9 +152,6 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
AH->ClonePtr = NULL;
AH->DeClonePtr = NULL;
AH->MasterStartParallelItemPtr = NULL;
AH->MasterEndParallelItemPtr = NULL;
AH->WorkerJobDumpPtr = NULL;
AH->WorkerJobRestorePtr = NULL;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment