diff -durpN postgresql.1/doc/src/sgml/ref/pg_basebackup.sgml postgresql.2/doc/src/sgml/ref/pg_basebackup.sgml --- postgresql.1/doc/src/sgml/ref/pg_basebackup.sgml 2012-11-08 13:13:04.151630632 +0100 +++ postgresql.2/doc/src/sgml/ref/pg_basebackup.sgml 2012-11-20 12:56:32.891843707 +0100 @@ -189,6 +189,21 @@ PostgreSQL documentation + + + + + + Write a minimal recovery.conf into the output directory (or into + the base archive file if was specified) + using the connection parameters from the command line to ease + setting up the standby. + + + + + + diff -durpN postgresql.1/src/bin/pg_basebackup/pg_basebackup.c postgresql.2/src/bin/pg_basebackup/pg_basebackup.c --- postgresql.1/src/bin/pg_basebackup/pg_basebackup.c 2012-10-03 10:40:48.297207389 +0200 +++ postgresql.2/src/bin/pg_basebackup/pg_basebackup.c 2012-11-20 16:40:52.329063335 +0100 @@ -19,12 +19,14 @@ #define FRONTEND 1 #include "postgres.h" #include "libpq-fe.h" +#include "pqexpbuffer.h" #include #include #include #include #include +#include #ifdef HAVE_LIBZ #include @@ -46,6 +48,7 @@ int compresslevel = 0; bool includewal = false; bool streamwal = false; bool fastcheckpoint = false; +bool writerecoveryconf = false; int standby_message_timeout = 10 * 1000; /* 10 sec = default */ /* Progress counters */ @@ -70,6 +73,10 @@ static int has_xlogendptr = 0; static volatile LONG has_xlogendptr = 0; #endif +/* Don't ever change this value, the TAR file format requires it. */ +#define TARCHUNKSZ 512 +PQExpBuffer rcExpBuf = NULL; + /* Function headers */ static void usage(void); static void verify_dir_is_empty_or_create(char *dirname); @@ -77,6 +84,13 @@ static void progress_report(int tablespa static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum); static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum); +static char *escape_quotes(const char *src); +static void CreateRecoveryConf(PGconn *conn); +static void WriteRecoveryConf(void); +static int _tarChecksum(char *header); +static void print_val(char *s, uint64 val, unsigned int base, size_t len); +static void scan_val(char *s, uint64 *val, unsigned int base, size_t len); +static void _tarCreateHeader(char *header, char *filename, size_t filesize); static void BaseBackup(void); static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline, @@ -107,6 +121,8 @@ usage(void) printf(_("\nOptions controlling the output:\n")); printf(_(" -D, --pgdata=DIRECTORY receive base backup into directory\n")); printf(_(" -F, --format=p|t output format (plain (default), tar)\n")); + printf(_(" -R, --write-recovery-conf\n" + " write recovery.conf after backup\n")); printf(_(" -x, --xlog include required WAL files in backup (fetch mode)\n")); printf(_(" -X, --xlog-method=fetch|stream\n" " include required WAL files with specified method\n")); @@ -452,6 +468,45 @@ progress_report(int tablespacenum, const /* + * Write a piece of tar data + */ +static void +writeTarData( +#ifdef HAVE_LIBZ + gzFile ztarfile, +#endif + FILE *tarfile, char *buf, int r, char *current_file) +{ +#ifdef HAVE_LIBZ + if (ztarfile != NULL) + { + if (gzwrite(ztarfile, buf, r) != r) + { + fprintf(stderr, + _("%s: could not write to compressed file \"%s\": %s\n"), + progname, current_file, get_gz_error(ztarfile)); + disconnect_and_exit(1); + } + } + else +#endif + { + if (fwrite(buf, r, 1, tarfile) != 1) + { + fprintf(stderr, _("%s: could not write to file \"%s\": %s\n"), + progname, current_file, strerror(errno)); + disconnect_and_exit(1); + } + } +} + +#ifdef HAVE_LIBZ +#define WRITE_TAR_DATA(buf, sz) writeTarData(ztarfile, tarfile, buf, sz, filename) +#else +#define WRITE_TAR_DATA(buf, sz) writeTarData(tarfile, buf, sz, filename) +#endif + +/* * Receive a tar format file from the connection to the server, and write * the data from this file directly into a tar file. If compression is * enabled, the data will be compressed while written to the file. @@ -467,12 +522,17 @@ ReceiveTarFile(PGconn *conn, PGresult *r char filename[MAXPGPATH]; char *copybuf = NULL; FILE *tarfile = NULL; + char tarhdr[TARCHUNKSZ]; + bool basetablespace = PQgetisnull(res, rownum, 0); + bool in_tarhdr, skip_file; + size_t tarhdrsz; + uint64 filesz; #ifdef HAVE_LIBZ gzFile ztarfile = NULL; #endif - if (PQgetisnull(res, rownum, 0)) + if (basetablespace) { /* * Base tablespaces @@ -584,6 +644,16 @@ ReceiveTarFile(PGconn *conn, PGresult *r disconnect_and_exit(1); } + /* + * Initialize our variables for tracking + * individual files inside the TAR stream. + * For more detailed explanation, see below. + */ + in_tarhdr = true; + skip_file = false; + tarhdrsz = 0; + filesz = 0; + while (1) { int r; @@ -598,38 +668,36 @@ ReceiveTarFile(PGconn *conn, PGresult *r if (r == -1) { /* - * End of chunk. Close file (but not stdout). + * End of chunk. Write recovery.conf into the tar file (if it + * was requested) and close file (but not stdout). * * Also, write two completely empty blocks at the end of the tar * file, as required by some tar programs. */ - char zerobuf[1024]; + char zerobuf[2*TARCHUNKSZ]; MemSet(zerobuf, 0, sizeof(zerobuf)); -#ifdef HAVE_LIBZ - if (ztarfile != NULL) - { - if (gzwrite(ztarfile, zerobuf, sizeof(zerobuf)) != - sizeof(zerobuf)) - { - fprintf(stderr, - _("%s: could not write to compressed file \"%s\": %s\n"), - progname, filename, get_gz_error(ztarfile)); - disconnect_and_exit(1); - } - } - else -#endif + + if (basetablespace && writerecoveryconf) { - if (fwrite(zerobuf, sizeof(zerobuf), 1, tarfile) != 1) - { - fprintf(stderr, - _("%s: could not write to file \"%s\": %s\n"), - progname, filename, strerror(errno)); - disconnect_and_exit(1); - } + char header[TARCHUNKSZ]; + int padding; + + _tarCreateHeader(header, "recovery.conf", rcExpBuf->len); + + padding = ((rcExpBuf->len + (TARCHUNKSZ-1)) & ~(TARCHUNKSZ-1)) - rcExpBuf->len; + + WRITE_TAR_DATA(header, sizeof(header)); + WRITE_TAR_DATA(rcExpBuf->data, rcExpBuf->len); + if (padding) + WRITE_TAR_DATA(zerobuf, padding); + + if (verbose) + fprintf(stderr, _("%s: recovery.conf written into '%s'\n"), progname, filename); } + WRITE_TAR_DATA(zerobuf, sizeof(zerobuf)); + #ifdef HAVE_LIBZ if (ztarfile != NULL) { @@ -665,25 +733,124 @@ ReceiveTarFile(PGconn *conn, PGresult *r disconnect_and_exit(1); } -#ifdef HAVE_LIBZ - if (ztarfile != NULL) + if (!writerecoveryconf || !basetablespace) { - if (gzwrite(ztarfile, copybuf, r) != r) - { - fprintf(stderr, - _("%s: could not write to compressed file \"%s\": %s\n"), - progname, filename, get_gz_error(ztarfile)); - disconnect_and_exit(1); - } + /* + * If --write-recovery-conf was not requested or this + * is not the base tablespace, simply pass the received + * data into the TAR file, either compressed or not. + */ + + WRITE_TAR_DATA(copybuf, r); } else -#endif { - if (fwrite(copybuf, r, 1, tarfile) != 1) + /* + * If --write-recovery-conf was requested AND this + * is the base tablespace, the TAR stream may contain + * a recovery.conf file if the backup is coming from + * a standby server. We have to skip this file in + * the stream and add a new one constructed by + * CreateRecoveryConf() at the end of the stream. + * + * To do this, we have to process the individual files + * inside the TAR stream. The stream consists of a header + * and zero or more chunks, all 512 bytes long. The stream + * from the server is broken up into smaller pieces, so + * we have to track the size of the files to find the next + * header structure. + */ + int rr = r; /* Save the value returned by PQgetCopyData */ + int pos = 0; + + while (rr > 0) { - fprintf(stderr, _("%s: could not write to file \"%s\": %s\n"), - progname, filename, strerror(errno)); - disconnect_and_exit(1); + if (in_tarhdr) + { + /* + * We're currently reading a header structure + * inside the TAR stream, i.e. the file metadata. + */ + if (tarhdrsz < TARCHUNKSZ) + { + /* + * Copy the header structure into tarhdr[] + * in case the header is not aligned to 512 bytes + * or it's not returned in whole by the last + * PQgetCopyData call. + */ + int hdrleft, bytes2copy; + + hdrleft = TARCHUNKSZ - tarhdrsz; + bytes2copy = (rr > hdrleft ? hdrleft : rr); + + memcpy(&tarhdr[tarhdrsz], copybuf + pos, bytes2copy); + + rr -= bytes2copy; + pos += bytes2copy; + tarhdrsz += bytes2copy; + } + else + { + /* + * We have the whole header structure in tarhdr[], + * look at the file metadata: + * - the subsequent file contents have to be skipped + * if the filename is recovery.conf + * - find out the size of the file padded to the next + * multiple of 512 + */ + int64 padding; + + skip_file = (strcmp(&tarhdr[0], "recovery.conf") == 0); + + scan_val(&tarhdr[124], &filesz, 8, 11); + + padding = ((filesz + (TARCHUNKSZ-1)) & ~(TARCHUNKSZ-1)) - filesz; + filesz += padding; + + /* Indicate that the subsequent data is the file content. */ + in_tarhdr = false; + + if (!skip_file) + WRITE_TAR_DATA(tarhdr, TARCHUNKSZ); + } + } + else + { + /* + * We're processing a file's contents. + */ + if (filesz > 0) + { + /* + * We still have data to read (and possibly write). + */ + int bytes2write; + + bytes2write = (filesz > rr ? rr : filesz); + + if (!skip_file) + WRITE_TAR_DATA(copybuf + pos, bytes2write); + + rr -= bytes2write; + pos += bytes2write; + filesz -= bytes2write; + } + else + { + /* + * No more data in the current file, + * the next piece of data (if any) will + * be a new file header structure. + * Reinitialize all our variables. + */ + in_tarhdr = true; + skip_file = false; + tarhdrsz = 0; + filesz = 0; + } + } } } totaldone += r; @@ -712,10 +879,11 @@ ReceiveAndUnpackTarFile(PGconn *conn, PG char filename[MAXPGPATH]; int current_len_left; int current_padding = 0; + bool basetablespace = PQgetisnull(res, rownum, 0); char *copybuf = NULL; FILE *file = NULL; - if (PQgetisnull(res, rownum, 0)) + if (basetablespace) strcpy(current_path, basedir); else strcpy(current_path, PQgetvalue(res, rownum, 1)); @@ -767,13 +935,13 @@ ReceiveAndUnpackTarFile(PGconn *conn, PG /* * No current file, so this must be the header for a new file */ - if (r != 512) + if (r != TARCHUNKSZ) { fprintf(stderr, _("%s: invalid tar block header size: %d\n"), progname, r); disconnect_and_exit(1); } - totaldone += 512; + totaldone += TARCHUNKSZ; if (sscanf(copybuf + 124, "%11o", ¤t_len_left) != 1) { @@ -794,7 +962,7 @@ ReceiveAndUnpackTarFile(PGconn *conn, PG * All files are padded up to 512 bytes */ current_padding = - ((current_len_left + 511) & ~511) - current_len_left; + ((current_len_left + (TARCHUNKSZ-1)) & ~(TARCHUNKSZ-1)) - current_len_left; /* * First part of header is zero terminated filename @@ -937,6 +1105,258 @@ ReceiveAndUnpackTarFile(PGconn *conn, PG if (copybuf != NULL) PQfreemem(copybuf); + + if (basetablespace) + WriteRecoveryConf(); +} + +static int +_tarChecksum(char *header) +{ + int i, + sum; + + /* + * Per POSIX, the checksum is the simple sum of all bytes in the header, + * treating the bytes as unsigned, and treating the checksum field (at + * offset 148) as though it contained 8 spaces. + */ + sum = 8 * ' '; /* presumed value for checksum field */ + for (i = 0; i < TARCHUNKSZ; i++) + if (i < 148 || i >= 156) + sum += 0xFF & header[i]; + return sum; +} + + +/* + * Utility routine to print possibly larger than 32 bit integers in a + * portable fashion. Filled with zeros. + */ +static void +print_val(char *s, uint64 val, unsigned int base, size_t len) +{ + int i; + + for (i = len; i > 0; i--) + { + int digit = val % base; + + s[i - 1] = '0' + digit; + val = val / base; + } +} + + +/* + * Inverse for print_val() + */ +static void +scan_val(char *s, uint64 *val, unsigned int base, size_t len) +{ + uint64 tmp = 0; + int i; + + for (i = 0; i < len; i++) + { + int digit = s[i] - '0'; + + tmp = tmp * base + digit; + } + + *val = tmp; +} + + +static void +_tarCreateHeader(char *header, char *filename, size_t filesize) +{ + /* + * Note: most of the fields in a tar header are not supposed to be + * null-terminated. We use sprintf, which will write a null after the + * required bytes; that null goes into the first byte of the next field. + * This is okay as long as we fill the fields in order. + */ + memset(header, 0, TARCHUNKSZ /* sizeof the tar header */); + + /* Name 100 */ + sprintf(&header[0], "%.99s", filename); + + /* Mode 8 */ + sprintf(&header[100], "0000600 "); + + /* User ID 8 */ + sprintf(&header[108], "0004000 "); + + /* Group 8 */ + sprintf(&header[116], "0002000 "); + + /* File size 12 - 11 digits, 1 space; use print_val for 64 bit support */ + print_val(&header[124], filesize, 8, 11); + sprintf(&header[135], " "); + + /* Mod Time 12 */ + sprintf(&header[136], "%011o ", (int) time(NULL)); + + /* Checksum 8 cannot be calculated until we've filled all other fields */ + + /* Type - regular file */ + sprintf(&header[156], "0"); + + /* Link Name 100 (leave as nulls) */ + + /* Magic 6 */ + sprintf(&header[257], "ustar"); + + /* Version 2 */ + sprintf(&header[263], "00"); + + /* User 32 */ + /* XXX: Do we need to care about setting correct username? */ + sprintf(&header[265], "%.31s", "postgres"); + + /* Group 32 */ + /* XXX: Do we need to care about setting correct group name? */ + sprintf(&header[297], "%.31s", "postgres"); + + /* Major Dev 8 */ + sprintf(&header[329], "%07o ", 0); + + /* Minor Dev 8 */ + sprintf(&header[337], "%07o ", 0); + + /* Prefix 155 - not used, leave as nulls */ + + /* + * We mustn't overwrite the next field while inserting the checksum. + * Fortunately, the checksum can't exceed 6 octal digits, so we just write + * 6 digits, a space, and a null, which is legal per POSIX. + */ + sprintf(&header[148], "%06o ", _tarChecksum(header)); +} + +/* + * Escape single quotes in a string + */ +static char * +escape_quotes(const char *src) +{ + int len = strlen(src), + i, + j; + char *result = pg_malloc(len * 2 + 1); + + for (i = 0, j = 0; i < len; i++) + { + if (SQL_STR_DOUBLE(src[i], true)) + result[j++] = src[i]; + result[j++] = src[i]; + } + result[j] = '\0'; + return result; +} + +/* + * Try to create recovery.conf in memory and set the length to write later. + */ +static void +CreateRecoveryConf(PGconn *conn) +{ + PQconninfoOption *connOptions; + PQconninfoOption *option; + + if (!writerecoveryconf) + return; + + connOptions = PQconninfo(conn, PG_CONNINFO_REPLICATION); + if (connOptions == NULL) + { + fprintf(stderr, _("%s: out of memory"), progname); + disconnect_and_exit(1); + } + + appendPQExpBufferStr(rcExpBuf, "standby_mode = 'on'\n"); + if (PQExpBufferBroken(rcExpBuf)) + { + fprintf(stderr, _("%s: out of memory"), progname); + disconnect_and_exit(1); + } + + appendPQExpBufferStr(rcExpBuf, "primary_conninfo = '"); + if (PQExpBufferBroken(rcExpBuf)) + { + fprintf(stderr, _("%s: out of memory"), progname); + disconnect_and_exit(1); + } + + for (option = connOptions; option && option->keyword; option++) + { + char *escaped; + + /* + * Do not emit this setting if not set or empty. + */ + if ((option->val == NULL) || + (option->val != NULL && option->val[0] == '\0')) + continue; + + /* + * Write "keyword='value'" pieces, the value string is escaped + * if necessary and doubled single quotes around the value string. + */ + escaped = escape_quotes(option->val); + + appendPQExpBuffer(rcExpBuf, "%s=''%s'' ", option->keyword, escaped); + if (PQExpBufferBroken(rcExpBuf)) + { + fprintf(stderr, _("%s: out of memory"), progname); + disconnect_and_exit(1); + } + + free(escaped); + } + + appendPQExpBufferStr(rcExpBuf, "'\n"); + if (PQExpBufferBroken(rcExpBuf)) + { + fprintf(stderr, _("%s: out of memory"), progname); + disconnect_and_exit(1); + } + + PQconninfoFree(connOptions); +} + + +static void +WriteRecoveryConf(void) +{ + char filename[MAXPGPATH]; + FILE *cf; + + if (!writerecoveryconf) + return; + + sprintf(filename, "%s/recovery.conf", basedir); + + cf = fopen(filename, "w"); + if (cf == NULL) + { + fprintf(stderr, _("%s: cannot create %s: %s"), progname, filename, strerror(errno)); + disconnect_and_exit(1); + } + + if (fwrite(rcExpBuf->data, rcExpBuf->len, 1, cf) != 1) + { + fprintf(stderr, + _("%s: could not write to file \"%s\": %s\n"), + progname, filename, strerror(errno)); + disconnect_and_exit(1); + } + + fclose(cf); + + if (verbose) + fprintf(stderr, _("%s: recovery.conf written.\n"), progname); } @@ -960,6 +1380,15 @@ BaseBackup(void) /* Error message already written in GetConnection() */ exit(1); + rcExpBuf = createPQExpBuffer(); + if (!rcExpBuf) + { + fprintf(stderr, _("%s: out of memory"), progname); + disconnect_and_exit(1); + } + + CreateRecoveryConf(conn); + /* * Run IDENTIFY_SYSTEM so we can get the timeline */ @@ -1223,6 +1652,9 @@ BaseBackup(void) #endif } + /* Free the recovery.conf contents */ + destroyPQExpBuffer(rcExpBuf); + /* * End of copy data. Final result is already checked inside the loop. */ @@ -1243,6 +1675,7 @@ main(int argc, char **argv) {"pgdata", required_argument, NULL, 'D'}, {"format", required_argument, NULL, 'F'}, {"checkpoint", required_argument, NULL, 'c'}, + {"write-recovery-conf", no_argument, NULL, 'R'}, {"xlog", no_argument, NULL, 'x'}, {"xlog-method", required_argument, NULL, 'X'}, {"gzip", no_argument, NULL, 'z'}, @@ -1280,7 +1713,7 @@ main(int argc, char **argv) } } - while ((c = getopt_long(argc, argv, "D:F:xX:l:zZ:c:h:p:U:s:wWvP", + while ((c = getopt_long(argc, argv, "D:F:RxX:l:zZ:c:h:p:U:s:wWvP", long_options, &option_index)) != -1) { switch (c) @@ -1301,6 +1734,9 @@ main(int argc, char **argv) exit(1); } break; + case 'R': + writerecoveryconf = true; + break; case 'x': if (includewal) { diff -durpN postgresql.1/src/bin/pg_basebackup/streamutil.c postgresql.2/src/bin/pg_basebackup/streamutil.c --- postgresql.1/src/bin/pg_basebackup/streamutil.c 2012-10-03 10:40:48.298207395 +0200 +++ postgresql.2/src/bin/pg_basebackup/streamutil.c 2012-11-20 14:29:11.815346687 +0100 @@ -44,13 +44,13 @@ pg_strdup(const char *s) if (!result) { fprintf(stderr, _("%s: out of memory\n"), progname); - exit(1); + disconnect_and_exit(1); } return result; } void * -pg_malloc0(size_t size) +pg_malloc(size_t size) { void *result; @@ -61,13 +61,24 @@ pg_malloc0(size_t size) if (!result) { fprintf(stderr, _("%s: out of memory\n"), progname); - exit(1); + disconnect_and_exit(1); } MemSet(result, 0, size); return result; } +void * +pg_malloc0(size_t size) +{ + void *tmp; + + tmp = pg_malloc(size); + MemSet(tmp, 0, size); + return tmp; +} + + /* * Connect to the server. Returns a valid PGconn pointer if connected, * or NULL on non-permanent error. On permanent error, the function will diff -durpN postgresql.1/src/bin/pg_basebackup/streamutil.h postgresql.2/src/bin/pg_basebackup/streamutil.h --- postgresql.1/src/bin/pg_basebackup/streamutil.h 2012-10-03 10:40:48.299207401 +0200 +++ postgresql.2/src/bin/pg_basebackup/streamutil.h 2012-11-20 14:24:29.517573163 +0100 @@ -17,6 +17,7 @@ extern PGconn *conn; extern char *pg_strdup(const char *s); +extern void *pg_malloc(size_t size); extern void *pg_malloc0(size_t size); extern PGconn *GetConnection(void);