diff -r 699924885324 -r 8c43292e0012 doc/src/sgml/ref/pg_rewind.sgml --- a/doc/src/sgml/ref/pg_rewind.sgml Sun Sep 03 11:12:29 2017 -0400 +++ b/doc/src/sgml/ref/pg_rewind.sgml Fri Oct 27 18:47:42 2017 +0200 @@ -48,16 +48,26 @@ - The result is equivalent to replacing the target data directory with the - source one. Only changed blocks from relation files are copied; - all other files are copied in full, including configuration files. The - advantage of pg_rewind over taking a new base backup, or - tools like rsync, is that pg_rewind does - not require reading through unchanged blocks in the cluster. This makes + The result is equivalent to replacing the data-related files in the target + data directory with the source one. Only changed blocks from relation files + are copied; all other files relating to control or WAL information are copied + in full. The advantage of pg_rewind over taking a new base + backup, or tools like rsync, is that pg_rewind + does not require reading through unchanged blocks in the cluster. This makes it a lot faster when the database is large and only a small fraction of blocks differ between the clusters. + + A second advantage is predictability. pg_rewind is aware of + what directories are relevant to restoring replication and which ones are not. + The result is that you get something of a guaranteed state at the end. Log + files on the source are unlikely to clobber those on the client. The + postgresql.conf.auto is unlikely to be copied over. Replication + slot information is removed (and must be added again), and so forth. Your + system is as it had been before, but the data is synchronized from the master. + + pg_rewind examines the timeline histories of the source and target clusters to determine the point where they diverged, and @@ -84,7 +94,9 @@ pg_rewind session, it must be made available when the target server is started. This can be done by creating a recovery.conf file in the target data directory with a - suitable restore_command. + suitable restore_command. Alternatively replication slots can + be set just before promotion to ensure that the wal files have not been + removed. @@ -206,7 +218,7 @@ The basic idea is to copy all file system-level changes from the source - cluster to the target cluster: + cluster to the target cluster if they implicate the data stored: @@ -229,9 +241,7 @@ - Copy all other files such as pg_xact and - configuration files from the source cluster to the target cluster - (everything except the relation files). + Copy wal-related files such as those in pg_xact. diff -r 699924885324 -r 8c43292e0012 src/bin/pg_rewind/RewindTest.pm --- a/src/bin/pg_rewind/RewindTest.pm Sun Sep 03 11:12:29 2017 -0400 +++ b/src/bin/pg_rewind/RewindTest.pm Fri Oct 27 18:47:42 2017 +0200 @@ -1,4 +1,5 @@ package RewindTest; +use Carp::Always; # Test driver for pg_rewind. Each test consists of a cycle where a new cluster # is first created with initdb, and a streaming replication standby is set up diff -r 699924885324 -r 8c43292e0012 src/bin/pg_rewind/copy_fetch.c --- a/src/bin/pg_rewind/copy_fetch.c Sun Sep 03 11:12:29 2017 -0400 +++ b/src/bin/pg_rewind/copy_fetch.c Fri Oct 27 18:47:42 2017 +0200 @@ -26,6 +26,30 @@ static void recurse_dir(const char *datadir, const char *path, process_file_callback_t callback); +/* List of directories to synchronize: + * base data dirs (and ablespaces) + * wal/transaction data + * and that is it. + * + * This array is null-terminated to make + * it easy to expand + */ + +const char *rewind_dirs[] = { + "base", + "global", + "pg_commit_ts", + "pg_logical", + "pg_multixact", + "pg_serial", + "pg_subtrans", + "pg_tblspc", + "pg_twophase", + "pg_wal", + "pg_xact", + NULL +}; + static void execute_pagemap(datapagemap_t *pagemap, const char *path); /* @@ -38,6 +62,15 @@ recurse_dir(datadir, NULL, callback); } +void +traverse_rewinddirs(const char *datadir, process_file_callback_t callback) +{ + int i; + for(i = 0; rewind_dirs[i] != NULL; i++){ + recurse_dir(datadir, rewind_dirs[i], callback); + } +} + /* * recursive part of traverse_datadir * diff -r 699924885324 -r 8c43292e0012 src/bin/pg_rewind/fetch.c --- a/src/bin/pg_rewind/fetch.c Sun Sep 03 11:12:29 2017 -0400 +++ b/src/bin/pg_rewind/fetch.c Fri Oct 27 18:47:42 2017 +0200 @@ -28,7 +28,7 @@ fetchSourceFileList(void) { if (datadir_source) - traverse_datadir(datadir_source, &process_source_file); + traverse_rewinddirs(datadir_source, &process_source_file); else libpqProcessFileList(); } diff -r 699924885324 -r 8c43292e0012 src/bin/pg_rewind/fetch.h --- a/src/bin/pg_rewind/fetch.h Sun Sep 03 11:12:29 2017 -0400 +++ b/src/bin/pg_rewind/fetch.h Fri Oct 27 18:47:42 2017 +0200 @@ -36,9 +36,11 @@ extern XLogRecPtr libpqGetCurrentXlogInsertLocation(void); /* in copy_fetch.c */ +extern char *rewind_dirs[]; extern void copy_executeFileMap(filemap_t *map); typedef void (*process_file_callback_t) (const char *path, file_type_t type, size_t size, const char *link_target); extern void traverse_datadir(const char *datadir, process_file_callback_t callback); +extern void traverse_rewinddirs(const char *datadir, process_file_callback_t callback); #endif /* FETCH_H */ diff -r 699924885324 -r 8c43292e0012 src/bin/pg_rewind/filemap.c --- a/src/bin/pg_rewind/filemap.c Sun Sep 03 11:12:29 2017 -0400 +++ b/src/bin/pg_rewind/filemap.c Fri Oct 27 18:47:42 2017 +0200 @@ -95,6 +95,7 @@ if (strstr(path, "/" PG_TEMP_FILES_DIR "/") != NULL) return; + /* * sanity check: a filename that looks like a data file better be a * regular file diff -r 699924885324 -r 8c43292e0012 src/bin/pg_rewind/libpq_fetch.c --- a/src/bin/pg_rewind/libpq_fetch.c Sun Sep 03 11:12:29 2017 -0400 +++ b/src/bin/pg_rewind/libpq_fetch.c Fri Oct 27 18:47:42 2017 +0200 @@ -150,7 +150,7 @@ { PGresult *res; const char *sql; - int i; + int i, p; /* * Create a recursive directory listing of the whole data directory. @@ -166,7 +166,8 @@ "WITH RECURSIVE files (path, filename, size, isdir) AS (\n" " SELECT '' AS path, filename, size, isdir FROM\n" " (SELECT pg_ls_dir('.', true, false) AS filename) AS fn,\n" - " pg_stat_file(fn.filename, true) AS this\n" + " LATERAL pg_stat_file(fn.filename, true) AS this\n" + " WHERE filename = $1\n" " UNION ALL\n" " SELECT parent.path || parent.filename || '/' AS path,\n" " fn, this.size, this.isdir\n" @@ -180,44 +181,48 @@ "FROM files\n" "LEFT OUTER JOIN pg_tablespace ON files.path = 'pg_tblspc/'\n" " AND oid::text = files.filename\n"; - res = PQexec(conn, sql); + for (p = 0; rewind_dirs[p] != NULL; ++p) + { + const char *paths[1]; + paths[0] = rewind_dirs[p]; + res = PQexecParams(conn, sql, 1, NULL, paths, NULL, NULL, 0); - if (PQresultStatus(res) != PGRES_TUPLES_OK) - pg_fatal("could not fetch file list: %s", + if (PQresultStatus(res) != PGRES_TUPLES_OK) + pg_fatal("could not fetch file list: %s", PQresultErrorMessage(res)); - /* sanity check the result set */ - if (PQnfields(res) != 4) - pg_fatal("unexpected result set while fetching file list\n"); + /* sanity check the result set */ + if (PQnfields(res) != 4) + pg_fatal("unexpected result set while fetching file list\n"); - /* Read result to local variables */ - for (i = 0; i < PQntuples(res); i++) - { - char *path = PQgetvalue(res, i, 0); - int64 filesize = atol(PQgetvalue(res, i, 1)); - bool isdir = (strcmp(PQgetvalue(res, i, 2), "t") == 0); - char *link_target = PQgetvalue(res, i, 3); - file_type_t type; - - if (PQgetisnull(res, 0, 1)) + /* Read result to local variables */ + for (i = 0; i < PQntuples(res); i++) { - /* - * The file was removed from the server while the query was - * running. Ignore it. - */ - continue; - } + char *path = PQgetvalue(res, i, 0); + int64 filesize = atol(PQgetvalue(res, i, 1)); + bool isdir = (strcmp(PQgetvalue(res, i, 2), "t") == 0); + char *link_target = PQgetvalue(res, i, 3); + file_type_t type; - if (link_target[0]) - type = FILE_TYPE_SYMLINK; - else if (isdir) - type = FILE_TYPE_DIRECTORY; - else - type = FILE_TYPE_REGULAR; + if (PQgetisnull(res, 0, 1)) + { + /* + * The file was removed from the server while the query was + * running. Ignore it. + */ + continue; + } - process_source_file(path, type, filesize, link_target); + if (link_target[0]) + type = FILE_TYPE_SYMLINK; + else if (isdir) + type = FILE_TYPE_DIRECTORY; + else + type = FILE_TYPE_REGULAR; + process_source_file(path, type, filesize, link_target); + } + PQclear(res); } - PQclear(res); } /* diff -r 699924885324 -r 8c43292e0012 src/bin/pg_rewind/pg_rewind.c --- a/src/bin/pg_rewind/pg_rewind.c Sun Sep 03 11:12:29 2017 -0400 +++ b/src/bin/pg_rewind/pg_rewind.c Fri Oct 27 18:47:42 2017 +0200 @@ -286,7 +286,7 @@ pg_log(PG_PROGRESS, "reading source file list\n"); fetchSourceFileList(); pg_log(PG_PROGRESS, "reading target file list\n"); - traverse_datadir(datadir_target, &process_target_file); + traverse_rewinddirs(datadir_target, &process_target_file); /* * Read the target WAL from last checkpoint before the point of fork, to diff -r 699924885324 -r 8c43292e0012 src/bin/pg_rewind/t/001_basic.pl --- a/src/bin/pg_rewind/t/001_basic.pl Sun Sep 03 11:12:29 2017 -0400 +++ b/src/bin/pg_rewind/t/001_basic.pl Fri Oct 27 18:47:42 2017 +0200 @@ -8,7 +8,6 @@ sub run_test { my $test_mode = shift; - RewindTest::setup_cluster(); RewindTest::start_master(); diff -r 699924885324 -r 8c43292e0012 src/bin/pg_rewind/t/003_extrafiles.pl --- a/src/bin/pg_rewind/t/003_extrafiles.pl Sun Sep 03 11:12:29 2017 -0400 +++ b/src/bin/pg_rewind/t/003_extrafiles.pl Fri Oct 27 18:47:42 2017 +0200 @@ -71,12 +71,12 @@ "$test_master_datadir/tst_both_dir/both_file2", "$test_master_datadir/tst_both_dir/both_subdir", "$test_master_datadir/tst_both_dir/both_subdir/both_file3", - "$test_master_datadir/tst_standby_dir", - "$test_master_datadir/tst_standby_dir/standby_file1", - "$test_master_datadir/tst_standby_dir/standby_file2", - "$test_master_datadir/tst_standby_dir/standby_subdir", -"$test_master_datadir/tst_standby_dir/standby_subdir/standby_file3" ], - "file lists match"); + "$test_master_datadir/tst_master_dir", + "$test_master_datadir/tst_master_dir/master_file1", + "$test_master_datadir/tst_master_dir/master_file2", + "$test_master_datadir/tst_master_dir/master_subdir", + "$test_master_datadir/tst_master_dir/master_subdir/master_file3", ], + "file lists match") or (diag("Files found:"), diag(explain(\@paths))); RewindTest::clean_rewind_test(); } diff -r 699924885324 -r 8c43292e0012 src/test/perl/TestLib.pm --- a/src/test/perl/TestLib.pm Sun Sep 03 11:12:29 2017 -0400 +++ b/src/test/perl/TestLib.pm Fri Oct 27 18:47:42 2017 +0200 @@ -10,6 +10,7 @@ use strict; use warnings; +use Carp::Always; use Config; use Exporter 'import'; use File::Basename;