diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 9d84f8b..25b3d96 100644
*** a/doc/src/sgml/ref/pg_dump.sgml
--- b/doc/src/sgml/ref/pg_dump.sgml
*************** PostgreSQL documentation
*** 272,283 ****
Output a tar-format archive suitable for input
! into pg_restore. The tar-format is
! compatible with the directory-format; extracting a tar-format
archive produces a valid directory-format archive.
! However, the tar-format does not support compression and has a
! limit of 8 GB on the size of individual tables. Also, the relative
! order of table data items cannot be changed during restore.
--- 272,283 ----
Output a tar-format archive suitable for input
! into pg_restore. The tar format is
! compatible with the directory format: extracting a tar-format
archive produces a valid directory-format archive.
! However, the tar format does not support compression. Also, when
! using tar format the relative order of table data items cannot be
! changed during restore.
*************** CREATE DATABASE foo WITH TEMPLATE templa
*** 1141,1155 ****
- Members of tar archives are limited to a size less than 8 GB.
- (This is an inherent limitation of the tar file format.) Therefore
- this format cannot be used if the textual representation of any one table
- exceeds that size. The total size of a tar archive and any of the
- other output formats is not limited, except possibly by the
- operating system.
-
-
-
The dump file produced by pg_dump
does not contain the statistics used by the optimizer to make
query planning decisions. Therefore, it is wise to run
--- 1141,1146 ----
diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c
index 1af011e..6120c8f 100644
*** a/src/backend/replication/basebackup.c
--- b/src/backend/replication/basebackup.c
*************** SendBackupHeader(List *tablespaces)
*** 698,704 ****
}
else
{
! Size len;
len = strlen(ti->oid);
pq_sendint(&buf, len, 4);
--- 698,704 ----
}
else
{
! Size len;
len = strlen(ti->oid);
pq_sendint(&buf, len, 4);
*************** sendDir(char *path, int basepathlen, boo
*** 1132,1144 ****
/*
- * Maximum file size for a tar member: The limit inherent in the
- * format is 2^33-1 bytes (nearly 8 GB). But we don't want to exceed
- * what we can represent in pgoff_t.
- */
- #define MAX_TAR_MEMBER_FILELEN (((int64) 1 << Min(33, sizeof(pgoff_t)*8 - 1)) - 1)
-
- /*
* Given the member, write the TAR header & send the file.
*
* If 'missing_ok' is true, will not throw an error if the file is not found.
--- 1132,1137 ----
*************** sendFile(char *readfilename, char *tarfi
*** 1166,1180 ****
errmsg("could not open file \"%s\": %m", readfilename)));
}
- /*
- * Some compilers will throw a warning knowing this test can never be true
- * because pgoff_t can't exceed the compared maximum on their platform.
- */
- if (statbuf->st_size > MAX_TAR_MEMBER_FILELEN)
- ereport(ERROR,
- (errmsg("archive member \"%s\" too large for tar format",
- tarfilename)));
-
_tarWriteHeader(tarfilename, NULL, statbuf);
while ((cnt = fread(buf, 1, Min(sizeof(buf), statbuf->st_size - len), fp)) > 0)
--- 1159,1164 ----
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index 80de882..8c4dffe 100644
*** a/src/bin/pg_basebackup/pg_basebackup.c
--- b/src/bin/pg_basebackup/pg_basebackup.c
*************** ReceiveTarFile(PGconn *conn, PGresult *r
*** 781,787 ****
bool in_tarhdr = true;
bool skip_file = false;
size_t tarhdrsz = 0;
! size_t filesz = 0;
#ifdef HAVE_LIBZ
gzFile ztarfile = NULL;
--- 781,787 ----
bool in_tarhdr = true;
bool skip_file = false;
size_t tarhdrsz = 0;
! pgoff_t filesz = 0;
#ifdef HAVE_LIBZ
gzFile ztarfile = NULL;
*************** ReceiveTarFile(PGconn *conn, PGresult *r
*** 1046,1052 ****
skip_file = (strcmp(&tarhdr[0], "recovery.conf") == 0);
! sscanf(&tarhdr[124], "%11o", (unsigned int *) &filesz);
padding = ((filesz + 511) & ~511) - filesz;
filesz += padding;
--- 1046,1052 ----
skip_file = (strcmp(&tarhdr[0], "recovery.conf") == 0);
! filesz = read_tar_number(&tarhdr[124], 12);
padding = ((filesz + 511) & ~511) - filesz;
filesz += padding;
*************** ReceiveAndUnpackTarFile(PGconn *conn, PG
*** 1139,1145 ****
char current_path[MAXPGPATH];
char filename[MAXPGPATH];
const char *mapped_tblspc_path;
! int current_len_left;
int current_padding = 0;
bool basetablespace;
char *copybuf = NULL;
--- 1139,1145 ----
char current_path[MAXPGPATH];
char filename[MAXPGPATH];
const char *mapped_tblspc_path;
! pgoff_t current_len_left = 0;
int current_padding = 0;
bool basetablespace;
char *copybuf = NULL;
*************** ReceiveAndUnpackTarFile(PGconn *conn, PG
*** 1208,1227 ****
}
totaldone += 512;
! if (sscanf(copybuf + 124, "%11o", ¤t_len_left) != 1)
! {
! fprintf(stderr, _("%s: could not parse file size\n"),
! progname);
! disconnect_and_exit(1);
! }
/* Set permissions on the file */
! if (sscanf(©buf[100], "%07o ", &filemode) != 1)
! {
! fprintf(stderr, _("%s: could not parse file mode\n"),
! progname);
! disconnect_and_exit(1);
! }
/*
* All files are padded up to 512 bytes
--- 1208,1217 ----
}
totaldone += 512;
! current_len_left = read_tar_number(©buf[124], 12);
/* Set permissions on the file */
! filemode = read_tar_number(©buf[100], 8);
/*
* All files are padded up to 512 bytes
*************** main(int argc, char **argv)
*** 2180,2186 ****
if (replication_slot && !streamwal)
{
fprintf(stderr,
! _("%s: replication slots can only be used with WAL streaming\n"),
progname);
fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
progname);
--- 2170,2176 ----
if (replication_slot && !streamwal)
{
fprintf(stderr,
! _("%s: replication slots can only be used with WAL streaming\n"),
progname);
fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
progname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 532eacc..c40dfe5 100644
*** a/src/bin/pg_dump/pg_backup_tar.c
--- b/src/bin/pg_dump/pg_backup_tar.c
*************** typedef struct
*** 78,90 ****
ArchiveHandle *AH;
} TAR_MEMBER;
- /*
- * Maximum file size for a tar member: The limit inherent in the
- * format is 2^33-1 bytes (nearly 8 GB). But we don't want to exceed
- * what we can represent in pgoff_t.
- */
- #define MAX_TAR_MEMBER_FILELEN (((int64) 1 << Min(33, sizeof(pgoff_t)*8 - 1)) - 1)
-
typedef struct
{
int hasSeek;
--- 78,83 ----
*************** isValidTarHeader(char *header)
*** 1049,1055 ****
int sum;
int chk = tarChecksum(header);
! sscanf(&header[148], "%8o", &sum);
if (sum != chk)
return false;
--- 1042,1048 ----
int sum;
int chk = tarChecksum(header);
! sum = read_tar_number(&header[148], 8);
if (sum != chk)
return false;
*************** _tarAddFile(ArchiveHandle *AH, TAR_MEMBE
*** 1091,1103 ****
strerror(errno));
fseeko(tmp, 0, SEEK_SET);
- /*
- * Some compilers will throw a warning knowing this test can never be true
- * because pgoff_t can't exceed the compared maximum on their platform.
- */
- if (th->fileLen > MAX_TAR_MEMBER_FILELEN)
- exit_horribly(modulename, "archive member too large for tar format\n");
-
_tarWriteHeader(th);
while ((cnt = fread(buf, 1, sizeof(buf), tmp)) > 0)
--- 1084,1089 ----
*************** _tarGetHeader(ArchiveHandle *AH, TAR_MEM
*** 1222,1232 ****
{
lclContext *ctx = (lclContext *) AH->formatData;
char h[512];
! char tag[100];
int sum,
chk;
! size_t len;
! unsigned long ullen;
pgoff_t hPos;
bool gotBlock = false;
--- 1208,1217 ----
{
lclContext *ctx = (lclContext *) AH->formatData;
char h[512];
! char tag[100 + 1];
int sum,
chk;
! pgoff_t len;
pgoff_t hPos;
bool gotBlock = false;
*************** _tarGetHeader(ArchiveHandle *AH, TAR_MEM
*** 1249,1255 ****
/* Calc checksum */
chk = tarChecksum(h);
! sscanf(&h[148], "%8o", &sum);
/*
* If the checksum failed, see if it is a null block. If so, silently
--- 1234,1240 ----
/* Calc checksum */
chk = tarChecksum(h);
! sum = read_tar_number(&h[148], 8);
/*
* If the checksum failed, see if it is a null block. If so, silently
*************** _tarGetHeader(ArchiveHandle *AH, TAR_MEM
*** 1272,1298 ****
}
}
! sscanf(&h[0], "%99s", tag);
! sscanf(&h[124], "%12lo", &ullen);
! len = (size_t) ullen;
{
! char buf[100];
! snprintf(buf, sizeof(buf), INT64_FORMAT, (int64) hPos);
! ahlog(AH, 3, "TOC Entry %s at %s (length %lu, checksum %d)\n",
! tag, buf, (unsigned long) len, sum);
}
if (chk != sum)
{
! char buf[100];
! snprintf(buf, sizeof(buf), INT64_FORMAT, (int64) ftello(ctx->tarFH));
exit_horribly(modulename,
"corrupt tar header found in %s "
"(expected %d, computed %d) file position %s\n",
! tag, sum, chk, buf);
}
th->targetFile = pg_strdup(tag);
--- 1257,1287 ----
}
}
! /* Name field is 100 bytes, might not be null-terminated */
! strlcpy(tag, &h[0], 100 + 1);
!
! len = read_tar_number(&h[124], 12);
{
! char posbuf[32];
! char lenbuf[32];
! snprintf(posbuf, sizeof(posbuf), UINT64_FORMAT, (uint64) hPos);
! snprintf(lenbuf, sizeof(lenbuf), UINT64_FORMAT, (uint64) len);
! ahlog(AH, 3, "TOC Entry %s at %s (length %s, checksum %d)\n",
! tag, posbuf, lenbuf, sum);
}
if (chk != sum)
{
! char posbuf[32];
! snprintf(posbuf, sizeof(posbuf), UINT64_FORMAT,
! (uint64) ftello(ctx->tarFH));
exit_horribly(modulename,
"corrupt tar header found in %s "
"(expected %d, computed %d) file position %s\n",
! tag, sum, chk, posbuf);
}
th->targetFile = pg_strdup(tag);
*************** _tarWriteHeader(TAR_MEMBER *th)
*** 1307,1313 ****
{
char h[512];
! tarCreateHeader(h, th->targetFile, NULL, th->fileLen, 0600, 04000, 02000, time(NULL));
/* Now write the completed header. */
if (fwrite(h, 1, 512, th->tarFH) != 512)
--- 1296,1303 ----
{
char h[512];
! tarCreateHeader(h, th->targetFile, NULL, th->fileLen,
! 0600, 04000, 02000, time(NULL));
/* Now write the completed header. */
if (fwrite(h, 1, 512, th->tarFH) != 512)
diff --git a/src/include/pgtar.h b/src/include/pgtar.h
index 906db7c..9c94a58 100644
*** a/src/include/pgtar.h
--- b/src/include/pgtar.h
*************** enum tarError
*** 19,23 ****
TAR_SYMLINK_TOO_LONG
};
! extern enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, size_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime);
extern int tarChecksum(char *header);
--- 19,25 ----
TAR_SYMLINK_TOO_LONG
};
! extern enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget,
! pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime);
! extern uint64 read_tar_number(const char *s, int len);
extern int tarChecksum(char *header);
diff --git a/src/port/tar.c b/src/port/tar.c
index 72fd4e1..52a2113 100644
*** a/src/port/tar.c
--- b/src/port/tar.c
***************
*** 3,23 ****
#include
/*
! * Utility routine to print possibly larger than 32 bit integers in a
! * portable fashion. Filled with zeros.
*/
static void
! print_val(char *s, uint64 val, unsigned int base, size_t len)
{
! int i;
!
! for (i = len; i > 0; i--)
{
! int digit = val % base;
! s[i - 1] = '0' + digit;
! val = val / base;
}
}
--- 3,82 ----
#include
/*
! * Print a numeric field in a tar header. The field starts at *s and is of
! * length len; val is the value to be written.
! *
! * Per POSIX, the way to write a number is in octal with leading zeroes and
! * one trailing space (or NUL, but we use space) at the end of the specified
! * field width.
! *
! * However, the given value may not fit in the available space in octal form.
! * If that's true, we use the GNU extension of writing \200 followed by the
! * number in base-256 form (ie, stored in binary MSB-first). (Note: here we
! * support only non-negative numbers, so we don't worry about the GNU rules
! * for handling negative numbers.)
*/
static void
! print_tar_number(char *s, int len, uint64 val)
{
! if (val < (((uint64) 1) << ((len - 1) * 3)))
{
! /* Use octal with trailing space */
! s[--len] = ' ';
! while (len)
! {
! s[--len] = (val & 7) + '0';
! val >>= 3;
! }
! }
! else
! {
! /* Use base-256 with leading \200 */
! s[0] = '\200';
! while (len > 1)
! {
! s[--len] = (val & 255);
! val >>= 8;
! }
! }
! }
!
! /*
! * Read a numeric field in a tar header. The field starts at *s and is of
! * length len.
! *
! * The POSIX-approved format for a number is octal, ending with a space or
! * NUL. However, for values that don't fit, we recognize the GNU extension
! * of \200 followed by the number in base-256 form (ie, stored in binary
! * MSB-first). (Note: here we support only non-negative numbers, so we don't
! * worry about the GNU rules for handling negative numbers.)
! */
! uint64
! read_tar_number(const char *s, int len)
! {
! uint64 result = 0;
!
! if (*s == '\200')
! {
! /* base-256 */
! while (--len)
! {
! result <<= 8;
! result |= (unsigned char) (*++s);
! }
! }
! else
! {
! /* octal */
! while (len-- && *s >= '0' && *s <= '7')
! {
! result <<= 3;
! result |= (*s - '0');
! s++;
! }
}
+ return result;
}
*************** tarChecksum(char *header)
*** 46,57 ****
/*
* Fill in the buffer pointed to by h with a tar format header. This buffer
! * must always have space for 512 characters, which is a requirement by
* the tar format.
*/
enum tarError
tarCreateHeader(char *h, const char *filename, const char *linktarget,
! size_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
{
if (strlen(filename) > 99)
return TAR_NAME_TOO_LONG;
--- 105,116 ----
/*
* Fill in the buffer pointed to by h with a tar format header. This buffer
! * must always have space for 512 characters, which is a requirement of
* the tar format.
*/
enum tarError
tarCreateHeader(char *h, const char *filename, const char *linktarget,
! pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
{
if (strlen(filename) > 99)
return TAR_NAME_TOO_LONG;
*************** tarCreateHeader(char *h, const char *fil
*** 59,70 ****
if (linktarget && strlen(linktarget) > 99)
return TAR_SYMLINK_TOO_LONG;
- /*
- * Note: most of the fields in a tar header are not supposed to be
- * null-terminated. We use sprintf, which will write a null after the
- * required bytes; that null goes into the first byte of the next field.
- * This is okay as long as we fill the fields in order.
- */
memset(h, 0, 512); /* assume tar header size */
/* Name 100 */
--- 118,123 ----
*************** tarCreateHeader(char *h, const char *fil
*** 84,129 ****
}
/* Mode 8 - this doesn't include the file type bits (S_IFMT) */
! sprintf(&h[100], "%07o ", (int) (mode & 07777));
/* User ID 8 */
! sprintf(&h[108], "%07o ", (int) uid);
/* Group 8 */
! sprintf(&h[116], "%07o ", (int) gid);
! /* File size 12 - 11 digits, 1 space; use print_val for 64 bit support */
if (linktarget != NULL || S_ISDIR(mode))
/* Symbolic link or directory has size zero */
! print_val(&h[124], 0, 8, 11);
else
! print_val(&h[124], size, 8, 11);
! sprintf(&h[135], " ");
/* Mod Time 12 */
! sprintf(&h[136], "%011o ", (int) mtime);
/* Checksum 8 cannot be calculated until we've filled all other fields */
if (linktarget != NULL)
{
/* Type - Symbolic link */
! sprintf(&h[156], "2");
/* Link Name 100 */
strlcpy(&h[157], linktarget, 100);
}
else if (S_ISDIR(mode))
/* Type - directory */
! sprintf(&h[156], "5");
else
/* Type - regular file */
! sprintf(&h[156], "0");
/* Magic 6 */
! sprintf(&h[257], "ustar");
/* Version 2 */
! sprintf(&h[263], "00");
/* User 32 */
/* XXX: Do we need to care about setting correct username? */
--- 137,185 ----
}
/* Mode 8 - this doesn't include the file type bits (S_IFMT) */
! print_tar_number(&h[100], 8, (mode & 07777));
/* User ID 8 */
! print_tar_number(&h[108], 8, uid);
/* Group 8 */
! print_tar_number(&h[116], 8, gid);
! /* File size 12 */
if (linktarget != NULL || S_ISDIR(mode))
/* Symbolic link or directory has size zero */
! print_tar_number(&h[124], 12, 0);
else
! print_tar_number(&h[124], 12, size);
/* Mod Time 12 */
! print_tar_number(&h[136], 12, mtime);
/* Checksum 8 cannot be calculated until we've filled all other fields */
if (linktarget != NULL)
{
/* Type - Symbolic link */
! h[156] = '2';
/* Link Name 100 */
strlcpy(&h[157], linktarget, 100);
}
else if (S_ISDIR(mode))
+ {
/* Type - directory */
! h[156] = '5';
! }
else
+ {
/* Type - regular file */
! h[156] = '0';
! }
/* Magic 6 */
! strcpy(&h[257], "ustar");
/* Version 2 */
! memcpy(&h[263], "00", 2);
/* User 32 */
/* XXX: Do we need to care about setting correct username? */
*************** tarCreateHeader(char *h, const char *fil
*** 134,152 ****
strlcpy(&h[297], "postgres", 32);
/* Major Dev 8 */
! sprintf(&h[329], "%07o ", 0);
/* Minor Dev 8 */
! sprintf(&h[337], "%07o ", 0);
/* Prefix 155 - not used, leave as nulls */
! /*
! * We mustn't overwrite the next field while inserting the checksum.
! * Fortunately, the checksum can't exceed 6 octal digits, so we just write
! * 6 digits, a space, and a null, which is legal per POSIX.
! */
! sprintf(&h[148], "%06o ", tarChecksum(h));
return TAR_OK;
}
--- 190,204 ----
strlcpy(&h[297], "postgres", 32);
/* Major Dev 8 */
! print_tar_number(&h[329], 8, 0);
/* Minor Dev 8 */
! print_tar_number(&h[337], 8, 0);
/* Prefix 155 - not used, leave as nulls */
! /* Finally, compute and insert the checksum */
! print_tar_number(&h[148], 8, tarChecksum(h));
return TAR_OK;
}