From 24f424fe5088407de70b03850a754010aebdca1e Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Wed, 11 Jul 2012 15:18:50 +0200 Subject: [PATCH 2/2] add simple xlogdump tool --- src/bin/Makefile | 2 +- src/bin/xlogdump/Makefile | 25 ++++ src/bin/xlogdump/xlogdump.c | 334 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 360 insertions(+), 1 deletion(-) create mode 100644 src/bin/xlogdump/Makefile create mode 100644 src/bin/xlogdump/xlogdump.c diff --git a/src/bin/Makefile b/src/bin/Makefile index b4dfdba..9992f7a 100644 --- a/src/bin/Makefile +++ b/src/bin/Makefile @@ -14,7 +14,7 @@ top_builddir = ../.. include $(top_builddir)/src/Makefile.global SUBDIRS = initdb pg_ctl pg_dump \ - psql scripts pg_config pg_controldata pg_resetxlog pg_basebackup + psql scripts pg_config pg_controldata pg_resetxlog pg_basebackup xlogdump ifeq ($(PORTNAME), win32) SUBDIRS += pgevent diff --git a/src/bin/xlogdump/Makefile b/src/bin/xlogdump/Makefile new file mode 100644 index 0000000..d54640a --- /dev/null +++ b/src/bin/xlogdump/Makefile @@ -0,0 +1,25 @@ +#------------------------------------------------------------------------- +# +# Makefile for src/bin/xlogdump +# +# Copyright (c) 1998-2012, PostgreSQL Global Development Group +# +# src/bin/pg_resetxlog/Makefile +# +#------------------------------------------------------------------------- + +PGFILEDESC = "xlogdump" +PGAPPICON=win32 + +subdir = src/bin/xlogdump +top_builddir = ../../.. +include $(top_builddir)/src/Makefile.global + +OBJS= xlogdump.o \ + $(WIN32RES) + +all: xlogdump + + +xlogdump: $(OBJS) $(shell find ../../backend ../../timezone -name objfiles.txt|xargs cat|tr -s " " "\012"|grep -v /main.o|sed 's/^/..\/..\/..\//') + $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) diff --git a/src/bin/xlogdump/xlogdump.c b/src/bin/xlogdump/xlogdump.c new file mode 100644 index 0000000..8e13193 --- /dev/null +++ b/src/bin/xlogdump/xlogdump.c @@ -0,0 +1,334 @@ +#include "postgres.h" + +#include + +#include "access/xlogreader.h" +#include "access/rmgr.h" +#include "miscadmin.h" +#include "storage/ipc.h" +#include "utils/memutils.h" +#include "utils/guc.h" + +/* + * needs to be declared because otherwise its defined in main.c which we cannot + * link from here. + */ +const char *progname = "xlogdump"; + +static void +XLogDumpXLogRead(char *buf, TimeLineID timeline_id, XLogRecPtr startptr, Size count); + +static void +XLogDumpXLogWrite(const char *directory, TimeLineID timeline_id, XLogRecPtr startptr, + char *buf, Size count); + +#define XLogFilePathWrite(path, base, tli, logSegNo) \ + snprintf(path, MAXPGPATH, "%s/%08X%08X%08X", base, tli, \ + (uint32) ((logSegNo) / XLogSegmentsPerXLogId), \ + (uint32) ((logSegNo) % XLogSegmentsPerXLogId)) + +static void +XLogDumpXLogWrite(const char *directory, TimeLineID timeline_id, XLogRecPtr startptr, + char *buf, Size count) +{ + char *p; + XLogRecPtr recptr; + Size nbytes; + + static int sendFile = -1; + static XLogSegNo sendSegNo = 0; + static uint32 sendOff = 0; + + p = buf; + recptr = startptr; + nbytes = count; + + while (nbytes > 0) + { + uint32 startoff; + int segbytes; + int writebytes; + + startoff = recptr % XLogSegSize; + + if (sendFile < 0 || !XLByteInSeg(recptr, sendSegNo)) + { + char path[MAXPGPATH]; + + /* Switch to another logfile segment */ + if (sendFile >= 0) + close(sendFile); + + XLByteToSeg(recptr, sendSegNo); + XLogFilePathWrite(path, directory, timeline_id, sendSegNo); + + sendFile = open(path, O_WRONLY|O_CREAT, S_IRUSR | S_IWUSR); + if (sendFile < 0) + { + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", + path))); + } + sendOff = 0; + } + + /* Need to seek in the file? */ + if (sendOff != startoff) + { + if (lseek(sendFile, (off_t) startoff, SEEK_SET) < 0){ + char fname[MAXPGPATH]; + XLogFileName(fname, timeline_id, sendSegNo); + + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not seek in log segment %s to offset %u: %m", + fname, + startoff))); + } + sendOff = startoff; + } + + /* How many bytes are within this segment? */ + if (nbytes > (XLogSegSize - startoff)) + segbytes = XLogSegSize - startoff; + else + segbytes = nbytes; + + writebytes = write(sendFile, p, segbytes); + if (writebytes <= 0) + { + char fname[MAXPGPATH]; + XLogFileName(fname, timeline_id, sendSegNo); + + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write to log segment %s, offset %u, length %lu: %m", + fname, + sendOff, (unsigned long) segbytes))); + } + + /* Update state for read */ + XLByteAdvance(recptr, writebytes); + + sendOff += writebytes; + nbytes -= writebytes; + p += writebytes; + } +} + +/* this should probably be put in a general implementation */ +static void +XLogDumpXLogRead(char *buf, TimeLineID timeline_id, XLogRecPtr startptr, Size count) +{ + char *p; + XLogRecPtr recptr; + Size nbytes; + + static int sendFile = -1; + static XLogSegNo sendSegNo = 0; + static uint32 sendOff = 0; + + p = buf; + recptr = startptr; + nbytes = count; + + while (nbytes > 0) + { + uint32 startoff; + int segbytes; + int readbytes; + + startoff = recptr % XLogSegSize; + + if (sendFile < 0 || !XLByteInSeg(recptr, sendSegNo)) + { + char path[MAXPGPATH]; + + /* Switch to another logfile segment */ + if (sendFile >= 0) + close(sendFile); + + XLByteToSeg(recptr, sendSegNo); + XLogFilePath(path, timeline_id, sendSegNo); + + sendFile = open(path, O_RDONLY, 0); + if (sendFile < 0) + { + char fname[MAXPGPATH]; + XLogFileName(fname, timeline_id, sendSegNo); + /* + * If the file is not found, assume it's because the standby + * asked for a too old WAL segment that has already been + * removed or recycled. + */ + if (errno == ENOENT) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("requested WAL segment %s has already been removed", + fname))); + else + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", + path))); + } + sendOff = 0; + } + + /* Need to seek in the file? */ + if (sendOff != startoff) + { + if (lseek(sendFile, (off_t) startoff, SEEK_SET) < 0){ + char fname[MAXPGPATH]; + XLogFileName(fname, timeline_id, sendSegNo); + + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not seek in log segment %s to offset %u: %m", + fname, + startoff))); + } + sendOff = startoff; + } + + /* How many bytes are within this segment? */ + if (nbytes > (XLogSegSize - startoff)) + segbytes = XLogSegSize - startoff; + else + segbytes = nbytes; + + readbytes = read(sendFile, p, segbytes); + if (readbytes <= 0) + { + char fname[MAXPGPATH]; + XLogFileName(fname, timeline_id, sendSegNo); + + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read from log segment %s, offset %u, length %lu: %m", + fname, + sendOff, (unsigned long) segbytes))); + } + + /* Update state for read */ + XLByteAdvance(recptr, readbytes); + + sendOff += readbytes; + nbytes -= readbytes; + p += readbytes; + } +} + +static void +XLogDumpReadPage(XLogReaderState* state, char* cur_page, XLogRecPtr startptr) +{ + XLogPageHeader page_header; + Assert((startptr % XLOG_BLCKSZ) == 0); + + /* FIXME: more sensible/efficient implementation */ + XLogDumpXLogRead(cur_page, 1, startptr, XLOG_BLCKSZ); + + page_header = (XLogPageHeader)cur_page; + + if (page_header->xlp_magic != XLOG_PAGE_MAGIC) + { + elog(FATAL, "page header magic %x, should be %x at %X/%X", page_header->xlp_magic, + XLOG_PAGE_MAGIC, (uint32)(startptr << 32), (uint32)startptr); + } +} + +static void +XLogDumpWrite(XLogReaderState* state, char* data, Size len) +{ + static char zero[XLOG_BLCKSZ]; + if(data == NULL) + data = zero; + + XLogDumpXLogWrite("/tmp/xlog", 1 /* FIXME */, state->curptr, + data, len); +} + +static void +XLogDumpFinishedRecord(XLogReaderState* state, XLogRecordBuffer* buf) +{ + XLogRecord *record = &buf->record; + const RmgrData *rmgr = &RmgrTable[record->xl_rmid]; + + StringInfo str = makeStringInfo(); + initStringInfo(str); + + rmgr->rm_desc(str, state->buf.record.xl_info, buf->record_data); + + fprintf(stderr, "xlog record: rmgr: %-11s, record_len: %6u, tot_len: %6u, tx: %10u, lsn: %X/%-8X, prev %X/%-8X, bkp: %u%u%u%u, desc: %s\n", + rmgr->rm_name, + record->xl_len, record->xl_tot_len, + record->xl_xid, + (uint32)(buf->origptr >> 32), (uint32)buf->origptr, + (uint32)(record->xl_prev >> 32), (uint32)record->xl_prev, + !!(XLR_BKP_BLOCK_1 & buf->record.xl_info), + !!(XLR_BKP_BLOCK_2 & buf->record.xl_info), + !!(XLR_BKP_BLOCK_3 & buf->record.xl_info), + !!(XLR_BKP_BLOCK_4 & buf->record.xl_info), + str->data); + +} + + +static void init() +{ + MemoryContextInit(); + IsPostmasterEnvironment = false; + log_min_messages = DEBUG1; + Log_error_verbosity = PGERROR_TERSE; + pg_timezone_initialize(); +} + +int main(int argc, char **argv) +{ + uint32 xlogid; + uint32 xrecoff; + XLogReaderState *xlogreader_state; + XLogRecPtr from, to; + + init(); + + /* FIXME: should use getopt */ + if (argc < 4) + elog(ERROR, "xlogdump timeline_id start finish"); + + if (sscanf(argv[2], "%X/%X", &xlogid, &xrecoff) != 2) + elog(ERROR, "couldn't parse argv[2]"); + + from = (((uint64)xlogid) << 32) | xrecoff; + + if (sscanf(argv[3], "%X/%X", &xlogid, &xrecoff) != 2) + elog(ERROR, "couldn't parse argv[2]"); + + to = (uint64)xlogid << 32 | xrecoff; + + xlogreader_state = XLogReaderAllocate(); + + /* + * not set because we want all records, perhaps we want filtering later? + * xlogreader_state->is_record_interesting = + */ + xlogreader_state->finished_record = XLogDumpFinishedRecord; + + /* + * not set because we do not want to copy data to somewhere yet + * xlogreader_state->writeout_data = ; + */ + xlogreader_state->writeout_data = XLogDumpWrite; + + xlogreader_state->read_page = XLogDumpReadPage; + + xlogreader_state->private_data = NULL; + + xlogreader_state->startptr = from; + xlogreader_state->endptr = to; + + XLogReaderRead(xlogreader_state); + XLogReaderFree(xlogreader_state); + return 0; +} -- 1.7.10.rc3.3.g19a6c.dirty