opensourcemirror
/
postgresql
mirror of git://git.postgresql.org/git/postgresql.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Revert "Allow on-line enabling and disabling of data checksums" 

This reverts the backend sides of commit 1fde38beaa.
I have, at least for now, left the pg_verify_checksums tool in place, as
this tool can be very valuable without the rest of the patch as well,
and since it's a read-only tool that only runs when the cluster is down
it should be a lot safer.
This commit is contained in:
Magnus Hagander 2018-04-09 19:02:42 +02:00
parent 03c11796a9
commit a228cc13ae
39 changed files with 34 additions and 1658 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
doc/src/sgml/func.sgml
View File

@ -19566,71 +19566,6 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup());
</sect2>
<sect2 id="functions-admin-checksum">
<title>Data Checksum Functions</title>
<para>
The functions shown in <xref linkend="functions-checksums-table" /> can
be used to enable or disable data checksums in a running cluster.
See <xref linkend="checksums" /> for details.
</para>
<table id="functions-checksums-table">
<title>Checksum <acronym>SQL</acronym> Functions</title>
<tgroup cols="3">
<thead>
<row>
<entry>Function</entry>
<entry>Return Type</entry>
<entry>Description</entry>
</row>
</thead>
<tbody>
<row>
<entry>
<indexterm>
<primary>pg_enable_data_checksums</primary>
</indexterm>
<literal><function>pg_enable_data_checksums(<optional><parameter>cost_delay</parameter> <type>int</type>, <parameter>cost_limit</parameter> <type>int</type></optional>)</function></literal>
</entry>
<entry>
void
</entry>
<entry>
<para>
Initiates data checksums for the cluster. This will switch the data checksums mode
to <literal>in progress</literal> and start a background worker that will process
all data in the database and enable checksums for it. When all data pages have had
checksums enabled, the cluster will automatically switch to checksums
<literal>on</literal>.
</para>
<para>
If <parameter>cost_delay</parameter> and <parameter>cost_limit</parameter> are
specified, the speed of the process is throttled using the same principles as
<link linkend="runtime-config-resource-vacuum-cost">Cost-based Vacuum Delay</link>.
</para>
</entry>
</row>
<row>
<entry>
<indexterm>
<primary>pg_disable_data_checksums</primary>
</indexterm>
<literal><function>pg_disable_data_checksums()</function></literal>
</entry>
<entry>
void
</entry>
<entry>
Disables data checksums for the cluster.
</entry>
</row>
</tbody>
</tgroup>
</table>
</sect2>
<sect2 id="functions-admin-dbobject">
<title>Database Object Management Functions</title>

6
doc/src/sgml/ref/initdb.sgml
View File

@ -214,9 +214,9 @@ PostgreSQL documentation
<para>
Use checksums on data pages to help detect corruption by the
I/O system that would otherwise be silent. Enabling checksums
may incur a noticeable performance penalty. If set, checksums
are calculated for all objects, in all databases. See
<xref linkend="checksums" /> for details.
may incur a noticeable performance penalty. This option can only
be set during initialization, and cannot be changed later. If
set, checksums are calculated for all objects, in all databases.
</para>
</listitem>
</varlistentry>

1
doc/src/sgml/reference.sgml
View File

@ -284,7 +284,6 @@
&pgtestfsync;
&pgtesttiming;
&pgupgrade;
&pgVerifyChecksums;
&pgwaldump;
&postgres;
&postmaster;

81
doc/src/sgml/wal.sgml
View File

@ -230,87 +230,6 @@
</para>
</sect1>
<sect1 id="checksums">
<title>Data checksums</title>
<indexterm>
<primary>checksums</primary>
</indexterm>
<para>
Data pages are not checksum protected by default, but this can optionally be enabled for a cluster.
When enabled, each data page will be assigned a checksum that is updated when the page is
written and verified every time the page is read. Only data pages are protected by checksums,
internal data structures and temporary files are not.
</para>
<para>
Checksums are normally enabled when the cluster is initialized using
<link linkend="app-initdb-data-checksums"><application>initdb</application></link>. They
can also be enabled or disabled at runtime. In all cases, checksums are enabled or disabled
at the full cluster level, and cannot be specified individually for databases or tables.
</para>
<para>
The current state of checksums in the cluster can be verified by viewing the value
of the read-only configuration variable <xref linkend="guc-data-checksums" /> by
issuing the command <command>SHOW data_checksums</command>.
</para>
<para>
When attempting to recover from corrupt data it may be necessary to bypass the checksum
protection in order to recover data. To do this, temporarily set the configuration parameter
<xref linkend="guc-ignore-checksum-failure" />.
</para>
<sect2 id="checksums-enable-disable">
<title>On-line enabling of checksums</title>
<para>
Checksums can be enabled or disabled online, by calling the appropriate
<link linkend="functions-admin-checksum">functions</link>.
Disabling of checksums takes effect immediately when the function is called.
</para>
<para>
Enabling checksums will put the cluster in <literal>inprogress</literal> mode.
During this time, checksums will be written but not verified. In addition to
this, a background worker process is started that enables checksums on all
existing data in the cluster. Once this worker has completed processing all
databases in the cluster, the checksum mode will automatically switch to
<literal>on</literal>.
</para>
<para>
The process will initially wait for all open transactions to finish before
it starts, so that it can be certain that there are no tables that have been
created inside a transaction that has not committed yet and thus would not
be visible to the process enabling checksums. It will also, for each database,
wait for all pre-existing temporary tables to get removed before it finishes.
If long-lived temporary tables are used in the application it may be necessary
to terminate these application connections to allow the process to complete.
Information about open transactions and connections with temporary tables is
written to log.
</para>
<para>
If the cluster is stopped while in <literal>inprogress</literal> mode, for
any reason, then this process must be restarted manually. To do this,
re-execute the function <function>pg_enable_data_checksums()</function>
once the cluster has been restarted. It is not possible to resume the work,
the process has to start over and re-process the cluster.
</para>
<note>
<para>
Enabling checksums can cause significant I/O to the system, as most of the
database pages will need to be rewritten, and will be written both to the
data files and the WAL.
</para>
</note>
</sect2>
</sect1>
<sect1 id="wal-intro">
<title>Write-Ahead Logging (<acronym>WAL</acronym>)</title>

16
src/backend/access/rmgrdesc/xlogdesc.c
View File

@ -17,7 +17,6 @@
#include "access/xlog.h"
#include "access/xlog_internal.h"
#include "catalog/pg_control.h"
#include "storage/bufpage.h"
#include "utils/guc.h"
#include "utils/timestamp.h"
@ -138,18 +137,6 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
xlrec.ThisTimeLineID, xlrec.PrevTimeLineID,
timestamptz_to_str(xlrec.end_time));
}
else if (info == XLOG_CHECKSUMS)
{
xl_checksum_state xlrec;
memcpy(&xlrec, rec, sizeof(xl_checksum_state));
if (xlrec.new_checksumtype == PG_DATA_CHECKSUM_VERSION)
appendStringInfo(buf, "on");
else if (xlrec.new_checksumtype == PG_DATA_CHECKSUM_INPROGRESS_VERSION)
appendStringInfo(buf, "inprogress");
else
appendStringInfo(buf, "off");
}
}
const char *
@ -195,9 +182,6 @@ xlog_identify(uint8 info)
case XLOG_FPI_FOR_HINT:
id = "FPI_FOR_HINT";
break;
case XLOG_CHECKSUMS:
id = "CHECKSUMS";
break;
}
return id;

124
src/backend/access/transam/xlog.c
View File

@ -856,7 +856,6 @@ static void SetLatestXTime(TimestampTz xtime);
static void SetCurrentChunkStartTime(TimestampTz xtime);
static void CheckRequiredParameterValues(void);
static void XLogReportParameters(void);
static void XlogChecksums(ChecksumType new_type);
static void checkTimeLineSwitch(XLogRecPtr lsn, TimeLineID newTLI,
TimeLineID prevTLI);
static void LocalSetXLogInsertAllowed(void);
@ -1034,7 +1033,7 @@ XLogInsertRecord(XLogRecData *rdata,
Assert(RedoRecPtr < Insert->RedoRecPtr);
RedoRecPtr = Insert->RedoRecPtr;
}
doPageWrites = (Insert->fullPageWrites || Insert->forcePageWrites || DataChecksumsInProgress());
doPageWrites = (Insert->fullPageWrites || Insert->forcePageWrites);
if (fpw_lsn != InvalidXLogRecPtr && fpw_lsn <= RedoRecPtr && doPageWrites)
{
@ -4674,6 +4673,10 @@ ReadControlFile(void)
(SizeOfXLogLongPHD - SizeOfXLogShortPHD);
CalculateCheckpointSegments();
/* Make the initdb settings visible as GUC variables, too */
SetConfigOption("data_checksums", DataChecksumsEnabled() ? "yes" : "no",
PGC_INTERNAL, PGC_S_OVERRIDE);
}
void
@ -4745,90 +4748,12 @@ GetMockAuthenticationNonce(void)
* Are checksums enabled for data pages?
*/
bool
DataChecksumsNeedWrite(void)
DataChecksumsEnabled(void)
{
Assert(ControlFile != NULL);
return (ControlFile->data_checksum_version > 0);
}
bool
DataChecksumsNeedVerify(void)
{
Assert(ControlFile != NULL);
/*
* Only verify checksums if they are fully enabled in the cluster. In
* inprogress state they are only updated, not verified.
*/
return (ControlFile->data_checksum_version == PG_DATA_CHECKSUM_VERSION);
}
bool
DataChecksumsInProgress(void)
{
Assert(ControlFile != NULL);
return (ControlFile->data_checksum_version == PG_DATA_CHECKSUM_INPROGRESS_VERSION);
}
void
SetDataChecksumsInProgress(void)
{
Assert(ControlFile != NULL);
if (ControlFile->data_checksum_version > 0)
return;
XlogChecksums(PG_DATA_CHECKSUM_INPROGRESS_VERSION);
LWLockAcquire(ControlFileLock, LW_EXCLUSIVE);
ControlFile->data_checksum_version = PG_DATA_CHECKSUM_INPROGRESS_VERSION;
UpdateControlFile();
LWLockRelease(ControlFileLock);
}
void
SetDataChecksumsOn(void)
{
Assert(ControlFile != NULL);
LWLockAcquire(ControlFileLock, LW_EXCLUSIVE);
if (ControlFile->data_checksum_version != PG_DATA_CHECKSUM_INPROGRESS_VERSION)
{
LWLockRelease(ControlFileLock);
elog(ERROR, "Checksums not in inprogress mode");
}
ControlFile->data_checksum_version = PG_DATA_CHECKSUM_VERSION;
UpdateControlFile();
LWLockRelease(ControlFileLock);
XlogChecksums(PG_DATA_CHECKSUM_VERSION);
}
void
SetDataChecksumsOff(void)
{
LWLockAcquire(ControlFileLock, LW_EXCLUSIVE);
ControlFile->data_checksum_version = 0;
UpdateControlFile();
LWLockRelease(ControlFileLock);
XlogChecksums(0);
}
/* guc hook */
const char *
show_data_checksums(void)
{
if (ControlFile->data_checksum_version == PG_DATA_CHECKSUM_VERSION)
return "on";
else if (ControlFile->data_checksum_version == PG_DATA_CHECKSUM_INPROGRESS_VERSION)
return "inprogress";
else
return "off";
}
/*
* Returns a fake LSN for unlogged relations.
*
@ -7863,16 +7788,6 @@ StartupXLOG(void)
*/
CompleteCommitTsInitialization();
/*
* If we reach this point with checksums in inprogress state, we notify
* the user that they need to manually restart the process to enable
* checksums.
*/
if (ControlFile->data_checksum_version == PG_DATA_CHECKSUM_INPROGRESS_VERSION)
ereport(WARNING,
(errmsg("checksum state is \"inprogress\" with no worker"),
errhint("Either disable or enable checksums by calling the pg_disable_data_checksums() or pg_enable_data_checksums() functions.")));
/*
* All done with end-of-recovery actions.
*
@ -9626,22 +9541,6 @@ XLogReportParameters(void)
}
}
/*
* Log the new state of checksums
*/
static void
XlogChecksums(ChecksumType new_type)
{
xl_checksum_state xlrec;
xlrec.new_checksumtype = new_type;
XLogBeginInsert();
XLogRegisterData((char *) &xlrec, sizeof(xl_checksum_state));
XLogInsert(RM_XLOG_ID, XLOG_CHECKSUMS);
}
/*
* Update full_page_writes in shared memory, and write an
* XLOG_FPW_CHANGE record if necessary.
@ -10070,17 +9969,6 @@ xlog_redo(XLogReaderState *record)
/* Keep track of full_page_writes */
lastFullPageWrites = fpw;
}
else if (info == XLOG_CHECKSUMS)
{
xl_checksum_state state;
memcpy(&state, XLogRecGetData(record), sizeof(xl_checksum_state));
LWLockAcquire(ControlFileLock, LW_EXCLUSIVE);
ControlFile->data_checksum_version = state.new_checksumtype;
UpdateControlFile();
LWLockRelease(ControlFileLock);
}
}
#ifdef WAL_DEBUG

59
src/backend/access/transam/xlogfuncs.c
View File

@ -24,7 +24,6 @@
#include "catalog/pg_type.h"
#include "funcapi.h"
#include "miscadmin.h"
#include "postmaster/checksumhelper.h"
#include "replication/walreceiver.h"
#include "storage/smgr.h"
#include "utils/builtins.h"
@ -699,61 +698,3 @@ pg_backup_start_time(PG_FUNCTION_ARGS)
PG_RETURN_DATUM(xtime);
}
/*
* Disables checksums for the cluster, unless already disabled.
*
* Has immediate effect - the checksums are set to off right away.
*/
Datum
disable_data_checksums(PG_FUNCTION_ARGS)
{
/*
* If we don't need to write new checksums, then clearly they are already
* disabled.
*/
if (!DataChecksumsNeedWrite())
ereport(ERROR,
(errmsg("data checksums already disabled")));
ShutdownChecksumHelperIfRunning();
SetDataChecksumsOff();
PG_RETURN_VOID();
}
/*
* Enables checksums for the cluster, unless already enabled.
*
* Supports vacuum-like cost-based throttling, to limit system load.
* Starts a background worker that updates checksums on existing data.
*/
Datum
enable_data_checksums(PG_FUNCTION_ARGS)
{
int cost_delay = PG_GETARG_INT32(0);
int cost_limit = PG_GETARG_INT32(1);
if (cost_delay < 0)
ereport(ERROR,
(errmsg("cost delay cannot be less than zero")));
if (cost_limit <= 0)
ereport(ERROR,
(errmsg("cost limit must be a positive value")));
/*
* Allow state change from "off" or from "inprogress", since this is how
* we restart the worker if necessary.
*/
if (DataChecksumsNeedVerify())
ereport(ERROR,
(errmsg("data checksums already enabled")));
SetDataChecksumsInProgress();
if (!StartChecksumHelperLauncher(cost_delay, cost_limit))
ereport(ERROR,
(errmsg("failed to start checksum helper process")));
PG_RETURN_VOID();
}

5
src/backend/catalog/system_views.sql
View File

@ -1027,11 +1027,6 @@ CREATE OR REPLACE FUNCTION pg_stop_backup (
RETURNS SETOF record STRICT VOLATILE LANGUAGE internal as 'pg_stop_backup_v2'
PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION pg_enable_data_checksums (
cost_delay int DEFAULT 0, cost_limit int DEFAULT 100)
RETURNS void STRICT VOLATILE LANGUAGE internal AS 'enable_data_checksums'
PARALLEL RESTRICTED;
-- legacy definition for compatibility with 9.3
CREATE OR REPLACE FUNCTION
json_populate_record(base anyelement, from_json json, use_json_as_text boolean DEFAULT false)

5
src/backend/postmaster/Makefile
View File

@ -12,8 +12,7 @@ subdir = src/backend/postmaster
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
OBJS = autovacuum.o bgworker.o bgwriter.o checkpointer.o checksumhelper.o \
fork_process.o pgarch.o pgstat.o postmaster.o startup.o syslogger.o \
walwriter.o
OBJS = autovacuum.o bgworker.o bgwriter.o checkpointer.o fork_process.o \
pgarch.o pgstat.o postmaster.o startup.o syslogger.o walwriter.o
include $(top_srcdir)/src/backend/common.mk

7
src/backend/postmaster/bgworker.c
View File

@ -20,7 +20,6 @@
#include "pgstat.h"
#include "port/atomics.h"
#include "postmaster/bgworker_internals.h"
#include "postmaster/checksumhelper.h"
#include "postmaster/postmaster.h"
#include "replication/logicallauncher.h"
#include "replication/logicalworker.h"
@ -130,12 +129,6 @@ static const struct
},
{
"ApplyWorkerMain", ApplyWorkerMain
},
{
"ChecksumHelperLauncherMain", ChecksumHelperLauncherMain
},
{
"ChecksumHelperWorkerMain", ChecksumHelperWorkerMain
}
};

855
src/backend/postmaster/checksumhelper.c
View File

@ -1,855 +0,0 @@
/*-------------------------------------------------------------------------
*
* checksumhelper.c
* Background worker to walk the database and write checksums to pages
*
* When enabling data checksums on a database at initdb time, no extra process
* is required as each page is checksummed, and verified, at accesses. When
* enabling checksums on an already running cluster, which was not initialized
* with checksums, this helper worker will ensure that all pages are
* checksummed before verification of the checksums is turned on.
*
* Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
*
*
* IDENTIFICATION
* src/backend/postmaster/checksumhelper.c
*
*-------------------------------------------------------------------------
*/
#include "postgres.h"
#include "access/heapam.h"
#include "access/htup_details.h"
#include "access/xact.h"
#include "catalog/pg_database.h"
#include "commands/vacuum.h"
#include "common/relpath.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "postmaster/bgworker.h"
#include "postmaster/bgwriter.h"
#include "postmaster/checksumhelper.h"
#include "storage/bufmgr.h"
#include "storage/checksum.h"
#include "storage/lmgr.h"
#include "storage/ipc.h"
#include "storage/procarray.h"
#include "storage/smgr.h"
#include "tcop/tcopprot.h"
#include "utils/lsyscache.h"
#include "utils/ps_status.h"
typedef enum
{
SUCCESSFUL = 0,
ABORTED,
FAILED
} ChecksumHelperResult;
typedef struct ChecksumHelperShmemStruct
{
pg_atomic_flag launcher_started;
ChecksumHelperResult success;
bool process_shared_catalogs;
bool abort;
/* Parameter values set on start */
int cost_delay;
int cost_limit;
} ChecksumHelperShmemStruct;
/* Shared memory segment for checksumhelper */
static ChecksumHelperShmemStruct * ChecksumHelperShmem;
/* Bookkeeping for work to do */
typedef struct ChecksumHelperDatabase
{
Oid dboid;
char *dbname;
} ChecksumHelperDatabase;
typedef struct ChecksumHelperRelation
{
Oid reloid;
char relkind;
} ChecksumHelperRelation;
/* Prototypes */
static List *BuildDatabaseList(void);
static List *BuildRelationList(bool include_shared);
static List *BuildTempTableList(void);
static ChecksumHelperResult ProcessDatabase(ChecksumHelperDatabase * db);
static void launcher_cancel_handler(SIGNAL_ARGS);
/*
* Main entry point for checksumhelper launcher process.
*/
bool
StartChecksumHelperLauncher(int cost_delay, int cost_limit)
{
BackgroundWorker bgw;
BackgroundWorkerHandle *bgw_handle;
if (ChecksumHelperShmem->abort)
{
ereport(ERROR,
(errmsg("could not start checksumhelper: has been cancelled")));
}
ChecksumHelperShmem->cost_delay = cost_delay;
ChecksumHelperShmem->cost_limit = cost_limit;
memset(&bgw, 0, sizeof(bgw));
bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION;
bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ChecksumHelperLauncherMain");
snprintf(bgw.bgw_name, BGW_MAXLEN, "checksumhelper launcher");
snprintf(bgw.bgw_type, BGW_MAXLEN, "checksumhelper launcher");
bgw.bgw_restart_time = BGW_NEVER_RESTART;
bgw.bgw_notify_pid = MyProcPid;
bgw.bgw_main_arg = (Datum) 0;
if (!pg_atomic_test_set_flag(&ChecksumHelperShmem->launcher_started))
{
/* Failed to set means somebody else started */
ereport(ERROR,
(errmsg("could not start checksumhelper: already running")));
}
if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
{
pg_atomic_clear_flag(&ChecksumHelperShmem->launcher_started);
return false;
}
return true;
}
/*
* ShutdownChecksumHelperIfRunning
* Request shutdown of the checksumhelper
*
* This does not turn off processing immediately, it signals the checksum
* process to end when done with the current block.
*/
void
ShutdownChecksumHelperIfRunning(void)
{
/* If the launcher isn't started, there is nothing to shut down */
if (pg_atomic_unlocked_test_flag(&ChecksumHelperShmem->launcher_started))
return;
/*
* We don't need an atomic variable for aborting, setting it multiple
* times will not change the handling.
*/
ChecksumHelperShmem->abort = true;
}
/*
* ProcessSingleRelationFork
* Enable checksums in a single relation/fork.
*
* Returns true if successful, and false if *aborted*. On error, an actual
* error is raised in the lower levels.
*/
static bool
ProcessSingleRelationFork(Relation reln, ForkNumber forkNum, BufferAccessStrategy strategy)
{
BlockNumber numblocks = RelationGetNumberOfBlocksInFork(reln, forkNum);
BlockNumber b;
char activity[NAMEDATALEN * 2 + 128];
for (b = 0; b < numblocks; b++)
{
Buffer buf = ReadBufferExtended(reln, forkNum, b, RBM_NORMAL, strategy);
/*
* Report to pgstat every 100 blocks (so as not to "spam")
*/
if ((b % 100) == 0)
{
snprintf(activity, sizeof(activity) - 1, "processing: %s.%s (%s block %d/%d)",
get_namespace_name(RelationGetNamespace(reln)), RelationGetRelationName(reln),
forkNames[forkNum], b, numblocks);
pgstat_report_activity(STATE_RUNNING, activity);
}
/* Need to get an exclusive lock before we can flag as dirty */
LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
/*
* Mark the buffer as dirty and force a full page write. We have to
* re-write the page to WAL even if the checksum hasn't changed,
* because if there is a replica it might have a slightly different
* version of the page with an invalid checksum, caused by unlogged
* changes (e.g. hintbits) on the master happening while checksums
* were off. This can happen if there was a valid checksum on the page
* at one point in the past, so only when checksums are first on, then
* off, and then turned on again.
*/
START_CRIT_SECTION();
MarkBufferDirty(buf);
log_newpage_buffer(buf, false);
END_CRIT_SECTION();
UnlockReleaseBuffer(buf);
/*
* This is the only place where we check if we are asked to abort, the
* abortion will bubble up from here.
*/
if (ChecksumHelperShmem->abort)
return false;
vacuum_delay_point();
}
return true;
}
/*
* ProcessSingleRelationByOid
* Process a single relation based on oid.
*
* Returns true if successful, and false if *aborted*. On error, an actual error
* is raised in the lower levels.
*/
static bool
ProcessSingleRelationByOid(Oid relationId, BufferAccessStrategy strategy)
{
Relation rel;
ForkNumber fnum;
bool aborted = false;
StartTransactionCommand();
elog(DEBUG2, "Checksumhelper starting to process relation %d", relationId);
rel = try_relation_open(relationId, AccessShareLock);
if (rel == NULL)
{
/*
* Relation no longer exist. We consider this a success, since there
* are no pages in it that need checksums, and thus return true.
*/
elog(DEBUG1, "Checksumhelper skipping relation %d as it no longer exists", relationId);
CommitTransactionCommand();
pgstat_report_activity(STATE_IDLE, NULL);
return true;
}
RelationOpenSmgr(rel);
for (fnum = 0; fnum <= MAX_FORKNUM; fnum++)
{
if (smgrexists(rel->rd_smgr, fnum))
{
if (!ProcessSingleRelationFork(rel, fnum, strategy))
{
aborted = true;
break;
}
}
}
relation_close(rel, AccessShareLock);
elog(DEBUG2, "Checksumhelper done with relation %d: %s",
relationId, (aborted ? "aborted" : "finished"));
CommitTransactionCommand();
pgstat_report_activity(STATE_IDLE, NULL);
return !aborted;
}
/*
* ProcessDatabase
* Enable checksums in a single database.
*
* We do this by launching a dynamic background worker into this database, and
* waiting for it to finish. We have to do this in a separate worker, since
* each process can only be connected to one database during its lifetime.
*/
static ChecksumHelperResult
ProcessDatabase(ChecksumHelperDatabase * db)
{
BackgroundWorker bgw;
BackgroundWorkerHandle *bgw_handle;
BgwHandleStatus status;
pid_t pid;
char activity[NAMEDATALEN + 64];
ChecksumHelperShmem->success = FAILED;
memset(&bgw, 0, sizeof(bgw));
bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION;
bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ChecksumHelperWorkerMain");
snprintf(bgw.bgw_name, BGW_MAXLEN, "checksumhelper worker");
snprintf(bgw.bgw_type, BGW_MAXLEN, "checksumhelper worker");
bgw.bgw_restart_time = BGW_NEVER_RESTART;
bgw.bgw_notify_pid = MyProcPid;
bgw.bgw_main_arg = ObjectIdGetDatum(db->dboid);
if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
{
ereport(LOG,
(errmsg("failed to start worker for checksumhelper in \"%s\"",
db->dbname)));
return FAILED;
}
status = WaitForBackgroundWorkerStartup(bgw_handle, &pid);
if (status != BGWH_STARTED)
{
ereport(LOG,
(errmsg("failed to wait for worker startup for checksumhelper in \"%s\"",
db->dbname)));
return FAILED;
}
ereport(DEBUG1,
(errmsg("started background worker for checksums in \"%s\"",
db->dbname)));
snprintf(activity, sizeof(activity) - 1,
"Waiting for worker in database %s (pid %d)", db->dbname, pid);
pgstat_report_activity(STATE_RUNNING, activity);
status = WaitForBackgroundWorkerShutdown(bgw_handle);
if (status != BGWH_STOPPED)
{
ereport(LOG,
(errmsg("failed to wait for worker shutdown for checksumhelper in \"%s\"",
db->dbname)));
return FAILED;
}
if (ChecksumHelperShmem->success == ABORTED)
ereport(LOG,
(errmsg("checksumhelper was aborted during processing in \"%s\"",
db->dbname)));
ereport(DEBUG1,
(errmsg("background worker for checksums in \"%s\" completed",
db->dbname)));
pgstat_report_activity(STATE_IDLE, NULL);
return ChecksumHelperShmem->success;
}
static void
launcher_exit(int code, Datum arg)
{
ChecksumHelperShmem->abort = false;
pg_atomic_clear_flag(&ChecksumHelperShmem->launcher_started);
}
static void
launcher_cancel_handler(SIGNAL_ARGS)
{
ChecksumHelperShmem->abort = true;
}
static void
WaitForAllTransactionsToFinish(void)
{
TransactionId waitforxid;
LWLockAcquire(XidGenLock, LW_SHARED);
waitforxid = ShmemVariableCache->nextXid;
LWLockRelease(XidGenLock);
while (true)
{
TransactionId oldestxid = GetOldestActiveTransactionId();
elog(DEBUG1, "Checking old transactions");
if (TransactionIdPrecedes(oldestxid, waitforxid))
{
char activity[64];
/* Oldest running xid is older than us, so wait */
snprintf(activity, sizeof(activity), "Waiting for current transactions to finish (waiting for %d)", waitforxid);
pgstat_report_activity(STATE_RUNNING, activity);
/* Retry every 5 seconds */
ResetLatch(MyLatch);
(void) WaitLatch(MyLatch,
WL_LATCH_SET | WL_TIMEOUT,
5000,
WAIT_EVENT_PG_SLEEP);
}
else
{
pgstat_report_activity(STATE_IDLE, NULL);
return;
}
}
}
void
ChecksumHelperLauncherMain(Datum arg)
{
List *DatabaseList;
List *remaining = NIL;
ListCell *lc,
*lc2;
List *CurrentDatabases = NIL;
bool found_failed = false;
on_shmem_exit(launcher_exit, 0);
ereport(DEBUG1,
(errmsg("checksumhelper launcher started")));
pqsignal(SIGTERM, die);
pqsignal(SIGINT, launcher_cancel_handler);
BackgroundWorkerUnblockSignals();
init_ps_display(pgstat_get_backend_desc(B_CHECKSUMHELPER_LAUNCHER), "", "", "");
/*
* Initialize a connection to shared catalogs only.
*/
BackgroundWorkerInitializeConnection(NULL, NULL, 0);
/*
* Set up so first run processes shared catalogs, but not once in every
* db.
*/
ChecksumHelperShmem->process_shared_catalogs = true;
/*
* Wait for all existing transactions to finish. This will make sure that
* we can see all tables all databases, so we don't miss any. Anything
* created after this point is known to have checksums on all pages
* already, so we don't have to care about those.
*/
WaitForAllTransactionsToFinish();
/*
* Create a database list. We don't need to concern ourselves with
* rebuilding this list during runtime since any database created after
* this process started will be running with checksums turned on from the
* start.
*/
DatabaseList = BuildDatabaseList();
/*
* If there are no databases at all to checksum, we can exit immediately
* as there is no work to do.
*/
if (DatabaseList == NIL || list_length(DatabaseList) == 0)
return;
foreach(lc, DatabaseList)
{
ChecksumHelperDatabase *db = (ChecksumHelperDatabase *) lfirst(lc);
ChecksumHelperResult processing;
processing = ProcessDatabase(db);
if (processing == SUCCESSFUL)
{
pfree(db->dbname);
pfree(db);
if (ChecksumHelperShmem->process_shared_catalogs)
/*
* Now that one database has completed shared catalogs, we
* don't have to process them again.
*/
ChecksumHelperShmem->process_shared_catalogs = false;
}
else if (processing == FAILED)
{
/*
* Put failed databases on the remaining list.
*/
remaining = lappend(remaining, db);
}
else
/* aborted */
return;
}
list_free(DatabaseList);
/*
* remaining now has all databases not yet processed. This can be because
* they failed for some reason, or because the database was dropped
* between us getting the database list and trying to process it. Get a
* fresh list of databases to detect the second case where the database
* was dropped before we had started processing it. If a database still
* exists, but enabling checksums failed then we fail the entire
* checksumming process and exit with an error.
*/
CurrentDatabases = BuildDatabaseList();
foreach(lc, remaining)
{
ChecksumHelperDatabase *db = (ChecksumHelperDatabase *) lfirst(lc);
bool found = false;
foreach(lc2, CurrentDatabases)
{
ChecksumHelperDatabase *db2 = (ChecksumHelperDatabase *) lfirst(lc2);
if (db->dboid == db2->dboid)
{
found = true;
ereport(WARNING,
(errmsg("failed to enable checksums in \"%s\"",
db->dbname)));
break;
}
}
if (found)
found_failed = true;
else
{
ereport(LOG,
(errmsg("database \"%s\" has been dropped, skipping",
db->dbname)));
}
pfree(db->dbname);
pfree(db);
}
list_free(remaining);
/* Free the extra list of databases */
foreach(lc, CurrentDatabases)
{
ChecksumHelperDatabase *db = (ChecksumHelperDatabase *) lfirst(lc);
pfree(db->dbname);
pfree(db);
}
list_free(CurrentDatabases);
if (found_failed)
{
/* Disable checksums on cluster, because we failed */
SetDataChecksumsOff();
ereport(ERROR,
(errmsg("checksumhelper failed to enable checksums in all databases, aborting")));
}
/*
* Force a checkpoint to get everything out to disk.
*/
RequestCheckpoint(CHECKPOINT_FORCE | CHECKPOINT_WAIT | CHECKPOINT_IMMEDIATE);
/*
* Everything has been processed, so flag checksums enabled.
*/
SetDataChecksumsOn();
ereport(LOG,
(errmsg("checksums enabled, checksumhelper launcher shutting down")));
}
/*
* ChecksumHelperShmemSize
* Compute required space for checksumhelper-related shared memory
*/
Size
ChecksumHelperShmemSize(void)
{
Size size;
size = sizeof(ChecksumHelperShmemStruct);
size = MAXALIGN(size);
return size;
}
/*
* ChecksumHelperShmemInit
* Allocate and initialize checksumhelper-related shared memory
*/
void
ChecksumHelperShmemInit(void)
{
bool found;
ChecksumHelperShmem = (ChecksumHelperShmemStruct *)
ShmemInitStruct("ChecksumHelper Data",
ChecksumHelperShmemSize(),
&found);
if (!found)
{
MemSet(ChecksumHelperShmem, 0, ChecksumHelperShmemSize());
pg_atomic_init_flag(&ChecksumHelperShmem->launcher_started);
}
}
/*
* BuildDatabaseList
* Compile a list of all currently available databases in the cluster
*
* This creates the list of databases for the checksumhelper workers to add
* checksums to.
*/
static List *
BuildDatabaseList(void)
{
List *DatabaseList = NIL;
Relation rel;
HeapScanDesc scan;
HeapTuple tup;
MemoryContext ctx = CurrentMemoryContext;
MemoryContext oldctx;
StartTransactionCommand();
rel = heap_open(DatabaseRelationId, AccessShareLock);
scan = heap_beginscan_catalog(rel, 0, NULL);
while (HeapTupleIsValid(tup = heap_getnext(scan, ForwardScanDirection)))
{
Form_pg_database pgdb = (Form_pg_database) GETSTRUCT(tup);
ChecksumHelperDatabase *db;
oldctx = MemoryContextSwitchTo(ctx);
db = (ChecksumHelperDatabase *) palloc(sizeof(ChecksumHelperDatabase));
db->dboid = HeapTupleGetOid(tup);
db->dbname = pstrdup(NameStr(pgdb->datname));
DatabaseList = lappend(DatabaseList, db);
MemoryContextSwitchTo(oldctx);
}
heap_endscan(scan);
heap_close(rel, AccessShareLock);
CommitTransactionCommand();
return DatabaseList;
}
/*
* BuildRelationList
* Compile a list of all relations in the database
*
* If shared is true, both shared relations and local ones are returned, else
* all non-shared relations are returned.
* Temp tables are not included.
*/
static List *
BuildRelationList(bool include_shared)
{
List *RelationList = NIL;
Relation rel;
HeapScanDesc scan;
HeapTuple tup;
MemoryContext ctx = CurrentMemoryContext;
MemoryContext oldctx;
StartTransactionCommand();
rel = heap_open(RelationRelationId, AccessShareLock);
scan = heap_beginscan_catalog(rel, 0, NULL);
while (HeapTupleIsValid(tup = heap_getnext(scan, ForwardScanDirection)))
{
Form_pg_class pgc = (Form_pg_class) GETSTRUCT(tup);
ChecksumHelperRelation *relentry;
if (pgc->relpersistence == 't')
continue;
if (pgc->relisshared && !include_shared)
continue;
/*
* Foreign tables have by definition no local storage that can be
* checksummed, so skip.
*/
if (pgc->relkind == RELKIND_FOREIGN_TABLE)
continue;
oldctx = MemoryContextSwitchTo(ctx);
relentry = (ChecksumHelperRelation *) palloc(sizeof(ChecksumHelperRelation));
relentry->reloid = HeapTupleGetOid(tup);
relentry->relkind = pgc->relkind;
RelationList = lappend(RelationList, relentry);
MemoryContextSwitchTo(oldctx);
}
heap_endscan(scan);
heap_close(rel, AccessShareLock);
CommitTransactionCommand();
return RelationList;
}
/*
* BuildTempTableList
* Compile a list of all temporary tables in database
*
* Returns a List of oids.
*/
static List *
BuildTempTableList(void)
{
List *RelationList = NIL;
Relation rel;
HeapScanDesc scan;
HeapTuple tup;
MemoryContext ctx = CurrentMemoryContext;
MemoryContext oldctx;
StartTransactionCommand();
rel = heap_open(RelationRelationId, AccessShareLock);
scan = heap_beginscan_catalog(rel, 0, NULL);
while (HeapTupleIsValid(tup = heap_getnext(scan, ForwardScanDirection)))
{
Form_pg_class pgc = (Form_pg_class) GETSTRUCT(tup);
if (pgc->relpersistence != 't')
continue;
oldctx = MemoryContextSwitchTo(ctx);
RelationList = lappend_oid(RelationList, HeapTupleGetOid(tup));
MemoryContextSwitchTo(oldctx);
}
heap_endscan(scan);
heap_close(rel, AccessShareLock);
CommitTransactionCommand();
return RelationList;
}
/*
* Main function for enabling checksums in a single database
*/
void
ChecksumHelperWorkerMain(Datum arg)
{
Oid dboid = DatumGetObjectId(arg);
List *RelationList = NIL;
List *InitialTempTableList = NIL;
ListCell *lc;
BufferAccessStrategy strategy;
bool aborted = false;
pqsignal(SIGTERM, die);
BackgroundWorkerUnblockSignals();
init_ps_display(pgstat_get_backend_desc(B_CHECKSUMHELPER_WORKER), "", "", "");
ereport(DEBUG1,
(errmsg("checksum worker starting for database oid %d", dboid)));
BackgroundWorkerInitializeConnectionByOid(dboid, InvalidOid, BGWORKER_BYPASS_ALLOWCONN);
/*
* Get a list of all temp tables present as we start in this database. We
* need to wait until they are all gone until we are done, since we cannot
* access those files and modify them.
*/
InitialTempTableList = BuildTempTableList();
/*
* Enable vacuum cost delay, if any.
*/
VacuumCostDelay = ChecksumHelperShmem->cost_delay;
VacuumCostLimit = ChecksumHelperShmem->cost_limit;
VacuumCostActive = (VacuumCostDelay > 0);
VacuumCostBalance = 0;
VacuumPageHit = 0;
VacuumPageMiss = 0;
VacuumPageDirty = 0;
/*
* Create and set the vacuum strategy as our buffer strategy.
*/
strategy = GetAccessStrategy(BAS_VACUUM);
RelationList = BuildRelationList(ChecksumHelperShmem->process_shared_catalogs);
foreach(lc, RelationList)
{
ChecksumHelperRelation *rel = (ChecksumHelperRelation *) lfirst(lc);
if (!ProcessSingleRelationByOid(rel->reloid, strategy))
{
aborted = true;
break;
}
}
list_free_deep(RelationList);
if (aborted)
{
ChecksumHelperShmem->success = ABORTED;
ereport(DEBUG1,
(errmsg("checksum worker aborted in database oid %d", dboid)));
return;
}
/*
* Wait for all temp tables that existed when we started to go away. This
* is necessary since we cannot "reach" them to enable checksums. Any temp
* tables created after we started will already have checksums in them
* (due to the inprogress state), so those are safe.
*/
while (true)
{
List *CurrentTempTables;
ListCell *lc;
int numleft;
char activity[64];
CurrentTempTables = BuildTempTableList();
numleft = 0;
foreach(lc, InitialTempTableList)
{
if (list_member_oid(CurrentTempTables, lfirst_oid(lc)))
numleft++;
}
list_free(CurrentTempTables);
if (numleft == 0)
break;
/* At least one temp table left to wait for */
snprintf(activity, sizeof(activity), "Waiting for %d temp tables to be removed", numleft);
pgstat_report_activity(STATE_RUNNING, activity);
/* Retry every 5 seconds */
ResetLatch(MyLatch);
(void) WaitLatch(MyLatch,
WL_LATCH_SET | WL_TIMEOUT,
5000,
WAIT_EVENT_PG_SLEEP);
}
list_free(InitialTempTableList);
ChecksumHelperShmem->success = SUCCESSFUL;
ereport(DEBUG1,
(errmsg("checksum worker completed in database oid %d", dboid)));
}

5
src/backend/postmaster/pgstat.c
View File

@ -4125,11 +4125,6 @@ pgstat_get_backend_desc(BackendType backendType)
case B_WAL_WRITER:
backendDesc = "walwriter";
break;
case B_CHECKSUMHELPER_LAUNCHER:
backendDesc = "checksumhelper launcher";
break;
case B_CHECKSUMHELPER_WORKER:
backendDesc = "checksumhelper worker";
}
return backendDesc;

2
src/backend/replication/basebackup.c
View File

@ -1384,7 +1384,7 @@ sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf
_tarWriteHeader(tarfilename, NULL, statbuf, false);
if (!noverify_checksums && DataChecksumsNeedVerify())
if (!noverify_checksums && DataChecksumsEnabled())
{
char *filename;

1
src/backend/replication/logical/decode.c
View File

@ -199,7 +199,6 @@ DecodeXLogOp(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
case XLOG_FPW_CHANGE:
case XLOG_FPI_FOR_HINT:
case XLOG_FPI:
case XLOG_CHECKSUMS:
break;
default:
elog(ERROR, "unexpected RM_XLOG_ID record type: %u", info);

2
src/backend/storage/ipc/ipci.c
View File

@ -27,7 +27,6 @@
#include "postmaster/autovacuum.h"
#include "postmaster/bgworker_internals.h"
#include "postmaster/bgwriter.h"
#include "postmaster/checksumhelper.h"
#include "postmaster/postmaster.h"
#include "replication/logicallauncher.h"
#include "replication/slot.h"
@ -262,7 +261,6 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port)
WalSndShmemInit();
WalRcvShmemInit();
ApplyLauncherShmemInit();
ChecksumHelperShmemInit();
/*
* Set up other modules that need some shared memory space

3
src/backend/storage/page/README
View File

@ -9,8 +9,7 @@ have a very low measured incidence according to research on large server farms,
http://www.cs.toronto.edu/~bianca/papers/sigmetrics09.pdf, discussed
2010/12/22 on -hackers list.
Checksums can be enabled at initdb time, but can also be turned on and off
using pg_enable_data_checksums()/pg_disable_data_checksums() at runtime.
Current implementation requires this be enabled system-wide at initdb time.
The checksum is not valid at all times on a data page!!
The checksum is valid when the page leaves the shared pool and is checked

6
src/backend/storage/page/bufpage.c
View File

@ -93,7 +93,7 @@ PageIsVerified(Page page, BlockNumber blkno)
*/
if (!PageIsNew(page))
{
if (DataChecksumsNeedVerify())
if (DataChecksumsEnabled())
{
checksum = pg_checksum_page((char *) page, blkno);
@ -1168,7 +1168,7 @@ PageSetChecksumCopy(Page page, BlockNumber blkno)
static char *pageCopy = NULL;
/* If we don't need a checksum, just return the passed-in data */
if (PageIsNew(page) || !DataChecksumsNeedWrite())
if (PageIsNew(page) || !DataChecksumsEnabled())
return (char *) page;
/*
@ -1195,7 +1195,7 @@ void
PageSetChecksumInplace(Page page, BlockNumber blkno)
{
/* If we don't need a checksum, just return */
if (PageIsNew(page) || !DataChecksumsNeedWrite())
if (PageIsNew(page) || !DataChecksumsEnabled())
return;
((PageHeader) page)->pd_checksum = pg_checksum_page((char *) page, blkno);

37
src/backend/utils/misc/guc.c
View File

@ -32,7 +32,6 @@
#include "access/transam.h"
#include "access/twophase.h"
#include "access/xact.h"
#include "access/xlog.h"
#include "access/xlog_internal.h"
#include "catalog/namespace.h"
#include "catalog/pg_authid.h"
@ -69,7 +68,6 @@
#include "replication/walreceiver.h"
#include "replication/walsender.h"
#include "storage/bufmgr.h"
#include "storage/checksum.h"
#include "storage/dsm_impl.h"
#include "storage/standby.h"
#include "storage/fd.h"
@ -422,17 +420,6 @@ static const struct config_enum_entry password_encryption_options[] = {
{NULL, 0, false}
};
/*
* data_checksum used to be a boolean, but was only set by initdb so there is
* no need to support variants of boolean input.
*/
static const struct config_enum_entry data_checksum_options[] = {
{"on", DATA_CHECKSUMS_ON, true},
{"off", DATA_CHECKSUMS_OFF, true},
{"inprogress", DATA_CHECKSUMS_INPROGRESS, true},
{NULL, 0, false}
};
/*
* Options for enum values stored in other modules
*/
@ -528,7 +515,7 @@ static int max_identifier_length;
static int block_size;
static int segment_size;
static int wal_block_size;
static int data_checksums_tmp; /* only accessed locally! */
static bool data_checksums;
static bool integer_datetimes;
static bool assert_enabled;
@ -1697,6 +1684,17 @@ static struct config_bool ConfigureNamesBool[] =
NULL, NULL, NULL
},
{
{"data_checksums", PGC_INTERNAL, PRESET_OPTIONS,
gettext_noop("Shows whether data checksums are turned on for this cluster."),
NULL,
GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE
},
&data_checksums,
false,
NULL, NULL, NULL
},
{
{"syslog_sequence_numbers", PGC_SIGHUP, LOGGING_WHERE,
gettext_noop("Add sequence number to syslog messages to avoid duplicate suppression."),
@ -4129,17 +4127,6 @@ static struct config_enum ConfigureNamesEnum[] =
NULL, NULL, NULL
},
{
{"data_checksums", PGC_INTERNAL, PRESET_OPTIONS,
gettext_noop("Shows whether data checksums are turned on for this cluster."),
NULL,
GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE
},
&data_checksums_tmp,
DATA_CHECKSUMS_OFF, data_checksum_options,
NULL, NULL, show_data_checksums
},
/* End-of-list marker */
{
{NULL, 0, 0, NULL, NULL}, NULL, 0, NULL, NULL, NULL, NULL

9
src/bin/pg_upgrade/controldata.c
View File

@ -590,15 +590,6 @@ check_control_data(ControlData *oldctrl,
* check_for_isn_and_int8_passing_mismatch().
*/
/*
* If checksums have been turned on in the old cluster, but the
* checksumhelper have yet to finish, then disallow upgrading. The user
* should either let the process finish, or turn off checksums, before
* retrying.
*/
if (oldctrl->data_checksum_version == 2)
pg_fatal("transition to data checksums not completed in old cluster\n");
/*
* We might eventually allow upgrades from checksum to no-checksum
* clusters.

2
src/bin/pg_upgrade/pg_upgrade.h
View File

@ -226,7 +226,7 @@ typedef struct
uint32 large_object;
bool date_is_int;
bool float8_pass_by_value;
uint32 data_checksum_version;
bool data_checksum_version;
} ControlData;
/*

5
src/bin/pg_verify_checksums/pg_verify_checksums.c
View File

@ -307,10 +307,7 @@ main(int argc, char *argv[])
printf(_("Data checksum version: %d\n"), ControlFile->data_checksum_version);
printf(_("Files scanned: %" INT64_MODIFIER "d\n"), files);
printf(_("Blocks scanned: %" INT64_MODIFIER "d\n"), blocks);
if (ControlFile->data_checksum_version == PG_DATA_CHECKSUM_INPROGRESS_VERSION)
printf(_("Blocks left in progress: %" INT64_MODIFIER "d\n"), badblocks);
else
printf(_("Bad checksums: %" INT64_MODIFIER "d\n"), badblocks);
printf(_("Bad checksums: %" INT64_MODIFIER "d\n"), badblocks);
if (badblocks > 0)
return 1;

10
src/include/access/xlog.h
View File

@ -154,7 +154,7 @@ extern PGDLLIMPORT int wal_level;
* of the bits make it to disk, but the checksum wouldn't match. Also WAL-log
* them if forced by wal_log_hints=on.
*/
#define XLogHintBitIsNeeded() (DataChecksumsNeedWrite() || wal_log_hints)
#define XLogHintBitIsNeeded() (DataChecksumsEnabled() || wal_log_hints)
/* Do we need to WAL-log information required only for Hot Standby and logical replication? */
#define XLogStandbyInfoActive() (wal_level >= WAL_LEVEL_REPLICA)
@ -257,13 +257,7 @@ extern char *XLogFileNameP(TimeLineID tli, XLogSegNo segno);
extern void UpdateControlFile(void);
extern uint64 GetSystemIdentifier(void);
extern char *GetMockAuthenticationNonce(void);
extern bool DataChecksumsNeedWrite(void);
extern bool DataChecksumsNeedVerify(void);
extern bool DataChecksumsInProgress(void);
extern void SetDataChecksumsInProgress(void);
extern void SetDataChecksumsOn(void);
extern void SetDataChecksumsOff(void);
extern const char *show_data_checksums(void);
extern bool DataChecksumsEnabled(void);
extern XLogRecPtr GetFakeLSNForUnloggedRel(void);
extern Size XLOGShmemSize(void);
extern void XLOGShmemInit(void);

7
src/include/access/xlog_internal.h
View File

@ -25,7 +25,6 @@
#include "lib/stringinfo.h"
#include "pgtime.h"
#include "storage/block.h"
#include "storage/checksum.h"
#include "storage/relfilenode.h"
@ -241,12 +240,6 @@ typedef struct xl_restore_point
char rp_name[MAXFNAMELEN];
} xl_restore_point;
/* Information logged when checksum level is changed */
typedef struct xl_checksum_state
{
ChecksumType new_checksumtype;
} xl_checksum_state;
/* End of recovery mark, when we don't do an END_OF_RECOVERY checkpoint */
typedef struct xl_end_of_recovery
{

1
src/include/catalog/pg_control.h
View File

@ -76,7 +76,6 @@ typedef struct CheckPoint
#define XLOG_END_OF_RECOVERY 0x90
#define XLOG_FPI_FOR_HINT 0xA0
#define XLOG_FPI 0xB0
#define XLOG_CHECKSUMS 0xC0
/*

8
src/include/catalog/pg_proc.dat
View File

@ -10169,14 +10169,6 @@
proargnames => '{max_data_alignment,database_block_size,blocks_per_segment,wal_block_size,bytes_per_wal_segment,max_identifier_length,max_index_columns,max_toast_chunk_size,large_object_chunk_size,float4_pass_by_value,float8_pass_by_value,data_page_checksum_version}',
prosrc => 'pg_control_init' },
{ oid => '3996', descr => 'disable data checksums',
proname => 'pg_disable_data_checksums', provolatile => 'v',
prorettype => 'void', proargtypes => '', prosrc => 'disable_data_checksums' },
{ oid => '3998', descr => 'enable data checksums',
proname => 'pg_enable_data_checksums', provolatile => 'v',
prorettype => 'void', proargtypes => 'int4 int4',
proargnames => '{cost_delay,cost_limit}', prosrc => 'enable_data_checksums' },
# collation management functions
{ oid => '3445', descr => 'import collations from operating system',
proname => 'pg_import_system_collations', procost => '100',

4
src/include/pgstat.h
View File

@ -710,9 +710,7 @@ typedef enum BackendType
B_STARTUP,
B_WAL_RECEIVER,
B_WAL_SENDER,
B_WAL_WRITER,
B_CHECKSUMHELPER_LAUNCHER,
B_CHECKSUMHELPER_WORKER
B_WAL_WRITER
} BackendType;

31
src/include/postmaster/checksumhelper.h
View File

@ -1,31 +0,0 @@
/*-------------------------------------------------------------------------
*
* checksumhelper.h
* header file for checksum helper background worker
*
*
* Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
*
* src/include/postmaster/checksumhelper.h
*
*-------------------------------------------------------------------------
*/
#ifndef CHECKSUMHELPER_H
#define CHECKSUMHELPER_H
/* Shared memory */
extern Size ChecksumHelperShmemSize(void);
extern void ChecksumHelperShmemInit(void);
/* Start the background processes for enabling checksums */
bool StartChecksumHelperLauncher(int cost_delay, int cost_limit);
/* Shutdown the background processes, if any */
void ShutdownChecksumHelperIfRunning(void);
/* Background worker entrypoints */
void ChecksumHelperLauncherMain(Datum arg);
void ChecksumHelperWorkerMain(Datum arg);
#endif /* CHECKSUMHELPER_H */

1
src/include/storage/bufpage.h
View File

@ -194,7 +194,6 @@ typedef PageHeaderData *PageHeader;
*/
#define PG_PAGE_LAYOUT_VERSION 4
#define PG_DATA_CHECKSUM_VERSION 1
#define PG_DATA_CHECKSUM_INPROGRESS_VERSION 2
/* ----------------------------------------------------------------
* page support macros

7
src/include/storage/checksum.h
View File

@ -15,13 +15,6 @@
#include "storage/block.h"
typedef enum ChecksumType
{
DATA_CHECKSUMS_OFF = 0,
DATA_CHECKSUMS_ON,
DATA_CHECKSUMS_INPROGRESS
} ChecksumType;
/*
* Compute the checksum for a Postgres page. The page must be aligned on a
* 4-byte boundary.

3
src/test/Makefile
View File

@ -12,8 +12,7 @@ subdir = src/test
top_builddir = ../..
include $(top_builddir)/src/Makefile.global
SUBDIRS = perl regress isolation modules authentication recovery subscription \
checksum
SUBDIRS = perl regress isolation modules authentication recovery subscription
# Test suites that are not safe by default but can be run if selected
# by the user via the whitespace-separated list in variable

2
src/test/checksum/.gitignore vendored
View File

@ -1,2 +0,0 @@
# Generated by test suite
/tmp_check/

24
src/test/checksum/Makefile
View File

@ -1,24 +0,0 @@
#-------------------------------------------------------------------------
#
# Makefile for src/test/checksum
#
# Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group
# Portions Copyright (c) 1994, Regents of the University of California
#
# src/test/checksum/Makefile
#
#-------------------------------------------------------------------------
subdir = src/test/checksum
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
check:
$(prove_check)
installcheck:
$(prove_installcheck)
clean distclean maintainer-clean:
rm -rf tmp_check

22
src/test/checksum/README
View File

@ -1,22 +0,0 @@
src/test/checksum/README
Regression tests for data checksums
===================================
This directory contains a test suite for enabling data checksums
in a running cluster with streaming replication.
Running the tests
=================
make check
or
make installcheck
NOTE: This creates a temporary installation (in the case of "check"),
with multiple nodes, be they master or standby(s) for the purpose of
the tests.
NOTE: This requires the --enable-tap-tests argument to configure.

101
src/test/checksum/t/001_standby_checksum.pl
View File

@ -1,101 +0,0 @@
# Test suite for testing enabling data checksums with streaming replication
use strict;
use warnings;
use PostgresNode;
use TestLib;
use Test::More tests => 10;
my $MAX_TRIES = 30;
# Initialize master node
my $node_master = get_new_node('master');
$node_master->init(allows_streaming => 1);
$node_master->start;
my $backup_name = 'my_backup';
# Take backup
$node_master->backup($backup_name);
# Create streaming standby linking to master
my $node_standby_1 = get_new_node('standby_1');
$node_standby_1->init_from_backup($node_master, $backup_name,
has_streaming => 1);
$node_standby_1->start;
# Create some content on master to have un-checksummed data in the cluster
$node_master->safe_psql('postgres',
"CREATE TABLE t AS SELECT generate_series(1,10000) AS a;");
# Wait for standbys to catch up
$node_master->wait_for_catchup($node_standby_1, 'replay',
$node_master->lsn('insert'));
# Check that checksums are turned off
my $result = $node_master->safe_psql('postgres',
"SELECT setting FROM pg_catalog.pg_settings WHERE name = 'data_checksums';");
is($result, "off", 'ensure checksums are turned off on master');
$result = $node_standby_1->safe_psql('postgres',
"SELECT setting FROM pg_catalog.pg_settings WHERE name = 'data_checksums';");
is($result, "off", 'ensure checksums are turned off on standby_1');
# Enable checksums for the cluster
$node_master->safe_psql('postgres', "SELECT pg_enable_data_checksums();");
# Ensure that the master has switched to inprogress immediately
$result = $node_master->safe_psql('postgres',
"SELECT setting FROM pg_catalog.pg_settings WHERE name = 'data_checksums';");
is($result, "inprogress", 'ensure checksums are in progress on master');
# Wait for checksum enable to be replayed
$node_master->wait_for_catchup($node_standby_1, 'replay');
# Ensure that the standby has switched to inprogress
$result = $node_standby_1->safe_psql('postgres',
"SELECT setting FROM pg_catalog.pg_settings WHERE name = 'data_checksums';");
is($result, "inprogress", 'ensure checksums are in progress on standby_1');
# Insert some more data which should be checksummed on INSERT
$node_master->safe_psql('postgres',
"INSERT INTO t VALUES (generate_series(1,10000));");
# Wait for checksums enabled on the master
for (my $i = 0; $i < $MAX_TRIES; $i++)
{
$result = $node_master->safe_psql('postgres',
"SELECT setting FROM pg_catalog.pg_settings WHERE name = 'data_checksums';");
last if ($result eq 'on');
sleep(1);
}
is ($result, "on", 'ensure checksums are enabled on master');
# Wait for checksums enabled on the standby
for (my $i = 0; $i < $MAX_TRIES; $i++)
{
$result = $node_standby_1->safe_psql('postgres',
"SELECT setting FROM pg_catalog.pg_settings WHERE name = 'data_checksums';");
last if ($result eq 'on');
sleep(1);
}
is ($result, "on", 'ensure checksums are enabled on standby');
$result = $node_master->safe_psql('postgres', "SELECT count(a) FROM t");
is ($result, "20000", 'ensure we can safely read all data with checksums');
# Disable checksums and ensure it's propagated to standby and that we can
# still read all data
$node_master->safe_psql('postgres', "SELECT pg_disable_data_checksums();");
$result = $node_master->safe_psql('postgres',
"SELECT setting FROM pg_catalog.pg_settings WHERE name = 'data_checksums';");
is($result, "off", 'ensure checksums are in progress on master');
# Wait for checksum disable to be replayed
$node_master->wait_for_catchup($node_standby_1, 'replay');
# Ensure that the standby has switched to off
$result = $node_standby_1->safe_psql('postgres',
"SELECT setting FROM pg_catalog.pg_settings WHERE name = 'data_checksums';");
is($result, "off", 'ensure checksums are in progress on standby_1');
$result = $node_master->safe_psql('postgres', "SELECT count(a) FROM t");
is ($result, "20000", 'ensure we can safely read all data without checksums');

27
src/test/isolation/expected/checksum_cancel.out
View File

@ -1,27 +0,0 @@
Parsed test spec with 2 sessions
starting permutation: c_verify_checksums_off r_seqread c_enable_checksums c_verify_checksums_inprogress c_disable_checksums c_wait_checksums_off
step c_verify_checksums_off: SELECT setting = 'off' FROM pg_catalog.pg_settings WHERE name = 'data_checksums';
?column?
t
step r_seqread: SELECT * FROM reader_loop();
reader_loop
t
step c_enable_checksums: SELECT pg_enable_data_checksums(1000);
pg_enable_data_checksums
step c_verify_checksums_inprogress: SELECT setting = 'inprogress' FROM pg_catalog.pg_settings WHERE name = 'data_checksums';
?column?
t
step c_disable_checksums: SELECT pg_disable_data_checksums();
pg_disable_data_checksums
step c_wait_checksums_off: SELECT test_checksums_off();
test_checksums_off
t

27
src/test/isolation/expected/checksum_enable.out
View File

@ -1,27 +0,0 @@
Parsed test spec with 3 sessions
starting permutation: c_verify_checksums_off w_insert100k r_seqread c_enable_checksums c_wait_for_checksums c_verify_checksums_on
step c_verify_checksums_off: SELECT setting = 'off' FROM pg_catalog.pg_settings WHERE name = 'data_checksums';
?column?
t
step w_insert100k: SELECT insert_1k(100);
insert_1k
t
step r_seqread: SELECT * FROM reader_loop();
reader_loop
t
step c_enable_checksums: SELECT pg_enable_data_checksums();
pg_enable_data_checksums
step c_wait_for_checksums: SELECT test_checksums_on();
test_checksums_on
t
step c_verify_checksums_on: SELECT setting = 'on' FROM pg_catalog.pg_settings WHERE name = 'data_checksums';
?column?
t

4
src/test/isolation/isolation_schedule
View File

@ -76,7 +76,3 @@ test: predicate-gin
test: partition-key-update-1
test: partition-key-update-2
test: partition-key-update-3
# The checksum_enable suite will enable checksums for the cluster so should
# not run before anything expecting the cluster to have checksums turned off
# test: checksum_cancel
# test: checksum_enable

47
src/test/isolation/specs/checksum_cancel.spec
View File

@ -1,47 +0,0 @@
setup
{
CREATE TABLE t1 (a serial, b integer, c text);
INSERT INTO t1 (b, c) VALUES (generate_series(1,10000), 'starting values');
CREATE OR REPLACE FUNCTION test_checksums_off() RETURNS boolean AS $$
DECLARE
enabled boolean;
BEGIN
PERFORM pg_sleep(1);
SELECT setting = 'off' INTO enabled FROM pg_catalog.pg_settings WHERE name = 'data_checksums';
RETURN enabled;
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION reader_loop() RETURNS boolean AS $$
DECLARE
counter integer;
enabled boolean;
BEGIN
FOR counter IN 1..100 LOOP
PERFORM count(a) FROM t1;
END LOOP;
RETURN True;
END;
$$ LANGUAGE plpgsql;
}
teardown
{
DROP FUNCTION reader_loop();
DROP FUNCTION test_checksums_off();
DROP TABLE t1;
}
session "reader"
step "r_seqread" { SELECT * FROM reader_loop(); }
session "checksums"
step "c_verify_checksums_off" { SELECT setting = 'off' FROM pg_catalog.pg_settings WHERE name = 'data_checksums'; }
step "c_enable_checksums" { SELECT pg_enable_data_checksums(1000); }
step "c_disable_checksums" { SELECT pg_disable_data_checksums(); }
step "c_verify_checksums_inprogress" { SELECT setting = 'inprogress' FROM pg_catalog.pg_settings WHERE name = 'data_checksums'; }
step "c_wait_checksums_off" { SELECT test_checksums_off(); }
permutation "c_verify_checksums_off" "r_seqread" "c_enable_checksums" "c_verify_checksums_inprogress" "c_disable_checksums" "c_wait_checksums_off"

70
src/test/isolation/specs/checksum_enable.spec
View File

@ -1,70 +0,0 @@
setup
{
CREATE TABLE t1 (a serial, b integer, c text);
INSERT INTO t1 (b, c) VALUES (generate_series(1,10000), 'starting values');
CREATE OR REPLACE FUNCTION insert_1k(iterations int) RETURNS boolean AS $$
DECLARE
counter integer;
BEGIN
FOR counter IN 1..$1 LOOP
INSERT INTO t1 (b, c) VALUES (
generate_series(1, 1000),
array_to_string(array(select chr(97 + (random() * 25)::int) from generate_series(1,250)), '')
);
PERFORM pg_sleep(0.1);
END LOOP;
RETURN True;
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION test_checksums_on() RETURNS boolean AS $$
DECLARE
enabled boolean;
BEGIN
LOOP
SELECT setting = 'on' INTO enabled FROM pg_catalog.pg_settings WHERE name = 'data_checksums';
IF enabled THEN
EXIT;
END IF;
PERFORM pg_sleep(1);
END LOOP;
RETURN enabled;
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION reader_loop() RETURNS boolean AS $$
DECLARE
counter integer;
BEGIN
FOR counter IN 1..30 LOOP
PERFORM count(a) FROM t1;
PERFORM pg_sleep(0.2);
END LOOP;
RETURN True;
END;
$$ LANGUAGE plpgsql;
}
teardown
{
DROP FUNCTION reader_loop();
DROP FUNCTION test_checksums_on();
DROP FUNCTION insert_1k(int);
DROP TABLE t1;
}
session "writer"
step "w_insert100k" { SELECT insert_1k(100); }
session "reader"
step "r_seqread" { SELECT * FROM reader_loop(); }
session "checksums"
step "c_verify_checksums_off" { SELECT setting = 'off' FROM pg_catalog.pg_settings WHERE name = 'data_checksums'; }
step "c_enable_checksums" { SELECT pg_enable_data_checksums(); }
step "c_wait_for_checksums" { SELECT test_checksums_on(); }
step "c_verify_checksums_on" { SELECT setting = 'on' FROM pg_catalog.pg_settings WHERE name = 'data_checksums'; }
permutation "c_verify_checksums_off" "w_insert100k" "r_seqread" "c_enable_checksums" "c_wait_for_checksums" "c_verify_checksums_on"