2015-03-23 18:47:52 +01:00
|
|
|
/*-------------------------------------------------------------------------
|
|
|
|
*
|
|
|
|
* libpq_fetch.c
|
|
|
|
* Functions for fetching files from a remote server.
|
|
|
|
*
|
2019-01-02 18:44:25 +01:00
|
|
|
* Copyright (c) 2013-2019, PostgreSQL Global Development Group
|
2015-03-23 18:47:52 +01:00
|
|
|
*
|
|
|
|
*-------------------------------------------------------------------------
|
|
|
|
*/
|
|
|
|
#include "postgres_fe.h"
|
|
|
|
|
|
|
|
#include <sys/stat.h>
|
|
|
|
#include <dirent.h>
|
|
|
|
#include <fcntl.h>
|
|
|
|
#include <unistd.h>
|
|
|
|
|
2019-10-23 06:08:53 +02:00
|
|
|
#include "catalog/pg_type_d.h"
|
2015-03-23 18:47:52 +01:00
|
|
|
#include "datapagemap.h"
|
2019-10-23 06:08:53 +02:00
|
|
|
#include "fe_utils/connect.h"
|
2015-03-23 18:47:52 +01:00
|
|
|
#include "fetch.h"
|
|
|
|
#include "file_ops.h"
|
|
|
|
#include "filemap.h"
|
2019-10-23 06:08:53 +02:00
|
|
|
#include "pg_rewind.h"
|
2017-10-02 00:36:14 +02:00
|
|
|
#include "port/pg_bswap.h"
|
2015-03-23 18:47:52 +01:00
|
|
|
|
2019-09-30 17:57:35 +02:00
|
|
|
PGconn *conn = NULL;
|
2015-03-23 18:47:52 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Files are fetched max CHUNKSIZE bytes at a time.
|
|
|
|
*
|
|
|
|
* (This only applies to files that are copied in whole, or for truncated
|
|
|
|
* files where we copy the tail. Relation files, where we know the individual
|
|
|
|
* blocks that need to be fetched, are fetched in BLCKSZ chunks.)
|
|
|
|
*/
|
|
|
|
#define CHUNKSIZE 1000000
|
|
|
|
|
|
|
|
static void receiveFileChunks(const char *sql);
|
|
|
|
static void execute_pagemap(datapagemap_t *pagemap, const char *path);
|
|
|
|
static char *run_simple_query(const char *sql);
|
2019-08-28 04:47:35 +02:00
|
|
|
static void run_simple_command(const char *sql);
|
2015-03-23 18:47:52 +01:00
|
|
|
|
|
|
|
void
|
|
|
|
libpqConnect(const char *connstr)
|
|
|
|
{
|
|
|
|
char *str;
|
Disable synchronous commits in pg_rewind.
If you point pg_rewind to a server that is using synchronous replication,
with "pg_rewind --source-server=...", and the replication is not working
for some reason, pg_rewind will get stuck because it creates a temporary
table, which needs to be replicated. You could call broken replication a
pilot error, but pg_rewind is often used in special circumstances, when
there are changes to the replication setup.
We don't do any "real" updates, and we don't care about fsyncing or
replicating the operations on the temporary tables, so fix that by
setting synchronous_commit off.
Michael Banck, Michael Paquier. Backpatch to 9.5, where pg_rewind was
introduced.
Discussion: <20161005143938.GA12247@nighthawk.caipicrew.dd-dns.de>
2016-10-06 12:24:46 +02:00
|
|
|
PGresult *res;
|
2015-03-23 18:47:52 +01:00
|
|
|
|
|
|
|
conn = PQconnectdb(connstr);
|
|
|
|
if (PQstatus(conn) == CONNECTION_BAD)
|
2015-06-23 02:40:01 +02:00
|
|
|
pg_fatal("could not connect to server: %s",
|
2015-03-23 18:47:52 +01:00
|
|
|
PQerrorMessage(conn));
|
|
|
|
|
Unified logging system for command-line programs
This unifies the various ad hoc logging (message printing, error
printing) systems used throughout the command-line programs.
Features:
- Program name is automatically prefixed.
- Message string does not end with newline. This removes a common
source of inconsistencies and omissions.
- Additionally, a final newline is automatically stripped, simplifying
use of PQerrorMessage() etc., another common source of mistakes.
- I converted error message strings to use %m where possible.
- As a result of the above several points, more translatable message
strings can be shared between different components and between
frontends and backend, without gratuitous punctuation or whitespace
differences.
- There is support for setting a "log level". This is not meant to be
user-facing, but can be used internally to implement debug or
verbose modes.
- Lazy argument evaluation, so no significant overhead if logging at
some level is disabled.
- Some color in the messages, similar to gcc and clang. Set
PG_COLOR=auto to try it out. Some colors are predefined, but can be
customized by setting PG_COLORS.
- Common files (common/, fe_utils/, etc.) can handle logging much more
simply by just using one API without worrying too much about the
context of the calling program, requiring callbacks, or having to
pass "progname" around everywhere.
- Some programs called setvbuf() to make sure that stderr is
unbuffered, even on Windows. But not all programs did that. This
is now done centrally.
Soft goals:
- Reduces vertical space use and visual complexity of error reporting
in the source code.
- Encourages more deliberate classification of messages. For example,
in some cases it wasn't clear without analyzing the surrounding code
whether a message was meant as an error or just an info.
- Concepts and terms are vaguely aligned with popular logging
frameworks such as log4j and Python logging.
This is all just about printing stuff out. Nothing affects program
flow (e.g., fatal exits). The uses are just too varied to do that.
Some existing code had wrappers that do some kind of print-and-exit,
and I adapted those.
I tried to keep the output mostly the same, but there is a lot of
historical baggage to unwind and special cases to consider, and I
might not always have succeeded. One significant change is that
pg_rewind used to write all error messages to stdout. That is now
changed to stderr.
Reviewed-by: Donald Dong <xdong@csumb.edu>
Reviewed-by: Arthur Zakirov <a.zakirov@postgrespro.ru>
Discussion: https://www.postgresql.org/message-id/flat/6a609b43-4f57-7348-6480-bd022f924310@2ndquadrant.com
2019-04-01 14:24:37 +02:00
|
|
|
if (showprogress)
|
|
|
|
pg_log_info("connected to server");
|
2015-03-23 18:47:52 +01:00
|
|
|
|
2019-08-28 04:47:35 +02:00
|
|
|
/* disable all types of timeouts */
|
|
|
|
run_simple_command("SET statement_timeout = 0");
|
|
|
|
run_simple_command("SET lock_timeout = 0");
|
|
|
|
run_simple_command("SET idle_in_transaction_session_timeout = 0");
|
|
|
|
|
Empty search_path in Autovacuum and non-psql/pgbench clients.
This makes the client programs behave as documented regardless of the
connect-time search_path and regardless of user-created objects. Today,
a malicious user with CREATE permission on a search_path schema can take
control of certain of these clients' queries and invoke arbitrary SQL
functions under the client identity, often a superuser. This is
exploitable in the default configuration, where all users have CREATE
privilege on schema "public".
This changes behavior of user-defined code stored in the database, like
pg_index.indexprs and pg_extension_config_dump(). If they reach code
bearing unqualified names, "does not exist" or "no schema has been
selected to create in" errors might appear. Users may fix such errors
by schema-qualifying affected names. After upgrading, consider watching
server logs for these errors.
The --table arguments of src/bin/scripts clients have been lax; for
example, "vacuumdb -Zt pg_am\;CHECKPOINT" performed a checkpoint. That
now fails, but for now, "vacuumdb -Zt 'pg_am(amname);CHECKPOINT'" still
performs a checkpoint.
Back-patch to 9.3 (all supported versions).
Reviewed by Tom Lane, though this fix strategy was not his first choice.
Reported by Arseniy Sharoglazov.
Security: CVE-2018-1058
2018-02-26 16:39:44 +01:00
|
|
|
res = PQexec(conn, ALWAYS_SECURE_SEARCH_PATH_SQL);
|
|
|
|
if (PQresultStatus(res) != PGRES_TUPLES_OK)
|
|
|
|
pg_fatal("could not clear search_path: %s",
|
|
|
|
PQresultErrorMessage(res));
|
|
|
|
PQclear(res);
|
|
|
|
|
2015-03-23 18:47:52 +01:00
|
|
|
/*
|
|
|
|
* Check that the server is not in hot standby mode. There is no
|
|
|
|
* fundamental reason that couldn't be made to work, but it doesn't
|
|
|
|
* currently because we use a temporary table. Better to check for it
|
|
|
|
* explicitly than error out, for a better error message.
|
|
|
|
*/
|
|
|
|
str = run_simple_query("SELECT pg_is_in_recovery()");
|
|
|
|
if (strcmp(str, "f") != 0)
|
Unified logging system for command-line programs
This unifies the various ad hoc logging (message printing, error
printing) systems used throughout the command-line programs.
Features:
- Program name is automatically prefixed.
- Message string does not end with newline. This removes a common
source of inconsistencies and omissions.
- Additionally, a final newline is automatically stripped, simplifying
use of PQerrorMessage() etc., another common source of mistakes.
- I converted error message strings to use %m where possible.
- As a result of the above several points, more translatable message
strings can be shared between different components and between
frontends and backend, without gratuitous punctuation or whitespace
differences.
- There is support for setting a "log level". This is not meant to be
user-facing, but can be used internally to implement debug or
verbose modes.
- Lazy argument evaluation, so no significant overhead if logging at
some level is disabled.
- Some color in the messages, similar to gcc and clang. Set
PG_COLOR=auto to try it out. Some colors are predefined, but can be
customized by setting PG_COLORS.
- Common files (common/, fe_utils/, etc.) can handle logging much more
simply by just using one API without worrying too much about the
context of the calling program, requiring callbacks, or having to
pass "progname" around everywhere.
- Some programs called setvbuf() to make sure that stderr is
unbuffered, even on Windows. But not all programs did that. This
is now done centrally.
Soft goals:
- Reduces vertical space use and visual complexity of error reporting
in the source code.
- Encourages more deliberate classification of messages. For example,
in some cases it wasn't clear without analyzing the surrounding code
whether a message was meant as an error or just an info.
- Concepts and terms are vaguely aligned with popular logging
frameworks such as log4j and Python logging.
This is all just about printing stuff out. Nothing affects program
flow (e.g., fatal exits). The uses are just too varied to do that.
Some existing code had wrappers that do some kind of print-and-exit,
and I adapted those.
I tried to keep the output mostly the same, but there is a lot of
historical baggage to unwind and special cases to consider, and I
might not always have succeeded. One significant change is that
pg_rewind used to write all error messages to stdout. That is now
changed to stderr.
Reviewed-by: Donald Dong <xdong@csumb.edu>
Reviewed-by: Arthur Zakirov <a.zakirov@postgrespro.ru>
Discussion: https://www.postgresql.org/message-id/flat/6a609b43-4f57-7348-6480-bd022f924310@2ndquadrant.com
2019-04-01 14:24:37 +02:00
|
|
|
pg_fatal("source server must not be in recovery mode");
|
2015-03-23 18:47:52 +01:00
|
|
|
pg_free(str);
|
|
|
|
|
|
|
|
/*
|
2015-07-12 22:25:51 +02:00
|
|
|
* Also check that full_page_writes is enabled. We can get torn pages if
|
2015-03-23 18:47:52 +01:00
|
|
|
* a page is modified while we read it with pg_read_binary_file(), and we
|
|
|
|
* rely on full page images to fix them.
|
|
|
|
*/
|
|
|
|
str = run_simple_query("SHOW full_page_writes");
|
|
|
|
if (strcmp(str, "on") != 0)
|
Unified logging system for command-line programs
This unifies the various ad hoc logging (message printing, error
printing) systems used throughout the command-line programs.
Features:
- Program name is automatically prefixed.
- Message string does not end with newline. This removes a common
source of inconsistencies and omissions.
- Additionally, a final newline is automatically stripped, simplifying
use of PQerrorMessage() etc., another common source of mistakes.
- I converted error message strings to use %m where possible.
- As a result of the above several points, more translatable message
strings can be shared between different components and between
frontends and backend, without gratuitous punctuation or whitespace
differences.
- There is support for setting a "log level". This is not meant to be
user-facing, but can be used internally to implement debug or
verbose modes.
- Lazy argument evaluation, so no significant overhead if logging at
some level is disabled.
- Some color in the messages, similar to gcc and clang. Set
PG_COLOR=auto to try it out. Some colors are predefined, but can be
customized by setting PG_COLORS.
- Common files (common/, fe_utils/, etc.) can handle logging much more
simply by just using one API without worrying too much about the
context of the calling program, requiring callbacks, or having to
pass "progname" around everywhere.
- Some programs called setvbuf() to make sure that stderr is
unbuffered, even on Windows. But not all programs did that. This
is now done centrally.
Soft goals:
- Reduces vertical space use and visual complexity of error reporting
in the source code.
- Encourages more deliberate classification of messages. For example,
in some cases it wasn't clear without analyzing the surrounding code
whether a message was meant as an error or just an info.
- Concepts and terms are vaguely aligned with popular logging
frameworks such as log4j and Python logging.
This is all just about printing stuff out. Nothing affects program
flow (e.g., fatal exits). The uses are just too varied to do that.
Some existing code had wrappers that do some kind of print-and-exit,
and I adapted those.
I tried to keep the output mostly the same, but there is a lot of
historical baggage to unwind and special cases to consider, and I
might not always have succeeded. One significant change is that
pg_rewind used to write all error messages to stdout. That is now
changed to stderr.
Reviewed-by: Donald Dong <xdong@csumb.edu>
Reviewed-by: Arthur Zakirov <a.zakirov@postgrespro.ru>
Discussion: https://www.postgresql.org/message-id/flat/6a609b43-4f57-7348-6480-bd022f924310@2ndquadrant.com
2019-04-01 14:24:37 +02:00
|
|
|
pg_fatal("full_page_writes must be enabled in the source server");
|
2015-03-23 18:47:52 +01:00
|
|
|
pg_free(str);
|
Disable synchronous commits in pg_rewind.
If you point pg_rewind to a server that is using synchronous replication,
with "pg_rewind --source-server=...", and the replication is not working
for some reason, pg_rewind will get stuck because it creates a temporary
table, which needs to be replicated. You could call broken replication a
pilot error, but pg_rewind is often used in special circumstances, when
there are changes to the replication setup.
We don't do any "real" updates, and we don't care about fsyncing or
replicating the operations on the temporary tables, so fix that by
setting synchronous_commit off.
Michael Banck, Michael Paquier. Backpatch to 9.5, where pg_rewind was
introduced.
Discussion: <20161005143938.GA12247@nighthawk.caipicrew.dd-dns.de>
2016-10-06 12:24:46 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Although we don't do any "real" updates, we do work with a temporary
|
|
|
|
* table. We don't care about synchronous commit for that. It doesn't
|
|
|
|
* otherwise matter much, but if the server is using synchronous
|
|
|
|
* replication, and replication isn't working for some reason, we don't
|
|
|
|
* want to get stuck, waiting for it to start working again.
|
|
|
|
*/
|
2019-08-28 04:47:35 +02:00
|
|
|
run_simple_command("SET synchronous_commit = off");
|
2015-03-23 18:47:52 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Runs a query that returns a single value.
|
2015-07-12 22:25:51 +02:00
|
|
|
* The result should be pg_free'd after use.
|
2015-03-23 18:47:52 +01:00
|
|
|
*/
|
|
|
|
static char *
|
|
|
|
run_simple_query(const char *sql)
|
|
|
|
{
|
|
|
|
PGresult *res;
|
|
|
|
char *result;
|
|
|
|
|
|
|
|
res = PQexec(conn, sql);
|
|
|
|
|
|
|
|
if (PQresultStatus(res) != PGRES_TUPLES_OK)
|
2015-06-23 02:40:01 +02:00
|
|
|
pg_fatal("error running query (%s) in source server: %s",
|
2015-03-23 18:47:52 +01:00
|
|
|
sql, PQresultErrorMessage(res));
|
|
|
|
|
|
|
|
/* sanity check the result set */
|
|
|
|
if (PQnfields(res) != 1 || PQntuples(res) != 1 || PQgetisnull(res, 0, 0))
|
Unified logging system for command-line programs
This unifies the various ad hoc logging (message printing, error
printing) systems used throughout the command-line programs.
Features:
- Program name is automatically prefixed.
- Message string does not end with newline. This removes a common
source of inconsistencies and omissions.
- Additionally, a final newline is automatically stripped, simplifying
use of PQerrorMessage() etc., another common source of mistakes.
- I converted error message strings to use %m where possible.
- As a result of the above several points, more translatable message
strings can be shared between different components and between
frontends and backend, without gratuitous punctuation or whitespace
differences.
- There is support for setting a "log level". This is not meant to be
user-facing, but can be used internally to implement debug or
verbose modes.
- Lazy argument evaluation, so no significant overhead if logging at
some level is disabled.
- Some color in the messages, similar to gcc and clang. Set
PG_COLOR=auto to try it out. Some colors are predefined, but can be
customized by setting PG_COLORS.
- Common files (common/, fe_utils/, etc.) can handle logging much more
simply by just using one API without worrying too much about the
context of the calling program, requiring callbacks, or having to
pass "progname" around everywhere.
- Some programs called setvbuf() to make sure that stderr is
unbuffered, even on Windows. But not all programs did that. This
is now done centrally.
Soft goals:
- Reduces vertical space use and visual complexity of error reporting
in the source code.
- Encourages more deliberate classification of messages. For example,
in some cases it wasn't clear without analyzing the surrounding code
whether a message was meant as an error or just an info.
- Concepts and terms are vaguely aligned with popular logging
frameworks such as log4j and Python logging.
This is all just about printing stuff out. Nothing affects program
flow (e.g., fatal exits). The uses are just too varied to do that.
Some existing code had wrappers that do some kind of print-and-exit,
and I adapted those.
I tried to keep the output mostly the same, but there is a lot of
historical baggage to unwind and special cases to consider, and I
might not always have succeeded. One significant change is that
pg_rewind used to write all error messages to stdout. That is now
changed to stderr.
Reviewed-by: Donald Dong <xdong@csumb.edu>
Reviewed-by: Arthur Zakirov <a.zakirov@postgrespro.ru>
Discussion: https://www.postgresql.org/message-id/flat/6a609b43-4f57-7348-6480-bd022f924310@2ndquadrant.com
2019-04-01 14:24:37 +02:00
|
|
|
pg_fatal("unexpected result set from query");
|
2015-03-23 18:47:52 +01:00
|
|
|
|
|
|
|
result = pg_strdup(PQgetvalue(res, 0, 0));
|
|
|
|
|
|
|
|
PQclear(res);
|
|
|
|
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2019-08-28 04:47:35 +02:00
|
|
|
/*
|
|
|
|
* Runs a command.
|
|
|
|
* In the event of a failure, exit immediately.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
run_simple_command(const char *sql)
|
|
|
|
{
|
|
|
|
PGresult *res;
|
|
|
|
|
|
|
|
res = PQexec(conn, sql);
|
|
|
|
|
|
|
|
if (PQresultStatus(res) != PGRES_COMMAND_OK)
|
|
|
|
pg_fatal("error running query (%s) in source server: %s",
|
|
|
|
sql, PQresultErrorMessage(res));
|
|
|
|
|
|
|
|
PQclear(res);
|
|
|
|
}
|
|
|
|
|
2015-03-23 18:47:52 +01:00
|
|
|
/*
|
2017-05-11 17:49:59 +02:00
|
|
|
* Calls pg_current_wal_insert_lsn() function
|
2015-03-23 18:47:52 +01:00
|
|
|
*/
|
|
|
|
XLogRecPtr
|
|
|
|
libpqGetCurrentXlogInsertLocation(void)
|
|
|
|
{
|
|
|
|
XLogRecPtr result;
|
|
|
|
uint32 hi;
|
|
|
|
uint32 lo;
|
|
|
|
char *val;
|
|
|
|
|
2017-05-11 17:49:59 +02:00
|
|
|
val = run_simple_query("SELECT pg_current_wal_insert_lsn()");
|
2015-03-23 18:47:52 +01:00
|
|
|
|
|
|
|
if (sscanf(val, "%X/%X", &hi, &lo) != 2)
|
Unified logging system for command-line programs
This unifies the various ad hoc logging (message printing, error
printing) systems used throughout the command-line programs.
Features:
- Program name is automatically prefixed.
- Message string does not end with newline. This removes a common
source of inconsistencies and omissions.
- Additionally, a final newline is automatically stripped, simplifying
use of PQerrorMessage() etc., another common source of mistakes.
- I converted error message strings to use %m where possible.
- As a result of the above several points, more translatable message
strings can be shared between different components and between
frontends and backend, without gratuitous punctuation or whitespace
differences.
- There is support for setting a "log level". This is not meant to be
user-facing, but can be used internally to implement debug or
verbose modes.
- Lazy argument evaluation, so no significant overhead if logging at
some level is disabled.
- Some color in the messages, similar to gcc and clang. Set
PG_COLOR=auto to try it out. Some colors are predefined, but can be
customized by setting PG_COLORS.
- Common files (common/, fe_utils/, etc.) can handle logging much more
simply by just using one API without worrying too much about the
context of the calling program, requiring callbacks, or having to
pass "progname" around everywhere.
- Some programs called setvbuf() to make sure that stderr is
unbuffered, even on Windows. But not all programs did that. This
is now done centrally.
Soft goals:
- Reduces vertical space use and visual complexity of error reporting
in the source code.
- Encourages more deliberate classification of messages. For example,
in some cases it wasn't clear without analyzing the surrounding code
whether a message was meant as an error or just an info.
- Concepts and terms are vaguely aligned with popular logging
frameworks such as log4j and Python logging.
This is all just about printing stuff out. Nothing affects program
flow (e.g., fatal exits). The uses are just too varied to do that.
Some existing code had wrappers that do some kind of print-and-exit,
and I adapted those.
I tried to keep the output mostly the same, but there is a lot of
historical baggage to unwind and special cases to consider, and I
might not always have succeeded. One significant change is that
pg_rewind used to write all error messages to stdout. That is now
changed to stderr.
Reviewed-by: Donald Dong <xdong@csumb.edu>
Reviewed-by: Arthur Zakirov <a.zakirov@postgrespro.ru>
Discussion: https://www.postgresql.org/message-id/flat/6a609b43-4f57-7348-6480-bd022f924310@2ndquadrant.com
2019-04-01 14:24:37 +02:00
|
|
|
pg_fatal("unrecognized result \"%s\" for current WAL insert location", val);
|
2015-03-23 18:47:52 +01:00
|
|
|
|
|
|
|
result = ((uint64) hi) << 32 | lo;
|
|
|
|
|
2015-07-12 22:25:51 +02:00
|
|
|
pg_free(val);
|
|
|
|
|
2015-03-23 18:47:52 +01:00
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Get a list of all files in the data directory.
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
libpqProcessFileList(void)
|
|
|
|
{
|
|
|
|
PGresult *res;
|
|
|
|
const char *sql;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Create a recursive directory listing of the whole data directory.
|
|
|
|
*
|
|
|
|
* The WITH RECURSIVE part does most of the work. The second part gets the
|
|
|
|
* targets of the symlinks in pg_tblspc directory.
|
|
|
|
*
|
|
|
|
* XXX: There is no backend function to get a symbolic link's target in
|
|
|
|
* general, so if the admin has put any custom symbolic links in the data
|
|
|
|
* directory, they won't be copied correctly.
|
|
|
|
*/
|
|
|
|
sql =
|
|
|
|
"WITH RECURSIVE files (path, filename, size, isdir) AS (\n"
|
|
|
|
" SELECT '' AS path, filename, size, isdir FROM\n"
|
2015-06-28 20:35:51 +02:00
|
|
|
" (SELECT pg_ls_dir('.', true, false) AS filename) AS fn,\n"
|
|
|
|
" pg_stat_file(fn.filename, true) AS this\n"
|
2015-03-23 18:47:52 +01:00
|
|
|
" UNION ALL\n"
|
|
|
|
" SELECT parent.path || parent.filename || '/' AS path,\n"
|
|
|
|
" fn, this.size, this.isdir\n"
|
|
|
|
" FROM files AS parent,\n"
|
2015-06-28 20:35:51 +02:00
|
|
|
" pg_ls_dir(parent.path || parent.filename, true, false) AS fn,\n"
|
|
|
|
" pg_stat_file(parent.path || parent.filename || '/' || fn, true) AS this\n"
|
2015-03-23 18:47:52 +01:00
|
|
|
" WHERE parent.isdir = 't'\n"
|
|
|
|
")\n"
|
|
|
|
"SELECT path || filename, size, isdir,\n"
|
|
|
|
" pg_tablespace_location(pg_tablespace.oid) AS link_target\n"
|
|
|
|
"FROM files\n"
|
|
|
|
"LEFT OUTER JOIN pg_tablespace ON files.path = 'pg_tblspc/'\n"
|
|
|
|
" AND oid::text = files.filename\n";
|
|
|
|
res = PQexec(conn, sql);
|
|
|
|
|
|
|
|
if (PQresultStatus(res) != PGRES_TUPLES_OK)
|
2015-06-23 02:40:01 +02:00
|
|
|
pg_fatal("could not fetch file list: %s",
|
2015-03-23 18:47:52 +01:00
|
|
|
PQresultErrorMessage(res));
|
|
|
|
|
|
|
|
/* sanity check the result set */
|
|
|
|
if (PQnfields(res) != 4)
|
Unified logging system for command-line programs
This unifies the various ad hoc logging (message printing, error
printing) systems used throughout the command-line programs.
Features:
- Program name is automatically prefixed.
- Message string does not end with newline. This removes a common
source of inconsistencies and omissions.
- Additionally, a final newline is automatically stripped, simplifying
use of PQerrorMessage() etc., another common source of mistakes.
- I converted error message strings to use %m where possible.
- As a result of the above several points, more translatable message
strings can be shared between different components and between
frontends and backend, without gratuitous punctuation or whitespace
differences.
- There is support for setting a "log level". This is not meant to be
user-facing, but can be used internally to implement debug or
verbose modes.
- Lazy argument evaluation, so no significant overhead if logging at
some level is disabled.
- Some color in the messages, similar to gcc and clang. Set
PG_COLOR=auto to try it out. Some colors are predefined, but can be
customized by setting PG_COLORS.
- Common files (common/, fe_utils/, etc.) can handle logging much more
simply by just using one API without worrying too much about the
context of the calling program, requiring callbacks, or having to
pass "progname" around everywhere.
- Some programs called setvbuf() to make sure that stderr is
unbuffered, even on Windows. But not all programs did that. This
is now done centrally.
Soft goals:
- Reduces vertical space use and visual complexity of error reporting
in the source code.
- Encourages more deliberate classification of messages. For example,
in some cases it wasn't clear without analyzing the surrounding code
whether a message was meant as an error or just an info.
- Concepts and terms are vaguely aligned with popular logging
frameworks such as log4j and Python logging.
This is all just about printing stuff out. Nothing affects program
flow (e.g., fatal exits). The uses are just too varied to do that.
Some existing code had wrappers that do some kind of print-and-exit,
and I adapted those.
I tried to keep the output mostly the same, but there is a lot of
historical baggage to unwind and special cases to consider, and I
might not always have succeeded. One significant change is that
pg_rewind used to write all error messages to stdout. That is now
changed to stderr.
Reviewed-by: Donald Dong <xdong@csumb.edu>
Reviewed-by: Arthur Zakirov <a.zakirov@postgrespro.ru>
Discussion: https://www.postgresql.org/message-id/flat/6a609b43-4f57-7348-6480-bd022f924310@2ndquadrant.com
2019-04-01 14:24:37 +02:00
|
|
|
pg_fatal("unexpected result set while fetching file list");
|
2015-03-23 18:47:52 +01:00
|
|
|
|
|
|
|
/* Read result to local variables */
|
|
|
|
for (i = 0; i < PQntuples(res); i++)
|
|
|
|
{
|
|
|
|
char *path = PQgetvalue(res, i, 0);
|
pg_rewind: Fix some problems when copying files >2GB.
When incrementally updating a file larger than 2GB, the old code could
either fail outright (if the client asked the server for bytes beyond
the 2GB boundary) or fail to copy all the blocks that had actually
been modified (if the server reported a file size to the client in
excess of 2GB), resulting in data corruption. Generally, such files
won't occur anyway, but they might if using a non-default segment size
or if there the directory contains stray files unrelated to
PostgreSQL. Fix by a more prudent choice of data types.
Even with these improvements, this code still uses a mix of different
types (off_t, size_t, uint64, int64) to represent file sizes and
offsets, not all of which necessarily have the same width or
signedness, so further cleanup might be in order here. However, at
least now they all have the potential to be 64 bits wide on 64-bit
platforms.
Kuntal Ghosh and Michael Paquier, with a tweak by me.
Discussion: http://postgr.es/m/CAGz5QC+8gbkz=Brp0TgoKNqHWTzonbPtPex80U0O6Uh_bevbaA@mail.gmail.com
2017-07-21 20:25:36 +02:00
|
|
|
int64 filesize = atol(PQgetvalue(res, i, 1));
|
2015-03-23 18:47:52 +01:00
|
|
|
bool isdir = (strcmp(PQgetvalue(res, i, 2), "t") == 0);
|
|
|
|
char *link_target = PQgetvalue(res, i, 3);
|
|
|
|
file_type_t type;
|
|
|
|
|
2015-06-28 20:35:51 +02:00
|
|
|
if (PQgetisnull(res, 0, 1))
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* The file was removed from the server while the query was
|
|
|
|
* running. Ignore it.
|
|
|
|
*/
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2015-03-23 18:47:52 +01:00
|
|
|
if (link_target[0])
|
|
|
|
type = FILE_TYPE_SYMLINK;
|
|
|
|
else if (isdir)
|
|
|
|
type = FILE_TYPE_DIRECTORY;
|
|
|
|
else
|
|
|
|
type = FILE_TYPE_REGULAR;
|
|
|
|
|
2015-04-15 21:52:00 +02:00
|
|
|
process_source_file(path, type, filesize, link_target);
|
2015-03-23 18:47:52 +01:00
|
|
|
}
|
2015-07-12 22:25:51 +02:00
|
|
|
PQclear(res);
|
2015-03-23 18:47:52 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/*----
|
|
|
|
* Runs a query, which returns pieces of files from the remote source data
|
|
|
|
* directory, and overwrites the corresponding parts of target files with
|
|
|
|
* the received parts. The result set is expected to be of format:
|
|
|
|
*
|
|
|
|
* path text -- path in the data directory, e.g "base/1/123"
|
pg_rewind: Fix some problems when copying files >2GB.
When incrementally updating a file larger than 2GB, the old code could
either fail outright (if the client asked the server for bytes beyond
the 2GB boundary) or fail to copy all the blocks that had actually
been modified (if the server reported a file size to the client in
excess of 2GB), resulting in data corruption. Generally, such files
won't occur anyway, but they might if using a non-default segment size
or if there the directory contains stray files unrelated to
PostgreSQL. Fix by a more prudent choice of data types.
Even with these improvements, this code still uses a mix of different
types (off_t, size_t, uint64, int64) to represent file sizes and
offsets, not all of which necessarily have the same width or
signedness, so further cleanup might be in order here. However, at
least now they all have the potential to be 64 bits wide on 64-bit
platforms.
Kuntal Ghosh and Michael Paquier, with a tweak by me.
Discussion: http://postgr.es/m/CAGz5QC+8gbkz=Brp0TgoKNqHWTzonbPtPex80U0O6Uh_bevbaA@mail.gmail.com
2017-07-21 20:25:36 +02:00
|
|
|
* begin int8 -- offset within the file
|
2015-03-23 18:47:52 +01:00
|
|
|
* chunk bytea -- file content
|
|
|
|
*----
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
receiveFileChunks(const char *sql)
|
|
|
|
{
|
|
|
|
PGresult *res;
|
|
|
|
|
|
|
|
if (PQsendQueryParams(conn, sql, 0, NULL, NULL, NULL, NULL, 1) != 1)
|
2015-06-23 02:40:01 +02:00
|
|
|
pg_fatal("could not send query: %s", PQerrorMessage(conn));
|
2015-03-23 18:47:52 +01:00
|
|
|
|
Unified logging system for command-line programs
This unifies the various ad hoc logging (message printing, error
printing) systems used throughout the command-line programs.
Features:
- Program name is automatically prefixed.
- Message string does not end with newline. This removes a common
source of inconsistencies and omissions.
- Additionally, a final newline is automatically stripped, simplifying
use of PQerrorMessage() etc., another common source of mistakes.
- I converted error message strings to use %m where possible.
- As a result of the above several points, more translatable message
strings can be shared between different components and between
frontends and backend, without gratuitous punctuation or whitespace
differences.
- There is support for setting a "log level". This is not meant to be
user-facing, but can be used internally to implement debug or
verbose modes.
- Lazy argument evaluation, so no significant overhead if logging at
some level is disabled.
- Some color in the messages, similar to gcc and clang. Set
PG_COLOR=auto to try it out. Some colors are predefined, but can be
customized by setting PG_COLORS.
- Common files (common/, fe_utils/, etc.) can handle logging much more
simply by just using one API without worrying too much about the
context of the calling program, requiring callbacks, or having to
pass "progname" around everywhere.
- Some programs called setvbuf() to make sure that stderr is
unbuffered, even on Windows. But not all programs did that. This
is now done centrally.
Soft goals:
- Reduces vertical space use and visual complexity of error reporting
in the source code.
- Encourages more deliberate classification of messages. For example,
in some cases it wasn't clear without analyzing the surrounding code
whether a message was meant as an error or just an info.
- Concepts and terms are vaguely aligned with popular logging
frameworks such as log4j and Python logging.
This is all just about printing stuff out. Nothing affects program
flow (e.g., fatal exits). The uses are just too varied to do that.
Some existing code had wrappers that do some kind of print-and-exit,
and I adapted those.
I tried to keep the output mostly the same, but there is a lot of
historical baggage to unwind and special cases to consider, and I
might not always have succeeded. One significant change is that
pg_rewind used to write all error messages to stdout. That is now
changed to stderr.
Reviewed-by: Donald Dong <xdong@csumb.edu>
Reviewed-by: Arthur Zakirov <a.zakirov@postgrespro.ru>
Discussion: https://www.postgresql.org/message-id/flat/6a609b43-4f57-7348-6480-bd022f924310@2ndquadrant.com
2019-04-01 14:24:37 +02:00
|
|
|
pg_log_debug("getting file chunks");
|
2015-03-23 18:47:52 +01:00
|
|
|
|
|
|
|
if (PQsetSingleRowMode(conn) != 1)
|
Unified logging system for command-line programs
This unifies the various ad hoc logging (message printing, error
printing) systems used throughout the command-line programs.
Features:
- Program name is automatically prefixed.
- Message string does not end with newline. This removes a common
source of inconsistencies and omissions.
- Additionally, a final newline is automatically stripped, simplifying
use of PQerrorMessage() etc., another common source of mistakes.
- I converted error message strings to use %m where possible.
- As a result of the above several points, more translatable message
strings can be shared between different components and between
frontends and backend, without gratuitous punctuation or whitespace
differences.
- There is support for setting a "log level". This is not meant to be
user-facing, but can be used internally to implement debug or
verbose modes.
- Lazy argument evaluation, so no significant overhead if logging at
some level is disabled.
- Some color in the messages, similar to gcc and clang. Set
PG_COLOR=auto to try it out. Some colors are predefined, but can be
customized by setting PG_COLORS.
- Common files (common/, fe_utils/, etc.) can handle logging much more
simply by just using one API without worrying too much about the
context of the calling program, requiring callbacks, or having to
pass "progname" around everywhere.
- Some programs called setvbuf() to make sure that stderr is
unbuffered, even on Windows. But not all programs did that. This
is now done centrally.
Soft goals:
- Reduces vertical space use and visual complexity of error reporting
in the source code.
- Encourages more deliberate classification of messages. For example,
in some cases it wasn't clear without analyzing the surrounding code
whether a message was meant as an error or just an info.
- Concepts and terms are vaguely aligned with popular logging
frameworks such as log4j and Python logging.
This is all just about printing stuff out. Nothing affects program
flow (e.g., fatal exits). The uses are just too varied to do that.
Some existing code had wrappers that do some kind of print-and-exit,
and I adapted those.
I tried to keep the output mostly the same, but there is a lot of
historical baggage to unwind and special cases to consider, and I
might not always have succeeded. One significant change is that
pg_rewind used to write all error messages to stdout. That is now
changed to stderr.
Reviewed-by: Donald Dong <xdong@csumb.edu>
Reviewed-by: Arthur Zakirov <a.zakirov@postgrespro.ru>
Discussion: https://www.postgresql.org/message-id/flat/6a609b43-4f57-7348-6480-bd022f924310@2ndquadrant.com
2019-04-01 14:24:37 +02:00
|
|
|
pg_fatal("could not set libpq connection to single row mode");
|
2015-03-23 18:47:52 +01:00
|
|
|
|
|
|
|
while ((res = PQgetResult(conn)) != NULL)
|
|
|
|
{
|
|
|
|
char *filename;
|
|
|
|
int filenamelen;
|
pg_rewind: Fix some problems when copying files >2GB.
When incrementally updating a file larger than 2GB, the old code could
either fail outright (if the client asked the server for bytes beyond
the 2GB boundary) or fail to copy all the blocks that had actually
been modified (if the server reported a file size to the client in
excess of 2GB), resulting in data corruption. Generally, such files
won't occur anyway, but they might if using a non-default segment size
or if there the directory contains stray files unrelated to
PostgreSQL. Fix by a more prudent choice of data types.
Even with these improvements, this code still uses a mix of different
types (off_t, size_t, uint64, int64) to represent file sizes and
offsets, not all of which necessarily have the same width or
signedness, so further cleanup might be in order here. However, at
least now they all have the potential to be 64 bits wide on 64-bit
platforms.
Kuntal Ghosh and Michael Paquier, with a tweak by me.
Discussion: http://postgr.es/m/CAGz5QC+8gbkz=Brp0TgoKNqHWTzonbPtPex80U0O6Uh_bevbaA@mail.gmail.com
2017-07-21 20:25:36 +02:00
|
|
|
int64 chunkoff;
|
2015-03-23 18:47:52 +01:00
|
|
|
int chunksize;
|
|
|
|
char *chunk;
|
|
|
|
|
|
|
|
switch (PQresultStatus(res))
|
|
|
|
{
|
|
|
|
case PGRES_SINGLE_TUPLE:
|
|
|
|
break;
|
|
|
|
|
|
|
|
case PGRES_TUPLES_OK:
|
2015-04-12 21:42:01 +02:00
|
|
|
PQclear(res);
|
2015-03-23 18:47:52 +01:00
|
|
|
continue; /* final zero-row result */
|
|
|
|
|
|
|
|
default:
|
2015-10-02 03:42:00 +02:00
|
|
|
pg_fatal("unexpected result while fetching remote files: %s",
|
2015-03-23 18:47:52 +01:00
|
|
|
PQresultErrorMessage(res));
|
|
|
|
}
|
|
|
|
|
|
|
|
/* sanity check the result set */
|
|
|
|
if (PQnfields(res) != 3 || PQntuples(res) != 1)
|
Unified logging system for command-line programs
This unifies the various ad hoc logging (message printing, error
printing) systems used throughout the command-line programs.
Features:
- Program name is automatically prefixed.
- Message string does not end with newline. This removes a common
source of inconsistencies and omissions.
- Additionally, a final newline is automatically stripped, simplifying
use of PQerrorMessage() etc., another common source of mistakes.
- I converted error message strings to use %m where possible.
- As a result of the above several points, more translatable message
strings can be shared between different components and between
frontends and backend, without gratuitous punctuation or whitespace
differences.
- There is support for setting a "log level". This is not meant to be
user-facing, but can be used internally to implement debug or
verbose modes.
- Lazy argument evaluation, so no significant overhead if logging at
some level is disabled.
- Some color in the messages, similar to gcc and clang. Set
PG_COLOR=auto to try it out. Some colors are predefined, but can be
customized by setting PG_COLORS.
- Common files (common/, fe_utils/, etc.) can handle logging much more
simply by just using one API without worrying too much about the
context of the calling program, requiring callbacks, or having to
pass "progname" around everywhere.
- Some programs called setvbuf() to make sure that stderr is
unbuffered, even on Windows. But not all programs did that. This
is now done centrally.
Soft goals:
- Reduces vertical space use and visual complexity of error reporting
in the source code.
- Encourages more deliberate classification of messages. For example,
in some cases it wasn't clear without analyzing the surrounding code
whether a message was meant as an error or just an info.
- Concepts and terms are vaguely aligned with popular logging
frameworks such as log4j and Python logging.
This is all just about printing stuff out. Nothing affects program
flow (e.g., fatal exits). The uses are just too varied to do that.
Some existing code had wrappers that do some kind of print-and-exit,
and I adapted those.
I tried to keep the output mostly the same, but there is a lot of
historical baggage to unwind and special cases to consider, and I
might not always have succeeded. One significant change is that
pg_rewind used to write all error messages to stdout. That is now
changed to stderr.
Reviewed-by: Donald Dong <xdong@csumb.edu>
Reviewed-by: Arthur Zakirov <a.zakirov@postgrespro.ru>
Discussion: https://www.postgresql.org/message-id/flat/6a609b43-4f57-7348-6480-bd022f924310@2ndquadrant.com
2019-04-01 14:24:37 +02:00
|
|
|
pg_fatal("unexpected result set size while fetching remote files");
|
2015-03-23 18:47:52 +01:00
|
|
|
|
2017-07-21 18:48:22 +02:00
|
|
|
if (PQftype(res, 0) != TEXTOID ||
|
pg_rewind: Fix some problems when copying files >2GB.
When incrementally updating a file larger than 2GB, the old code could
either fail outright (if the client asked the server for bytes beyond
the 2GB boundary) or fail to copy all the blocks that had actually
been modified (if the server reported a file size to the client in
excess of 2GB), resulting in data corruption. Generally, such files
won't occur anyway, but they might if using a non-default segment size
or if there the directory contains stray files unrelated to
PostgreSQL. Fix by a more prudent choice of data types.
Even with these improvements, this code still uses a mix of different
types (off_t, size_t, uint64, int64) to represent file sizes and
offsets, not all of which necessarily have the same width or
signedness, so further cleanup might be in order here. However, at
least now they all have the potential to be 64 bits wide on 64-bit
platforms.
Kuntal Ghosh and Michael Paquier, with a tweak by me.
Discussion: http://postgr.es/m/CAGz5QC+8gbkz=Brp0TgoKNqHWTzonbPtPex80U0O6Uh_bevbaA@mail.gmail.com
2017-07-21 20:25:36 +02:00
|
|
|
PQftype(res, 1) != INT8OID ||
|
2015-03-23 18:47:52 +01:00
|
|
|
PQftype(res, 2) != BYTEAOID)
|
|
|
|
{
|
Unified logging system for command-line programs
This unifies the various ad hoc logging (message printing, error
printing) systems used throughout the command-line programs.
Features:
- Program name is automatically prefixed.
- Message string does not end with newline. This removes a common
source of inconsistencies and omissions.
- Additionally, a final newline is automatically stripped, simplifying
use of PQerrorMessage() etc., another common source of mistakes.
- I converted error message strings to use %m where possible.
- As a result of the above several points, more translatable message
strings can be shared between different components and between
frontends and backend, without gratuitous punctuation or whitespace
differences.
- There is support for setting a "log level". This is not meant to be
user-facing, but can be used internally to implement debug or
verbose modes.
- Lazy argument evaluation, so no significant overhead if logging at
some level is disabled.
- Some color in the messages, similar to gcc and clang. Set
PG_COLOR=auto to try it out. Some colors are predefined, but can be
customized by setting PG_COLORS.
- Common files (common/, fe_utils/, etc.) can handle logging much more
simply by just using one API without worrying too much about the
context of the calling program, requiring callbacks, or having to
pass "progname" around everywhere.
- Some programs called setvbuf() to make sure that stderr is
unbuffered, even on Windows. But not all programs did that. This
is now done centrally.
Soft goals:
- Reduces vertical space use and visual complexity of error reporting
in the source code.
- Encourages more deliberate classification of messages. For example,
in some cases it wasn't clear without analyzing the surrounding code
whether a message was meant as an error or just an info.
- Concepts and terms are vaguely aligned with popular logging
frameworks such as log4j and Python logging.
This is all just about printing stuff out. Nothing affects program
flow (e.g., fatal exits). The uses are just too varied to do that.
Some existing code had wrappers that do some kind of print-and-exit,
and I adapted those.
I tried to keep the output mostly the same, but there is a lot of
historical baggage to unwind and special cases to consider, and I
might not always have succeeded. One significant change is that
pg_rewind used to write all error messages to stdout. That is now
changed to stderr.
Reviewed-by: Donald Dong <xdong@csumb.edu>
Reviewed-by: Arthur Zakirov <a.zakirov@postgrespro.ru>
Discussion: https://www.postgresql.org/message-id/flat/6a609b43-4f57-7348-6480-bd022f924310@2ndquadrant.com
2019-04-01 14:24:37 +02:00
|
|
|
pg_fatal("unexpected data types in result set while fetching remote files: %u %u %u",
|
2015-03-23 18:47:52 +01:00
|
|
|
PQftype(res, 0), PQftype(res, 1), PQftype(res, 2));
|
|
|
|
}
|
|
|
|
|
|
|
|
if (PQfformat(res, 0) != 1 &&
|
|
|
|
PQfformat(res, 1) != 1 &&
|
|
|
|
PQfformat(res, 2) != 1)
|
|
|
|
{
|
Unified logging system for command-line programs
This unifies the various ad hoc logging (message printing, error
printing) systems used throughout the command-line programs.
Features:
- Program name is automatically prefixed.
- Message string does not end with newline. This removes a common
source of inconsistencies and omissions.
- Additionally, a final newline is automatically stripped, simplifying
use of PQerrorMessage() etc., another common source of mistakes.
- I converted error message strings to use %m where possible.
- As a result of the above several points, more translatable message
strings can be shared between different components and between
frontends and backend, without gratuitous punctuation or whitespace
differences.
- There is support for setting a "log level". This is not meant to be
user-facing, but can be used internally to implement debug or
verbose modes.
- Lazy argument evaluation, so no significant overhead if logging at
some level is disabled.
- Some color in the messages, similar to gcc and clang. Set
PG_COLOR=auto to try it out. Some colors are predefined, but can be
customized by setting PG_COLORS.
- Common files (common/, fe_utils/, etc.) can handle logging much more
simply by just using one API without worrying too much about the
context of the calling program, requiring callbacks, or having to
pass "progname" around everywhere.
- Some programs called setvbuf() to make sure that stderr is
unbuffered, even on Windows. But not all programs did that. This
is now done centrally.
Soft goals:
- Reduces vertical space use and visual complexity of error reporting
in the source code.
- Encourages more deliberate classification of messages. For example,
in some cases it wasn't clear without analyzing the surrounding code
whether a message was meant as an error or just an info.
- Concepts and terms are vaguely aligned with popular logging
frameworks such as log4j and Python logging.
This is all just about printing stuff out. Nothing affects program
flow (e.g., fatal exits). The uses are just too varied to do that.
Some existing code had wrappers that do some kind of print-and-exit,
and I adapted those.
I tried to keep the output mostly the same, but there is a lot of
historical baggage to unwind and special cases to consider, and I
might not always have succeeded. One significant change is that
pg_rewind used to write all error messages to stdout. That is now
changed to stderr.
Reviewed-by: Donald Dong <xdong@csumb.edu>
Reviewed-by: Arthur Zakirov <a.zakirov@postgrespro.ru>
Discussion: https://www.postgresql.org/message-id/flat/6a609b43-4f57-7348-6480-bd022f924310@2ndquadrant.com
2019-04-01 14:24:37 +02:00
|
|
|
pg_fatal("unexpected result format while fetching remote files");
|
2015-03-23 18:47:52 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
if (PQgetisnull(res, 0, 0) ||
|
2015-06-28 20:35:51 +02:00
|
|
|
PQgetisnull(res, 0, 1))
|
2015-03-23 18:47:52 +01:00
|
|
|
{
|
Unified logging system for command-line programs
This unifies the various ad hoc logging (message printing, error
printing) systems used throughout the command-line programs.
Features:
- Program name is automatically prefixed.
- Message string does not end with newline. This removes a common
source of inconsistencies and omissions.
- Additionally, a final newline is automatically stripped, simplifying
use of PQerrorMessage() etc., another common source of mistakes.
- I converted error message strings to use %m where possible.
- As a result of the above several points, more translatable message
strings can be shared between different components and between
frontends and backend, without gratuitous punctuation or whitespace
differences.
- There is support for setting a "log level". This is not meant to be
user-facing, but can be used internally to implement debug or
verbose modes.
- Lazy argument evaluation, so no significant overhead if logging at
some level is disabled.
- Some color in the messages, similar to gcc and clang. Set
PG_COLOR=auto to try it out. Some colors are predefined, but can be
customized by setting PG_COLORS.
- Common files (common/, fe_utils/, etc.) can handle logging much more
simply by just using one API without worrying too much about the
context of the calling program, requiring callbacks, or having to
pass "progname" around everywhere.
- Some programs called setvbuf() to make sure that stderr is
unbuffered, even on Windows. But not all programs did that. This
is now done centrally.
Soft goals:
- Reduces vertical space use and visual complexity of error reporting
in the source code.
- Encourages more deliberate classification of messages. For example,
in some cases it wasn't clear without analyzing the surrounding code
whether a message was meant as an error or just an info.
- Concepts and terms are vaguely aligned with popular logging
frameworks such as log4j and Python logging.
This is all just about printing stuff out. Nothing affects program
flow (e.g., fatal exits). The uses are just too varied to do that.
Some existing code had wrappers that do some kind of print-and-exit,
and I adapted those.
I tried to keep the output mostly the same, but there is a lot of
historical baggage to unwind and special cases to consider, and I
might not always have succeeded. One significant change is that
pg_rewind used to write all error messages to stdout. That is now
changed to stderr.
Reviewed-by: Donald Dong <xdong@csumb.edu>
Reviewed-by: Arthur Zakirov <a.zakirov@postgrespro.ru>
Discussion: https://www.postgresql.org/message-id/flat/6a609b43-4f57-7348-6480-bd022f924310@2ndquadrant.com
2019-04-01 14:24:37 +02:00
|
|
|
pg_fatal("unexpected null values in result while fetching remote files");
|
2015-03-23 18:47:52 +01:00
|
|
|
}
|
|
|
|
|
pg_rewind: Fix some problems when copying files >2GB.
When incrementally updating a file larger than 2GB, the old code could
either fail outright (if the client asked the server for bytes beyond
the 2GB boundary) or fail to copy all the blocks that had actually
been modified (if the server reported a file size to the client in
excess of 2GB), resulting in data corruption. Generally, such files
won't occur anyway, but they might if using a non-default segment size
or if there the directory contains stray files unrelated to
PostgreSQL. Fix by a more prudent choice of data types.
Even with these improvements, this code still uses a mix of different
types (off_t, size_t, uint64, int64) to represent file sizes and
offsets, not all of which necessarily have the same width or
signedness, so further cleanup might be in order here. However, at
least now they all have the potential to be 64 bits wide on 64-bit
platforms.
Kuntal Ghosh and Michael Paquier, with a tweak by me.
Discussion: http://postgr.es/m/CAGz5QC+8gbkz=Brp0TgoKNqHWTzonbPtPex80U0O6Uh_bevbaA@mail.gmail.com
2017-07-21 20:25:36 +02:00
|
|
|
if (PQgetlength(res, 0, 1) != sizeof(int64))
|
Unified logging system for command-line programs
This unifies the various ad hoc logging (message printing, error
printing) systems used throughout the command-line programs.
Features:
- Program name is automatically prefixed.
- Message string does not end with newline. This removes a common
source of inconsistencies and omissions.
- Additionally, a final newline is automatically stripped, simplifying
use of PQerrorMessage() etc., another common source of mistakes.
- I converted error message strings to use %m where possible.
- As a result of the above several points, more translatable message
strings can be shared between different components and between
frontends and backend, without gratuitous punctuation or whitespace
differences.
- There is support for setting a "log level". This is not meant to be
user-facing, but can be used internally to implement debug or
verbose modes.
- Lazy argument evaluation, so no significant overhead if logging at
some level is disabled.
- Some color in the messages, similar to gcc and clang. Set
PG_COLOR=auto to try it out. Some colors are predefined, but can be
customized by setting PG_COLORS.
- Common files (common/, fe_utils/, etc.) can handle logging much more
simply by just using one API without worrying too much about the
context of the calling program, requiring callbacks, or having to
pass "progname" around everywhere.
- Some programs called setvbuf() to make sure that stderr is
unbuffered, even on Windows. But not all programs did that. This
is now done centrally.
Soft goals:
- Reduces vertical space use and visual complexity of error reporting
in the source code.
- Encourages more deliberate classification of messages. For example,
in some cases it wasn't clear without analyzing the surrounding code
whether a message was meant as an error or just an info.
- Concepts and terms are vaguely aligned with popular logging
frameworks such as log4j and Python logging.
This is all just about printing stuff out. Nothing affects program
flow (e.g., fatal exits). The uses are just too varied to do that.
Some existing code had wrappers that do some kind of print-and-exit,
and I adapted those.
I tried to keep the output mostly the same, but there is a lot of
historical baggage to unwind and special cases to consider, and I
might not always have succeeded. One significant change is that
pg_rewind used to write all error messages to stdout. That is now
changed to stderr.
Reviewed-by: Donald Dong <xdong@csumb.edu>
Reviewed-by: Arthur Zakirov <a.zakirov@postgrespro.ru>
Discussion: https://www.postgresql.org/message-id/flat/6a609b43-4f57-7348-6480-bd022f924310@2ndquadrant.com
2019-04-01 14:24:37 +02:00
|
|
|
pg_fatal("unexpected result length while fetching remote files");
|
2015-03-23 18:47:52 +01:00
|
|
|
|
|
|
|
/* Read result set to local variables */
|
pg_rewind: Fix some problems when copying files >2GB.
When incrementally updating a file larger than 2GB, the old code could
either fail outright (if the client asked the server for bytes beyond
the 2GB boundary) or fail to copy all the blocks that had actually
been modified (if the server reported a file size to the client in
excess of 2GB), resulting in data corruption. Generally, such files
won't occur anyway, but they might if using a non-default segment size
or if there the directory contains stray files unrelated to
PostgreSQL. Fix by a more prudent choice of data types.
Even with these improvements, this code still uses a mix of different
types (off_t, size_t, uint64, int64) to represent file sizes and
offsets, not all of which necessarily have the same width or
signedness, so further cleanup might be in order here. However, at
least now they all have the potential to be 64 bits wide on 64-bit
platforms.
Kuntal Ghosh and Michael Paquier, with a tweak by me.
Discussion: http://postgr.es/m/CAGz5QC+8gbkz=Brp0TgoKNqHWTzonbPtPex80U0O6Uh_bevbaA@mail.gmail.com
2017-07-21 20:25:36 +02:00
|
|
|
memcpy(&chunkoff, PQgetvalue(res, 0, 1), sizeof(int64));
|
2017-10-02 00:36:14 +02:00
|
|
|
chunkoff = pg_ntoh64(chunkoff);
|
2015-03-23 18:47:52 +01:00
|
|
|
chunksize = PQgetlength(res, 0, 2);
|
|
|
|
|
|
|
|
filenamelen = PQgetlength(res, 0, 0);
|
|
|
|
filename = pg_malloc(filenamelen + 1);
|
|
|
|
memcpy(filename, PQgetvalue(res, 0, 0), filenamelen);
|
|
|
|
filename[filenamelen] = '\0';
|
|
|
|
|
|
|
|
chunk = PQgetvalue(res, 0, 2);
|
|
|
|
|
2015-06-28 20:35:51 +02:00
|
|
|
/*
|
2018-03-28 21:00:21 +02:00
|
|
|
* If a file has been deleted on the source, remove it on the target
|
|
|
|
* as well. Note that multiple unlink() calls may happen on the same
|
|
|
|
* file if multiple data chunks are associated with it, hence ignore
|
|
|
|
* unconditionally anything missing. If this file is not a relation
|
|
|
|
* data file, then it has been already truncated when creating the
|
|
|
|
* file chunk list at the previous execution of the filemap.
|
2015-06-28 20:35:51 +02:00
|
|
|
*/
|
|
|
|
if (PQgetisnull(res, 0, 2))
|
|
|
|
{
|
Unified logging system for command-line programs
This unifies the various ad hoc logging (message printing, error
printing) systems used throughout the command-line programs.
Features:
- Program name is automatically prefixed.
- Message string does not end with newline. This removes a common
source of inconsistencies and omissions.
- Additionally, a final newline is automatically stripped, simplifying
use of PQerrorMessage() etc., another common source of mistakes.
- I converted error message strings to use %m where possible.
- As a result of the above several points, more translatable message
strings can be shared between different components and between
frontends and backend, without gratuitous punctuation or whitespace
differences.
- There is support for setting a "log level". This is not meant to be
user-facing, but can be used internally to implement debug or
verbose modes.
- Lazy argument evaluation, so no significant overhead if logging at
some level is disabled.
- Some color in the messages, similar to gcc and clang. Set
PG_COLOR=auto to try it out. Some colors are predefined, but can be
customized by setting PG_COLORS.
- Common files (common/, fe_utils/, etc.) can handle logging much more
simply by just using one API without worrying too much about the
context of the calling program, requiring callbacks, or having to
pass "progname" around everywhere.
- Some programs called setvbuf() to make sure that stderr is
unbuffered, even on Windows. But not all programs did that. This
is now done centrally.
Soft goals:
- Reduces vertical space use and visual complexity of error reporting
in the source code.
- Encourages more deliberate classification of messages. For example,
in some cases it wasn't clear without analyzing the surrounding code
whether a message was meant as an error or just an info.
- Concepts and terms are vaguely aligned with popular logging
frameworks such as log4j and Python logging.
This is all just about printing stuff out. Nothing affects program
flow (e.g., fatal exits). The uses are just too varied to do that.
Some existing code had wrappers that do some kind of print-and-exit,
and I adapted those.
I tried to keep the output mostly the same, but there is a lot of
historical baggage to unwind and special cases to consider, and I
might not always have succeeded. One significant change is that
pg_rewind used to write all error messages to stdout. That is now
changed to stderr.
Reviewed-by: Donald Dong <xdong@csumb.edu>
Reviewed-by: Arthur Zakirov <a.zakirov@postgrespro.ru>
Discussion: https://www.postgresql.org/message-id/flat/6a609b43-4f57-7348-6480-bd022f924310@2ndquadrant.com
2019-04-01 14:24:37 +02:00
|
|
|
pg_log_debug("received null value for chunk for file \"%s\", file has been deleted",
|
2019-05-22 18:55:34 +02:00
|
|
|
filename);
|
2018-03-28 21:00:21 +02:00
|
|
|
remove_target_file(filename, true);
|
2015-06-28 20:35:51 +02:00
|
|
|
pg_free(filename);
|
|
|
|
PQclear(res);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2019-06-06 14:14:29 +02:00
|
|
|
pg_log_debug("received chunk for file \"%s\", offset %lld, size %d",
|
|
|
|
filename, (long long int) chunkoff, chunksize);
|
2015-03-23 18:47:52 +01:00
|
|
|
|
|
|
|
open_target_file(filename, false);
|
|
|
|
|
|
|
|
write_target_range(chunk, chunkoff, chunksize);
|
2015-03-30 02:02:14 +02:00
|
|
|
|
|
|
|
pg_free(filename);
|
|
|
|
|
|
|
|
PQclear(res);
|
2015-03-23 18:47:52 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Receive a single file as a malloc'd buffer.
|
|
|
|
*/
|
|
|
|
char *
|
|
|
|
libpqGetFile(const char *filename, size_t *filesize)
|
|
|
|
{
|
|
|
|
PGresult *res;
|
|
|
|
char *result;
|
|
|
|
int len;
|
|
|
|
const char *paramValues[1];
|
|
|
|
|
|
|
|
paramValues[0] = filename;
|
|
|
|
res = PQexecParams(conn, "SELECT pg_read_binary_file($1)",
|
|
|
|
1, NULL, paramValues, NULL, NULL, 1);
|
|
|
|
|
|
|
|
if (PQresultStatus(res) != PGRES_TUPLES_OK)
|
2015-06-23 02:40:01 +02:00
|
|
|
pg_fatal("could not fetch remote file \"%s\": %s",
|
2015-03-23 18:47:52 +01:00
|
|
|
filename, PQresultErrorMessage(res));
|
|
|
|
|
|
|
|
/* sanity check the result set */
|
|
|
|
if (PQntuples(res) != 1 || PQgetisnull(res, 0, 0))
|
Unified logging system for command-line programs
This unifies the various ad hoc logging (message printing, error
printing) systems used throughout the command-line programs.
Features:
- Program name is automatically prefixed.
- Message string does not end with newline. This removes a common
source of inconsistencies and omissions.
- Additionally, a final newline is automatically stripped, simplifying
use of PQerrorMessage() etc., another common source of mistakes.
- I converted error message strings to use %m where possible.
- As a result of the above several points, more translatable message
strings can be shared between different components and between
frontends and backend, without gratuitous punctuation or whitespace
differences.
- There is support for setting a "log level". This is not meant to be
user-facing, but can be used internally to implement debug or
verbose modes.
- Lazy argument evaluation, so no significant overhead if logging at
some level is disabled.
- Some color in the messages, similar to gcc and clang. Set
PG_COLOR=auto to try it out. Some colors are predefined, but can be
customized by setting PG_COLORS.
- Common files (common/, fe_utils/, etc.) can handle logging much more
simply by just using one API without worrying too much about the
context of the calling program, requiring callbacks, or having to
pass "progname" around everywhere.
- Some programs called setvbuf() to make sure that stderr is
unbuffered, even on Windows. But not all programs did that. This
is now done centrally.
Soft goals:
- Reduces vertical space use and visual complexity of error reporting
in the source code.
- Encourages more deliberate classification of messages. For example,
in some cases it wasn't clear without analyzing the surrounding code
whether a message was meant as an error or just an info.
- Concepts and terms are vaguely aligned with popular logging
frameworks such as log4j and Python logging.
This is all just about printing stuff out. Nothing affects program
flow (e.g., fatal exits). The uses are just too varied to do that.
Some existing code had wrappers that do some kind of print-and-exit,
and I adapted those.
I tried to keep the output mostly the same, but there is a lot of
historical baggage to unwind and special cases to consider, and I
might not always have succeeded. One significant change is that
pg_rewind used to write all error messages to stdout. That is now
changed to stderr.
Reviewed-by: Donald Dong <xdong@csumb.edu>
Reviewed-by: Arthur Zakirov <a.zakirov@postgrespro.ru>
Discussion: https://www.postgresql.org/message-id/flat/6a609b43-4f57-7348-6480-bd022f924310@2ndquadrant.com
2019-04-01 14:24:37 +02:00
|
|
|
pg_fatal("unexpected result set while fetching remote file \"%s\"",
|
2015-03-23 18:47:52 +01:00
|
|
|
filename);
|
|
|
|
|
|
|
|
/* Read result to local variables */
|
|
|
|
len = PQgetlength(res, 0, 0);
|
|
|
|
result = pg_malloc(len + 1);
|
|
|
|
memcpy(result, PQgetvalue(res, 0, 0), len);
|
|
|
|
result[len] = '\0';
|
|
|
|
|
2015-07-27 19:38:44 +02:00
|
|
|
PQclear(res);
|
|
|
|
|
Unified logging system for command-line programs
This unifies the various ad hoc logging (message printing, error
printing) systems used throughout the command-line programs.
Features:
- Program name is automatically prefixed.
- Message string does not end with newline. This removes a common
source of inconsistencies and omissions.
- Additionally, a final newline is automatically stripped, simplifying
use of PQerrorMessage() etc., another common source of mistakes.
- I converted error message strings to use %m where possible.
- As a result of the above several points, more translatable message
strings can be shared between different components and between
frontends and backend, without gratuitous punctuation or whitespace
differences.
- There is support for setting a "log level". This is not meant to be
user-facing, but can be used internally to implement debug or
verbose modes.
- Lazy argument evaluation, so no significant overhead if logging at
some level is disabled.
- Some color in the messages, similar to gcc and clang. Set
PG_COLOR=auto to try it out. Some colors are predefined, but can be
customized by setting PG_COLORS.
- Common files (common/, fe_utils/, etc.) can handle logging much more
simply by just using one API without worrying too much about the
context of the calling program, requiring callbacks, or having to
pass "progname" around everywhere.
- Some programs called setvbuf() to make sure that stderr is
unbuffered, even on Windows. But not all programs did that. This
is now done centrally.
Soft goals:
- Reduces vertical space use and visual complexity of error reporting
in the source code.
- Encourages more deliberate classification of messages. For example,
in some cases it wasn't clear without analyzing the surrounding code
whether a message was meant as an error or just an info.
- Concepts and terms are vaguely aligned with popular logging
frameworks such as log4j and Python logging.
This is all just about printing stuff out. Nothing affects program
flow (e.g., fatal exits). The uses are just too varied to do that.
Some existing code had wrappers that do some kind of print-and-exit,
and I adapted those.
I tried to keep the output mostly the same, but there is a lot of
historical baggage to unwind and special cases to consider, and I
might not always have succeeded. One significant change is that
pg_rewind used to write all error messages to stdout. That is now
changed to stderr.
Reviewed-by: Donald Dong <xdong@csumb.edu>
Reviewed-by: Arthur Zakirov <a.zakirov@postgrespro.ru>
Discussion: https://www.postgresql.org/message-id/flat/6a609b43-4f57-7348-6480-bd022f924310@2ndquadrant.com
2019-04-01 14:24:37 +02:00
|
|
|
pg_log_debug("fetched file \"%s\", length %d", filename, len);
|
2015-03-23 18:47:52 +01:00
|
|
|
|
|
|
|
if (filesize)
|
|
|
|
*filesize = len;
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Write a file range to a temporary table in the server.
|
|
|
|
*
|
|
|
|
* The range is sent to the server as a COPY formatted line, to be inserted
|
|
|
|
* into the 'fetchchunks' temporary table. It is used in receiveFileChunks()
|
|
|
|
* function to actually fetch the data.
|
|
|
|
*/
|
|
|
|
static void
|
pg_rewind: Fix some problems when copying files >2GB.
When incrementally updating a file larger than 2GB, the old code could
either fail outright (if the client asked the server for bytes beyond
the 2GB boundary) or fail to copy all the blocks that had actually
been modified (if the server reported a file size to the client in
excess of 2GB), resulting in data corruption. Generally, such files
won't occur anyway, but they might if using a non-default segment size
or if there the directory contains stray files unrelated to
PostgreSQL. Fix by a more prudent choice of data types.
Even with these improvements, this code still uses a mix of different
types (off_t, size_t, uint64, int64) to represent file sizes and
offsets, not all of which necessarily have the same width or
signedness, so further cleanup might be in order here. However, at
least now they all have the potential to be 64 bits wide on 64-bit
platforms.
Kuntal Ghosh and Michael Paquier, with a tweak by me.
Discussion: http://postgr.es/m/CAGz5QC+8gbkz=Brp0TgoKNqHWTzonbPtPex80U0O6Uh_bevbaA@mail.gmail.com
2017-07-21 20:25:36 +02:00
|
|
|
fetch_file_range(const char *path, uint64 begin, uint64 end)
|
2015-03-23 18:47:52 +01:00
|
|
|
{
|
|
|
|
char linebuf[MAXPGPATH + 23];
|
|
|
|
|
|
|
|
/* Split the range into CHUNKSIZE chunks */
|
|
|
|
while (end - begin > 0)
|
|
|
|
{
|
|
|
|
unsigned int len;
|
|
|
|
|
pg_rewind: Fix some problems when copying files >2GB.
When incrementally updating a file larger than 2GB, the old code could
either fail outright (if the client asked the server for bytes beyond
the 2GB boundary) or fail to copy all the blocks that had actually
been modified (if the server reported a file size to the client in
excess of 2GB), resulting in data corruption. Generally, such files
won't occur anyway, but they might if using a non-default segment size
or if there the directory contains stray files unrelated to
PostgreSQL. Fix by a more prudent choice of data types.
Even with these improvements, this code still uses a mix of different
types (off_t, size_t, uint64, int64) to represent file sizes and
offsets, not all of which necessarily have the same width or
signedness, so further cleanup might be in order here. However, at
least now they all have the potential to be 64 bits wide on 64-bit
platforms.
Kuntal Ghosh and Michael Paquier, with a tweak by me.
Discussion: http://postgr.es/m/CAGz5QC+8gbkz=Brp0TgoKNqHWTzonbPtPex80U0O6Uh_bevbaA@mail.gmail.com
2017-07-21 20:25:36 +02:00
|
|
|
/* Fine as long as CHUNKSIZE is not bigger than UINT32_MAX */
|
2015-03-23 18:47:52 +01:00
|
|
|
if (end - begin > CHUNKSIZE)
|
|
|
|
len = CHUNKSIZE;
|
|
|
|
else
|
pg_rewind: Fix some problems when copying files >2GB.
When incrementally updating a file larger than 2GB, the old code could
either fail outright (if the client asked the server for bytes beyond
the 2GB boundary) or fail to copy all the blocks that had actually
been modified (if the server reported a file size to the client in
excess of 2GB), resulting in data corruption. Generally, such files
won't occur anyway, but they might if using a non-default segment size
or if there the directory contains stray files unrelated to
PostgreSQL. Fix by a more prudent choice of data types.
Even with these improvements, this code still uses a mix of different
types (off_t, size_t, uint64, int64) to represent file sizes and
offsets, not all of which necessarily have the same width or
signedness, so further cleanup might be in order here. However, at
least now they all have the potential to be 64 bits wide on 64-bit
platforms.
Kuntal Ghosh and Michael Paquier, with a tweak by me.
Discussion: http://postgr.es/m/CAGz5QC+8gbkz=Brp0TgoKNqHWTzonbPtPex80U0O6Uh_bevbaA@mail.gmail.com
2017-07-21 20:25:36 +02:00
|
|
|
len = (unsigned int) (end - begin);
|
2015-03-23 18:47:52 +01:00
|
|
|
|
pg_rewind: Fix some problems when copying files >2GB.
When incrementally updating a file larger than 2GB, the old code could
either fail outright (if the client asked the server for bytes beyond
the 2GB boundary) or fail to copy all the blocks that had actually
been modified (if the server reported a file size to the client in
excess of 2GB), resulting in data corruption. Generally, such files
won't occur anyway, but they might if using a non-default segment size
or if there the directory contains stray files unrelated to
PostgreSQL. Fix by a more prudent choice of data types.
Even with these improvements, this code still uses a mix of different
types (off_t, size_t, uint64, int64) to represent file sizes and
offsets, not all of which necessarily have the same width or
signedness, so further cleanup might be in order here. However, at
least now they all have the potential to be 64 bits wide on 64-bit
platforms.
Kuntal Ghosh and Michael Paquier, with a tweak by me.
Discussion: http://postgr.es/m/CAGz5QC+8gbkz=Brp0TgoKNqHWTzonbPtPex80U0O6Uh_bevbaA@mail.gmail.com
2017-07-21 20:25:36 +02:00
|
|
|
snprintf(linebuf, sizeof(linebuf), "%s\t" UINT64_FORMAT "\t%u\n", path, begin, len);
|
2015-03-23 18:47:52 +01:00
|
|
|
|
|
|
|
if (PQputCopyData(conn, linebuf, strlen(linebuf)) != 1)
|
2015-06-23 02:40:01 +02:00
|
|
|
pg_fatal("could not send COPY data: %s",
|
2015-03-23 18:47:52 +01:00
|
|
|
PQerrorMessage(conn));
|
|
|
|
|
|
|
|
begin += len;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Fetch all changed blocks from remote source data directory.
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
libpq_executeFileMap(filemap_t *map)
|
|
|
|
{
|
|
|
|
file_entry_t *entry;
|
|
|
|
const char *sql;
|
|
|
|
PGresult *res;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* First create a temporary table, and load it with the blocks that we
|
|
|
|
* need to fetch.
|
|
|
|
*/
|
pg_rewind: Fix some problems when copying files >2GB.
When incrementally updating a file larger than 2GB, the old code could
either fail outright (if the client asked the server for bytes beyond
the 2GB boundary) or fail to copy all the blocks that had actually
been modified (if the server reported a file size to the client in
excess of 2GB), resulting in data corruption. Generally, such files
won't occur anyway, but they might if using a non-default segment size
or if there the directory contains stray files unrelated to
PostgreSQL. Fix by a more prudent choice of data types.
Even with these improvements, this code still uses a mix of different
types (off_t, size_t, uint64, int64) to represent file sizes and
offsets, not all of which necessarily have the same width or
signedness, so further cleanup might be in order here. However, at
least now they all have the potential to be 64 bits wide on 64-bit
platforms.
Kuntal Ghosh and Michael Paquier, with a tweak by me.
Discussion: http://postgr.es/m/CAGz5QC+8gbkz=Brp0TgoKNqHWTzonbPtPex80U0O6Uh_bevbaA@mail.gmail.com
2017-07-21 20:25:36 +02:00
|
|
|
sql = "CREATE TEMPORARY TABLE fetchchunks(path text, begin int8, len int4);";
|
2019-08-28 04:47:35 +02:00
|
|
|
run_simple_command(sql);
|
2015-03-23 18:47:52 +01:00
|
|
|
|
|
|
|
sql = "COPY fetchchunks FROM STDIN";
|
|
|
|
res = PQexec(conn, sql);
|
|
|
|
|
|
|
|
if (PQresultStatus(res) != PGRES_COPY_IN)
|
2015-06-23 02:40:01 +02:00
|
|
|
pg_fatal("could not send file list: %s",
|
2015-03-23 18:47:52 +01:00
|
|
|
PQresultErrorMessage(res));
|
2015-07-27 19:38:44 +02:00
|
|
|
PQclear(res);
|
2015-03-23 18:47:52 +01:00
|
|
|
|
|
|
|
for (i = 0; i < map->narray; i++)
|
|
|
|
{
|
|
|
|
entry = map->array[i];
|
|
|
|
|
|
|
|
/* If this is a relation file, copy the modified blocks */
|
|
|
|
execute_pagemap(&entry->pagemap, entry->path);
|
|
|
|
|
|
|
|
switch (entry->action)
|
|
|
|
{
|
|
|
|
case FILE_ACTION_NONE:
|
|
|
|
/* nothing else to do */
|
|
|
|
break;
|
|
|
|
|
|
|
|
case FILE_ACTION_COPY:
|
|
|
|
/* Truncate the old file out of the way, if any */
|
|
|
|
open_target_file(entry->path, true);
|
|
|
|
fetch_file_range(entry->path, 0, entry->newsize);
|
|
|
|
break;
|
|
|
|
|
|
|
|
case FILE_ACTION_TRUNCATE:
|
|
|
|
truncate_target_file(entry->path, entry->newsize);
|
|
|
|
break;
|
|
|
|
|
|
|
|
case FILE_ACTION_COPY_TAIL:
|
|
|
|
fetch_file_range(entry->path, entry->oldsize, entry->newsize);
|
|
|
|
break;
|
|
|
|
|
|
|
|
case FILE_ACTION_REMOVE:
|
|
|
|
remove_target(entry);
|
|
|
|
break;
|
|
|
|
|
|
|
|
case FILE_ACTION_CREATE:
|
|
|
|
create_target(entry);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (PQputCopyEnd(conn, NULL) != 1)
|
2015-06-23 02:40:01 +02:00
|
|
|
pg_fatal("could not send end-of-COPY: %s",
|
2015-03-23 18:47:52 +01:00
|
|
|
PQerrorMessage(conn));
|
|
|
|
|
|
|
|
while ((res = PQgetResult(conn)) != NULL)
|
|
|
|
{
|
|
|
|
if (PQresultStatus(res) != PGRES_COMMAND_OK)
|
2015-06-23 02:40:01 +02:00
|
|
|
pg_fatal("unexpected result while sending file list: %s",
|
2015-03-23 18:47:52 +01:00
|
|
|
PQresultErrorMessage(res));
|
2015-07-27 19:38:44 +02:00
|
|
|
PQclear(res);
|
2015-03-23 18:47:52 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We've now copied the list of file ranges that we need to fetch to the
|
|
|
|
* temporary table. Now, actually fetch all of those ranges.
|
|
|
|
*/
|
|
|
|
sql =
|
2017-04-14 05:47:46 +02:00
|
|
|
"SELECT path, begin,\n"
|
2015-06-28 20:35:51 +02:00
|
|
|
" pg_read_binary_file(path, begin, len, true) AS chunk\n"
|
2015-03-23 18:47:52 +01:00
|
|
|
"FROM fetchchunks\n";
|
|
|
|
|
|
|
|
receiveFileChunks(sql);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
execute_pagemap(datapagemap_t *pagemap, const char *path)
|
|
|
|
{
|
|
|
|
datapagemap_iterator_t *iter;
|
|
|
|
BlockNumber blkno;
|
|
|
|
off_t offset;
|
|
|
|
|
|
|
|
iter = datapagemap_iterate(pagemap);
|
|
|
|
while (datapagemap_next(iter, &blkno))
|
|
|
|
{
|
|
|
|
offset = blkno * BLCKSZ;
|
|
|
|
|
|
|
|
fetch_file_range(path, offset, offset + BLCKSZ);
|
|
|
|
}
|
2015-06-11 15:31:18 +02:00
|
|
|
pg_free(iter);
|
2015-03-23 18:47:52 +01:00
|
|
|
}
|