From bf5bb2e85b6492c7245b9446efaf43d52a98db13 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 29 Sep 2016 12:00:00 -0400 Subject: [PATCH] Move fsync routines of initdb into src/common/ The intention is to used those in other utilities such as pg_basebackup and pg_receivexlog. From: Michael Paquier --- src/bin/initdb/initdb.c | 270 ++----------------------------- src/bin/initdb/nls.mk | 2 +- src/bin/pg_basebackup/nls.mk | 2 +- src/common/Makefile | 2 +- src/common/file_utils.c | 276 ++++++++++++++++++++++++++++++++ src/include/common/file_utils.h | 22 +++ src/tools/msvc/Mkvcbuild.pm | 2 +- 7 files changed, 313 insertions(+), 263 deletions(-) create mode 100644 src/common/file_utils.c create mode 100644 src/include/common/file_utils.h diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 3350e13059..e52e67df61 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -61,6 +61,7 @@ #endif #include "catalog/catalog.h" +#include "common/file_utils.h" #include "common/restricted_token.h" #include "common/username.h" #include "mb/pg_wchar.h" @@ -70,13 +71,6 @@ #include "fe_utils/string_utils.h" -/* Define PG_FLUSH_DATA_WORKS if we have an implementation for pg_flush_data */ -#if defined(HAVE_SYNC_FILE_RANGE) -#define PG_FLUSH_DATA_WORKS 1 -#elif defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED) -#define PG_FLUSH_DATA_WORKS 1 -#endif - /* Ideally this would be in a .h file, but it hardly seems worth the trouble */ extern const char *select_default_timezone(const char *share_path); @@ -237,13 +231,6 @@ static char **filter_lines_with_token(char **lines, const char *token); #endif static char **readfile(const char *path); static void writefile(char *path, char **lines); -static void walkdir(const char *path, - void (*action) (const char *fname, bool isdir), - bool process_symlinks); -#ifdef PG_FLUSH_DATA_WORKS -static void pre_sync_fname(const char *fname, bool isdir); -#endif -static void fsync_fname_ext(const char *fname, bool isdir); static FILE *popen_check(const char *command, const char *mode); static void exit_nicely(void); static char *get_id(void); @@ -270,7 +257,6 @@ static void load_plpgsql(FILE *cmdfd); static void vacuum_db(FILE *cmdfd); static void make_template0(FILE *cmdfd); static void make_postgres(FILE *cmdfd); -static void fsync_pgdata(void); static void trapsig(int signum); static void check_ok(void); static char *escape_quotes(const char *src); @@ -528,177 +514,6 @@ writefile(char *path, char **lines) } } -/* - * walkdir: recursively walk a directory, applying the action to each - * regular file and directory (including the named directory itself). - * - * If process_symlinks is true, the action and recursion are also applied - * to regular files and directories that are pointed to by symlinks in the - * given directory; otherwise symlinks are ignored. Symlinks are always - * ignored in subdirectories, ie we intentionally don't pass down the - * process_symlinks flag to recursive calls. - * - * Errors are reported but not considered fatal. - * - * See also walkdir in fd.c, which is a backend version of this logic. - */ -static void -walkdir(const char *path, - void (*action) (const char *fname, bool isdir), - bool process_symlinks) -{ - DIR *dir; - struct dirent *de; - - dir = opendir(path); - if (dir == NULL) - { - fprintf(stderr, _("%s: could not open directory \"%s\": %s\n"), - progname, path, strerror(errno)); - return; - } - - while (errno = 0, (de = readdir(dir)) != NULL) - { - char subpath[MAXPGPATH]; - struct stat fst; - int sret; - - if (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0) - continue; - - snprintf(subpath, MAXPGPATH, "%s/%s", path, de->d_name); - - if (process_symlinks) - sret = stat(subpath, &fst); - else - sret = lstat(subpath, &fst); - - if (sret < 0) - { - fprintf(stderr, _("%s: could not stat file \"%s\": %s\n"), - progname, subpath, strerror(errno)); - continue; - } - - if (S_ISREG(fst.st_mode)) - (*action) (subpath, false); - else if (S_ISDIR(fst.st_mode)) - walkdir(subpath, action, false); - } - - if (errno) - fprintf(stderr, _("%s: could not read directory \"%s\": %s\n"), - progname, path, strerror(errno)); - - (void) closedir(dir); - - /* - * It's important to fsync the destination directory itself as individual - * file fsyncs don't guarantee that the directory entry for the file is - * synced. Recent versions of ext4 have made the window much wider but - * it's been an issue for ext3 and other filesystems in the past. - */ - (*action) (path, true); -} - -/* - * Hint to the OS that it should get ready to fsync() this file. - * - * Ignores errors trying to open unreadable files, and reports other errors - * non-fatally. - */ -#ifdef PG_FLUSH_DATA_WORKS - -static void -pre_sync_fname(const char *fname, bool isdir) -{ - int fd; - - fd = open(fname, O_RDONLY | PG_BINARY); - - if (fd < 0) - { - if (errno == EACCES || (isdir && errno == EISDIR)) - return; - fprintf(stderr, _("%s: could not open file \"%s\": %s\n"), - progname, fname, strerror(errno)); - return; - } - - /* - * We do what pg_flush_data() would do in the backend: prefer to use - * sync_file_range, but fall back to posix_fadvise. We ignore errors - * because this is only a hint. - */ -#if defined(HAVE_SYNC_FILE_RANGE) - (void) sync_file_range(fd, 0, 0, SYNC_FILE_RANGE_WRITE); -#elif defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED) - (void) posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED); -#else -#error PG_FLUSH_DATA_WORKS should not have been defined -#endif - - (void) close(fd); -} - -#endif /* PG_FLUSH_DATA_WORKS */ - -/* - * fsync_fname_ext -- Try to fsync a file or directory - * - * Ignores errors trying to open unreadable files, or trying to fsync - * directories on systems where that isn't allowed/required. Reports - * other errors non-fatally. - */ -static void -fsync_fname_ext(const char *fname, bool isdir) -{ - int fd; - int flags; - int returncode; - - /* - * Some OSs require directories to be opened read-only whereas other - * systems don't allow us to fsync files opened read-only; so we need both - * cases here. Using O_RDWR will cause us to fail to fsync files that are - * not writable by our userid, but we assume that's OK. - */ - flags = PG_BINARY; - if (!isdir) - flags |= O_RDWR; - else - flags |= O_RDONLY; - - /* - * Open the file, silently ignoring errors about unreadable files (or - * unsupported operations, e.g. opening a directory under Windows), and - * logging others. - */ - fd = open(fname, flags); - if (fd < 0) - { - if (errno == EACCES || (isdir && errno == EISDIR)) - return; - fprintf(stderr, _("%s: could not open file \"%s\": %s\n"), - progname, fname, strerror(errno)); - return; - } - - returncode = fsync(fd); - - /* - * Some OSes don't allow us to fsync directories at all, so we can ignore - * those errors. Anything else needs to be reported. - */ - if (returncode != 0 && !(isdir && errno == EBADF)) - fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), - progname, fname, strerror(errno)); - - (void) close(fd); -} - /* * Open a subcommand with suitable error messaging */ @@ -2276,77 +2091,6 @@ make_postgres(FILE *cmdfd) PG_CMD_PUTS(*line); } -/* - * Issue fsync recursively on PGDATA and all its contents. - * - * We fsync regular files and directories wherever they are, but we - * follow symlinks only for pg_xlog and immediately under pg_tblspc. - * Other symlinks are presumed to point at files we're not responsible - * for fsyncing, and might not have privileges to write at all. - * - * Errors are reported but not considered fatal. - */ -static void -fsync_pgdata(void) -{ - bool xlog_is_symlink; - char pg_xlog[MAXPGPATH]; - char pg_tblspc[MAXPGPATH]; - - fputs(_("syncing data to disk ... "), stdout); - fflush(stdout); - - snprintf(pg_xlog, MAXPGPATH, "%s/pg_xlog", pg_data); - snprintf(pg_tblspc, MAXPGPATH, "%s/pg_tblspc", pg_data); - - /* - * If pg_xlog is a symlink, we'll need to recurse into it separately, - * because the first walkdir below will ignore it. - */ - xlog_is_symlink = false; - -#ifndef WIN32 - { - struct stat st; - - if (lstat(pg_xlog, &st) < 0) - fprintf(stderr, _("%s: could not stat file \"%s\": %s\n"), - progname, pg_xlog, strerror(errno)); - else if (S_ISLNK(st.st_mode)) - xlog_is_symlink = true; - } -#else - if (pgwin32_is_junction(pg_xlog)) - xlog_is_symlink = true; -#endif - - /* - * If possible, hint to the kernel that we're soon going to fsync the data - * directory and its contents. - */ -#ifdef PG_FLUSH_DATA_WORKS - walkdir(pg_data, pre_sync_fname, false); - if (xlog_is_symlink) - walkdir(pg_xlog, pre_sync_fname, false); - walkdir(pg_tblspc, pre_sync_fname, true); -#endif - - /* - * Now we do the fsync()s in the same order. - * - * The main call ignores symlinks, so in addition to specially processing - * pg_xlog if it's a symlink, pg_tblspc has to be visited separately with - * process_symlinks = true. Note that if there are any plain directories - * in pg_tblspc, they'll get fsync'd twice. That's not an expected case - * so we don't worry about optimizing it. - */ - walkdir(pg_data, fsync_fname_ext, false); - if (xlog_is_symlink) - walkdir(pg_xlog, fsync_fname_ext, false); - walkdir(pg_tblspc, fsync_fname_ext, true); - - check_ok(); -} /* @@ -3512,7 +3256,10 @@ main(int argc, char *argv[]) exit_nicely(); } - fsync_pgdata(); + fputs(_("syncing data to disk ... "), stdout); + fflush(stdout); + fsync_pgdata(pg_data, progname); + check_ok(); return 0; } @@ -3574,7 +3321,12 @@ main(int argc, char *argv[]) initialize_data_directory(); if (do_sync) - fsync_pgdata(); + { + fputs(_("syncing data to disk ... "), stdout); + fflush(stdout); + fsync_pgdata(pg_data, progname); + check_ok(); + } else printf(_("\nSync to disk skipped.\nThe data directory might become corrupt if the operating system crashes.\n")); diff --git a/src/bin/initdb/nls.mk b/src/bin/initdb/nls.mk index 0d5368304e..7a12daa6fb 100644 --- a/src/bin/initdb/nls.mk +++ b/src/bin/initdb/nls.mk @@ -1,5 +1,5 @@ # src/bin/initdb/nls.mk CATALOG_NAME = initdb AVAIL_LANGUAGES = cs de es fr it ja ko pl pt_BR ru sv zh_CN -GETTEXT_FILES = findtimezone.c initdb.c ../../common/exec.c ../../common/fe_memutils.c ../../common/pgfnames.c ../../common/restricted_token.c ../../common/rmtree.c ../../common/username.c ../../common/wait_error.c ../../port/dirmod.c +GETTEXT_FILES = findtimezone.c initdb.c ../../common/exec.c ../../common/fe_memutils.c ../../common/file_utils.c ../../common/pgfnames.c ../../common/restricted_token.c ../../common/rmtree.c ../../common/username.c ../../common/wait_error.c ../../port/dirmod.c GETTEXT_TRIGGERS = simple_prompt diff --git a/src/bin/pg_basebackup/nls.mk b/src/bin/pg_basebackup/nls.mk index a34ca3d268..dba43b857e 100644 --- a/src/bin/pg_basebackup/nls.mk +++ b/src/bin/pg_basebackup/nls.mk @@ -1,5 +1,5 @@ # src/bin/pg_basebackup/nls.mk CATALOG_NAME = pg_basebackup AVAIL_LANGUAGES = de es fr it ko pl pt_BR ru zh_CN -GETTEXT_FILES = pg_basebackup.c pg_receivexlog.c pg_recvlogical.c receivelog.c streamutil.c ../../common/fe_memutils.c +GETTEXT_FILES = pg_basebackup.c pg_receivexlog.c pg_recvlogical.c receivelog.c streamutil.c ../../common/fe_memutils.c ../../common/file_utils.c GETTEXT_TRIGGERS = simple_prompt diff --git a/src/common/Makefile b/src/common/Makefile index a5fa649766..03dfaa19c4 100644 --- a/src/common/Makefile +++ b/src/common/Makefile @@ -44,7 +44,7 @@ OBJS_COMMON = config_info.o controldata_utils.o exec.o ip.o keywords.o \ md5.o pg_lzcompress.o pgfnames.o psprintf.o relpath.o rmtree.o \ string.o username.o wait_error.o -OBJS_FRONTEND = $(OBJS_COMMON) fe_memutils.o restricted_token.o +OBJS_FRONTEND = $(OBJS_COMMON) fe_memutils.o file_utils.o restricted_token.o OBJS_SRV = $(OBJS_COMMON:%.o=%_srv.o) diff --git a/src/common/file_utils.c b/src/common/file_utils.c new file mode 100644 index 0000000000..b6f62f7bf1 --- /dev/null +++ b/src/common/file_utils.c @@ -0,0 +1,276 @@ +/*------------------------------------------------------------------------- + * + * File-processing utility routines. + * + * Assorted utility functions to work on files. + * + * + * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/common/file_utils.c + * + *------------------------------------------------------------------------- + */ +#include "postgres_fe.h" + +#include +#include +#include +#include + +#include "common/file_utils.h" + + +/* Define PG_FLUSH_DATA_WORKS if we have an implementation for pg_flush_data */ +#if defined(HAVE_SYNC_FILE_RANGE) +#define PG_FLUSH_DATA_WORKS 1 +#elif defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED) +#define PG_FLUSH_DATA_WORKS 1 +#endif + +#ifdef PG_FLUSH_DATA_WORKS +static void pre_sync_fname(const char *fname, bool isdir, + const char *progname); +#endif +static void walkdir(const char *path, + void (*action) (const char *fname, bool isdir, const char *progname), + bool process_symlinks, const char *progname); + +/* + * Issue fsync recursively on PGDATA and all its contents. + * + * We fsync regular files and directories wherever they are, but we + * follow symlinks only for pg_xlog and immediately under pg_tblspc. + * Other symlinks are presumed to point at files we're not responsible + * for fsyncing, and might not have privileges to write at all. + * + * Errors are reported but not considered fatal. + */ +void +fsync_pgdata(const char *pg_data, const char *progname) +{ + bool xlog_is_symlink; + char pg_xlog[MAXPGPATH]; + char pg_tblspc[MAXPGPATH]; + + snprintf(pg_xlog, MAXPGPATH, "%s/pg_xlog", pg_data); + snprintf(pg_tblspc, MAXPGPATH, "%s/pg_tblspc", pg_data); + + /* + * If pg_xlog is a symlink, we'll need to recurse into it separately, + * because the first walkdir below will ignore it. + */ + xlog_is_symlink = false; + +#ifndef WIN32 + { + struct stat st; + + if (lstat(pg_xlog, &st) < 0) + fprintf(stderr, _("%s: could not stat file \"%s\": %s\n"), + progname, pg_xlog, strerror(errno)); + else if (S_ISLNK(st.st_mode)) + xlog_is_symlink = true; + } +#else + if (pgwin32_is_junction(pg_xlog)) + xlog_is_symlink = true; +#endif + + /* + * If possible, hint to the kernel that we're soon going to fsync the data + * directory and its contents. + */ +#ifdef PG_FLUSH_DATA_WORKS + walkdir(pg_data, pre_sync_fname, false, progname); + if (xlog_is_symlink) + walkdir(pg_xlog, pre_sync_fname, false, progname); + walkdir(pg_tblspc, pre_sync_fname, true, progname); +#endif + + /* + * Now we do the fsync()s in the same order. + * + * The main call ignores symlinks, so in addition to specially processing + * pg_xlog if it's a symlink, pg_tblspc has to be visited separately with + * process_symlinks = true. Note that if there are any plain directories + * in pg_tblspc, they'll get fsync'd twice. That's not an expected case + * so we don't worry about optimizing it. + */ + walkdir(pg_data, fsync_fname, false, progname); + if (xlog_is_symlink) + walkdir(pg_xlog, fsync_fname, false, progname); + walkdir(pg_tblspc, fsync_fname, true, progname); +} + +/* + * walkdir: recursively walk a directory, applying the action to each + * regular file and directory (including the named directory itself). + * + * If process_symlinks is true, the action and recursion are also applied + * to regular files and directories that are pointed to by symlinks in the + * given directory; otherwise symlinks are ignored. Symlinks are always + * ignored in subdirectories, ie we intentionally don't pass down the + * process_symlinks flag to recursive calls. + * + * Errors are reported but not considered fatal. + * + * See also walkdir in fd.c, which is a backend version of this logic. + */ +static void +walkdir(const char *path, + void (*action) (const char *fname, bool isdir, const char *progname), + bool process_symlinks, const char *progname) +{ + DIR *dir; + struct dirent *de; + + dir = opendir(path); + if (dir == NULL) + { + fprintf(stderr, _("%s: could not open directory \"%s\": %s\n"), + progname, path, strerror(errno)); + return; + } + + while (errno = 0, (de = readdir(dir)) != NULL) + { + char subpath[MAXPGPATH]; + struct stat fst; + int sret; + + if (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0) + continue; + + snprintf(subpath, MAXPGPATH, "%s/%s", path, de->d_name); + + if (process_symlinks) + sret = stat(subpath, &fst); + else + sret = lstat(subpath, &fst); + + if (sret < 0) + { + fprintf(stderr, _("%s: could not stat file \"%s\": %s\n"), + progname, subpath, strerror(errno)); + continue; + } + + if (S_ISREG(fst.st_mode)) + (*action) (subpath, false, progname); + else if (S_ISDIR(fst.st_mode)) + walkdir(subpath, action, false, progname); + } + + if (errno) + fprintf(stderr, _("%s: could not read directory \"%s\": %s\n"), + progname, path, strerror(errno)); + + (void) closedir(dir); + + /* + * It's important to fsync the destination directory itself as individual + * file fsyncs don't guarantee that the directory entry for the file is + * synced. Recent versions of ext4 have made the window much wider but + * it's been an issue for ext3 and other filesystems in the past. + */ + (*action) (path, true, progname); +} + +/* + * Hint to the OS that it should get ready to fsync() this file. + * + * Ignores errors trying to open unreadable files, and reports other errors + * non-fatally. + */ +#ifdef PG_FLUSH_DATA_WORKS + +static void +pre_sync_fname(const char *fname, bool isdir, const char *progname) +{ + int fd; + + fd = open(fname, O_RDONLY | PG_BINARY); + + if (fd < 0) + { + if (errno == EACCES || (isdir && errno == EISDIR)) + return; + fprintf(stderr, _("%s: could not open file \"%s\": %s\n"), + progname, fname, strerror(errno)); + return; + } + + /* + * We do what pg_flush_data() would do in the backend: prefer to use + * sync_file_range, but fall back to posix_fadvise. We ignore errors + * because this is only a hint. + */ +#if defined(HAVE_SYNC_FILE_RANGE) + (void) sync_file_range(fd, 0, 0, SYNC_FILE_RANGE_WRITE); +#elif defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED) + (void) posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED); +#else +#error PG_FLUSH_DATA_WORKS should not have been defined +#endif + + (void) close(fd); +} + +#endif /* PG_FLUSH_DATA_WORKS */ + +/* + * fsync_fname -- Try to fsync a file or directory + * + * Ignores errors trying to open unreadable files, or trying to fsync + * directories on systems where that isn't allowed/required. Reports + * other errors non-fatally. + */ +void +fsync_fname(const char *fname, bool isdir, const char *progname) +{ + int fd; + int flags; + int returncode; + + /* + * Some OSs require directories to be opened read-only whereas other + * systems don't allow us to fsync files opened read-only; so we need both + * cases here. Using O_RDWR will cause us to fail to fsync files that are + * not writable by our userid, but we assume that's OK. + */ + flags = PG_BINARY; + if (!isdir) + flags |= O_RDWR; + else + flags |= O_RDONLY; + + /* + * Open the file, silently ignoring errors about unreadable files (or + * unsupported operations, e.g. opening a directory under Windows), and + * logging others. + */ + fd = open(fname, flags); + if (fd < 0) + { + if (errno == EACCES || (isdir && errno == EISDIR)) + return; + fprintf(stderr, _("%s: could not open file \"%s\": %s\n"), + progname, fname, strerror(errno)); + return; + } + + returncode = fsync(fd); + + /* + * Some OSes don't allow us to fsync directories at all, so we can ignore + * those errors. Anything else needs to be reported. + */ + if (returncode != 0 && !(isdir && errno == EBADF)) + fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), + progname, fname, strerror(errno)); + + (void) close(fd); +} diff --git a/src/include/common/file_utils.h b/src/include/common/file_utils.h new file mode 100644 index 0000000000..d3794df7a0 --- /dev/null +++ b/src/include/common/file_utils.h @@ -0,0 +1,22 @@ +/*------------------------------------------------------------------------- + * + * File-processing utility routines for frontend code + * + * Assorted utility functions to work on files. + * + * + * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/common/file_utils.h + * + *------------------------------------------------------------------------- + */ +#ifndef FILE_UTILS_H +#define FILE_UTILS_H + +extern void fsync_fname(const char *fname, bool isdir, + const char *progname); +extern void fsync_pgdata(const char *pg_data, const char *progname); + +#endif /* FILE_UTILS_H */ diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index 07c91d1849..5add0b8a91 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -115,7 +115,7 @@ sub mkvcbuild string.c username.c wait_error.c); our @pgcommonfrontendfiles = ( - @pgcommonallfiles, qw(fe_memutils.c + @pgcommonallfiles, qw(fe_memutils.c file_utils.c restricted_token.c)); our @pgcommonbkndfiles = @pgcommonallfiles;