Skip unnecessary stat() calls in walkdir().

Some kernels can tell us the type of a "dirent", so we can avoid a call
to stat() or lstat() in many cases.  Define a new function
get_dirent_type() to contain that logic, for use by the backend and
frontend versions of walkdir(), and perhaps other callers in future.

Reviewed-by: Tom Lane <tgl@sss.pgh.pa.us>
Reviewed-by: Juan José Santamaría Flecha <juanjo.santamaria@gmail.com>
Discussion: https://postgr.es/m/CA%2BhUKG%2BFzxupGGN4GpUdbzZN%2Btn6FQPHo8w0Q%2BAPH5Wz8RG%2Bww%40mail.gmail.com
This commit is contained in:
Thomas Munro 2020-09-07 18:11:46 +12:00
parent f0942b1327
commit 861c6e7c8e
6 changed files with 129 additions and 40 deletions

View File

@ -89,6 +89,7 @@
#include "access/xlog.h"
#include "catalog/pg_tablespace.h"
#include "common/file_perm.h"
#include "common/file_utils.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "portability/mem.h"
@ -3340,8 +3341,6 @@ walkdir(const char *path,
while ((de = ReadDirExtended(dir, path, elevel)) != NULL)
{
char subpath[MAXPGPATH * 2];
struct stat fst;
int sret;
CHECK_FOR_INTERRUPTS();
@ -3351,23 +3350,23 @@ walkdir(const char *path,
snprintf(subpath, sizeof(subpath), "%s/%s", path, de->d_name);
if (process_symlinks)
sret = stat(subpath, &fst);
else
sret = lstat(subpath, &fst);
if (sret < 0)
switch (get_dirent_type(subpath, de, process_symlinks, elevel))
{
ereport(elevel,
(errcode_for_file_access(),
errmsg("could not stat file \"%s\": %m", subpath)));
continue;
}
case PGFILETYPE_REG:
(*action) (subpath, false, elevel);
break;
case PGFILETYPE_DIR:
walkdir(subpath, action, false, elevel);
break;
default:
if (S_ISREG(fst.st_mode))
(*action) (subpath, false, elevel);
else if (S_ISDIR(fst.st_mode))
walkdir(subpath, action, false, elevel);
/*
* Errors are already reported directly by get_dirent_type(),
* and any remaining symlinks and unknown file types are
* ignored.
*/
break;
}
}
FreeDir(dir); /* we ignore any error here */

View File

@ -56,6 +56,7 @@ OBJS_COMMON = \
exec.o \
f2s.o \
file_perm.o \
file_utils.o \
hashfn.o \
ip.o \
jsonapi.o \
@ -91,7 +92,6 @@ endif
OBJS_FRONTEND = \
$(OBJS_COMMON) \
fe_memutils.o \
file_utils.o \
logging.o \
restricted_token.o \
sprompt.o

View File

@ -14,10 +14,10 @@
*/
#ifndef FRONTEND
#error "This file is not expected to be compiled for backend code"
#endif
#include "postgres.h"
#else
#include "postgres_fe.h"
#endif
#include <dirent.h>
#include <fcntl.h>
@ -25,8 +25,11 @@
#include <unistd.h>
#include "common/file_utils.h"
#ifdef FRONTEND
#include "common/logging.h"
#endif
#ifdef FRONTEND
/* Define PG_FLUSH_DATA_WORKS if we have an implementation for pg_flush_data */
#if defined(HAVE_SYNC_FILE_RANGE)
@ -167,8 +170,6 @@ walkdir(const char *path,
while (errno = 0, (de = readdir(dir)) != NULL)
{
char subpath[MAXPGPATH * 2];
struct stat fst;
int sret;
if (strcmp(de->d_name, ".") == 0 ||
strcmp(de->d_name, "..") == 0)
@ -176,21 +177,23 @@ walkdir(const char *path,
snprintf(subpath, sizeof(subpath), "%s/%s", path, de->d_name);
if (process_symlinks)
sret = stat(subpath, &fst);
else
sret = lstat(subpath, &fst);
if (sret < 0)
switch (get_dirent_type(subpath, de, process_symlinks, PG_LOG_ERROR))
{
pg_log_error("could not stat file \"%s\": %m", subpath);
continue;
}
case PGFILETYPE_REG:
(*action) (subpath, false);
break;
case PGFILETYPE_DIR:
walkdir(subpath, action, false);
break;
default:
if (S_ISREG(fst.st_mode))
(*action) (subpath, false);
else if (S_ISDIR(fst.st_mode))
walkdir(subpath, action, false);
/*
* Errors are already reported directly by get_dirent_type(),
* and any remaining symlinks and unknown file types are
* ignored.
*/
break;
}
}
if (errno)
@ -394,3 +397,73 @@ durable_rename(const char *oldfile, const char *newfile)
return 0;
}
#endif /* FRONTEND */
/*
* Return the type of a directory entry.
*
* In frontend code, elevel should be a level from logging.h; in backend code
* it should be a level from elog.h.
*/
PGFileType
get_dirent_type(const char *path,
const struct dirent *de,
bool look_through_symlinks,
int elevel)
{
PGFileType result;
/*
* Some systems tell us the type directly in the dirent struct, but that's
* a BSD and Linux extension not required by POSIX. Even when the
* interface is present, sometimes the type is unknown, depending on the
* filesystem.
*/
#if defined(DT_REG) && defined(DT_DIR) && defined(DT_LNK)
if (de->d_type == DT_REG)
result = PGFILETYPE_REG;
else if (de->d_type == DT_DIR)
result = PGFILETYPE_DIR;
else if (de->d_type == DT_LNK && !look_through_symlinks)
result = PGFILETYPE_LNK;
else
result = PGFILETYPE_UNKNOWN;
#else
result = PGFILETYPE_UNKNOWN;
#endif
if (result == PGFILETYPE_UNKNOWN)
{
struct stat fst;
int sret;
if (look_through_symlinks)
sret = stat(path, &fst);
else
sret = lstat(path, &fst);
if (sret < 0)
{
result = PGFILETYPE_ERROR;
#ifdef FRONTEND
pg_log_generic(elevel, "could not stat file \"%s\": %m", path);
#else
ereport(elevel,
(errcode_for_file_access(),
errmsg("could not stat file \"%s\": %m", path)));
#endif
}
else if (S_ISREG(fst.st_mode))
result = PGFILETYPE_REG;
else if (S_ISDIR(fst.st_mode))
result = PGFILETYPE_DIR;
#ifdef S_ISLNK
else if (S_ISLNK(fst.st_mode))
result = PGFILETYPE_LNK;
#endif
}
return result;
}

View File

@ -1,6 +1,4 @@
/*-------------------------------------------------------------------------
*
* File-processing utility routines for frontend code
*
* Assorted utility functions to work on files.
*
@ -15,10 +13,28 @@
#ifndef FILE_UTILS_H
#define FILE_UTILS_H
#include <dirent.h>
typedef enum PGFileType
{
PGFILETYPE_ERROR,
PGFILETYPE_UNKNOWN,
PGFILETYPE_REG,
PGFILETYPE_DIR,
PGFILETYPE_LNK
} PGFileType;
#ifdef FRONTEND
extern int fsync_fname(const char *fname, bool isdir);
extern void fsync_pgdata(const char *pg_data, int serverVersion);
extern void fsync_dir_recurse(const char *dir);
extern int durable_rename(const char *oldfile, const char *newfile);
extern int fsync_parent_path(const char *fname);
#endif
extern PGFileType get_dirent_type(const char *path,
const struct dirent *de,
bool look_through_symlinks,
int elevel);
#endif /* FILE_UTILS_H */

View File

@ -121,7 +121,7 @@ sub mkvcbuild
our @pgcommonallfiles = qw(
archive.c base64.c checksum_helper.c
config_info.c controldata_utils.c d2s.c encnames.c exec.c
f2s.c file_perm.c hashfn.c ip.c jsonapi.c
f2s.c file_perm.c file_utils.c hashfn.c ip.c jsonapi.c
keywords.c kwlookup.c link-canary.c md5.c
pg_get_line.c pg_lzcompress.c pgfnames.c psprintf.c relpath.c rmtree.c
saslprep.c scram-common.c string.c stringinfo.c unicode_norm.c username.c
@ -138,7 +138,7 @@ sub mkvcbuild
}
our @pgcommonfrontendfiles = (
@pgcommonallfiles, qw(fe_memutils.c file_utils.c
@pgcommonallfiles, qw(fe_memutils.c
logging.c restricted_token.c sprompt.c));
our @pgcommonbkndfiles = @pgcommonallfiles;

View File

@ -1515,6 +1515,7 @@ PGEventResultCopy
PGEventResultCreate
PGEventResultDestroy
PGFInfoFunction
PGFileType
PGFunction
PGLZ_HistEntry
PGLZ_Strategy