tests: disable fsync everywhere

The "GIT_TEST_FSYNC" environment variable now exists for
disabling fsync() even on packfiles and other "critical" data.

Running "make test -j8 NO_SVN_TESTS=1" on a noisy 8-core system
on an HDD, test runtime drops from ~4 minutes down to ~3 minutes.
Using "GIT_TEST_FSYNC=1" re-enables fsync() for comparison
purposes.

SVN interopability tests are minimally affected since SVN will
still use fsync in various places.

This will also be useful for 3rd-party tools which create
throwaway git repositories of temporary data, but remains
undocumented for end users.

Signed-off-by: Eric Wong <e@80x24.org>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
This commit is contained in:
Eric Wong 2021-10-29 00:15:52 +00:00 committed by Junio C Hamano
parent af6d1d602a
commit 412e4caee3
6 changed files with 48 additions and 2 deletions

View File

@ -985,6 +985,7 @@ extern int read_replace_refs;
extern char *git_replace_ref_base; extern char *git_replace_ref_base;
extern int fsync_object_files; extern int fsync_object_files;
extern int use_fsync;
extern int core_preload_index; extern int core_preload_index;
extern int precomposed_unicode; extern int precomposed_unicode;
extern int protect_hfs; extern int protect_hfs;

View File

@ -43,6 +43,7 @@ const char *git_hooks_path;
int zlib_compression_level = Z_BEST_SPEED; int zlib_compression_level = Z_BEST_SPEED;
int pack_compression_level = Z_DEFAULT_COMPRESSION; int pack_compression_level = Z_DEFAULT_COMPRESSION;
int fsync_object_files; int fsync_object_files;
int use_fsync = -1;
size_t packed_git_window_size = DEFAULT_PACKED_GIT_WINDOW_SIZE; size_t packed_git_window_size = DEFAULT_PACKED_GIT_WINDOW_SIZE;
size_t packed_git_limit = DEFAULT_PACKED_GIT_LIMIT; size_t packed_git_limit = DEFAULT_PACKED_GIT_LIMIT;
size_t delta_base_cache_limit = 96 * 1024 * 1024; size_t delta_base_cache_limit = 96 * 1024 * 1024;

View File

@ -3607,6 +3607,22 @@ package GITCVS::updater;
use strict; use strict;
use warnings; use warnings;
use DBI; use DBI;
our $_use_fsync;
# n.b. consider using Git.pm
sub use_fsync {
if (!defined($_use_fsync)) {
my $x = $ENV{GIT_TEST_FSYNC};
if (defined $x) {
local $ENV{GIT_CONFIG};
delete $ENV{GIT_CONFIG};
my $v = ::safe_pipe_capture('git', '-c', "test.fsync=$x",
qw(config --type=bool test.fsync));
$_use_fsync = defined($v) ? ($v eq "true\n") : 1;
}
}
$_use_fsync;
}
=head1 METHODS =head1 METHODS
@ -3676,6 +3692,9 @@ sub new
$self->{dbuser}, $self->{dbuser},
$self->{dbpass}); $self->{dbpass});
die "Error connecting to database\n" unless defined $self->{dbh}; die "Error connecting to database\n" unless defined $self->{dbh};
if ($self->{dbdriver} eq 'SQLite' && !use_fsync()) {
$self->{dbh}->do('PRAGMA synchronous = OFF');
}
$self->{tables} = {}; $self->{tables} = {};
foreach my $table ( keys %{$self->{dbh}->table_info(undef,undef,undef,'TABLE')->fetchall_hashref('TABLE_NAME')} ) foreach my $table ( keys %{$self->{dbh}->table_info(undef,undef,undef,'TABLE')->fetchall_hashref('TABLE_NAME')} )

View File

@ -6,7 +6,7 @@ use constant rev_map_fmt => 'NH*';
use vars qw/$_no_metadata use vars qw/$_no_metadata
$_repack $_repack_flags $_use_svm_props $_head $_repack $_repack_flags $_use_svm_props $_head
$_use_svnsync_props $no_reuse_existing $_use_svnsync_props $no_reuse_existing
$_use_log_author $_add_author_from $_localtime/; $_use_log_author $_add_author_from $_localtime $_use_fsync/;
use Carp qw/croak/; use Carp qw/croak/;
use File::Path qw/mkpath/; use File::Path qw/mkpath/;
use IPC::Open3; use IPC::Open3;
@ -2269,6 +2269,19 @@ sub mkfile {
} }
} }
# TODO: move this to Git.pm?
sub use_fsync {
if (!defined($_use_fsync)) {
my $x = $ENV{GIT_TEST_FSYNC};
if (defined $x) {
my $v = command_oneline('-c', "test.fsync=$x",
qw(config --type=bool test.fsync));
$_use_fsync = defined($v) ? ($v eq "true\n") : 1;
}
}
$_use_fsync;
}
sub rev_map_set { sub rev_map_set {
my ($self, $rev, $commit, $update_ref, $uuid) = @_; my ($self, $rev, $commit, $update_ref, $uuid) = @_;
defined $commit or die "missing arg3\n"; defined $commit or die "missing arg3\n";
@ -2290,7 +2303,7 @@ sub rev_map_set {
my $sync; my $sync;
# both of these options make our .rev_db file very, very important # both of these options make our .rev_db file very, very important
# and we can't afford to lose it because rebuild() won't work # and we can't afford to lose it because rebuild() won't work
if ($self->use_svm_props || $self->no_metadata) { if (($self->use_svm_props || $self->no_metadata) && use_fsync()) {
require File::Copy; require File::Copy;
$sync = 1; $sync = 1;
File::Copy::copy($db, $db_lock) or die "rev_map_set(@_): ", File::Copy::copy($db, $db_lock) or die "rev_map_set(@_): ",

View File

@ -489,6 +489,13 @@ then
export GIT_PERL_FATAL_WARNINGS export GIT_PERL_FATAL_WARNINGS
fi fi
case $GIT_TEST_FSYNC in
'')
GIT_TEST_FSYNC=0
export GIT_TEST_FSYNC
;;
esac
# Add libc MALLOC and MALLOC_PERTURB test # Add libc MALLOC and MALLOC_PERTURB test
# only if we are not executing the test with valgrind # only if we are not executing the test with valgrind
if test -n "$valgrind" || if test -n "$valgrind" ||

View File

@ -1,4 +1,5 @@
#include "cache.h" #include "cache.h"
#include "config.h"
#include "run-command.h" #include "run-command.h"
/* /*
@ -57,6 +58,10 @@ void fprintf_or_die(FILE *f, const char *fmt, ...)
void fsync_or_die(int fd, const char *msg) void fsync_or_die(int fd, const char *msg)
{ {
if (use_fsync < 0)
use_fsync = git_env_bool("GIT_TEST_FSYNC", 1);
if (!use_fsync)
return;
while (fsync(fd) < 0) { while (fsync(fd) < 0) {
if (errno != EINTR) if (errno != EINTR)
die_errno("fsync error on '%s'", msg); die_errno("fsync error on '%s'", msg);