Merge branch 'ds/sparse-sparse-checkout'

"sparse-checkout" learns to work well with the sparse-index
feature.

* ds/sparse-sparse-checkout:
  sparse-checkout: integrate with sparse index
  p2000: add test for 'git sparse-checkout [add|set]'
  sparse-index: complete partial expansion
  sparse-index: partially expand directories
  sparse-checkout: --no-sparse-index needs a full index
  cache-tree: implement cache_tree_find_path()
  sparse-index: introduce partially-sparse indexes
  sparse-index: create expand_index()
  t1092: stress test 'git sparse-checkout set'
  t1092: refactor 'sparse-index contents' test
This commit is contained in:
Junio C Hamano 2022-06-03 14:30:35 -07:00
commit c276c21da6
10 changed files with 279 additions and 50 deletions

View File

@ -128,7 +128,7 @@ static void clean_tracked_sparse_directories(struct repository *r)
* sparse index will not delete directories that contain
* conflicted entries or submodules.
*/
if (!r->index->sparse_index) {
if (r->index->sparse_index == INDEX_EXPANDED) {
/*
* If something, such as a merge conflict or other concern,
* prevents us from converting to a sparse index, then do
@ -413,6 +413,9 @@ static int update_modes(int *cone_mode, int *sparse_index)
/* force an index rewrite */
repo_read_index(the_repository);
the_repository->index->updated_workdir = 1;
if (!*sparse_index)
ensure_full_index(the_repository->index);
}
return 0;
@ -934,6 +937,9 @@ int cmd_sparse_checkout(int argc, const char **argv, const char *prefix)
git_config(git_default_config, NULL);
prepare_repo_settings(the_repository);
the_repository->settings.command_requires_full_index = 0;
if (argc > 0) {
if (!strcmp(argv[0], "list"))
return sparse_checkout_list(argc, argv);

View File

@ -101,6 +101,33 @@ struct cache_tree_sub *cache_tree_sub(struct cache_tree *it, const char *path)
return find_subtree(it, path, pathlen, 1);
}
struct cache_tree *cache_tree_find_path(struct cache_tree *it, const char *path)
{
const char *slash;
int namelen;
struct cache_tree_sub it_sub = {
.cache_tree = it,
};
struct cache_tree_sub *down = &it_sub;
while (down) {
slash = strchrnul(path, '/');
namelen = slash - path;
down->cache_tree->entry_count = -1;
if (!*slash) {
int pos;
pos = cache_tree_subtree_pos(down->cache_tree, path, namelen);
if (0 <= pos)
return down->cache_tree->down[pos]->cache_tree;
return NULL;
}
down = find_subtree(it, path, namelen, 0);
path = slash + 1;
}
return NULL;
}
static int do_invalidate_path(struct cache_tree *it, const char *path)
{
/* a/b/c

View File

@ -29,6 +29,8 @@ struct cache_tree_sub *cache_tree_sub(struct cache_tree *, const char *);
int cache_tree_subtree_pos(struct cache_tree *it, const char *path, int pathlen);
struct cache_tree *cache_tree_find_path(struct cache_tree *it, const char *path);
void cache_tree_write(struct strbuf *, struct cache_tree *root);
struct cache_tree *cache_tree_read(const char *buffer, unsigned long size);

33
cache.h
View File

@ -310,6 +310,29 @@ struct untracked_cache;
struct progress;
struct pattern_list;
enum sparse_index_mode {
/*
* There are no sparse directories in the index at all.
*
* Repositories that don't use cone-mode sparse-checkout will
* always have their indexes in this mode.
*/
INDEX_EXPANDED = 0,
/*
* The index has already been collapsed to sparse directories
* whereever possible.
*/
INDEX_COLLAPSED,
/*
* The sparse directories that exist are outside the
* sparse-checkout boundary, but it is possible that some file
* entries could collapse to sparse directory entries.
*/
INDEX_PARTIALLY_SPARSE,
};
struct index_state {
struct cache_entry **cache;
unsigned int version;
@ -323,14 +346,8 @@ struct index_state {
drop_cache_tree : 1,
updated_workdir : 1,
updated_skipworktree : 1,
fsmonitor_has_run_once : 1,
/*
* sparse_index == 1 when sparse-directory
* entries exist. Requires sparse-checkout
* in cone mode.
*/
sparse_index : 1;
fsmonitor_has_run_once : 1;
enum sparse_index_mode sparse_index;
struct hashmap name_hash;
struct hashmap dir_hash;
struct object_id oid;

View File

@ -112,7 +112,7 @@ static const char *alternate_index_output;
static void set_index_entry(struct index_state *istate, int nr, struct cache_entry *ce)
{
if (S_ISSPARSEDIR(ce->ce_mode))
istate->sparse_index = 1;
istate->sparse_index = INDEX_COLLAPSED;
istate->cache[nr] = ce;
add_name_hash(istate, ce);
@ -1856,7 +1856,7 @@ static int read_index_extension(struct index_state *istate,
break;
case CACHE_EXT_SPARSE_DIRECTORIES:
/* no content, only an indicator */
istate->sparse_index = 1;
istate->sparse_index = INDEX_COLLAPSED;
break;
default:
if (*ext < 'A' || 'Z' < *ext)
@ -3165,7 +3165,7 @@ static int do_write_locked_index(struct index_state *istate, struct lock_file *l
unsigned flags)
{
int ret;
int was_full = !istate->sparse_index;
int was_full = istate->sparse_index == INDEX_EXPANDED;
ret = convert_to_sparse(istate, 0);

View File

@ -9,6 +9,11 @@
#include "dir.h"
#include "fsmonitor.h"
struct modify_index_context {
struct index_state *write;
struct pattern_list *pl;
};
static struct cache_entry *construct_sparse_dir_entry(
struct index_state *istate,
const char *sparse_dir,
@ -173,7 +178,7 @@ int convert_to_sparse(struct index_state *istate, int flags)
* If the index is already sparse, empty, or otherwise
* cannot be converted to sparse, do not convert.
*/
if (istate->sparse_index || !istate->cache_nr ||
if (istate->sparse_index == INDEX_COLLAPSED || !istate->cache_nr ||
!is_sparse_index_allowed(istate, flags))
return 0;
@ -214,7 +219,7 @@ int convert_to_sparse(struct index_state *istate, int flags)
FREE_AND_NULL(istate->fsmonitor_dirty);
FREE_AND_NULL(istate->fsmonitor_last_update);
istate->sparse_index = 1;
istate->sparse_index = INDEX_COLLAPSED;
trace2_region_leave("index", "convert_to_sparse", istate->repo);
return 0;
}
@ -231,56 +236,148 @@ static int add_path_to_index(const struct object_id *oid,
struct strbuf *base, const char *path,
unsigned int mode, void *context)
{
struct index_state *istate = (struct index_state *)context;
struct modify_index_context *ctx = (struct modify_index_context *)context;
struct cache_entry *ce;
size_t len = base->len;
if (S_ISDIR(mode))
return READ_TREE_RECURSIVE;
if (S_ISDIR(mode)) {
int dtype;
size_t baselen = base->len;
if (!ctx->pl)
return READ_TREE_RECURSIVE;
strbuf_addstr(base, path);
/*
* Have we expanded to a point outside of the sparse-checkout?
*
* Artificially pad the path name with a slash "/" to
* indicate it as a directory, and add an arbitrary file
* name ("-") so we can consider base->buf as a file name
* to match against the cone-mode patterns.
*
* If we compared just "path", then we would expand more
* than we should. Since every file at root is always
* included, we would expand every directory at root at
* least one level deep instead of using sparse directory
* entries.
*/
strbuf_addstr(base, path);
strbuf_add(base, "/-", 2);
ce = make_cache_entry(istate, mode, oid, base->buf, 0, 0);
if (path_matches_pattern_list(base->buf, base->len,
NULL, &dtype,
ctx->pl, ctx->write)) {
strbuf_setlen(base, baselen);
return READ_TREE_RECURSIVE;
}
/*
* The path "{base}{path}/" is a sparse directory. Create the correct
* name for inserting the entry into the index.
*/
strbuf_setlen(base, base->len - 1);
} else {
strbuf_addstr(base, path);
}
ce = make_cache_entry(ctx->write, mode, oid, base->buf, 0, 0);
ce->ce_flags |= CE_SKIP_WORKTREE | CE_EXTENDED;
set_index_entry(istate, istate->cache_nr++, ce);
set_index_entry(ctx->write, ctx->write->cache_nr++, ce);
strbuf_setlen(base, len);
return 0;
}
void ensure_full_index(struct index_state *istate)
void expand_index(struct index_state *istate, struct pattern_list *pl)
{
int i;
struct index_state *full;
struct strbuf base = STRBUF_INIT;
const char *tr_region;
struct modify_index_context ctx;
if (!istate || !istate->sparse_index)
/*
* If the index is already full, then keep it full. We will convert
* it to a sparse index on write, if possible.
*/
if (!istate || istate->sparse_index == INDEX_EXPANDED)
return;
/*
* If our index is sparse, but our new pattern set does not use
* cone mode patterns, then we need to expand the index before we
* continue. A NULL pattern set indicates a full expansion to a
* full index.
*/
if (pl && !pl->use_cone_patterns) {
pl = NULL;
} else {
/*
* We might contract file entries into sparse-directory
* entries, and for that we will need the cache tree to
* be recomputed.
*/
cache_tree_free(&istate->cache_tree);
/*
* If there is a problem creating the cache tree, then we
* need to expand to a full index since we cannot satisfy
* the current request as a sparse index.
*/
if (cache_tree_update(istate, 0))
pl = NULL;
}
if (!istate->repo)
istate->repo = the_repository;
trace2_region_enter("index", "ensure_full_index", istate->repo);
/*
* A NULL pattern set indicates we are expanding a full index, so
* we use a special region name that indicates the full expansion.
* This is used by test cases, but also helps to differentiate the
* two cases.
*/
tr_region = pl ? "expand_index" : "ensure_full_index";
trace2_region_enter("index", tr_region, istate->repo);
/* initialize basics of new index */
full = xcalloc(1, sizeof(struct index_state));
memcpy(full, istate, sizeof(struct index_state));
/*
* This slightly-misnamed 'full' index might still be sparse if we
* are only modifying the list of sparse directories. This hinges
* on whether we have a non-NULL pattern list.
*/
full->sparse_index = pl ? INDEX_PARTIALLY_SPARSE : INDEX_EXPANDED;
/* then change the necessary things */
full->sparse_index = 0;
full->cache_alloc = (3 * istate->cache_alloc) / 2;
full->cache_nr = 0;
ALLOC_ARRAY(full->cache, full->cache_alloc);
ctx.write = full;
ctx.pl = pl;
for (i = 0; i < istate->cache_nr; i++) {
struct cache_entry *ce = istate->cache[i];
struct tree *tree;
struct pathspec ps;
int dtype;
if (!S_ISSPARSEDIR(ce->ce_mode)) {
set_index_entry(full, full->cache_nr++, ce);
continue;
}
/* We now have a sparse directory entry. Should we expand? */
if (pl &&
path_matches_pattern_list(ce->name, ce->ce_namelen,
NULL, &dtype,
pl, istate) == NOT_MATCHED) {
set_index_entry(full, full->cache_nr++, ce);
continue;
}
if (!(ce->ce_flags & CE_SKIP_WORKTREE))
warning(_("index entry is a directory, but not sparse (%08x)"),
ce->ce_flags);
@ -297,7 +394,7 @@ void ensure_full_index(struct index_state *istate)
strbuf_add(&base, ce->name, strlen(ce->name));
read_tree_at(istate->repo, tree, &base, &ps,
add_path_to_index, full);
add_path_to_index, &ctx);
/* free directory entries. full entries are re-used */
discard_cache_entry(ce);
@ -306,7 +403,7 @@ void ensure_full_index(struct index_state *istate)
/* Copy back into original index. */
memcpy(&istate->name_hash, &full->name_hash, sizeof(full->name_hash));
memcpy(&istate->dir_hash, &full->dir_hash, sizeof(full->dir_hash));
istate->sparse_index = 0;
istate->sparse_index = pl ? INDEX_PARTIALLY_SPARSE : INDEX_EXPANDED;
free(istate->cache);
istate->cache = full->cache;
istate->cache_nr = full->cache_nr;
@ -322,7 +419,12 @@ void ensure_full_index(struct index_state *istate)
cache_tree_free(&istate->cache_tree);
cache_tree_update(istate, 0);
trace2_region_leave("index", "ensure_full_index", istate->repo);
trace2_region_leave("index", tr_region, istate->repo);
}
void ensure_full_index(struct index_state *istate)
{
expand_index(istate, NULL);
}
void ensure_correct_sparsity(struct index_state *istate)

View File

@ -24,4 +24,17 @@ void expand_to_path(struct index_state *istate,
struct repository;
int set_sparse_index_config(struct repository *repo, int enable);
struct pattern_list;
/**
* Scan the given index and compare its entries to the given pattern list.
* If the index is sparse and the pattern list uses cone mode patterns,
* then modify the index to contain the all of the file entries within that
* new pattern list. This expands sparse directories only as far as needed.
*
* If the pattern list is NULL or does not use cone mode patterns, then the
* index is expanded to a full index.
*/
void expand_index(struct index_state *istate, struct pattern_list *pl);
#endif

View File

@ -112,6 +112,7 @@ test_perf_on_all git add -A
test_perf_on_all git add .
test_perf_on_all git commit -a -m A
test_perf_on_all git checkout -f -
test_perf_on_all "git sparse-checkout add f2/f3/f1 && git sparse-checkout set $SPARSE_CONE"
test_perf_on_all git reset
test_perf_on_all git reset --hard
test_perf_on_all git reset -- does-not-exist

View File

@ -205,36 +205,68 @@ test_sparse_unstaged () {
done
}
# Usage: test_sprase_checkout_set "<c1> ... <cN>" "<s1> ... <sM>"
# Verifies that "git sparse-checkout set <c1> ... <cN>" succeeds and
# leaves the sparse index in a state where <s1> ... <sM> are sparse
# directories (and <c1> ... <cN> are not).
test_sparse_checkout_set () {
CONE_DIRS=$1 &&
SPARSE_DIRS=$2 &&
git -C sparse-index sparse-checkout set --skip-checks $CONE_DIRS &&
git -C sparse-index ls-files --sparse --stage >cache &&
# Check that the directories outside of the sparse-checkout cone
# have sparse directory entries.
for dir in $SPARSE_DIRS
do
TREE=$(git -C sparse-index rev-parse HEAD:$dir) &&
grep "040000 $TREE 0 $dir/" cache \
|| return 1
done &&
# Check that the directories in the sparse-checkout cone
# are not sparse directory entries.
for dir in $CONE_DIRS
do
# Allow TREE to not exist because
# $dir does not exist at HEAD.
TREE=$(git -C sparse-index rev-parse HEAD:$dir) ||
! grep "040000 $TREE 0 $dir/" cache \
|| return 1
done
}
test_expect_success 'sparse-index contents' '
init_repos &&
git -C sparse-index ls-files --sparse --stage >cache &&
for dir in folder1 folder2 x
do
TREE=$(git -C sparse-index rev-parse HEAD:$dir) &&
grep "040000 $TREE 0 $dir/" cache \
|| return 1
done &&
# Remove deep, add three other directories.
test_sparse_checkout_set \
"folder1 folder2 x" \
"before deep" &&
git -C sparse-index sparse-checkout set folder1 &&
# Remove folder1, add deep
test_sparse_checkout_set \
"deep folder2 x" \
"before folder1" &&
git -C sparse-index ls-files --sparse --stage >cache &&
for dir in deep folder2 x
do
TREE=$(git -C sparse-index rev-parse HEAD:$dir) &&
grep "040000 $TREE 0 $dir/" cache \
|| return 1
done &&
# Replace deep with deep/deeper2 (dropping deep/deeper1)
# Add folder1
test_sparse_checkout_set \
"deep/deeper2 folder1 folder2 x" \
"before deep/deeper1" &&
git -C sparse-index sparse-checkout set deep/deeper1 &&
# Replace deep/deeper2 with deep/deeper1
# Replace folder1 with folder1/0/0
# Replace folder2 with non-existent folder2/2/3
# Add non-existent "bogus"
test_sparse_checkout_set \
"bogus deep/deeper1 folder1/0/0 folder2/2/3 x" \
"before deep/deeper2 folder2/0" &&
git -C sparse-index ls-files --sparse --stage >cache &&
for dir in deep/deeper2 folder1 folder2 x
do
TREE=$(git -C sparse-index rev-parse HEAD:$dir) &&
grep "040000 $TREE 0 $dir/" cache \
|| return 1
done &&
# Drop down to only files at root
test_sparse_checkout_set \
"" \
"before deep folder1 folder2 x" &&
# Disabling the sparse-index replaces tree entries with full ones
git -C sparse-index sparse-checkout init --no-sparse-index &&
@ -1628,6 +1660,31 @@ test_expect_success 'ls-files' '
ensure_not_expanded ls-files --sparse
'
test_expect_success 'sparse index is not expanded: sparse-checkout' '
init_repos &&
ensure_not_expanded sparse-checkout set deep/deeper2 &&
ensure_not_expanded sparse-checkout set deep/deeper1 &&
ensure_not_expanded sparse-checkout set deep &&
ensure_not_expanded sparse-checkout add folder1 &&
ensure_not_expanded sparse-checkout set deep/deeper1 &&
ensure_not_expanded sparse-checkout set folder2 &&
# Demonstrate that the checks that "folder1/a" is a file
# do not cause a sparse-index expansion (since it is in the
# sparse-checkout cone).
echo >>sparse-index/folder2/a &&
git -C sparse-index add folder2/a &&
ensure_not_expanded sparse-checkout add folder1 &&
# Skip checks here, since deep/deeper1 is inside a sparse directory
# that must be expanded to check whether `deep/deeper1` is a file
# or not.
ensure_not_expanded sparse-checkout set --skip-checks deep/deeper1 &&
ensure_not_expanded sparse-checkout set
'
# NEEDSWORK: a sparse-checkout behaves differently from a full checkout
# in this scenario, but it shouldn't.
test_expect_success 'reset mixed and checkout orphan' '

View File

@ -19,6 +19,7 @@
#include "promisor-remote.h"
#include "entry.h"
#include "parallel-checkout.h"
#include "sparse-index.h"
/*
* Error messages expected by scripts out of plumbing commands such as
@ -2024,6 +2025,9 @@ enum update_sparsity_result update_sparsity(struct unpack_trees_options *o)
goto skip_sparse_checkout;
}
/* Expand sparse directories as needed */
expand_index(o->src_index, o->pl);
/* Set NEW_SKIP_WORKTREE on existing entries. */
mark_all_ce_unused(o->src_index);
mark_new_skip_worktree(o->pl, o->src_index, 0,