Merge pull request #839 from littlefs-project/configurable-disk-version

Add support for writing previous on-disk minor versions
This commit is contained in:
Christopher Haster 2023-06-30 00:26:29 -05:00 committed by GitHub
commit 5db368c0a2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 254 additions and 63 deletions

View File

@ -102,7 +102,7 @@ jobs:
# sizes table
i=0
j=0
for c in "" readonly threadsafe migrate error-asserts
for c in "" readonly threadsafe multiversion migrate error-asserts
do
# per-config results
c_or_default=${c:-default}

View File

@ -170,6 +170,27 @@ jobs:
cp lfs.data.csv sizes/${{matrix.arch}}-threadsafe.data.csv
cp lfs.stack.csv sizes/${{matrix.arch}}-threadsafe.stack.csv
cp lfs.structs.csv sizes/${{matrix.arch}}-threadsafe.structs.csv
- name: sizes-multiversion
run: |
make clean
CFLAGS="$CFLAGS \
-DLFS_NO_ASSERT \
-DLFS_NO_DEBUG \
-DLFS_NO_WARN \
-DLFS_NO_ERROR \
-DLFS_MULTIVERSION" \
make lfs.code.csv lfs.data.csv lfs.stack.csv lfs.structs.csv
./scripts/structs.py -u lfs.structs.csv
./scripts/summary.py lfs.code.csv lfs.data.csv lfs.stack.csv \
-bfunction \
-fcode=code_size \
-fdata=data_size \
-fstack=stack_limit --max=stack_limit
mkdir -p sizes
cp lfs.code.csv sizes/${{matrix.arch}}-multiversion.code.csv
cp lfs.data.csv sizes/${{matrix.arch}}-multiversion.data.csv
cp lfs.stack.csv sizes/${{matrix.arch}}-multiversion.stack.csv
cp lfs.structs.csv sizes/${{matrix.arch}}-multiversion.structs.csv
- name: sizes-migrate
run: |
make clean
@ -353,6 +374,42 @@ jobs:
run: |
CFLAGS="$CFLAGS -DLFS_NO_INTRINSICS" make test
# run LFS_MULTIVERSION tests
test-multiversion:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v2
- name: install
run: |
# need a few things
sudo apt-get update -qq
sudo apt-get install -qq gcc python3 python3-pip
pip3 install toml
gcc --version
python3 --version
- name: test-multiversion
run: |
CFLAGS="$CFLAGS -DLFS_MULTIVERSION" make test
# run tests on the older version lfs2.0
test-lfs2_0:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v2
- name: install
run: |
# need a few things
sudo apt-get update -qq
sudo apt-get install -qq gcc python3 python3-pip
pip3 install toml
gcc --version
python3 --version
- name: test-lfs2_0
run: |
CFLAGS="$CFLAGS -DLFS_MULTIVERSION" \
TESTFLAGS="$TESTFLAGS -DDISK_VERSION=0x00020000" \
make test
# run under Valgrind to check for memory errors
test-valgrind:
runs-on: ubuntu-22.04
@ -685,7 +742,7 @@ jobs:
# sizes table
i=0
j=0
for c in "" readonly threadsafe migrate error-asserts
for c in "" readonly threadsafe multiversion migrate error-asserts
do
# per-config results
c_or_default=${c:-default}

144
lfs.c
View File

@ -518,6 +518,28 @@ static void lfs_mlist_append(lfs_t *lfs, struct lfs_mlist *mlist) {
lfs->mlist = mlist;
}
// some other filesystem operations
static uint32_t lfs_fs_disk_version(lfs_t *lfs) {
(void)lfs;
#ifdef LFS_MULTIVERSION
if (lfs->cfg->disk_version) {
return lfs->cfg->disk_version;
} else
#endif
{
return LFS_DISK_VERSION;
}
}
static uint16_t lfs_fs_disk_version_major(lfs_t *lfs) {
return 0xffff & (lfs_fs_disk_version(lfs) >> 16);
}
static uint16_t lfs_fs_disk_version_minor(lfs_t *lfs) {
return 0xffff & (lfs_fs_disk_version(lfs) >> 0);
}
/// Internal operations predeclared here ///
#ifndef LFS_READONLY
@ -1111,7 +1133,8 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
// next commit not yet programmed?
if (!lfs_tag_isvalid(tag)) {
maybeerased = true;
// we only might be erased if the last tag was a crc
maybeerased = (lfs_tag_type2(ptag) == LFS_TYPE_CCRC);
break;
// out of range?
} else if (off + lfs_tag_dsize(tag) > lfs->cfg->block_size) {
@ -1156,14 +1179,11 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
dir->tail[1] = temptail[1];
dir->split = tempsplit;
// reset crc
// reset crc, hasfcrc
crc = 0xffffffff;
continue;
}
// fcrc is only valid when last tag was a crc
hasfcrc = false;
// crc the entry first, hopefully leaving it in the cache
err = lfs_bd_crc(lfs,
NULL, &lfs->rcache, lfs->cfg->block_size,
@ -1257,20 +1277,33 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
// did we end on a valid commit? we may have an erased block
dir->erased = false;
if (maybeerased && hasfcrc && dir->off % lfs->cfg->prog_size == 0) {
// check for an fcrc matching the next prog's erased state, if
// this failed most likely a previous prog was interrupted, we
// need a new erase
uint32_t fcrc_ = 0xffffffff;
int err = lfs_bd_crc(lfs,
NULL, &lfs->rcache, lfs->cfg->block_size,
dir->pair[0], dir->off, fcrc.size, &fcrc_);
if (err && err != LFS_ERR_CORRUPT) {
return err;
}
if (maybeerased && dir->off % lfs->cfg->prog_size == 0) {
#ifdef LFS_MULTIVERSION
// note versions < lfs2.1 did not have fcrc tags, if
// we're < lfs2.1 treat missing fcrc as erased data
//
// we don't strictly need to do this, but otherwise writing
// to lfs2.0 disks becomes very inefficient
if (lfs_fs_disk_version(lfs) < 0x00020001) {
dir->erased = true;
// found beginning of erased part?
dir->erased = (fcrc_ == fcrc.crc);
} else
#endif
if (hasfcrc) {
// check for an fcrc matching the next prog's erased state, if
// this failed most likely a previous prog was interrupted, we
// need a new erase
uint32_t fcrc_ = 0xffffffff;
int err = lfs_bd_crc(lfs,
NULL, &lfs->rcache, lfs->cfg->block_size,
dir->pair[0], dir->off, fcrc.size, &fcrc_);
if (err && err != LFS_ERR_CORRUPT) {
return err;
}
// found beginning of erased part?
dir->erased = (fcrc_ == fcrc.crc);
}
}
// synthetic move
@ -1606,22 +1639,34 @@ static int lfs_dir_commitcrc(lfs_t *lfs, struct lfs_commit *commit) {
return err;
}
// find the expected fcrc, don't bother avoiding a reread
// of the eperturb, it should still be in our cache
struct lfs_fcrc fcrc = {.size=lfs->cfg->prog_size, .crc=0xffffffff};
err = lfs_bd_crc(lfs,
NULL, &lfs->rcache, lfs->cfg->prog_size,
commit->block, noff, fcrc.size, &fcrc.crc);
if (err && err != LFS_ERR_CORRUPT) {
return err;
}
#ifdef LFS_MULTIVERSION
// unfortunately fcrcs break mdir fetching < lfs2.1, so only write
// these if we're a >= lfs2.1 filesystem
if (lfs_fs_disk_version(lfs) <= 0x00020000) {
// don't write fcrc
} else
#endif
{
// find the expected fcrc, don't bother avoiding a reread
// of the eperturb, it should still be in our cache
struct lfs_fcrc fcrc = {
.size = lfs->cfg->prog_size,
.crc = 0xffffffff
};
err = lfs_bd_crc(lfs,
NULL, &lfs->rcache, lfs->cfg->prog_size,
commit->block, noff, fcrc.size, &fcrc.crc);
if (err && err != LFS_ERR_CORRUPT) {
return err;
}
lfs_fcrc_tole32(&fcrc);
err = lfs_dir_commitattr(lfs, commit,
LFS_MKTAG(LFS_TYPE_FCRC, 0x3ff, sizeof(struct lfs_fcrc)),
&fcrc);
if (err) {
return err;
lfs_fcrc_tole32(&fcrc);
err = lfs_dir_commitattr(lfs, commit,
LFS_MKTAG(LFS_TYPE_FCRC, 0x3ff, sizeof(struct lfs_fcrc)),
&fcrc);
if (err) {
return err;
}
}
}
@ -4052,6 +4097,15 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) {
lfs->cfg = cfg;
int err = 0;
#ifdef LFS_MULTIVERSION
// this driver only supports minor version < current minor version
LFS_ASSERT(!lfs->cfg->disk_version || (
(0xffff & (lfs->cfg->disk_version >> 16))
== LFS_DISK_VERSION_MAJOR
&& (0xffff & (lfs->cfg->disk_version >> 0))
<= LFS_DISK_VERSION_MINOR));
#endif
// check that bool is a truthy-preserving type
//
// note the most common reason for this failure is a before-c99 compiler,
@ -4209,7 +4263,7 @@ static int lfs_rawformat(lfs_t *lfs, const struct lfs_config *cfg) {
// write one superblock
lfs_superblock_t superblock = {
.version = LFS_DISK_VERSION,
.version = lfs_fs_disk_version(lfs),
.block_size = lfs->cfg->block_size,
.block_count = lfs->cfg->block_count,
.name_max = lfs->name_max,
@ -4307,12 +4361,14 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) {
// check version
uint16_t major_version = (0xffff & (superblock.version >> 16));
uint16_t minor_version = (0xffff & (superblock.version >> 0));
if ((major_version != LFS_DISK_VERSION_MAJOR ||
minor_version > LFS_DISK_VERSION_MINOR)) {
if (major_version != lfs_fs_disk_version_major(lfs)
|| minor_version > lfs_fs_disk_version_minor(lfs)) {
LFS_ERROR("Invalid version "
"v%"PRIu16".%"PRIu16" != v%"PRIu16".%"PRIu16,
major_version, minor_version,
LFS_DISK_VERSION_MAJOR, LFS_DISK_VERSION_MINOR);
major_version,
minor_version,
lfs_fs_disk_version_major(lfs),
lfs_fs_disk_version_minor(lfs));
err = LFS_ERR_INVAL;
goto cleanup;
}
@ -4320,11 +4376,13 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) {
// found older minor version? set an in-device only bit in the
// gstate so we know we need to rewrite the superblock before
// the first write
if (minor_version < LFS_DISK_VERSION_MINOR) {
if (minor_version < lfs_fs_disk_version_minor(lfs)) {
LFS_DEBUG("Found older minor version "
"v%"PRIu16".%"PRIu16" < v%"PRIu16".%"PRIu16,
major_version, minor_version,
LFS_DISK_VERSION_MAJOR, LFS_DISK_VERSION_MINOR);
major_version,
minor_version,
lfs_fs_disk_version_major(lfs),
lfs_fs_disk_version_minor(lfs));
// note this bit is reserved on disk, so fetching more gstate
// will not interfere here
lfs_fs_prepsuperblock(lfs, true);
@ -4424,7 +4482,7 @@ static int lfs_fs_rawstat(lfs_t *lfs, struct lfs_fsinfo *fsinfo) {
// if the superblock is up-to-date, we must be on the most recent
// minor version of littlefs
if (!lfs_gstate_needssuperblock(&lfs->gstate)) {
fsinfo->disk_version = LFS_DISK_VERSION;
fsinfo->disk_version = lfs_fs_disk_version(lfs);
// otherwise we need to read the minor version on disk
} else {
@ -4711,7 +4769,7 @@ static int lfs_fs_desuperblock(lfs_t *lfs) {
// write a new superblock
lfs_superblock_t superblock = {
.version = LFS_DISK_VERSION,
.version = lfs_fs_disk_version(lfs),
.block_size = lfs->cfg->block_size,
.block_count = lfs->cfg->block_count,
.name_max = lfs->name_max,

8
lfs.h
View File

@ -263,6 +263,14 @@ struct lfs_config {
// can help bound the metadata compaction time. Must be <= block_size.
// Defaults to block_size when zero.
lfs_size_t metadata_max;
#ifdef LFS_MULTIVERSION
// On-disk version to use when writing in the form of 16-bit major version
// + 16-bit minor version. This limiting metadata to what is supported by
// older minor versions. Note that some features will be lost. Defaults to
// to the most recent minor version when zero.
uint32_t disk_version;
#endif
};
// File info structure

View File

@ -1346,6 +1346,9 @@ static void run_powerloss_none(
.block_cycles = BLOCK_CYCLES,
.cache_size = CACHE_SIZE,
.lookahead_size = LOOKAHEAD_SIZE,
#ifdef LFS_MULTIVERSION
.disk_version = DISK_VERSION,
#endif
};
struct lfs_emubd_config bdcfg = {
@ -1415,6 +1418,9 @@ static void run_powerloss_linear(
.block_cycles = BLOCK_CYCLES,
.cache_size = CACHE_SIZE,
.lookahead_size = LOOKAHEAD_SIZE,
#ifdef LFS_MULTIVERSION
.disk_version = DISK_VERSION,
#endif
};
struct lfs_emubd_config bdcfg = {
@ -1501,6 +1507,9 @@ static void run_powerloss_log(
.block_cycles = BLOCK_CYCLES,
.cache_size = CACHE_SIZE,
.lookahead_size = LOOKAHEAD_SIZE,
#ifdef LFS_MULTIVERSION
.disk_version = DISK_VERSION,
#endif
};
struct lfs_emubd_config bdcfg = {
@ -1585,6 +1594,9 @@ static void run_powerloss_cycles(
.block_cycles = BLOCK_CYCLES,
.cache_size = CACHE_SIZE,
.lookahead_size = LOOKAHEAD_SIZE,
#ifdef LFS_MULTIVERSION
.disk_version = DISK_VERSION,
#endif
};
struct lfs_emubd_config bdcfg = {
@ -1767,6 +1779,9 @@ static void run_powerloss_exhaustive(
.block_cycles = BLOCK_CYCLES,
.cache_size = CACHE_SIZE,
.lookahead_size = LOOKAHEAD_SIZE,
#ifdef LFS_MULTIVERSION
.disk_version = DISK_VERSION,
#endif
};
struct lfs_emubd_config bdcfg = {

View File

@ -91,6 +91,7 @@ intmax_t test_define(size_t define);
#define ERASE_CYCLES_i 8
#define BADBLOCK_BEHAVIOR_i 9
#define POWERLOSS_BEHAVIOR_i 10
#define DISK_VERSION_i 11
#define READ_SIZE TEST_DEFINE(READ_SIZE_i)
#define PROG_SIZE TEST_DEFINE(PROG_SIZE_i)
@ -103,6 +104,7 @@ intmax_t test_define(size_t define);
#define ERASE_CYCLES TEST_DEFINE(ERASE_CYCLES_i)
#define BADBLOCK_BEHAVIOR TEST_DEFINE(BADBLOCK_BEHAVIOR_i)
#define POWERLOSS_BEHAVIOR TEST_DEFINE(POWERLOSS_BEHAVIOR_i)
#define DISK_VERSION TEST_DEFINE(DISK_VERSION_i)
#define TEST_IMPLICIT_DEFINES \
TEST_DEF(READ_SIZE, PROG_SIZE) \
@ -115,9 +117,10 @@ intmax_t test_define(size_t define);
TEST_DEF(ERASE_VALUE, 0xff) \
TEST_DEF(ERASE_CYCLES, 0) \
TEST_DEF(BADBLOCK_BEHAVIOR, LFS_EMUBD_BADBLOCK_PROGERROR) \
TEST_DEF(POWERLOSS_BEHAVIOR, LFS_EMUBD_POWERLOSS_NOOP)
TEST_DEF(POWERLOSS_BEHAVIOR, LFS_EMUBD_POWERLOSS_NOOP) \
TEST_DEF(DISK_VERSION, 0)
#define TEST_IMPLICIT_DEFINE_COUNT 11
#define TEST_IMPLICIT_DEFINE_COUNT 12
#define TEST_GEOMETRY_DEFINE_COUNT 4

View File

@ -60,7 +60,10 @@ code = '''
# test we can mount in a new version
[cases.test_compat_forward_mount]
if = 'LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR'
if = '''
LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR
&& DISK_VERSION == 0
'''
code = '''
// create the previous version
struct lfsp_config cfgp;
@ -88,7 +91,10 @@ code = '''
# test we can read dirs in a new version
[cases.test_compat_forward_read_dirs]
defines.COUNT = 5
if = 'LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR'
if = '''
LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR
&& DISK_VERSION == 0
'''
code = '''
// create the previous version
struct lfsp_config cfgp;
@ -145,7 +151,10 @@ code = '''
defines.COUNT = 5
defines.SIZE = [4, 32, 512, 8192]
defines.CHUNK = 4
if = 'LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR'
if = '''
LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR
&& DISK_VERSION == 0
'''
code = '''
// create the previous version
struct lfsp_config cfgp;
@ -232,7 +241,10 @@ code = '''
defines.COUNT = 5
defines.SIZE = [4, 32, 512, 8192]
defines.CHUNK = 4
if = 'LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR'
if = '''
LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR
&& DISK_VERSION == 0
'''
code = '''
// create the previous version
struct lfsp_config cfgp;
@ -344,7 +356,10 @@ code = '''
# test we can write dirs in a new version
[cases.test_compat_forward_write_dirs]
defines.COUNT = 10
if = 'LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR'
if = '''
LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR
&& DISK_VERSION == 0
'''
code = '''
// create the previous version
struct lfsp_config cfgp;
@ -408,7 +423,10 @@ code = '''
defines.COUNT = 5
defines.SIZE = [4, 32, 512, 8192]
defines.CHUNK = 2
if = 'LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR'
if = '''
LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR
&& DISK_VERSION == 0
'''
code = '''
// create the previous version
struct lfsp_config cfgp;
@ -527,7 +545,10 @@ code = '''
defines.COUNT = 5
defines.SIZE = [4, 32, 512, 8192]
defines.CHUNK = 2
if = 'LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR'
if = '''
LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR
&& DISK_VERSION == 0
'''
code = '''
// create the previous version
struct lfsp_config cfgp;
@ -674,7 +695,10 @@ code = '''
# test we can mount in an old version
[cases.test_compat_backward_mount]
if = 'LFS_DISK_VERSION == LFSP_DISK_VERSION'
if = '''
LFS_DISK_VERSION == LFSP_DISK_VERSION
&& DISK_VERSION == 0
'''
code = '''
// create the new version
lfs_t lfs;
@ -696,7 +720,10 @@ code = '''
# test we can read dirs in an old version
[cases.test_compat_backward_read_dirs]
defines.COUNT = 5
if = 'LFS_DISK_VERSION == LFSP_DISK_VERSION'
if = '''
LFS_DISK_VERSION == LFSP_DISK_VERSION
&& DISK_VERSION == 0
'''
code = '''
// create the new version
lfs_t lfs;
@ -748,7 +775,10 @@ code = '''
defines.COUNT = 5
defines.SIZE = [4, 32, 512, 8192]
defines.CHUNK = 4
if = 'LFS_DISK_VERSION == LFSP_DISK_VERSION'
if = '''
LFS_DISK_VERSION == LFSP_DISK_VERSION
&& DISK_VERSION == 0
'''
code = '''
// create the new version
lfs_t lfs;
@ -830,7 +860,10 @@ code = '''
defines.COUNT = 5
defines.SIZE = [4, 32, 512, 8192]
defines.CHUNK = 4
if = 'LFS_DISK_VERSION == LFSP_DISK_VERSION'
if = '''
LFS_DISK_VERSION == LFSP_DISK_VERSION
&& DISK_VERSION == 0
'''
code = '''
// create the new version
lfs_t lfs;
@ -937,7 +970,10 @@ code = '''
# test we can write dirs in an old version
[cases.test_compat_backward_write_dirs]
defines.COUNT = 10
if = 'LFS_DISK_VERSION == LFSP_DISK_VERSION'
if = '''
LFS_DISK_VERSION == LFSP_DISK_VERSION
&& DISK_VERSION == 0
'''
code = '''
// create the new version
lfs_t lfs;
@ -996,7 +1032,10 @@ code = '''
defines.COUNT = 5
defines.SIZE = [4, 32, 512, 8192]
defines.CHUNK = 2
if = 'LFS_DISK_VERSION == LFSP_DISK_VERSION'
if = '''
LFS_DISK_VERSION == LFSP_DISK_VERSION
&& DISK_VERSION == 0
'''
code = '''
// create the previous version
lfs_t lfs;
@ -1110,7 +1149,10 @@ code = '''
defines.COUNT = 5
defines.SIZE = [4, 32, 512, 8192]
defines.CHUNK = 2
if = 'LFS_DISK_VERSION == LFSP_DISK_VERSION'
if = '''
LFS_DISK_VERSION == LFSP_DISK_VERSION
&& DISK_VERSION == 0
'''
code = '''
// create the previous version
lfs_t lfs;
@ -1319,7 +1361,10 @@ code = '''
# test that we correctly bump the minor version
[cases.test_compat_minor_bump]
in = 'lfs.c'
if = 'LFS_DISK_VERSION_MINOR > 0'
if = '''
LFS_DISK_VERSION_MINOR > 0
&& DISK_VERSION == 0
'''
code = '''
// create a superblock
lfs_t lfs;

View File

@ -90,7 +90,10 @@ code = '''
# partial prog, may not be byte in order!
[cases.test_powerloss_partial_prog]
if = "PROG_SIZE < BLOCK_SIZE"
if = '''
PROG_SIZE < BLOCK_SIZE
&& (DISK_VERSION == 0 || DISK_VERSION >= 0x00020001)
'''
defines.BYTE_OFF = ["0", "PROG_SIZE-1", "PROG_SIZE/2"]
defines.BYTE_VALUE = [0x33, 0xcc]
in = "lfs.c"

View File

@ -36,6 +36,7 @@ code = '''
# test we can read superblock info through lfs_fs_stat
[cases.test_superblocks_stat]
if = 'DISK_VERSION == 0'
code = '''
lfs_t lfs;
lfs_format(&lfs, cfg) => 0;
@ -54,6 +55,7 @@ code = '''
'''
[cases.test_superblocks_stat_tweaked]
if = 'DISK_VERSION == 0'
defines.TWEAKED_NAME_MAX = 63
defines.TWEAKED_FILE_MAX = '(1 << 16)-1'
defines.TWEAKED_ATTR_MAX = 512