github-mirror/mergerfs
github-mirror/mergerfs
2
0
Fork 0
mirror of https://github.com/trapexit/mergerfs.git synced 2025-04-24 22:14:05 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Change the "devino" inode calculation

Browse Source 
Use hashing of branch path rather than use st_dev.
This commit is contained in:
Antonio SJ Musumeci 2025-04-06 01:03:26 -05:00
parent ad54c8be19
commit 77bf16e9ae
15 changed files with 1890 additions and 245 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
libfuse/include/fuse.h
View File

@ -11,6 +11,7 @@
#include "extern_c.h"
#include "fuse_common.h"
#include "fuse_kernel.h"
#include <fcntl.h>
#include <time.h>

6
src/fh.hpp
View File

@ -22,6 +22,12 @@
class FH
{
public:
FH(const std::string fusepath_)
: fusepath(std::move(fusepath_))
{
}
FH(const char *fusepath_)
: fusepath(fusepath_)
{

9
src/fileinfo.hpp
View File

@ -26,17 +26,20 @@
class FileInfo : public FH
{
public:
FileInfo(int const fd_,
char const *fusepath_,
bool const direct_io_)
FileInfo(const int fd_,
const std::string branchpath_,
const char *fusepath_,
const bool direct_io_)
: FH(fusepath_),
fd(fd_),
branchpath(std::move(branchpath_)),
direct_io(direct_io_)
{
}
public:
int fd;
std::string branchpath;
uint32_t direct_io:1;
std::mutex mutex;
};

173
src/fs_inode.cpp
View File

@ -29,9 +29,18 @@
#include <string.h>
#include <sys/stat.h>
typedef uint64_t (*inodefunc_t)(const char*,const uint64_t,const mode_t,const dev_t,const ino_t);
using namespace nonstd;
typedef uint64_t (*inodefunc_t)(const string_view,
const string_view,
const mode_t,
const ino_t);
static uint64_t hybrid_hash(const string_view,
const string_view,
const mode_t,
const ino_t);
static uint64_t hybrid_hash(const char*,const uint64_t,const mode_t,const dev_t,const ino_t);
static inodefunc_t g_func = hybrid_hash;
@ -47,40 +56,40 @@ h64_to_h32(uint64_t h_)
static
uint64_t
passthrough(const char *fusepath_,
const uint64_t fusepath_len_,
const mode_t mode_,
const dev_t dev_,
const ino_t ino_)
passthrough(const string_view branch_path_,
const string_view fusepath_,
const mode_t mode_,
const ino_t ino_)
{
return ino_;
}
static
uint64_t
path_hash(const char *fusepath_,
const uint64_t fusepath_len_,
const mode_t mode_,
const dev_t dev_,
const ino_t ino_)
path_hash(const string_view branch_path_,
const string_view fusepath_,
const mode_t mode_,
const ino_t ino_)
{
return rapidhash(fusepath_,fusepath_len_);
uint64_t seed;
seed = rapidhash(&fusepath_[0],fusepath_.size());
return seed;
}
static
uint64_t
path_hash32(const char *fusepath_,
const uint64_t fusepath_len_,
const mode_t mode_,
const dev_t dev_,
const ino_t ino_)
path_hash32(const string_view branch_path_,
const string_view fusepath_,
const mode_t mode_,
const ino_t ino_)
{
uint64_t h;
h = path_hash(fusepath_,
fusepath_len_,
h = path_hash(branch_path_,
fusepath_,
mode_,
dev_,
ino_);
return h64_to_h32(h);
@ -88,34 +97,31 @@ path_hash32(const char *fusepath_,
static
uint64_t
devino_hash(const char *fusepath_,
const uint64_t fusepath_len_,
const mode_t mode_,
const dev_t dev_,
const ino_t ino_)
devino_hash(const string_view branch_path_,
const string_view fusepath_,
const mode_t mode_,
const ino_t ino_)
{
uint64_t buf[2];
uint64_t seed;
buf[0] = dev_;
buf[1] = ino_;
seed = rapidhash(&branch_path_[0],branch_path_.size());
seed = rapidhash_withSeed(&ino_,sizeof(ino_),seed);
return rapidhash((void*)&buf[0],sizeof(buf));
return seed;
}
static
uint64_t
devino_hash32(const char *fusepath_,
const uint64_t fusepath_len_,
const mode_t mode_,
const dev_t dev_,
const ino_t ino_)
devino_hash32(const string_view branch_path_,
const string_view fusepath_,
const mode_t mode_,
const ino_t ino_)
{
uint64_t h;
h = devino_hash(fusepath_,
fusepath_len_,
h = devino_hash(branch_path_,
fusepath_,
mode_,
dev_,
ino_);
return h64_to_h32(h);
@ -123,28 +129,26 @@ devino_hash32(const char *fusepath_,
static
uint64_t
hybrid_hash(const char *fusepath_,
const uint64_t fusepath_len_,
const mode_t mode_,
const dev_t dev_,
const ino_t ino_)
hybrid_hash(const string_view branch_path_,
const string_view fusepath_,
const mode_t mode_,
const ino_t ino_)
{
return (S_ISDIR(mode_) ?
path_hash(fusepath_,fusepath_len_,mode_,dev_,ino_) :
devino_hash(fusepath_,fusepath_len_,mode_,dev_,ino_));
path_hash(branch_path_,fusepath_,mode_,ino_) :
devino_hash(branch_path_,fusepath_,mode_,ino_));
}
static
uint64_t
hybrid_hash32(const char *fusepath_,
const uint64_t fusepath_len_,
const mode_t mode_,
const dev_t dev_,
const ino_t ino_)
hybrid_hash32(const string_view branch_path_,
const string_view fusepath_,
const mode_t mode_,
const ino_t ino_)
{
return (S_ISDIR(mode_) ?
path_hash32(fusepath_,fusepath_len_,mode_,dev_,ino_) :
devino_hash32(fusepath_,fusepath_len_,mode_,dev_,ino_));
path_hash32(branch_path_,fusepath_,mode_,ino_) :
devino_hash32(branch_path_,fusepath_,mode_,ino_));
}
namespace fs
@ -196,71 +200,34 @@ namespace fs
}
uint64_t
calc(const char *fusepath_,
const uint64_t fusepath_len_,
const mode_t mode_,
const dev_t dev_,
const ino_t ino_)
calc(const string_view branch_path_,
const string_view fusepath_,
const mode_t mode_,
const ino_t ino_)
{
return g_func(fusepath_,fusepath_len_,mode_,dev_,ino_);
}
uint64_t
calc(std::string const &fusepath_,
const mode_t mode_,
const dev_t dev_,
const ino_t ino_)
{
return calc(fusepath_.c_str(),
fusepath_.size(),
mode_,
dev_,
ino_);
return g_func(branch_path_,fusepath_,mode_,ino_);
}
void
calc(const char *fusepath_,
const uint64_t fusepath_len_,
struct stat *st_)
calc(const string_view branch_path_,
const string_view fusepath_,
struct stat *st_)
{
st_->st_ino = calc(fusepath_,
fusepath_len_,
st_->st_ino = calc(branch_path_,
fusepath_,
st_->st_mode,
st_->st_dev,
st_->st_ino);
}
void
calc(const char *fusepath_,
const uint64_t fusepath_len_,
calc(const string_view branch_path_,
const string_view fusepath_,
struct fuse_statx *st_)
{
st_->ino = calc(fusepath_,
fusepath_len_,
st_->ino = calc(branch_path_,
fusepath_,
st_->mode,
st_->dev_major ^ st_->dev_minor,
st_->ino);
}
void
calc(const char *fusepath_,
struct stat *st_)
{
calc(fusepath_,strlen(fusepath_),st_);
}
void
calc(const char *fusepath_,
struct fuse_statx *st_)
{
calc(fusepath_,strlen(fusepath_),st_);
}
void
calc(const std::string &fusepath_,
struct stat *st_)
{
calc(fusepath_.c_str(),fusepath_.size(),st_);
}
}
}

38
src/fs_inode.hpp
View File

@ -20,6 +20,9 @@
#include "fuse_kernel.h"
#include "nonstd/string_view.hpp"
#include "ghc/filesystem.hpp"
#include <cstdint>
#include <string>
@ -30,34 +33,19 @@ namespace fs
{
namespace inode
{
static const uint64_t MAGIC = 0x7472617065786974;
int set_algo(const std::string &s);
std::string get_algo(void);
uint64_t calc(const char *fusepath,
const uint64_t fusepath_len,
const mode_t mode,
const dev_t dev,
const ino_t ino);
uint64_t calc(std::string const &fusepath,
mode_t const mode,
dev_t const dev,
ino_t ino);
uint64_t calc(const nonstd::string_view basepath,
const nonstd::string_view fusepath,
const mode_t mode,
const ino_t ino);
void calc(const char *fusepath,
const uint64_t fusepath_len,
struct stat *st);
void calc(const char *fusepath,
const uint64_t fusepath_len,
struct fuse_statx *st);
void calc(const char *fusepath,
struct stat *st);
void calc(const char *fusepath,
struct fuse_statx *st);
void calc(const std::string &fusepath,
struct stat *st);
void calc(const nonstd::string_view basepath,
const nonstd::string_view fusepath,
struct stat *st);
void calc(const nonstd::string_view basepath,
const nonstd::string_view fusepath,
struct fuse_statx *st);
}
}

2
src/fuse_create.cpp
View File

@ -163,7 +163,7 @@ namespace l
if(rv == -1)
return -errno;
fi = new FileInfo(rv,fusepath_,ffi_->direct_io);
fi = new FileInfo(rv,createpath_,fusepath_,ffi_->direct_io);
ffi_->fh = reinterpret_cast<uint64_t>(fi);

28
src/fuse_fgetattr.cpp
View File

@ -23,26 +23,6 @@
#include "fuse.h"
namespace l
{
static
int
fgetattr(const int fd_,
const std::string &fusepath_,
struct stat *st_)
{
int rv;
rv = fs::fstat(fd_,st_);
if(rv == -1)
return -errno;
fs::inode::calc(fusepath_,st_);
return 0;
}
}
namespace FUSE
{
int
@ -54,7 +34,13 @@ namespace FUSE
Config::Read cfg;
FileInfo *fi = reinterpret_cast<FileInfo*>(ffi_->fh);
rv = l::fgetattr(fi->fd,fi->fusepath,st_);
rv = fs::fstat(fi->fd,st_);
if(rv == -1)
return -errno;
fs::inode::calc(fi->branchpath,
fi->fusepath,
st_);
timeout_->entry = ((rv >= 0) ?
cfg->cache_entry :

4
src/fuse_getattr.cpp
View File

@ -77,7 +77,7 @@ namespace l
static const time_t now = ::time(NULL);
st_->st_dev = 0;
st_->st_ino = fs::inode::MAGIC;
st_->st_ino = 0;
st_->st_mode = (S_IFREG|S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH);
st_->st_nlink = 1;
st_->st_uid = uid;
@ -141,7 +141,7 @@ namespace l
if(symlinkify_ && symlinkify::can_be_symlink(*st_,symlinkify_timeout_))
symlinkify::convert(fullpath,st_);
fs::inode::calc(fusepath_,st_);
fs::inode::calc(basepaths[0],fusepath_,st_);
return 0;
}

2
src/fuse_open.cpp
View File

@ -211,7 +211,7 @@ namespace l
if(fd == -1)
return -errno;
fi = new FileInfo(fd,fusepath_,ffi_->direct_io);
fi = new FileInfo(fd,basepath_,fusepath_,ffi_->direct_io);
ffi_->fh = reinterpret_cast<uint64_t>(fi);

80
src/fuse_readdir_cor.cpp
View File

@ -20,7 +20,6 @@
#include "dirinfo.hpp"
#include "errno.hpp"
#include "fs_close.hpp"
#include "fs_devid.hpp"
#include "fs_getdents64.hpp"
#include "fs_inode.hpp"
#include "fs_open.hpp"
@ -77,24 +76,25 @@ namespace l
static
inline
int
readdir(std::string basepath_,
HashSet &names_,
fuse_dirents_t *buf_,
std::mutex &mutex_)
readdir(const std::string &branch_path_,
const std::string &rel_dirpath_,
HashSet &names_,
fuse_dirents_t *buf_,
std::mutex &mutex_)
{
int rv;
int dfd;
dev_t dev;
std::string filepath;
std::string rel_filepath;
std::string abs_dirpath;
dfd = fs::open_dir_ro(basepath_);
abs_dirpath = fs::path::make(branch_path_,rel_dirpath_);
dfd = fs::open_dir_ro(abs_dirpath);
if(dfd == -1)
return errno;
DEFER{ fs::close(dfd); };
dev = fs::devid(dfd);
rv = 0;
for(;;)
{
@ -121,10 +121,10 @@ namespace l
if(rv == 0)
continue;
filepath = fs::path::make(basepath_,d->name);
d->ino = fs::inode::calc(filepath,
rel_filepath = fs::path::make(rel_dirpath_,d->name);
d->ino = fs::inode::calc(branch_path_,
rel_filepath,
DTTOIF(d->type),
dev,
d->ino);
rv = fuse_dirents_add_linux(buf_,d,namelen);
@ -142,27 +142,31 @@ namespace l
int
concurrent_readdir(ThreadPool &tp_,
const Branches::CPtr &branches_,
const char *dirname_,
const std::string &rel_dirpath_,
fuse_dirents_t *buf_,
uid_t const uid_,
gid_t const gid_)
const uid_t uid_,
const gid_t gid_)
{
HashSet names;
std::mutex mutex;
std::vector<std::future<int>> futures;
fuse_dirents_reset(buf_);
futures.reserve(branches_->size());
for(auto const &branch : *branches_)
for(const auto &branch : *branches_)
{
auto func = [&,dirname_,buf_,uid_,gid_]()
{
std::string basepath;
ugid::Set const ugid(uid_,gid_);
auto func =
[&,buf_,uid_,gid_]()
{
const ugid::Set ugid(uid_,gid_);
basepath = fs::path::make(branch.path,dirname_);
return l::readdir(basepath,names,buf_,mutex);
};
return l::readdir(branch.path,
rel_dirpath_,
names,
buf_,
mutex);
};
auto rv = tp_.enqueue_task(func);
@ -175,20 +179,6 @@ namespace l
return -error;
}
static
int
readdir(ThreadPool &tp_,
const Branches::CPtr &branches_,
const char *dirname_,
fuse_dirents_t *buf_,
uid_t const uid_,
gid_t const gid_)
{
fuse_dirents_reset(buf_);
return l::concurrent_readdir(tp_,branches_,dirname_,buf_,uid_,gid_);
}
}
int
@ -199,10 +189,10 @@ FUSE::ReadDirCOR::operator()(fuse_file_info_t const *ffi_,
DirInfo *di = reinterpret_cast<DirInfo*>(ffi_->fh);
const fuse_context *fc = fuse_get_context();
return l::readdir(_tp,
cfg->branches,
di->fusepath.c_str(),
buf_,
fc->uid,
fc->gid);
return l::concurrent_readdir(_tp,
cfg->branches,
di->fusepath,
buf_,
fc->uid,
fc->gid);
}

53
src/fuse_readdir_cosr.cpp
View File

@ -20,7 +20,6 @@
#include "dirinfo.hpp"
#include "errno.hpp"
#include "fs_closedir.hpp"
#include "fs_devid.hpp"
#include "fs_dirfd.hpp"
#include "fs_inode.hpp"
#include "fs_opendir.hpp"
@ -50,6 +49,7 @@ namespace l
{
struct DirRV
{
const std::string *branch_path;
DIR *dir;
int err;
};
@ -99,29 +99,30 @@ namespace l
std::vector<std::future<DirRV>>
opendir(ThreadPool &tp_,
const Branches::CPtr &branches_,
char const *dirname_,
const std::string &rel_dirpath_,
uid_t const uid_,
gid_t const gid_)
{
std::vector<std::future<DirRV>> futures;
futures.reserve(branches_->size());
for(auto const &branch : *branches_)
{
auto func = [&branch,dirname_,uid_,gid_]()
{
DirRV rv;
std::string basepath;
ugid::Set const ugid(uid_,gid_);
auto func =
[&branch,&rel_dirpath_,uid_,gid_]()
{
DIR *dir;
std::string abs_dirpath;
const ugid::Set ugid(uid_,gid_);
basepath = fs::path::make(branch.path,dirname_);
abs_dirpath = fs::path::make(branch.path,rel_dirpath_);
errno = 0;
rv.dir = fs::opendir(basepath);
rv.err = errno;
errno = 0;
dir = fs::opendir(abs_dirpath);
return rv;
};
return DirRV{&branch.path,dir,errno};
};
auto rv = tp_.enqueue_task(func);
@ -135,28 +136,26 @@ namespace l
inline
int
readdir(std::vector<std::future<DirRV>> &dh_futures_,
char const *dirname_,
const std::string &rel_dirpath_,
fuse_dirents_t *buf_)
{
Error error;
HashSet names;
std::string fullpath;
std::string rel_filepath;
for(auto &dh_future : dh_futures_)
{
int rv;
dev_t dev;
DirRV dirrv;
dirrv = dh_future.get();
error = dirrv.err;
if(dirrv.dir == NULL)
continue;
DEFER { fs::closedir(dirrv.dir); };
dev = fs::devid(dirrv.dir);
rv = 0;
for(dirent *de = fs::readdir(dirrv.dir); de && !rv; de = fs::readdir(dirrv.dir))
{
@ -168,11 +167,11 @@ namespace l
if(rv == 0)
continue;
fullpath = fs::path::make(dirname_,de->d_name);
de->d_ino = fs::inode::calc(fullpath,
DTTOIF(de->d_type),
dev,
de->d_ino);
rel_filepath = fs::path::make(rel_dirpath_,de->d_name);
de->d_ino = fs::inode::calc(*dirrv.branch_path,
rel_filepath,
DTTOIF(de->d_type),
de->d_ino);
rv = fuse_dirents_add(buf_,de,namelen);
if(rv == 0)
@ -190,7 +189,7 @@ namespace l
int
readdir(ThreadPool &tp_,
const Branches::CPtr &branches_,
const char *dirname_,
const std::string &rel_dirpath_,
fuse_dirents_t *buf_,
uid_t const uid_,
gid_t const gid_)
@ -200,8 +199,8 @@ namespace l
fuse_dirents_reset(buf_);
futures = l::opendir(tp_,branches_,dirname_,uid_,gid_);
rv = l::readdir(futures,dirname_,buf_);
futures = l::opendir(tp_,branches_,rel_dirpath_,uid_,gid_);
rv = l::readdir(futures,rel_dirpath_,buf_);
return rv;
}
@ -217,7 +216,7 @@ FUSE::ReadDirCOSR::operator()(fuse_file_info_t const *ffi_,
return l::readdir(_tp,
cfg->branches,
di->fusepath.c_str(),
di->fusepath,
buf_,
fc->uid,
fc->gid);

24
src/fuse_readdir_seq.cpp
View File

@ -23,7 +23,6 @@
#include "dirinfo.hpp"
#include "errno.hpp"
#include "fs_closedir.hpp"
#include "fs_devid.hpp"
#include "fs_dirfd.hpp"
#include "fs_inode.hpp"
#include "fs_opendir.hpp"
@ -85,34 +84,31 @@ namespace l
static
int
readdir(const Branches::CPtr &branches_,
const char *dirname_,
const std::string &rel_dirpath_,
fuse_dirents_t *buf_)
{
Error error;
HashSet names;
std::string basepath;
std::string fullpath;
std::string rel_filepath;
std::string abs_dirpath;
fuse_dirents_reset(buf_);
for(auto const &branch : *branches_)
for(const auto &branch : *branches_)
{
int rv;
DIR *dh;
dev_t dev;
basepath = fs::path::make(branch.path,dirname_);
abs_dirpath = fs::path::make(branch.path,rel_dirpath_);
errno = 0;
dh = fs::opendir(basepath);
dh = fs::opendir(abs_dirpath);
error = errno;
if(!dh)
continue;
DEFER{ fs::closedir(dh); };
dev = fs::devid(dh);
rv = 0;
for(dirent *de = fs::readdir(dh); de && !rv; de = fs::readdir(dh))
{
@ -124,10 +120,10 @@ namespace l
if(rv == 0)
continue;
fullpath = fs::path::make(dirname_,de->d_name);
de->d_ino = fs::inode::calc(fullpath,
rel_filepath = fs::path::make(rel_dirpath_,de->d_name);
de->d_ino = fs::inode::calc(branch.path,
rel_filepath,
DTTOIF(de->d_type),
dev,
de->d_ino);
rv = fuse_dirents_add(buf_,de,namelen);
@ -150,6 +146,6 @@ FUSE::ReadDirSeq::operator()(fuse_file_info_t const *ffi_,
const ugid::Set ugid(fc->uid,fc->gid);
return l::readdir(cfg->branches,
di->fusepath.c_str(),
di->fusepath,
buf_);
}

4
src/fuse_statx.cpp
View File

@ -60,7 +60,7 @@ _statx_controlfile(struct fuse_statx *st_)
static const gid_t gid = ::getgid();
static const time_t now = ::time(NULL);
st_->ino = fs::inode::MAGIC;
st_->ino = 0;
st_->mode = (S_IFREG|S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH);
st_->nlink = 1;
st_->uid = uid;
@ -124,7 +124,7 @@ _statx(const Policy::Search &searchFunc_,
if(symlinkify_ && symlinkify::can_be_symlink(*st_,symlinkify_timeout_))
symlinkify::convert(fullpath,st_);
fs::inode::calc(fusepath_,st_);
fs::inode::calc(basepaths[0],fusepath_,st_);
return 0;
}

2
src/fuse_symlink.cpp
View File

@ -74,7 +74,7 @@ namespace l
{
fs::lstat(fullnewpath,st_);
if(st_->st_ino != 0)
fs::inode::calc(linkpath_,st_);
fs::inode::calc(newbasepath_,linkpath_,st_);
}
return error::calc(rv,error_,errno);

1709
src/nonstd/string_view.hpp Normal file
View File

File diff suppressed because it is too large Load Diff