github-mirror/mergerfs
github-mirror/mergerfs
2
0
Fork 0
mirror of https://github.com/trapexit/mergerfs.git synced 2025-04-24 22:14:05 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #677 from trapexit/moveonenospc

Browse Source 
fix short writes on >2GB files when cloning file
This commit is contained in:
trapexit 2019-10-14 23:34:36 -04:00 committed by GitHub
commit 6058ac0716
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 286 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
src/fs_clonefile.cpp
View File

@ -26,56 +26,56 @@
#include "fs_base_stat.hpp"
#include "fs_base_utime.hpp"
#include "fs_copy_file_range.hpp"
#include "fs_copyfile.hpp"
#include "fs_copydata_copy_file_range.hpp"
#include "fs_copydata_readwrite.hpp"
#include "fs_ficlone.hpp"
#include "fs_sendfile.hpp"
#include "fs_xattr.hpp"
static
int
copydata(const int src_fd_,
const int dst_fd_,
const size_t count_)
namespace l
{
int rv;
static
int
copydata(const int src_fd_,
const int dst_fd_,
const size_t count_)
{
int rv;
rv = fs::ftruncate(dst_fd_,count_);
if(rv == -1)
return -1;
rv = fs::ftruncate(dst_fd_,count_);
if(rv == -1)
return -1;
rv = fs::ficlone(src_fd_,dst_fd_);
if(rv != -1)
return rv;
rv = fs::ficlone(src_fd_,dst_fd_);
if(rv != -1)
return rv;
fs::fadvise_willneed(src_fd_,0,count_);
fs::fadvise_sequential(src_fd_,0,count_);
fs::fadvise_willneed(src_fd_,0,count_);
fs::fadvise_sequential(src_fd_,0,count_);
rv = fs::copy_file_range(src_fd_,dst_fd_,count_);
if(rv != -1)
return rv;
rv = fs::copydata_copy_file_range(src_fd_,dst_fd_);
if(rv != -1)
return rv;
rv = fs::sendfile(src_fd_,dst_fd_,count_);
if(rv != -1)
return rv;
return fs::copydata_readwrite(src_fd_,dst_fd_);
}
return fs::copyfile(src_fd_,dst_fd_);
}
static
bool
ignorable_error(const int err_)
{
switch(err_)
{
case ENOTTY:
case ENOTSUP:
static
bool
ignorable_error(const int err_)
{
switch(err_)
{
case ENOTTY:
case ENOTSUP:
#if ENOTSUP != EOPNOTSUPP
case EOPNOTSUPP:
case EOPNOTSUPP:
#endif
return true;
}
return true;
}
return false;
return false;
}
}
namespace fs
@ -91,16 +91,16 @@ namespace fs
if(rv == -1)
return -1;
rv = ::copydata(src_fd_,dst_fd_,src_st.st_size);
rv = l::copydata(src_fd_,dst_fd_,src_st.st_size);
if(rv == -1)
return -1;
rv = fs::attr::copy(src_fd_,dst_fd_);
if((rv == -1) && !ignorable_error(errno))
if((rv == -1) && !l::ignorable_error(errno))
return -1;
rv = fs::xattr::copy(src_fd_,dst_fd_);
if((rv == -1) && !ignorable_error(errno))
if((rv == -1) && !l::ignorable_error(errno))
return -1;
rv = fs::fchown_check_on_error(dst_fd_,src_st);

20
src/fs_copy_file_range.hpp
View File

@ -24,17 +24,11 @@
namespace fs
{
ssize_t
copy_file_range(const int fd_in_,
int64_t *off_in_,
const int fd_out_,
int64_t *off_out_,
const size_t len_,
const unsigned int flags_);
ssize_t
copy_file_range(const int fd_in_,
const int fd_out_,
const size_t len_,
const unsigned int flags_ = 0);
int64_t
copy_file_range(const int fd_in,
int64_t *off_in,
const int fd_out,
int64_t *off_out,
const uint64_t len,
const unsigned int flags);
}

82
src/fs_copy_file_range_linux.icpp
View File

@ -32,33 +32,39 @@
namespace l
{
static
ssize_t
copy_file_range_(int fd_in_,
int64_t *off_in_,
int fd_out_,
int64_t *off_out_,
size_t len_,
unsigned int flags_)
int64_t
copy_file_range_(int src_fd_,
int64_t *src_off_,
int tgt_fd_,
int64_t *tgt_off_,
const uint64_t len_,
const unsigned int flags_)
{
#ifdef SYS_copy_file_range
ssize_t rv;
loff_t off_in;
loff_t off_out;
int64_t rv;
loff_t src_off;
loff_t tgt_off;
loff_t *src_off_ptr;
loff_t *tgt_off_ptr;
off_in = *off_in_;
off_out = *off_out_;
src_off = ((src_off_ == NULL) ? 0 : *src_off_);
tgt_off = ((tgt_off_ == NULL) ? 0 : *tgt_off_);
src_off_ptr = ((src_off_ == NULL) ? NULL : &src_off);
tgt_off_ptr = ((tgt_off_ == NULL) ? NULL : &tgt_off);
rv = ::syscall(SYS_copy_file_range,
fd_in_,
&off_in,
fd_out_,
&off_out,
src_fd_,
src_off_ptr,
tgt_fd_,
tgt_off_ptr,
len_,
flags_);
if(rv != -1)
{
*off_in_ = off_in;
*off_out_ = off_out;
if(src_off_ != NULL)
*src_off_ = src_off;
if(tgt_off_ != NULL)
*tgt_off_ = tgt_off;
}
return rv;
@ -70,38 +76,18 @@ namespace l
namespace fs
{
ssize_t
copy_file_range(const int fd_in_,
int64_t *off_in_,
const int fd_out_,
int64_t *off_out_,
const size_t len_,
int64_t
copy_file_range(const int src_fd_,
int64_t *src_off_,
const int tgt_fd_,
int64_t *tgt_off_,
const uint64_t len_,
const unsigned int flags_)
{
return l::copy_file_range_(fd_in_,
off_in_,
fd_out_,
off_out_,
len_,
flags_);
}
ssize_t
copy_file_range(const int fd_in_,
const int fd_out_,
const size_t len_,
const unsigned int flags_)
{
int64_t off_in;
int64_t off_out;
off_in = 0;
off_out = 0;
return fs::copy_file_range(fd_in_,
&off_in,
fd_out_,
&off_out,
return l::copy_file_range_(src_fd_,
src_off_,
tgt_fd_,
tgt_off_,
len_,
flags_);
}

67
src/fs_copydata_copy_file_range.cpp Normal file
View File

@ -0,0 +1,67 @@
/*
Copyright (c) 2019, Antonio SJ Musumeci <trapexit@spawn.link>
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include "errno.hpp"
#include "fs_base_stat.hpp"
#include "fs_copy_file_range.hpp"
namespace l
{
int64_t
copydata_copy_file_range(const int src_fd_,
const int dst_fd_,
uint64_t size_)
{
int64_t rv;
uint64_t len;
int64_t src_off;
int64_t dst_off;
src_off = 0;
dst_off = 0;
len = size_;
do
{
rv = fs::copy_file_range(src_fd_,&src_off,dst_fd_,&dst_off,len,0);
if(rv == -1)
return -1;
len -= rv;
}
while((len > 0) && (rv > 0));
return size_;
}
}
namespace fs
{
int64_t
copydata_copy_file_range(const int src_fd_,
const int dst_fd_)
{
int rv;
struct stat st;
rv = fs::fstat(src_fd_,&st);
if(rv == -1)
return -1;
return l::copydata_copy_file_range(src_fd_,
dst_fd_,
st.st_size);
}
}

24
src/fs_copydata_copy_file_range.hpp Normal file
View File

@ -0,0 +1,24 @@
/*
Copyright (c) 2016, Antonio SJ Musumeci <trapexit@spawn.link>
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#pragma once
namespace fs
{
int
copydata_copy_file_range(const int src_fd,
const int dst_fd);
}

113
src/fs_copydata_readwrite.cpp Normal file
View File

@ -0,0 +1,113 @@
/*
Copyright (c) 2018, Antonio SJ Musumeci <trapexit@spawn.link>
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include "errno.hpp"
#include "fs_base_stat.hpp"
#include "fs_base_lseek.hpp"
#include "fs_base_read.hpp"
#include "fs_base_write.hpp"
#include <vector>
using std::vector;
namespace l
{
static
int
writen(const int fd_,
const char *buf_,
const size_t count_)
{
size_t nleft;
ssize_t nwritten;
nleft = count_;
do
{
nwritten = fs::write(fd_,buf_,nleft);
if((nwritten == -1) && (errno == EINTR))
continue;
if(nwritten == -1)
return -1;
nleft -= nwritten;
buf_ += nwritten;
}
while(nleft > 0);
return count_;
}
static
int
copydata_readwrite(const int src_fd_,
const int dst_fd_,
const size_t count_,
const size_t blocksize_)
{
ssize_t nr;
ssize_t nw;
ssize_t bufsize;
size_t totalwritten;
vector<char> buf;
bufsize = (blocksize_ * 16);
buf.resize(bufsize);
fs::lseek(src_fd_,0,SEEK_SET);
totalwritten = 0;
while(totalwritten < count_)
{
nr = fs::read(src_fd_,&buf[0],bufsize);
if(nr == 0)
return totalwritten;
if((nr == -1) && (errno == EINTR))
continue;
if(nr == -1)
return -1;
nw = l::writen(dst_fd_,&buf[0],nr);
if(nw == -1)
return -1;
totalwritten += nw;
}
return totalwritten;
}
}
namespace fs
{
int
copydata_readwrite(const int src_fd_,
const int dst_fd_)
{
int rv;
struct stat st;
rv = fs::fstat(src_fd_,&st);
if(rv == -1)
return -1;
return l::copydata_readwrite(src_fd_,
dst_fd_,
st.st_size,
st.st_blksize);
}
}

4
src/fs_copyfile.hpp → src/fs_copydata_readwrite.hpp
View File

@ -19,6 +19,6 @@
namespace fs
{
int
copyfile(const int src_fd_,
const int dst_fd_);
copydata_readwrite(const int src_fd,
const int dst_fd);
}

110
src/fs_copyfile.cpp
View File

@ -1,110 +0,0 @@
/*
Copyright (c) 2018, Antonio SJ Musumeci <trapexit@spawn.link>
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include "errno.hpp"
#include "fs_base_stat.hpp"
#include "fs_base_lseek.hpp"
#include "fs_base_read.hpp"
#include "fs_base_write.hpp"
#include <vector>
using std::vector;
static
int
writen(const int fd_,
const char *buf_,
const size_t count_)
{
size_t nleft;
ssize_t nwritten;
nleft = count_;
do
{
nwritten = fs::write(fd_,buf_,nleft);
if((nwritten == -1) && (errno == EINTR))
continue;
if(nwritten == -1)
return -1;
nleft -= nwritten;
buf_ += nwritten;
}
while(nleft > 0);
return count_;
}
static
int
copyfile(const int src_fd_,
const int dst_fd_,
const size_t count_,
const size_t blocksize_)
{
ssize_t nr;
ssize_t nw;
ssize_t bufsize;
size_t totalwritten;
vector<char> buf;
bufsize = (blocksize_ * 16);
buf.resize(bufsize);
fs::lseek(src_fd_,0,SEEK_SET);
totalwritten = 0;
while(totalwritten < count_)
{
nr = fs::read(src_fd_,&buf[0],bufsize);
if(nr == 0)
return totalwritten;
if((nr == -1) && (errno == EINTR))
continue;
if(nr == -1)
return -1;
nw = writen(dst_fd_,&buf[0],nr);
if(nw == -1)
return -1;
totalwritten += nw;
}
return totalwritten;
}
namespace fs
{
int
copyfile(const int src_fd_,
const int dst_fd_)
{
int rv;
struct stat st;
rv = fs::fstat(src_fd_,&st);
if(rv == -1)
return rv;
return ::copyfile(src_fd_,
dst_fd_,
st.st_size,
st.st_blksize);
}
}