database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Add UserFunctionCache to cache UDF's library (#453)

Browse Source 
* Add UserFunctionCache to cache UDF's library

This patch replace LibCache with UserFunctionCache. LibCache use HDFS
URL to identify a UDF's Library, and when BE process restart all of
downloaded library should be loaded another time. We use function id
corresponding to a library, and when process restart, all downloaded
libraries can be loaded without another downloading.

* update
This commit is contained in:
ZHAO Chun
2018-12-21 22:07:21 +08:00
committed by chenhao
parent 0341ffde67
commit 90d71508ff
38 changed files with 858 additions and 699 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
be/src/util/file_utils.cpp
View File

@ -94,19 +94,10 @@ Status FileUtils::scan_dir(
}
DeferOp close_dir(std::bind<void>(&closedir, dir));
struct dirent entry;
struct dirent* result = nullptr;
int64_t count = 0;
while (true) {
int ret = readdir_r(dir, &entry, &result);
if (ret != 0) {
char buf[64];
std::stringstream ss;
ss << "readdir(" << dir_path << ") failed, because: " << strerror_r(errno, buf, 64);
return Status(ss.str());
}
auto result = readdir(dir.get());
if (result == nullptr) {
// Over
break;
}
std::string file_name = result->d_name;
@ -127,6 +118,36 @@ Status FileUtils::scan_dir(
return Status::OK;
}
Status FileUtils::scan_dir(
const std::string& dir_path,
const std::function<bool(const std::string&, const std::string&)>& callback) {
auto dir_closer = [] (DIR* dir) { closedir(dir); };
std::unique_ptr<DIR, decltype(dir_closer)> dir(opendir(dir_path.c_str()), dir_closer);
if (dir == nullptr) {
char buf[64];
LOG(WARNING) << "fail to open dir, dir=" << dir_path << ", errmsg=" << strerror_r(errno, buf, 64);
return Status("fail to opendir");
}
struct dirent* result = nullptr;
while (true) {
auto result = readdir(dir.get());
if (result == nullptr) {
break;
}
std::string file_name = result->d_name;
if (file_name == "." || file_name == "..") {
continue;
}
auto is_continue = callback(dir_path, file_name);
if (!is_continue) {
break;
}
}
return Status::OK;
}
bool FileUtils::is_dir(const std::string& path) {
struct stat path_stat;
if (stat(path.c_str(), &path_stat) != 0) {