database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improvement](multi-catalog)add scanner isolation class loader (#22247)

Browse Source 
Add scanner isolation class loader to make each plugin non-conflicting.
The BE will get scanner classes by JNI call and use JniClassLoader load them.
In the last version，we always get canner classes from the system class path by default,
so it cannot isolate the classes for each scanner
This commit is contained in:
slothever
2023-08-10 10:02:46 +08:00
committed by GitHub
parent e6b835617b
commit 919bfd73f1
8 changed files with 277 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
be/src/util/jni-util.cpp
View File

@ -153,6 +153,8 @@ jmethodID JniUtil::throwable_to_stack_trace_id_ = NULL;
jmethodID JniUtil::get_jvm_metrics_id_ = NULL;
jmethodID JniUtil::get_jvm_threads_id_ = NULL;
jmethodID JniUtil::get_jmx_json_ = NULL;
jobject JniUtil::jni_scanner_loader_obj_ = NULL;
jmethodID JniUtil::jni_scanner_loader_method_ = NULL;
Status JniUtfCharGuard::create(JNIEnv* env, jstring jstr, JniUtfCharGuard* out) {
DCHECK(jstr != nullptr);
@ -340,12 +342,57 @@ Status JniUtil::LocalToGlobalRef(JNIEnv* env, jobject local_ref, jobject* global
return Status::OK();
}
Status JniUtil::init_jni_scanner_loader(JNIEnv* env) {
// Get scanner loader;
jclass jni_scanner_loader_cls;
std::string jni_scanner_loader_str = "org/apache/doris/common/classloader/ScannerLoader";
RETURN_IF_ERROR(JniUtil::GetGlobalClassRef(env, jni_scanner_loader_str.c_str(),
&jni_scanner_loader_cls));
jmethodID jni_scanner_loader_constructor =
env->GetMethodID(jni_scanner_loader_cls, "<init>", "()V");
RETURN_ERROR_IF_EXC(env);
jni_scanner_loader_method_ = env->GetMethodID(jni_scanner_loader_cls, "getLoadedClass",
"(Ljava/lang/String;)Ljava/lang/Class;");
if (jni_scanner_loader_method_ == NULL) {
if (env->ExceptionOccurred()) env->ExceptionDescribe();
return Status::InternalError("Failed to find ScannerLoader.getLoadedClass method.");
}
RETURN_ERROR_IF_EXC(env);
jmethodID load_jni_scanner =
env->GetMethodID(jni_scanner_loader_cls, "loadAllScannerJars", "()V");
RETURN_ERROR_IF_EXC(env);
jni_scanner_loader_obj_ =
env->NewObject(jni_scanner_loader_cls, jni_scanner_loader_constructor);
RETURN_ERROR_IF_EXC(env);
if (jni_scanner_loader_obj_ == NULL) {
if (env->ExceptionOccurred()) env->ExceptionDescribe();
return Status::InternalError("Failed to create ScannerLoader object.");
}
env->CallVoidMethod(jni_scanner_loader_obj_, load_jni_scanner);
RETURN_ERROR_IF_EXC(env);
return Status::OK();
}
Status JniUtil::get_jni_scanner_class(JNIEnv* env, std::string classname,
jclass* jni_scanner_class) {
// Get JNI scanner class by class name;
jobject loaded_class_obj =
env->CallObjectMethod(jni_scanner_loader_obj_, jni_scanner_loader_method_,
env->NewStringUTF(classname.c_str()));
RETURN_ERROR_IF_EXC(env);
*jni_scanner_class = reinterpret_cast<jclass>(env->NewGlobalRef(loaded_class_obj));
RETURN_ERROR_IF_EXC(env);
return Status::OK();
}
Status JniUtil::Init() {
RETURN_IF_ERROR(LibJVMLoader::instance().load());
// Get the JNIEnv* corresponding to current thread.
JNIEnv* env;
RETURN_IF_ERROR(JniUtil::GetJNIEnv(&env));
if (env == NULL) return Status::InternalError("Failed to get/create JVM");
// Find JniUtil class and create a global ref.
jclass local_jni_util_cl = env->FindClass("org/apache/doris/common/jni/utils/JniUtil");
@ -454,6 +501,7 @@ Status JniUtil::Init() {
if (env->ExceptionOccurred()) env->ExceptionDescribe();
return Status::InternalError("Failed to find JniUtil.getJMXJson method.");
}
RETURN_IF_ERROR(init_jni_scanner_loader(env));
jvm_inited_ = true;
return Status::OK();
}

9
be/src/util/jni-util.h
View File

@ -30,7 +30,7 @@
#include "util/thrift_util.h"
#ifdef USE_HADOOP_HDFS
// defined in hadoop_hdfs/hdfs.h
// defined in hadoop/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/jni_helper.c
extern "C" JNIEnv* getJNIEnv(void);
#endif
@ -74,12 +74,13 @@ public:
static inline int64_t IncreaseReservedBufferSize(int n) {
return INITIAL_RESERVED_BUFFER_SIZE << n;
}
static Status get_jni_scanner_class(JNIEnv* env, std::string classname, jclass* loaded_class);
static jobject convert_to_java_map(JNIEnv* env, const std::map<std::string, std::string>& map);
static std::map<std::string, std::string> convert_to_cpp_map(JNIEnv* env, jobject map);
private:
static Status GetJNIEnvSlowPath(JNIEnv** env);
static Status init_jni_scanner_loader(JNIEnv* env);
static bool jvm_inited_;
static jclass internal_exc_cl_;
@ -90,7 +91,9 @@ private:
static jmethodID get_jvm_metrics_id_;
static jmethodID get_jvm_threads_id_;
static jmethodID get_jmx_json_;
// JNI scanner loader
static jobject jni_scanner_loader_obj_;
static jmethodID jni_scanner_loader_method_;
// Thread-local cache of the JNIEnv for this thread.
static __thread JNIEnv* tls_env_;
};

7
be/src/vec/exec/jni_connector.cpp
View File

@ -201,7 +201,12 @@ Status JniConnector::close() {
}
Status JniConnector::_init_jni_scanner(JNIEnv* env, int batch_size) {
RETURN_IF_ERROR(JniUtil::GetGlobalClassRef(env, _connector_class.c_str(), &_jni_scanner_cls));
RETURN_IF_ERROR(JniUtil::get_jni_scanner_class(env, _connector_class, &_jni_scanner_cls));
if (_jni_scanner_cls == NULL) {
if (env->ExceptionOccurred()) env->ExceptionDescribe();
return Status::InternalError("Fail to get JniScanner class.");
}
RETURN_ERROR_IF_EXC(env);
jmethodID scanner_constructor =
env->GetMethodID(_jni_scanner_cls, "<init>", "(ILjava/util/Map;)V");
RETURN_ERROR_IF_EXC(env);

2
bin/start_be.sh
View File

@ -78,7 +78,7 @@ if [[ "${MAX_FILE_COUNT}" -lt 65536 ]]; then
fi
# add java libs
for f in "${DORIS_HOME}/lib/java_extensions"/*.jar; do
for f in "${DORIS_HOME}/lib/java_extensions/java-udf"/*.jar; do
if [[ -z "${DORIS_CLASSPATH}" ]]; then
export DORIS_CLASSPATH="${f}"
else

6
build.sh
View File

@ -667,8 +667,7 @@ EOF
cp -r -p "${DORIS_HOME}/bin/run-fs-benchmark.sh" "${DORIS_OUTPUT}/be/bin/"/
fi
extensions_modules=("")
extensions_modules+=("java-udf")
extensions_modules=("java-udf")
extensions_modules+=("jdbc-scanner")
extensions_modules+=("hudi-scanner")
extensions_modules+=("paimon-scanner")
@ -680,8 +679,9 @@ EOF
mkdir "${BE_JAVA_EXTENSIONS_DIR}"
for extensions_module in "${extensions_modules[@]}"; do
module_path="${DORIS_HOME}/fe/be-java-extensions/${extensions_module}/target/${extensions_module}-jar-with-dependencies.jar"
mkdir "${BE_JAVA_EXTENSIONS_DIR}"/"${extensions_module}"
if [[ -f "${module_path}" ]]; then
cp "${module_path}" "${BE_JAVA_EXTENSIONS_DIR}"/
cp "${module_path}" "${BE_JAVA_EXTENSIONS_DIR}"/"${extensions_module}"
fi
done

39
fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/classloader/JniScannerClassLoader.java Normal file
View File

@ -0,0 +1,39 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.common.classloader;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.List;
public class JniScannerClassLoader extends URLClassLoader {
private final String scannerName;
public JniScannerClassLoader(String scannerName, List<URL> urls) {
super(urls.toArray(new URL[0]), ClassLoader.getSystemClassLoader());
this.scannerName = scannerName;
}
@Override
public String toString() {
return "JniScannerClassLoader{"
+ "scannerName='" + scannerName
+ '}';
}
}

139
fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/classloader/ScannerLoader.java Normal file
View File

@ -0,0 +1,139 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.common.classloader;
import com.google.common.collect.Streams;
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.stream.Collectors;
/**
* BE will load scanners by JNI call, and then the JniConnector on BE will get scanner class by getLoadedClass.
*/
public class ScannerLoader {
private static final Map<String, Class<?>> loadedClasses = new HashMap<>();
private static final String CLASS_SUFFIX = ".class";
private static final String LOAD_PACKAGE = "org.apache.doris";
/**
* Load all classes from $DORIS_HOME/lib/java_extensions/*
*/
public void loadAllScannerJars() {
String basePath = System.getenv("DORIS_HOME");
File library = new File(basePath, "/lib/java_extensions/");
// TODO: add thread pool to load each scanner
listFiles(library).stream().filter(File::isDirectory).forEach(sd -> {
JniScannerClassLoader classLoader = new JniScannerClassLoader(sd.getName(), buildClassPath(sd));
try (ThreadClassLoaderContext ignored = new ThreadClassLoaderContext(classLoader)) {
loadJarClassFromDir(sd, classLoader);
}
});
}
/**
* Get loaded class for JNI scanners
* @param className JNI scanner class name
* @return scanner class object
* @throws ClassNotFoundException JNI scanner class not found
*/
public Class<?> getLoadedClass(String className) throws ClassNotFoundException {
String loadedClassName = getPackagePathName(className);
if (loadedClasses.containsKey(loadedClassName)) {
return loadedClasses.get(loadedClassName);
} else {
throw new ClassNotFoundException("JNI scanner has not been loaded or no such class: " + className);
}
}
private static List<URL> buildClassPath(File path) {
return listFiles(path).stream()
.map(ScannerLoader::classFileUrl)
.collect(Collectors.toList());
}
private static URL classFileUrl(File file) {
try {
return file.toURI().toURL();
} catch (MalformedURLException e) {
throw new UncheckedIOException(e);
}
}
public static List<File> listFiles(File library) {
try (DirectoryStream<Path> directoryStream = Files.newDirectoryStream(library.toPath())) {
return Streams.stream(directoryStream)
.map(Path::toFile)
.sorted()
.collect(Collectors.toList());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public static void loadJarClassFromDir(File dir, JniScannerClassLoader classLoader) {
listFiles(dir).forEach(file -> {
Enumeration<JarEntry> entryEnumeration;
List<String> loadClassNames = new ArrayList<>();
try {
try (JarFile jar = new JarFile(file)) {
entryEnumeration = jar.entries();
while (entryEnumeration.hasMoreElements()) {
JarEntry entry = entryEnumeration.nextElement();
String className = entry.getName();
if (!className.endsWith(CLASS_SUFFIX)) {
continue;
}
className = className.substring(0, className.length() - CLASS_SUFFIX.length());
String packageClassName = getPackagePathName(className);
if (needToLoad(packageClassName)) {
loadClassNames.add(packageClassName);
}
}
}
for (String className : loadClassNames) {
loadedClasses.putIfAbsent(className, classLoader.loadClass(className));
}
} catch (Exception e) {
throw new RuntimeException(e.getMessage(), e);
}
});
}
private static String getPackagePathName(String className) {
return className.replace("/", ".");
}
private static boolean needToLoad(String className) {
return className.contains(LOAD_PACKAGE) && !className.contains("$");
}
}

35
fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/classloader/ThreadClassLoaderContext.java Normal file
View File

@ -0,0 +1,35 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.common.classloader;
import java.io.Closeable;
public class ThreadClassLoaderContext implements Closeable {
private final ClassLoader originClassLoader;
public ThreadClassLoaderContext(ClassLoader contextClassLoader) {
this.originClassLoader = Thread.currentThread().getContextClassLoader();
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
@Override
public void close() {
Thread.currentThread().setContextClassLoader(originClassLoader);
}
}