database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Modify some thirdparties (#1228)

Browse Source 
1. Change Kafka java client from 2.0.0 to 0.10.1.1. Because high version client may not support low server server.
2. Enable SSL in librdkafka
This commit is contained in:
Mingyu Chen
2019-05-30 21:23:37 +08:00
committed by ZHAO Chun
parent fa4ac9f751
commit 180d8e5cbd
7 changed files with 9 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
be/CMakeLists.txt
View File

@ -440,6 +440,7 @@ set(DORIS_LINK_LIBS
# Set thirdparty libraries
set(DORIS_DEPENDENCIES
mysql
${WL_START_GROUP}
rocksdb
librdkafka_cpp
@ -455,7 +456,6 @@ set(DORIS_DEPENDENCIES
pprof
lz4
libevent
mysql
curl
${LIBZ}
${LIBBZ2}

3
be/src/runtime/routine_load/routine_load_task_executor.h
View File

@ -49,7 +49,8 @@ public:
}
~RoutineLoadTaskExecutor() {
_thread_pool.shutdown();
_thread_pool.join();
}
// submit a routine load task

2
fe/pom.xml
View File

@ -520,7 +520,7 @@ under the License.
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>2.0.0</version>
<version>0.10.1.1</version>
</dependency>
</dependencies>

4
fe/src/main/java/org/apache/doris/load/routineload/KafkaRoutineLoadJob.java
View File

@ -48,7 +48,6 @@ import org.apache.logging.log4j.Logger;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@ -280,8 +279,7 @@ public class KafkaRoutineLoadJob extends RoutineLoadJob {
private List<Integer> getAllKafkaPartitions() throws LoadException {
List<Integer> result = new ArrayList<>();
try {
List<PartitionInfo> partitionList = consumer.partitionsFor(topic,
Duration.ofSeconds(FETCH_PARTITIONS_TIMEOUT_SECOND));
List<PartitionInfo> partitionList = consumer.partitionsFor(topic);
for (PartitionInfo partitionInfo : partitionList) {
result.add(partitionInfo.partition());
}

3
fe/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java
View File

@ -52,7 +52,6 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@ -294,7 +293,7 @@ public class KafkaRoutineLoadJobTest {
result = tableId;
table.getType();
result = Table.TableType.OLAP;
kafkaConsumer.partitionsFor(anyString, (Duration) any);
kafkaConsumer.partitionsFor(anyString);
result = kafkaPartitionInfoList;
}
};

3
fe/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java
View File

@ -39,7 +39,6 @@ import org.apache.kafka.common.PartitionInfo;
import org.junit.Assert;
import org.junit.Test;
import java.time.Duration;
import java.util.List;
import java.util.Map;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@ -213,7 +212,7 @@ public class RoutineLoadJobTest {
result = database;
database.getTable(anyLong);
result = table;
kafkaConsumer.partitionsFor(anyString, (Duration) any);
kafkaConsumer.partitionsFor(anyString);
result = partitionInfoList;
partitionInfo.partition();
result = 1;

4
thirdparty/build-thirdparty.sh vendored
View File

@ -173,7 +173,7 @@ build_openssl() {
CFLAGS="-fPIC" \
LIBDIR="lib" \
./Configure --prefix=$TP_INSTALL_DIR -zlib -shared linux-x86_64
make -j$PARALLEL && make install
make && make install
if [ -f $TP_INSTALL_DIR/lib64/libcrypto.a ]; then
mkdir -p $TP_INSTALL_DIR/lib && \
cp $TP_INSTALL_DIR/lib64/libcrypto.a $TP_INSTALL_DIR/lib/libcrypto.a && \
@ -520,7 +520,7 @@ build_librdkafka() {
CPPFLAGS="-I${TP_INCLUDE_DIR}" \
LDFLAGS="-L${TP_LIB_DIR}" \
CFLAGS="-fPIC" \
./configure --prefix=$TP_INSTALL_DIR --enable-static --disable-ssl --disable-sasl
./configure --prefix=$TP_INSTALL_DIR --enable-static --disable-sasl
make -j$PARALLEL && make install
}