diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index d800a5eaef..2c3f5b7408 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -245,6 +245,9 @@ set_target_properties(brpc PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/ add_library(rocksdb STATIC IMPORTED) set_target_properties(rocksdb PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/librocksdb.a) +add_library(cyrus-sasl STATIC IMPORTED) +set_target_properties(cyrus-sasl PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libsasl2.a) + add_library(librdkafka_cpp STATIC IMPORTED) set_target_properties(librdkafka_cpp PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/librdkafka++.a) @@ -482,6 +485,7 @@ endif() # add it here first. set(COMMON_THIRDPARTY rocksdb + cyrus-sasl librdkafka_cpp librdkafka libs2 diff --git a/be/test/exprs/string_functions_test.cpp b/be/test/exprs/string_functions_test.cpp index 330d99c563..267d3efb2c 100644 --- a/be/test/exprs/string_functions_test.cpp +++ b/be/test/exprs/string_functions_test.cpp @@ -292,6 +292,7 @@ TEST_F(StringFunctionsTest, left) { ASSERT_EQ(AnyValUtil::from_string(ctx, std::string("")), StringFunctions::left(context, StringVal(""), 10)); + delete context; } TEST_F(StringFunctionsTest, substring) { diff --git a/be/test/util/counts_test.cpp b/be/test/util/counts_test.cpp index 2d683efa0a..db020a6b0d 100644 --- a/be/test/util/counts_test.cpp +++ b/be/test/util/counts_test.cpp @@ -59,7 +59,7 @@ TEST_F(TCountsTest, TotalTest) { counts.merge(&other1); // 1 1 1 1 2 5 7 7 9 9 10 19 50 50 50 99 99 100 100 100 EXPECT_EQ(counts.terminate(0.3).val, 6.4); - + delete [] writer; } } // namespace doris diff --git a/be/test/util/tuple_row_zorder_compare_test.cpp b/be/test/util/tuple_row_zorder_compare_test.cpp index b83b53cccc..11ece77ac9 100644 --- a/be/test/util/tuple_row_zorder_compare_test.cpp +++ b/be/test/util/tuple_row_zorder_compare_test.cpp @@ -34,23 +34,21 @@ namespace doris { class TupleRowZOrderCompareTest : public testing::Test { public: - TupleRowZOrderComparator _comparator; ObjectPool _agg_buffer_pool; + std::unique_ptr _mem_tracker; std::unique_ptr _buffer_mem_pool; - MemTracker* _mem_tracker; - Schema* _schema; - std::vector* _slot_descs; - public: TupleRowZOrderCompareTest() { - _mem_tracker = new MemTracker(-1); - _buffer_mem_pool.reset(new MemPool(_mem_tracker)); + _mem_tracker.reset(new MemTracker(-1)); + _buffer_mem_pool.reset(new MemPool(_mem_tracker.get())); } + ~TupleRowZOrderCompareTest() = default; + template int CompareInt8Test(T lval1, T lval2, T rval1, T rval2) { int col_num = 2; - std::vector slot_descs; + std::vector slot_descs; for (int i = 1; i <= col_num; i++) { TSlotDescriptor slot_desc; slot_desc.id = i; @@ -75,35 +73,30 @@ namespace doris { slot_desc.colName = ss.str(); slot_desc.slotIdx = i; slot_desc.isMaterialized = true; - SlotDescriptor* slot = new SlotDescriptor(slot_desc); - slot_descs.push_back(slot); + slot_descs.emplace_back(slot_desc); } - _slot_descs = &slot_descs; std::vector col_schemas; col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_TINYINT, true); col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_TINYINT, true); - Schema schema(col_schemas, col_num); - _schema = &schema; + Schema schema = Schema(col_schemas, col_num); TupleRowZOrderComparator comparator(&schema, 2); - _comparator = comparator; - Tuple* lhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(int8_t), _buffer_mem_pool.get()); if (IS_FIRST_SLOT_NULL) { lhs_tuple->set_null(NullIndicatorOffset(0, 1)); } FillMem(lhs_tuple, 1, lval1, lval2); uint8_t* lhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(int8_t)); - ContiguousRow lhs_row(_schema, lhs_tuple_buf); - tuple_to_row(lhs_tuple, &lhs_row, _buffer_mem_pool.get()); + ContiguousRow lhs_row(&schema, lhs_tuple_buf); + tuple_to_row(schema, slot_descs, lhs_tuple, &lhs_row, _buffer_mem_pool.get()); Tuple* rhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(int8_t), _buffer_mem_pool.get()); FillMem(rhs_tuple, 1, rval1, rval2); uint8_t* rhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(int8_t)); - ContiguousRow rhs_row(_schema, rhs_tuple_buf); - tuple_to_row(rhs_tuple, &rhs_row, _buffer_mem_pool.get()); - int result = _comparator.compare(reinterpret_cast(lhs_row.row_ptr()), + ContiguousRow rhs_row(&schema, rhs_tuple_buf); + tuple_to_row(schema, slot_descs, rhs_tuple, &rhs_row, _buffer_mem_pool.get()); + int result = comparator.compare(reinterpret_cast(lhs_row.row_ptr()), reinterpret_cast(rhs_row.row_ptr())); return result; } @@ -111,7 +104,7 @@ namespace doris { template int CompareInt16Test(T lval1, T lval2, T rval1, T rval2) { int col_num = 2; - std::vector slot_descs; + std::vector slot_descs; for (int i = 1; i <= col_num; i++) { TSlotDescriptor slot_desc; slot_desc.id = i; @@ -127,7 +120,7 @@ namespace doris { } slot_desc.slotType = type; slot_desc.columnPos = i; - slot_desc.byteOffset = (i-1)*sizeof(int16_t) + 1*i; + slot_desc.byteOffset = (i-1)*sizeof(int16_t) + 1 * i; slot_desc.nullIndicatorByte = 0+(i-1)*sizeof(int16_t) + 1*(i-1); slot_desc.nullIndicatorBit = 1; @@ -136,35 +129,31 @@ namespace doris { slot_desc.colName = ss.str(); slot_desc.slotIdx = i; slot_desc.isMaterialized = true; - SlotDescriptor* slot = new SlotDescriptor(slot_desc); - slot_descs.push_back(slot); + + slot_descs.emplace_back(slot_desc); } - _slot_descs = &slot_descs; std::vector col_schemas; col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_SMALLINT, true); col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_SMALLINT, true); - Schema schema(col_schemas, col_num); - _schema = &schema; + Schema schema = Schema(col_schemas, col_num); TupleRowZOrderComparator comparator(&schema, 2); - _comparator = comparator; - Tuple* lhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(int16_t), _buffer_mem_pool.get()); if (IS_FIRST_SLOT_NULL) { lhs_tuple->set_null(NullIndicatorOffset(0, 1)); } FillMem(lhs_tuple, 1, lval1, lval2); uint8_t* lhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(int16_t)); - ContiguousRow lhs_row(_schema, lhs_tuple_buf); - tuple_to_row(lhs_tuple, &lhs_row, _buffer_mem_pool.get()); + ContiguousRow lhs_row(&schema, lhs_tuple_buf); + tuple_to_row(schema, slot_descs, lhs_tuple, &lhs_row, _buffer_mem_pool.get()); Tuple* rhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(int16_t), _buffer_mem_pool.get()); FillMem(rhs_tuple, 1, rval1, rval2); uint8_t* rhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(int16_t)); - ContiguousRow rhs_row(_schema, rhs_tuple_buf); - tuple_to_row(rhs_tuple, &rhs_row, _buffer_mem_pool.get()); - int result = _comparator.compare(reinterpret_cast(lhs_row.row_ptr()), + ContiguousRow rhs_row(&schema, rhs_tuple_buf); + tuple_to_row(schema, slot_descs, rhs_tuple, &rhs_row, _buffer_mem_pool.get()); + int result = comparator.compare(reinterpret_cast(lhs_row.row_ptr()), reinterpret_cast(rhs_row.row_ptr())); return result; } @@ -172,7 +161,7 @@ namespace doris { template int CompareIntTest(T lval1, T lval2, T rval1, T rval2) { int col_num = 2; - std::vector slot_descs; + std::vector slot_descs; for (int i = 1; i <= col_num; i++) { TSlotDescriptor slot_desc; slot_desc.id = i; @@ -189,7 +178,6 @@ namespace doris { slot_desc.slotType = type; slot_desc.columnPos = i; slot_desc.byteOffset = (i-1)*sizeof(int32_t) + 1*i; - slot_desc.nullIndicatorByte = 0+(i-1)*sizeof(int32_t) + 1*(i-1); slot_desc.nullIndicatorBit = 1; std::ostringstream ss; @@ -197,35 +185,30 @@ namespace doris { slot_desc.colName = ss.str(); slot_desc.slotIdx = i; slot_desc.isMaterialized = true; - SlotDescriptor* slot = new SlotDescriptor(slot_desc); - slot_descs.push_back(slot); + slot_descs.emplace_back(slot_desc); } - _slot_descs = &slot_descs; std::vector col_schemas; col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_INT, true); col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_INT, true); - Schema schema(col_schemas, col_num); - _schema = &schema; + Schema schema = Schema(col_schemas, col_num); TupleRowZOrderComparator comparator(&schema, 2); - _comparator = comparator; - Tuple* lhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(int32_t), _buffer_mem_pool.get()); if (IS_FIRST_SLOT_NULL) { lhs_tuple->set_null(NullIndicatorOffset(0, 1)); } FillMem(lhs_tuple, 1, lval1, lval2); uint8_t* lhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(int32_t)); - ContiguousRow lhs_row(_schema, lhs_tuple_buf); - tuple_to_row(lhs_tuple, &lhs_row, _buffer_mem_pool.get()); + ContiguousRow lhs_row(&schema, lhs_tuple_buf); + tuple_to_row(schema, slot_descs, lhs_tuple, &lhs_row, _buffer_mem_pool.get()); Tuple* rhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(int32_t), _buffer_mem_pool.get()); FillMem(rhs_tuple, 1, rval1, rval2); uint8_t* rhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(int32_t)); - ContiguousRow rhs_row(_schema, rhs_tuple_buf); - tuple_to_row(rhs_tuple, &rhs_row, _buffer_mem_pool.get()); - int result = _comparator.compare(reinterpret_cast(lhs_row.row_ptr()), + ContiguousRow rhs_row(&schema, rhs_tuple_buf); + tuple_to_row(schema, slot_descs, rhs_tuple, &rhs_row, _buffer_mem_pool.get()); + int result = comparator.compare(reinterpret_cast(lhs_row.row_ptr()), reinterpret_cast(rhs_row.row_ptr())); return result; } @@ -233,7 +216,7 @@ namespace doris { template int CompareInt64Test(T lval1, T lval2, T rval1, T rval2) { int col_num = 2; - std::vector slot_descs; + std::vector slot_descs; for (int i = 1; i <= col_num; i++) { TSlotDescriptor slot_desc; slot_desc.id = i; @@ -250,7 +233,6 @@ namespace doris { slot_desc.slotType = type; slot_desc.columnPos = i; slot_desc.byteOffset = (i-1)*sizeof(int64_t) + 1*i; - slot_desc.nullIndicatorByte = 0+(i-1)*sizeof(int64_t) + 1*(i-1); slot_desc.nullIndicatorBit = 1; std::ostringstream ss; @@ -258,35 +240,30 @@ namespace doris { slot_desc.colName = ss.str(); slot_desc.slotIdx = i; slot_desc.isMaterialized = true; - SlotDescriptor* slot = new SlotDescriptor(slot_desc); - slot_descs.push_back(slot); + slot_descs.emplace_back(slot_desc); } - _slot_descs = &slot_descs; std::vector col_schemas; col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_BIGINT, true); col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_BIGINT, true); - Schema schema(col_schemas, col_num); - _schema = &schema; + Schema schema = Schema(col_schemas, col_num); TupleRowZOrderComparator comparator(&schema, 2); - _comparator = comparator; - Tuple* lhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(int64_t), _buffer_mem_pool.get()); if (IS_FIRST_SLOT_NULL) { lhs_tuple->set_null(NullIndicatorOffset(0, 1)); } FillMem(lhs_tuple, 1, lval1, lval2); uint8_t* lhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(int64_t)); - ContiguousRow lhs_row(_schema, lhs_tuple_buf); - tuple_to_row(lhs_tuple, &lhs_row, _buffer_mem_pool.get()); + ContiguousRow lhs_row(&schema, lhs_tuple_buf); + tuple_to_row(schema, slot_descs, lhs_tuple, &lhs_row, _buffer_mem_pool.get()); Tuple* rhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(int64_t), _buffer_mem_pool.get()); FillMem(rhs_tuple, 1, rval1, rval2); uint8_t* rhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(int64_t)); - ContiguousRow rhs_row(_schema, rhs_tuple_buf); - tuple_to_row(rhs_tuple, &rhs_row, _buffer_mem_pool.get()); - int result = _comparator.compare(reinterpret_cast(lhs_row.row_ptr()), + ContiguousRow rhs_row(&schema, rhs_tuple_buf); + tuple_to_row(schema, slot_descs, rhs_tuple, &rhs_row, _buffer_mem_pool.get()); + int result = comparator.compare(reinterpret_cast(lhs_row.row_ptr()), reinterpret_cast(rhs_row.row_ptr())); return result; } @@ -294,7 +271,7 @@ namespace doris { template int CompareInt128Test(T lval1, T lval2, T rval1, T rval2) { int col_num = 2; - std::vector slot_descs; + std::vector slot_descs; for (int i = 1; i <= col_num; i++) { TSlotDescriptor slot_desc; slot_desc.id = i; @@ -311,7 +288,6 @@ namespace doris { slot_desc.slotType = type; slot_desc.columnPos = i; slot_desc.byteOffset = (i-1)*sizeof(int128_t) + 1*i; - slot_desc.nullIndicatorByte = 0+(i-1)*sizeof(int128_t) + 1*(i-1); slot_desc.nullIndicatorBit = 1; std::ostringstream ss; @@ -319,36 +295,30 @@ namespace doris { slot_desc.colName = ss.str(); slot_desc.slotIdx = i; slot_desc.isMaterialized = true; - SlotDescriptor* slot = new SlotDescriptor(slot_desc); - slot_descs.push_back(slot); + slot_descs.emplace_back(slot_desc); } - _slot_descs = &slot_descs; std::vector col_schemas; col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_LARGEINT, true); col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_LARGEINT, true); - Schema schema(col_schemas, col_num); - _schema = &schema; + Schema schema = Schema(col_schemas, col_num); TupleRowZOrderComparator comparator(&schema, 2); - _comparator = comparator; - MemTable::RowCursorComparator row_comparator(&schema); - Tuple* lhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(int128_t), _buffer_mem_pool.get()); if (IS_FIRST_SLOT_NULL) { lhs_tuple->set_null(NullIndicatorOffset(0, 1)); } FillMem(lhs_tuple, 1, lval1, lval2); uint8_t* lhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(int128_t)); - ContiguousRow lhs_row(_schema, lhs_tuple_buf); - tuple_to_row(lhs_tuple, &lhs_row, _buffer_mem_pool.get()); + ContiguousRow lhs_row(&schema, lhs_tuple_buf); + tuple_to_row(schema, slot_descs, lhs_tuple, &lhs_row, _buffer_mem_pool.get()); Tuple* rhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(int128_t), _buffer_mem_pool.get()); FillMem(rhs_tuple, 1, rval1, rval2); uint8_t* rhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(int128_t)); - ContiguousRow rhs_row(_schema, rhs_tuple_buf); - tuple_to_row(rhs_tuple, &rhs_row, _buffer_mem_pool.get()); - int result = _comparator.compare(reinterpret_cast(lhs_row.row_ptr()), + ContiguousRow rhs_row(&schema, rhs_tuple_buf); + tuple_to_row(schema, slot_descs, rhs_tuple, &rhs_row, _buffer_mem_pool.get()); + int result = comparator.compare(reinterpret_cast(lhs_row.row_ptr()), reinterpret_cast(rhs_row.row_ptr())); return result; } @@ -356,7 +326,7 @@ namespace doris { template int CompareFloatTest(T lval1, T lval2, T rval1, T rval2) { int col_num = 2; - std::vector slot_descs; + std::vector slot_descs; for (int i = 1; i <= col_num; i++) { TSlotDescriptor slot_desc; slot_desc.id = i; @@ -373,7 +343,6 @@ namespace doris { slot_desc.slotType = type; slot_desc.columnPos = i; slot_desc.byteOffset = (i-1)*sizeof(float) + 1*i; - slot_desc.nullIndicatorByte = 0+(i-1)*sizeof(float) + 1*(i-1); slot_desc.nullIndicatorBit = 1; std::ostringstream ss; @@ -381,35 +350,30 @@ namespace doris { slot_desc.colName = ss.str(); slot_desc.slotIdx = i; slot_desc.isMaterialized = true; - SlotDescriptor* slot = new SlotDescriptor(slot_desc); - slot_descs.push_back(slot); + slot_descs.emplace_back(slot_desc); } - _slot_descs = &slot_descs; std::vector col_schemas; col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_FLOAT, true); col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_FLOAT, true); - Schema schema(col_schemas, col_num); - _schema = &schema; + Schema schema = Schema(col_schemas, col_num); TupleRowZOrderComparator comparator(&schema, 2); - _comparator = comparator; - Tuple* lhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(float), _buffer_mem_pool.get()); if (IS_FIRST_SLOT_NULL) { lhs_tuple->set_null(NullIndicatorOffset(0, 1)); } FillMem(lhs_tuple, 1, lval1, lval2); uint8_t* lhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(float)); - ContiguousRow lhs_row(_schema, lhs_tuple_buf); - tuple_to_row(lhs_tuple, &lhs_row, _buffer_mem_pool.get()); + ContiguousRow lhs_row(&schema, lhs_tuple_buf); + tuple_to_row(schema, slot_descs, lhs_tuple, &lhs_row, _buffer_mem_pool.get()); Tuple* rhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(float), _buffer_mem_pool.get()); FillMem(rhs_tuple, 1, rval1, rval2); uint8_t* rhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(float)); - ContiguousRow rhs_row(_schema, rhs_tuple_buf); - tuple_to_row(rhs_tuple, &rhs_row, _buffer_mem_pool.get()); - int result = _comparator.compare(reinterpret_cast(lhs_row.row_ptr()), + ContiguousRow rhs_row(&schema, rhs_tuple_buf); + tuple_to_row(schema, slot_descs, rhs_tuple, &rhs_row, _buffer_mem_pool.get()); + int result = comparator.compare(reinterpret_cast(lhs_row.row_ptr()), reinterpret_cast(rhs_row.row_ptr())); return result; } @@ -417,7 +381,7 @@ namespace doris { template int CompareDoubleTest(T lval1, T lval2, T rval1, T rval2) { int col_num = 2; - std::vector slot_descs; + std::vector slot_descs; for (int i = 1; i <= col_num; i++) { TSlotDescriptor slot_desc; slot_desc.id = i; @@ -434,7 +398,6 @@ namespace doris { slot_desc.slotType = type; slot_desc.columnPos = i; slot_desc.byteOffset = (i-1)*sizeof(double) + 1*i; - slot_desc.nullIndicatorByte = 0+(i-1)*sizeof(double) + 1*(i-1); slot_desc.nullIndicatorBit = 1; std::ostringstream ss; @@ -442,35 +405,31 @@ namespace doris { slot_desc.colName = ss.str(); slot_desc.slotIdx = i; slot_desc.isMaterialized = true; - SlotDescriptor* slot = new SlotDescriptor(slot_desc); - slot_descs.push_back(slot); + + slot_descs.emplace_back(slot_desc); } - _slot_descs = &slot_descs; std::vector col_schemas; col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_DOUBLE, true); col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_DOUBLE, true); - Schema schema(col_schemas, col_num); - _schema = &schema; + Schema schema = Schema(col_schemas, col_num); TupleRowZOrderComparator comparator(&schema, 2); - _comparator = comparator; - Tuple* lhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(double), _buffer_mem_pool.get()); if (IS_FIRST_SLOT_NULL) { lhs_tuple->set_null(NullIndicatorOffset(0, 1)); } FillMem(lhs_tuple, 1, lval1, lval2); uint8_t* lhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(double)); - ContiguousRow lhs_row(_schema, lhs_tuple_buf); - tuple_to_row(lhs_tuple, &lhs_row, _buffer_mem_pool.get()); + ContiguousRow lhs_row(&schema, lhs_tuple_buf); + tuple_to_row(schema, slot_descs, lhs_tuple, &lhs_row, _buffer_mem_pool.get()); Tuple* rhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(double), _buffer_mem_pool.get()); FillMem(rhs_tuple, 1, rval1, rval2); uint8_t* rhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(double)); - ContiguousRow rhs_row(_schema, rhs_tuple_buf); - tuple_to_row(rhs_tuple, &rhs_row, _buffer_mem_pool.get()); - int result = _comparator.compare(reinterpret_cast(lhs_row.row_ptr()), + ContiguousRow rhs_row(&schema, rhs_tuple_buf); + tuple_to_row(schema, slot_descs, rhs_tuple, &rhs_row, _buffer_mem_pool.get()); + int result = comparator.compare(reinterpret_cast(lhs_row.row_ptr()), reinterpret_cast(rhs_row.row_ptr())); return result; } @@ -478,7 +437,7 @@ namespace doris { template int CompareBoolTest(T lval1, T lval2, T rval1, T rval2) { int col_num = 2; - std::vector slot_descs; + std::vector slot_descs; for (int i = 1; i <= col_num; i++) { TSlotDescriptor slot_desc; slot_desc.id = i; @@ -502,35 +461,30 @@ namespace doris { slot_desc.colName = ss.str(); slot_desc.slotIdx = i; slot_desc.isMaterialized = true; - SlotDescriptor* slot = new SlotDescriptor(slot_desc); - slot_descs.push_back(slot); + slot_descs.emplace_back(slot_desc); } - _slot_descs = &slot_descs; std::vector col_schemas; col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_BOOL, true); col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_BOOL, true); - Schema schema(col_schemas, col_num); - _schema = &schema; + Schema schema = Schema(col_schemas, col_num); TupleRowZOrderComparator comparator(&schema, 2); - _comparator = comparator; - Tuple* lhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(bool), _buffer_mem_pool.get()); if (IS_FIRST_SLOT_NULL) { lhs_tuple->set_null(NullIndicatorOffset(0, 1)); } FillMem(lhs_tuple, 1, lval1, lval2); uint8_t* lhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(bool)); - ContiguousRow lhs_row(_schema, lhs_tuple_buf); - tuple_to_row(lhs_tuple, &lhs_row, _buffer_mem_pool.get()); + ContiguousRow lhs_row(&schema, lhs_tuple_buf); + tuple_to_row(schema, slot_descs, lhs_tuple, &lhs_row, _buffer_mem_pool.get()); Tuple* rhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(bool), _buffer_mem_pool.get()); FillMem(rhs_tuple, 1, rval1, rval2); uint8_t* rhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(bool)); - ContiguousRow rhs_row(_schema, rhs_tuple_buf); - tuple_to_row(rhs_tuple, &rhs_row, _buffer_mem_pool.get()); - int result = _comparator.compare(reinterpret_cast(lhs_row.row_ptr()), + ContiguousRow rhs_row(&schema, rhs_tuple_buf); + tuple_to_row(schema, slot_descs, rhs_tuple, &rhs_row, _buffer_mem_pool.get()); + int result = comparator.compare(reinterpret_cast(lhs_row.row_ptr()), reinterpret_cast(rhs_row.row_ptr())); return result; } @@ -538,7 +492,7 @@ namespace doris { template int CompareCharTest(T lval1, T lval2, T rval1, T rval2) { int col_num = 2; - std::vector slot_descs; + std::vector slot_descs; for (int i = 1; i <= col_num; i++) { TSlotDescriptor slot_desc; slot_desc.id = i; @@ -563,35 +517,30 @@ namespace doris { slot_desc.colName = ss.str(); slot_desc.slotIdx = i; slot_desc.isMaterialized = true; - SlotDescriptor* slot = new SlotDescriptor(slot_desc); - slot_descs.push_back(slot); + slot_descs.emplace_back(slot_desc); } - _slot_descs = &slot_descs; std::vector col_schemas; col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_VARCHAR, true); col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_VARCHAR, true); - Schema schema(col_schemas, col_num); - _schema = &schema; + Schema schema = Schema(col_schemas, col_num); TupleRowZOrderComparator comparator(&schema, 2); - _comparator = comparator; - Tuple* lhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(StringValue), _buffer_mem_pool.get()); if (IS_FIRST_SLOT_NULL) { lhs_tuple->set_null(NullIndicatorOffset(0, 1)); } FillMem(lhs_tuple, 1, lval1, lval2); uint8_t* lhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(StringValue)); - ContiguousRow lhs_row(_schema, lhs_tuple_buf); - tuple_to_row(lhs_tuple, &lhs_row, _buffer_mem_pool.get()); + ContiguousRow lhs_row(&schema, lhs_tuple_buf); + tuple_to_row(schema, slot_descs, lhs_tuple, &lhs_row, _buffer_mem_pool.get()); Tuple* rhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(StringValue), _buffer_mem_pool.get()); FillMem(rhs_tuple, 1, rval1, rval2); uint8_t* rhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(StringValue)); - ContiguousRow rhs_row(_schema, rhs_tuple_buf); - tuple_to_row(rhs_tuple, &rhs_row, _buffer_mem_pool.get()); - int result = _comparator.compare(reinterpret_cast(lhs_row.row_ptr()), + ContiguousRow rhs_row(&schema, rhs_tuple_buf); + tuple_to_row(schema, slot_descs, rhs_tuple, &rhs_row, _buffer_mem_pool.get()); + int result = comparator.compare(reinterpret_cast(lhs_row.row_ptr()), reinterpret_cast(rhs_row.row_ptr())); return result; } @@ -599,7 +548,7 @@ namespace doris { template int CompareDateTest(T lval1, T lval2, T rval1, T rval2) { int col_num = 2; - std::vector slot_descs; + std::vector slot_descs; for (int i = 1; i <= col_num; i++) { TSlotDescriptor slot_desc; slot_desc.id = i; @@ -623,35 +572,30 @@ namespace doris { slot_desc.colName = ss.str(); slot_desc.slotIdx = i; slot_desc.isMaterialized = true; - SlotDescriptor* slot = new SlotDescriptor(slot_desc); - slot_descs.push_back(slot); + slot_descs.emplace_back(slot_desc); } - _slot_descs = &slot_descs; std::vector col_schemas; col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_DATETIME, true); col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_DATETIME, true); - Schema schema(col_schemas, col_num); - _schema = &schema; + Schema schema = Schema(col_schemas, col_num); TupleRowZOrderComparator comparator(&schema, 2); - _comparator = comparator; - Tuple* lhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(DateTimeValue), _buffer_mem_pool.get()); if (IS_FIRST_SLOT_NULL) { lhs_tuple->set_null(NullIndicatorOffset(0, 1)); } FillMem(lhs_tuple, 1, lval1, lval2); uint8_t* lhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(DateTimeValue)); - ContiguousRow lhs_row(_schema, lhs_tuple_buf); - tuple_to_row(lhs_tuple, &lhs_row, _buffer_mem_pool.get()); + ContiguousRow lhs_row(&schema, lhs_tuple_buf); + tuple_to_row(schema, slot_descs, lhs_tuple, &lhs_row, _buffer_mem_pool.get()); Tuple* rhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(DateTimeValue), _buffer_mem_pool.get()); FillMem(rhs_tuple, 1, rval1, rval2); uint8_t* rhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(DateTimeValue)); - ContiguousRow rhs_row(_schema, rhs_tuple_buf); - tuple_to_row(rhs_tuple, &rhs_row, _buffer_mem_pool.get()); - int result = _comparator.compare(reinterpret_cast(lhs_row.row_ptr()), + ContiguousRow rhs_row(&schema, rhs_tuple_buf); + tuple_to_row(schema, slot_descs, rhs_tuple, &rhs_row, _buffer_mem_pool.get()); + int result = comparator.compare(reinterpret_cast(lhs_row.row_ptr()), reinterpret_cast(rhs_row.row_ptr())); return result; } @@ -659,7 +603,7 @@ namespace doris { template int CompareDecimalTest(T lval1, T lval2, T rval1, T rval2) { int col_num = 2; - std::vector slot_descs; + std::vector slot_descs; for (int i = 1; i <= col_num; i++) { TSlotDescriptor slot_desc; slot_desc.id = i; @@ -687,47 +631,43 @@ namespace doris { slot_desc.colName = ss.str(); slot_desc.slotIdx = i; slot_desc.isMaterialized = true; - SlotDescriptor* slot = new SlotDescriptor(slot_desc); - slot_descs.push_back(slot); + slot_descs.emplace_back(slot_desc); } - _slot_descs = &slot_descs; std::vector col_schemas; col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_DECIMAL, true); col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_DECIMAL, true); - Schema schema(col_schemas, col_num); - _schema = &schema; + Schema schema = Schema(col_schemas, col_num); TupleRowZOrderComparator comparator(&schema, 2); - _comparator = comparator; - Tuple* lhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(int128_t), _buffer_mem_pool.get()); if (IS_FIRST_SLOT_NULL) { lhs_tuple->set_null(NullIndicatorOffset(0, 1)); } FillMem(lhs_tuple, 1, lval1.value(), lval2.value()); uint8_t* lhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(int128_t)); - ContiguousRow lhs_row(_schema, lhs_tuple_buf); - tuple_to_row(lhs_tuple, &lhs_row, _buffer_mem_pool.get()); + ContiguousRow lhs_row(&schema, lhs_tuple_buf); + tuple_to_row(schema, slot_descs, lhs_tuple, &lhs_row, _buffer_mem_pool.get()); Tuple* rhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(int128_t), _buffer_mem_pool.get()); FillMem(rhs_tuple, 1, rval1.value(), rval2.value()); uint8_t* rhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(int128_t)); - ContiguousRow rhs_row(_schema, rhs_tuple_buf); - tuple_to_row(rhs_tuple, &rhs_row, _buffer_mem_pool.get()); - int result = _comparator.compare(reinterpret_cast(lhs_row.row_ptr()), + ContiguousRow rhs_row(&schema, rhs_tuple_buf); + tuple_to_row(schema, slot_descs, rhs_tuple, &rhs_row, _buffer_mem_pool.get()); + int result = comparator.compare(reinterpret_cast(lhs_row.row_ptr()), reinterpret_cast(rhs_row.row_ptr())); return result; } - void tuple_to_row(const Tuple* tuple, ContiguousRow* row, MemPool* mem_pool) { - for (size_t i = 0; i < _slot_descs->size(); ++i) { + void tuple_to_row(const Schema& schema, const vector& slot_descs, const Tuple* tuple, + ContiguousRow* row, MemPool* mem_pool) { + for (size_t i = 0; i < slot_descs.size(); ++i) { auto cell = row->cell(i); - const SlotDescriptor* slot = (*_slot_descs)[i]; - bool is_null = tuple->is_null(slot->null_indicator_offset()); - const void* value = tuple->get_slot(slot->tuple_offset()); - _schema->column(i)->consume(&cell, (const char*)value, is_null, mem_pool, + const SlotDescriptor& slot = slot_descs[i]; + bool is_null = tuple->is_null(slot.null_indicator_offset()); + const void* value = tuple->get_slot(slot.tuple_offset()); + schema.column(i)->consume(&cell, (const char*)value, is_null, mem_pool, &_agg_buffer_pool); } } diff --git a/fe/spark-dpp/pom.xml b/fe/spark-dpp/pom.xml index 1a44c1359c..b3396fffd6 100644 --- a/fe/spark-dpp/pom.xml +++ b/fe/spark-dpp/pom.xml @@ -177,7 +177,7 @@ under the License. not reuse forked jvm, so that each unit test will run in separate jvm. to avoid singleton confict<--> false - -javaagent:${settings.localRepository}/org/jmockit/jmockit/1.48/jmockit-1.48.jar + -javaagent:${settings.localRepository}/org/jmockit/jmockit/${jmockit.version}/jmockit-${jmockit.version}.jar diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index f6c0b74517..03bf9bc62e 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -521,6 +521,14 @@ build_rocksdb() { cp -r include/rocksdb ../../installed/include/ } +# cyrus_sasl +build_cyrus_sasl() { + check_if_source_exist $CYRUS_SASL_SOURCE + cd $TP_SOURCE_DIR/$CYRUS_SASL_SOURCE + ./configure --prefix=$TP_INSTALL_DIR --prefix=$TP_INSTALL_DIR --enable-static --enable-shared=no --with-openssl=$TP_INSTALL_DIR + make -j $PARALLEL && make install +} + # librdkafka build_librdkafka() { check_if_source_exist $LIBRDKAFKA_SOURCE @@ -739,6 +747,9 @@ build_orc() { -DLZ4_HOME=$TP_INSTALL_DIR \ -DLZ4_INCLUDE_DIR=$TP_INSTALL_DIR/include/lz4 \ -DZLIB_HOME=$TP_INSTALL_DIR \ + -DZSTD_HOME=$TP_INSTALL_DIR \ + -DZSTD_INCLUDE_DIR=$TP_INSTALL_DIR/include \ + -DZSTD_LIBRARIES=$TP_INSTALL_DIR/lib/libzstd.a \ -DBUILD_LIBHDFSPP=OFF \ -DBUILD_CPP_TESTS=OFF \ -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR @@ -933,6 +944,7 @@ build_thrift build_leveldb build_brpc build_rocksdb +build_cyrus_sasl build_librdkafka build_flatbuffers build_arrow diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index ae473836bc..f71e0ea1eb 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -186,6 +186,12 @@ ROCKSDB_NAME=rocksdb-5.14.2.tar.gz ROCKSDB_SOURCE=rocksdb-5.14.2 ROCKSDB_MD5SUM="b72720ea3b1e9ca9e4ed0febfef65b14" +#cyrus-sasl +CYRUS_SASL_DOWNLOAD="https://github.com/cyrusimap/cyrus-sasl/releases/download/cyrus-sasl-2.1.27/cyrus-sasl-2.1.27.tar.gz" +CYRUS_SASL_NAME=cyrus-sasl-2.1.27.tar.gz +CYRUS_SASL_SOURCE=cyrus-sasl-2.1.27 +CYRUS_SASL_MD5SUM="a33820c66e0622222c5aefafa1581083" + # librdkafka-1.8.0 LIBRDKAFKA_DOWNLOAD="https://github.com/edenhill/librdkafka/archive/v1.8.0.tar.gz" LIBRDKAFKA_NAME=librdkafka-1.8.0.tar.gz @@ -406,6 +412,7 @@ ODBC LEVELDB BRPC ROCKSDB +CYRUS_SASL LIBRDKAFKA FLATBUFFERS ARROW