24d38614a0
Upgrade the following dependecies: libevent -> 2.1.12 OpenSSL 1.0.2k -> 1.1.1l thrift 0.9.3 -> 0.13.0 protobuf 3.5.1 -> 3.14.0 gflags 2.2.0 -> 2.2.2 glog 0.3.3 -> 0.4.0 googletest 1.8.0 -> 1.10.0 snappy 1.1.7 -> 1.1.8 gperftools 2.7 -> 2.9.1 lz4 1.7.5 -> 1.9.3 curl 7.54.1 -> 7.79.0 re2 2017-05-01 -> 2021-02-02 zstd 1.3.7 -> 1.5.0 brotli 1.0.7 -> 1.0.9 flatbuffers 1.10.0 -> 2.0.0 apache-arrow 0.15.1 -> 5.0.0 CRoaring 0.2.60 -> 0.3.4 orc 1.5.8 -> 1.6.6 libdivide 4.0.0 -> 5.0 brpc 0.97 -> 1.0.0-rc02 librdkafka 1.7.0 -> 1.8.0 after this pr compile doris should use build-env:1.4.0
98 lines
3.2 KiB
C++
98 lines
3.2 KiB
C++
// Licensed to the Apache Software Foundation (ASF) under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing,
|
|
// software distributed under the License is distributed on an
|
|
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
// KIND, either express or implied. See the License for the
|
|
// specific language governing permissions and limitations
|
|
// under the License.
|
|
|
|
#include <gtest/gtest.h>
|
|
|
|
#include <sstream>
|
|
#include <string>
|
|
|
|
#include "common/logging.h"
|
|
#include "util/arrow/row_batch.h"
|
|
|
|
#define ARROW_UTIL_LOGGING_H
|
|
#include <arrow/buffer.h>
|
|
#include <arrow/json/api.h>
|
|
#include <arrow/json/test_common.h>
|
|
#include <arrow/pretty_print.h>
|
|
#include <arrow/result.h>
|
|
|
|
#include "common/object_pool.h"
|
|
#include "runtime/mem_tracker.h"
|
|
#include "runtime/row_batch.h"
|
|
#include "util/debug_util.h"
|
|
|
|
namespace doris {
|
|
|
|
class ArrowRowBatchTest : public testing::Test {};
|
|
|
|
std::string test_str() {
|
|
return R"(
|
|
{ "c1": 1, "c2": 1.1 }
|
|
{ "c1": 2, "c2": 2.2 }
|
|
{ "c1": 3, "c2": 3.3 }
|
|
)";
|
|
}
|
|
|
|
void MakeBuffer(const std::string& data, std::shared_ptr<arrow::Buffer>* out) {
|
|
auto res = arrow::AllocateBuffer(data.size(), arrow::default_memory_pool());
|
|
*out = std::move(res.ValueOrDie());
|
|
std::copy(std::begin(data), std::end(data), (*out)->mutable_data());
|
|
}
|
|
|
|
TEST_F(ArrowRowBatchTest, PrettyPrint) {
|
|
auto json = test_str();
|
|
std::shared_ptr<arrow::Buffer> buffer;
|
|
MakeBuffer(test_str(), &buffer);
|
|
arrow::json::ParseOptions parse_opts = arrow::json::ParseOptions::Defaults();
|
|
parse_opts.explicit_schema = arrow::schema({
|
|
arrow::field("c1", arrow::int64()),
|
|
});
|
|
|
|
auto arrow_st = arrow::json::ParseOne(parse_opts, buffer);
|
|
ASSERT_TRUE(arrow_st.ok());
|
|
std::shared_ptr<arrow::RecordBatch> record_batch = arrow_st.ValueOrDie();
|
|
|
|
ObjectPool obj_pool;
|
|
RowDescriptor* row_desc;
|
|
auto doris_st = convert_to_row_desc(&obj_pool, *record_batch->schema(), &row_desc);
|
|
ASSERT_TRUE(doris_st.ok());
|
|
auto tracker = std::make_shared<MemTracker>(-1, "PrettyPrintTest");
|
|
std::shared_ptr<RowBatch> row_batch;
|
|
doris_st = convert_to_row_batch(*record_batch, *row_desc, tracker, &row_batch);
|
|
ASSERT_TRUE(doris_st.ok());
|
|
|
|
{
|
|
std::shared_ptr<arrow::Schema> check_schema;
|
|
doris_st = convert_to_arrow_schema(*row_desc, &check_schema);
|
|
ASSERT_TRUE(doris_st.ok());
|
|
|
|
arrow::MemoryPool* pool = arrow::default_memory_pool();
|
|
std::shared_ptr<arrow::RecordBatch> check_batch;
|
|
doris_st = convert_to_arrow_batch(*row_batch, check_schema, pool, &check_batch);
|
|
ASSERT_TRUE(doris_st.ok());
|
|
ASSERT_EQ(3, check_batch->num_rows());
|
|
ASSERT_TRUE(record_batch->Equals(*check_batch));
|
|
}
|
|
}
|
|
|
|
} // namespace doris
|
|
|
|
int main(int argc, char** argv) {
|
|
::testing::InitGoogleTest(&argc, argv);
|
|
return RUN_ALL_TESTS();
|
|
}
|