1605 lines
65 KiB
Python
1605 lines
65 KiB
Python
#!/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
############################################################################
|
|
#
|
|
# @file test_sys_partition_load.py
|
|
# @date 2015/05/13 15:26:21
|
|
# @brief This file is a test file for palo data loading and verifying.
|
|
#
|
|
#############################################################################
|
|
|
|
"""
|
|
复合分区，导入操作测试
|
|
"""
|
|
|
|
import time
|
|
import threading
|
|
import pytest
|
|
from data import partition as DATA
|
|
from data import timestamp as DATA_TS
|
|
from lib import palo_config
|
|
from lib import palo_client
|
|
from lib import palo_task
|
|
from lib import util
|
|
|
|
LOG = palo_client.LOG
|
|
L = palo_client.L
|
|
config = palo_config.config
|
|
broker_info = palo_config.broker_info
|
|
|
|
|
|
class LoadThread(threading.Thread):
|
|
"""
|
|
导入线程
|
|
"""
|
|
def __init__(self, database_name, data_desc_list, label, max_filter_ratio=None):
|
|
threading.Thread.__init__(self)
|
|
self.database_name = database_name
|
|
self.data_desc_list = data_desc_list
|
|
self.label = label
|
|
self.max_filter_ratio = max_filter_ratio
|
|
|
|
def run(self):
|
|
"""
|
|
run
|
|
"""
|
|
thread_client = palo_client.get_client(config.fe_host, config.fe_query_port, self.database_name,
|
|
user=config.fe_user, password=config.fe_password,
|
|
http_port=config.fe_http_port)
|
|
thread_client.batch_load(self.label, self.data_desc_list, broker=broker_info,
|
|
max_filter_ratio=self.max_filter_ratio, is_wait=True)
|
|
|
|
|
|
def setup_module():
|
|
"""
|
|
setUp
|
|
"""
|
|
global client
|
|
client = palo_client.get_client(config.fe_host, config.fe_query_port, user=config.fe_user,
|
|
password=config.fe_password, http_port=config.fe_http_port)
|
|
|
|
|
|
def check_partition_list(table_name, partition_name_list):
|
|
"""
|
|
验证分区是否创建成功
|
|
"""
|
|
for partition_name in partition_name_list:
|
|
assert client.get_partition(table_name, partition_name)
|
|
|
|
|
|
def check_load_and_verify(table_name, load_partition_list=None):
|
|
"""
|
|
验证表是否创建成功，分区是否创建成功，导入数据，校验
|
|
"""
|
|
data_desc_list = palo_client.LoadDataInfo(DATA.file_path_1, \
|
|
table_name, load_partition_list)
|
|
assert client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info)
|
|
assert client.verify(DATA.expected_data_file_list_1, table_name)
|
|
assert client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info)
|
|
assert client.verify(list(DATA.expected_data_file_list_1) * 2, table_name)
|
|
|
|
|
|
def partition_check(table_name, column_name, partition_name_list, \
|
|
partition_value_list, distribution_type, bucket_num, storage_type):
|
|
"""
|
|
检查，验证
|
|
"""
|
|
partition_info = palo_client.PartitionInfo(column_name, \
|
|
partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo(distribution_type, bucket_num)
|
|
client.create_table(table_name, DATA.schema_1, partition_info,
|
|
distribution_info, keys_desc='AGGREGATE KEY (k1, k2, k3, k4, k5)')
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
|
|
def test_load_without_partition():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_load_without_partition",
|
|
"describe": "导入，不指定分区, 带rollup表",
|
|
"tag": "system,p1"
|
|
}
|
|
"""
|
|
"""
|
|
导入，不指定分区, 带rollup表
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['5', '20', '30', 'MAXVALUE']
|
|
|
|
partition_check(table_name, 'k1', \
|
|
partition_name_list, partition_value_list, \
|
|
'RANDOM', 13, 'row')
|
|
|
|
column_name_list = ['k1', 'k5', 'v4', 'v5']
|
|
assert client.create_rollup_table(table_name, \
|
|
index_name, column_name_list, is_wait=True)
|
|
|
|
check_load_and_verify(table_name)
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_load_with_partition():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_load_with_partition",
|
|
"describe": "导入，指定分区",
|
|
"tag": "system,p1"
|
|
}
|
|
"""
|
|
"""
|
|
导入，指定分区
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['5', '20', '31', 'MAXVALUE']
|
|
|
|
partition_check(table_name, 'k1', \
|
|
partition_name_list, partition_value_list, \
|
|
'RANDOM', 13, 'row')
|
|
check_load_and_verify(table_name, ['partition_a', 'partition_b', 'partition_c'])
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_multi_loading():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_multi_loading",
|
|
"describe": "并行导入",
|
|
"tag": "system,p1"
|
|
}
|
|
"""
|
|
"""
|
|
并行导入
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['5', '20', '31', 'MAXVALUE']
|
|
|
|
partition_check(table_name, 'k1', \
|
|
partition_name_list, partition_value_list, \
|
|
'RANDOM', 13, 'row')
|
|
|
|
data_desc_list_1 = palo_client.LoadDataInfo(DATA.file_path_1, \
|
|
table_name, ['partition_a', 'partition_b', 'partition_c'])
|
|
data_desc_list_2 = palo_client.LoadDataInfo(DATA.file_path_1, \
|
|
table_name, ['partition_a', 'partition_b', 'partition_c', 'partition_d'])
|
|
data_desc_list_3 = palo_client.LoadDataInfo(DATA.file_path_1, table_name)
|
|
data_desc_list_4 = palo_client.LoadDataInfo(DATA.file_path_1, table_name)
|
|
laod_thread_1 = LoadThread(database_name, data_desc_list_1, 'label_1')
|
|
laod_thread_2 = LoadThread(database_name, data_desc_list_2, 'label_2')
|
|
laod_thread_3 = LoadThread(database_name, data_desc_list_3, 'label_3')
|
|
laod_thread_4 = LoadThread(database_name, data_desc_list_4, 'label_4')
|
|
laod_thread_1.start()
|
|
laod_thread_2.start()
|
|
laod_thread_3.start()
|
|
laod_thread_4.start()
|
|
laod_thread_1.join()
|
|
laod_thread_2.join()
|
|
laod_thread_3.join()
|
|
laod_thread_4.join()
|
|
|
|
assert client.verify(list(DATA.expected_data_file_list_1) * 4, table_name)
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_multi_loading_for_one_database():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_multi_loading_for_one_database",
|
|
"describe": "同一库中的单分区表和复合分区表同时进行导入, #相同label导两次，失败",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
1. 同一库中的单分区表和复合分区表同时进行导入
|
|
2. #相同label导两次，失败
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
single_table_name = 'table_1'
|
|
client.create_table(single_table_name, DATA.schema_1)
|
|
assert client.show_tables(single_table_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['5', '20', '31', 'MAXVALUE']
|
|
|
|
partition_check(table_name, 'k1', \
|
|
partition_name_list, partition_value_list, \
|
|
'RANDOM', 13, 'row')
|
|
|
|
data_desc_list_1 = palo_client.LoadDataInfo(DATA.file_path_1, single_table_name)
|
|
data_desc_list_2 = palo_client.LoadDataInfo(DATA.file_path_1, \
|
|
table_name, ['partition_a', 'partition_b', 'partition_c', 'partition_d'])
|
|
laod_thread_1 = LoadThread(database_name, data_desc_list_1, 'label_1')
|
|
laod_thread_2 = LoadThread(database_name, data_desc_list_2, 'label_2')
|
|
laod_thread_1.start()
|
|
laod_thread_2.start()
|
|
laod_thread_1.join()
|
|
laod_thread_2.join()
|
|
assert client.verify(list(DATA.expected_data_file_list_1), single_table_name)
|
|
assert client.verify(list(DATA.expected_data_file_list_1), table_name)
|
|
|
|
#相同label导两次，失败
|
|
ret = False
|
|
try:
|
|
ret = client.batch_load('label_2', data_desc_list_2, is_wait=True, broker=broker_info)
|
|
except:
|
|
pass
|
|
assert not ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_cancel_load():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_cancel_load",
|
|
"describe": "取消导入",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
取消导入
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['5', '20', '31', 'MAXVALUE']
|
|
|
|
partition_check(table_name, 'k1', \
|
|
partition_name_list, partition_value_list, \
|
|
'RANDOM', 13, 'row')
|
|
|
|
data_desc_list_1 = palo_client.LoadDataInfo(DATA.file_path_1, \
|
|
table_name, ['partition_a', 'partition_b', 'partition_c'])
|
|
data_desc_list_2 = palo_client.LoadDataInfo(DATA.file_path_1, \
|
|
table_name, ['partition_a', 'partition_b', 'partition_c', 'partition_d'])
|
|
data_desc_list_3 = palo_client.LoadDataInfo(DATA.file_path_1, table_name)
|
|
data_desc_list_4 = palo_client.LoadDataInfo(DATA.file_path_1, table_name)
|
|
laod_thread_1 = LoadThread(database_name, data_desc_list_1, 'label_1')
|
|
laod_thread_2 = LoadThread(database_name, data_desc_list_2, 'label_2')
|
|
laod_thread_3 = LoadThread(database_name, data_desc_list_3, 'label_3')
|
|
laod_thread_4 = LoadThread(database_name, data_desc_list_4, 'label_4')
|
|
laod_thread_1.start()
|
|
laod_thread_2.start()
|
|
laod_thread_3.start()
|
|
laod_thread_4.start()
|
|
time.sleep(1)
|
|
try:
|
|
client.cancel_load('label_2')
|
|
client.cancel_load('label_3')
|
|
except Exception as e:
|
|
print(str(e))
|
|
raise pytest.skip('can not cancel load')
|
|
finally:
|
|
laod_thread_1.join()
|
|
laod_thread_2.join()
|
|
laod_thread_3.join()
|
|
laod_thread_4.join()
|
|
|
|
assert client.verify(list(DATA.expected_data_file_list_1) * 2, table_name)
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_add_partition_while_loading():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_add_partition_while_loading",
|
|
"describe": "导入过程中，增加分区",
|
|
"tag": "system,p1"
|
|
}
|
|
"""
|
|
"""
|
|
导入过程中，增加分区
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['5', '20', '31', '40']
|
|
|
|
partition_check(table_name, 'k1', \
|
|
partition_name_list, partition_value_list, \
|
|
'RANDOM', 13, 'row')
|
|
|
|
data_desc_list_1 = palo_client.LoadDataInfo(DATA.file_path_1, \
|
|
table_name, ['partition_a', 'partition_b', 'partition_c'])
|
|
data_desc_list_2 = palo_client.LoadDataInfo(DATA.file_path_1, \
|
|
table_name, ['partition_a', 'partition_b', 'partition_c', 'partition_d'])
|
|
data_desc_list_3 = palo_client.LoadDataInfo(DATA.file_path_1, table_name)
|
|
data_desc_list_4 = palo_client.LoadDataInfo(DATA.file_path_1, table_name)
|
|
laod_thread_1 = LoadThread(database_name, data_desc_list_1, 'label_1')
|
|
laod_thread_2 = LoadThread(database_name, data_desc_list_2, 'label_2')
|
|
laod_thread_3 = LoadThread(database_name, data_desc_list_3, 'label_3')
|
|
laod_thread_4 = LoadThread(database_name, data_desc_list_4, 'label_4')
|
|
laod_thread_1.start()
|
|
client.add_partition(table_name, 'partition_e', '50')
|
|
laod_thread_2.start()
|
|
timeout = 30
|
|
while not client.get_load_job_list(state="LOADING") and timeout > 0:
|
|
time.sleep(1)
|
|
timeout -= 1
|
|
client.add_partition(table_name, 'partition_f', '60')
|
|
laod_thread_3.start()
|
|
client.add_partition(table_name, 'partition_g', '70')
|
|
laod_thread_4.start()
|
|
client.add_partition(table_name, 'partition_h', '80')
|
|
laod_thread_1.join()
|
|
laod_thread_2.join()
|
|
laod_thread_3.join()
|
|
laod_thread_4.join()
|
|
|
|
assert client.verify(list(DATA.expected_data_file_list_1) * 4, table_name)
|
|
add_partition_list = ['partition_e', 'partition_f', 'partition_g', 'partition_h']
|
|
#FIXME
|
|
#check_partition_list(table_name, partition_name_list.extend(add_partition_list))
|
|
check_partition_list(table_name, partition_name_list + add_partition_list)
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_load_while_selecting():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_load_while_selecting",
|
|
"describe": "查询的同时导入",
|
|
"tag": "system,p1"
|
|
}
|
|
"""
|
|
"""
|
|
查询的同时导入
|
|
"""
|
|
|
|
class SelectTask(palo_task.PaloTask):
|
|
"""
|
|
导入线程
|
|
"""
|
|
def __init__(self, database_name, table_name):
|
|
self.database_name = database_name
|
|
self.table_name = table_name
|
|
self.thread_client = palo_client.PaloClient(palo_config.config.fe_host, \
|
|
palo_config.config.fe_query_port, database_name)
|
|
self.thread_client.init()
|
|
|
|
def do_task(self):
|
|
"""
|
|
run
|
|
"""
|
|
self.thread_client.select_all(self.table_name)
|
|
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['5', '20', '31', 'MAXVALUE']
|
|
|
|
partition_check(table_name, 'k1', \
|
|
partition_name_list, partition_value_list, \
|
|
'RANDOM', 13, 'row')
|
|
|
|
data_desc_list_1 = palo_client.LoadDataInfo(DATA.file_path_1, \
|
|
table_name, ['partition_a', 'partition_b', 'partition_c'])
|
|
data_desc_list_2 = palo_client.LoadDataInfo(DATA.file_path_1, \
|
|
table_name, ['partition_a', 'partition_b', 'partition_c', 'partition_d'])
|
|
data_desc_list_3 = palo_client.LoadDataInfo(DATA.file_path_1, table_name)
|
|
data_desc_list_4 = palo_client.LoadDataInfo(DATA.file_path_1, table_name)
|
|
laod_thread_1 = LoadThread(database_name, data_desc_list_1, 'label_1')
|
|
laod_thread_2 = LoadThread(database_name, data_desc_list_2, 'label_2')
|
|
laod_thread_3 = LoadThread(database_name, data_desc_list_3, 'label_3')
|
|
laod_thread_4 = LoadThread(database_name, data_desc_list_4, 'label_4')
|
|
laod_thread_1.start()
|
|
laod_thread_2.start()
|
|
timeout = 30
|
|
while not client.get_load_job_list(state="LOADING") and timeout > 0:
|
|
time.sleep(1)
|
|
timeout -= 1
|
|
select_task = SelectTask(database_name, table_name)
|
|
select_thread = palo_task.TaskThread(select_task)
|
|
select_thread.start()
|
|
laod_thread_3.start()
|
|
laod_thread_4.start()
|
|
laod_thread_1.join()
|
|
laod_thread_2.join()
|
|
laod_thread_3.join()
|
|
laod_thread_4.join()
|
|
select_thread.stop()
|
|
|
|
assert client.verify(list(DATA.expected_data_file_list_1) * 4, table_name)
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_load_while_deleting():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_load_while_deleting",
|
|
"describe": "同时向不同的分区导入和删除",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
同时向不同的分区导入和删除
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['5', '20', '31', 'MAXVALUE']
|
|
|
|
partition_check(table_name, 'k1', \
|
|
partition_name_list, partition_value_list, \
|
|
'RANDOM', 13, 'row')
|
|
|
|
data_desc_list_1 = palo_client.LoadDataInfo(DATA.file_path_1, table_name)
|
|
assert client.batch_load(util.get_label(), data_desc_list_1, is_wait=True, broker=broker_info)
|
|
assert client.verify(DATA.expected_data_file_list_1, table_name)
|
|
data_desc_list_2 = palo_client.LoadDataInfo(DATA.file_path_1, \
|
|
table_name, ['partition_b', 'partition_c'])
|
|
laod_thread_1 = LoadThread(database_name, data_desc_list_2, 'label_1', max_filter_ratio='0.5')
|
|
laod_thread_1.start()
|
|
timeout = 30
|
|
while not client.get_load_job_list(state="LOADING") and timeout > 0:
|
|
time.sleep(1)
|
|
timeout -= 1
|
|
client.delete(table_name, [('k1', '<', '5'),], 'partition_a')
|
|
laod_thread_1.join()
|
|
|
|
assert client.verify([DATA.expected_file_10] * 2, table_name)
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_load_while_deleting_to_same_partition():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_load_while_deleting_to_same_partition",
|
|
"describe": "同时向相同的分区导入和删除",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
同时向相同的分区导入和删除
|
|
流式导入一期：支持同时向相同的分区导入和删除
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['5', '20', '31', 'MAXVALUE']
|
|
|
|
partition_check(table_name, 'k1', \
|
|
partition_name_list, partition_value_list, \
|
|
'RANDOM', 13, 'row')
|
|
|
|
data_desc_list_1 = palo_client.LoadDataInfo(DATA.file_path_1, table_name)
|
|
assert client.batch_load(util.get_label(), data_desc_list_1, is_wait=True, broker=broker_info)
|
|
assert client.verify(DATA.expected_data_file_list_1, table_name)
|
|
data_desc_list_2 = palo_client.LoadDataInfo(DATA.file_path_1, \
|
|
table_name, ['partition_b', 'partition_c'])
|
|
load_thread_1 = LoadThread(database_name, data_desc_list_2, 'label_1', max_filter_ratio='0.5')
|
|
load_thread_1.start()
|
|
timeout = 30
|
|
while not client.get_load_job_list(state="LOADING") and timeout > 0:
|
|
time.sleep(1)
|
|
timeout -= 1
|
|
ret = False
|
|
try:
|
|
ret = client.delete(table_name, [('k1', '<', '20'),], 'partition_b')
|
|
except:
|
|
pass
|
|
load_thread_1.join()
|
|
# assert not ret
|
|
assert ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_load_set_with_partition():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_load_set_with_partition",
|
|
"describe": "SET, 导入，不指定分区",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
SET, 导入，不指定分区
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \
|
|
'2099-12-21 23:59:59', 'MAXVALUE']
|
|
partition_info = palo_client.PartitionInfo('k1', \
|
|
partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
client.create_table(table_name, DATA_TS.schema_1, \
|
|
partition_info, distribution_info)
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
set_list = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \
|
|
'k3 = strftime("%Y-%m-%d %H:%M:%S", tmp_k3)', \
|
|
'k4 = strftime("%Y-%m-%d %H:%M:%S", tmp_k4)', \
|
|
'v2 = strftime("%Y-%m-%d %H:%M:%S", tmp_v2)', \
|
|
'v4 = strftime("%Y-%m-%d %H:%M:%S", tmp_v4)', \
|
|
'v6 = strftime("%Y-%m-%d %H:%M:%S", tmp_v6)', \
|
|
'v7 = strftime("%Y-%m-%d %H:%M:%S", tmp_v7)']
|
|
|
|
column_name_list = ['tmp_k1', 'k2', 'tmp_k3', 'tmp_k4', \
|
|
'v1', 'tmp_v2', 'v3', 'tmp_v4', 'v5', 'tmp_v6', 'tmp_v7']
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_1, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list)
|
|
assert client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info, max_filter_ratio=0.05)
|
|
assert client.verify(DATA_TS.expected_data_file_list_1, table_name)
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_load_set_with_null_data():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_load_set_with_null_data",
|
|
"describe": "SET, 导入，有空数据",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
SET, 导入，有空数据
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \
|
|
'2099-12-21 23:59:59', 'MAXVALUE']
|
|
partition_info = palo_client.PartitionInfo('k1', \
|
|
partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
client.create_table(table_name, DATA_TS.schema_1, \
|
|
partition_info, distribution_info)
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
set_list = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \
|
|
'k3 = strftime("%Y-%m-%d %H:%M:%S", tmp_k3)', \
|
|
'k4 = strftime("%Y-%m-%d %H:%M:%S", tmp_k4)', \
|
|
'v2 = strftime("%Y-%m-%d %H:%M:%S", tmp_v2)', \
|
|
'v4 = strftime("%Y-%m-%d %H:%M:%S", tmp_v4)', \
|
|
'v6 = strftime("%Y-%m-%d %H:%M:%S", tmp_v6)', \
|
|
'v7 = strftime("%Y-%m-%d %H:%M:%S", tmp_v7)']
|
|
|
|
column_name_list = ['tmp_k1', 'k2', 'tmp_k3', 'tmp_k4', \
|
|
'v1', 'tmp_v2', 'v3', 'tmp_v4', 'v5', 'tmp_v6', 'tmp_v7']
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_2, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list)
|
|
assert client.batch_load(util.get_label(), data_desc_list, \
|
|
max_filter_ratio='0.05', is_wait=True, broker=broker_info)
|
|
assert client.verify(DATA_TS.expected_data_file_list_1, table_name)
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_load_set_out_of_range():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_load_set_out_of_range",
|
|
"describe": "SET, 导入，有超过范围的数据",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
SET, 导入，有超过范围的数据
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \
|
|
'2099-12-21 23:59:59', 'MAXVALUE']
|
|
partition_info = palo_client.PartitionInfo('k1', \
|
|
partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
client.create_table(table_name, DATA_TS.schema_1, \
|
|
partition_info, distribution_info)
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
set_list = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \
|
|
'k3 = strftime("%Y-%m-%d %H:%M:%S", tmp_k3)', \
|
|
'k4 = strftime("%Y-%m-%d %H:%M:%S", tmp_k4)', \
|
|
'v2 = strftime("%Y-%m-%d %H:%M:%S", tmp_v2)', \
|
|
'v4 = strftime("%Y-%m-%d %H:%M:%S", tmp_v4)', \
|
|
'v6 = strftime("%Y-%m-%d %H:%M:%S", tmp_v6)', \
|
|
'v7 = strftime("%Y-%m-%d %H:%M:%S", tmp_v7)']
|
|
|
|
column_name_list = ['tmp_k1', 'k2', 'tmp_k3', 'tmp_k4', \
|
|
'v1', 'tmp_v2', 'v3', 'tmp_v4', 'v5', 'tmp_v6', 'tmp_v7']
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_3, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list)
|
|
assert client.batch_load(util.get_label(), data_desc_list, \
|
|
max_filter_ratio='0.05', is_wait=True, broker=broker_info)
|
|
assert client.verify(DATA_TS.expected_data_file_list_1, table_name)
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_load_set_align():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_load_set_align",
|
|
"describe": "SET, 导入，对齐",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
SET, 导入，对齐
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['-300000', '0', '300000', 'MAXVALUE']
|
|
partition_info = palo_client.PartitionInfo('k1', \
|
|
partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
assert client.create_table(table_name, DATA_TS.schema_2, \
|
|
partition_info, distribution_info)
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
set_list = ['k1 = alignment_timestamp("hour", tmp_k1)', \
|
|
'k3 = alignment_timestamp("day", tmp_k3)', \
|
|
'k4 = alignment_timestamp("month", tmp_k4)', \
|
|
'k5 = alignment_timestamp("year", tmp_k5)', \
|
|
'v2 = alignment_timestamp("year", tmp_v2)', \
|
|
'v4 = alignment_timestamp("month", tmp_v4)', \
|
|
'v6 = alignment_timestamp("day", tmp_v6)', \
|
|
'v7 = alignment_timestamp("hour", tmp_v7)']
|
|
|
|
column_name_list = ['tmp_k1', 'k2', 'tmp_k3', 'tmp_k4', 'tmp_k5', \
|
|
'v1', 'tmp_v2', 'v3', 'tmp_v4', 'v5', 'tmp_v6', 'tmp_v7']
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_4, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list)
|
|
assert client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info, max_filter_ratio=0.05)
|
|
assert client.verify(DATA_TS.expected_data_file_list_2, table_name)
|
|
client.clean(database_name)
|
|
|
|
|
|
def load_set_align_out_range():
|
|
"""
|
|
SET, 导入，对齐, 对于year month有超过范围数据
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['-300000', '0', '300000', 'MAXVALUE']
|
|
partition_info = palo_client.PartitionInfo('k1', \
|
|
partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
assert client.create_table(table_name, DATA_TS.schema_2, \
|
|
partition_info, distribution_info)
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
set_list = ['k1 = alignment_timestamp("hour", tmp_k1)', \
|
|
'k3 = alignment_timestamp("day", tmp_k3)', \
|
|
'k4 = alignment_timestamp("month", tmp_k4)', \
|
|
'k5 = alignment_timestamp("year", tmp_k5)', \
|
|
'v2 = alignment_timestamp("year", tmp_v2)', \
|
|
'v4 = alignment_timestamp("month", tmp_v4)', \
|
|
'v6 = alignment_timestamp("day", tmp_v6)', \
|
|
'v7 = alignment_timestamp("hour", tmp_v7)']
|
|
|
|
column_name_list = ['tmp_k1', 'k2', 'tmp_k3', 'tmp_k4', 'tmp_k5', \
|
|
'v1', 'tmp_v2', 'v3', 'tmp_v4', 'v5', 'tmp_v6', 'tmp_v7']
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_5, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list)
|
|
assert client.batch_load(util.get_label(), data_desc_list, max_filter_ratio='0.5', is_wait=True,
|
|
broker=broker_info)
|
|
assert client.verify(DATA_TS.expected_data_file_list_2, table_name)
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_datetime_timestamp():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_datetime_timestamp",
|
|
"describe": "datetime timestamp",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
datetime timestamp
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \
|
|
'2099-12-21 23:59:59', 'MAXVALUE']
|
|
partition_info = palo_client.PartitionInfo('k1', \
|
|
partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
assert client.create_table(table_name, DATA_TS.schema_3, \
|
|
partition_info, distribution_info)
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
set_list = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \
|
|
'k3 = alignment_timestamp("hour", tmp_k3)', \
|
|
'k4 = alignment_timestamp("day", tmp_k4)', \
|
|
'k5 = strftime("%Y-%m-%d %H:%M:%S", tmp_k5)', \
|
|
'k6 = alignment_timestamp("month", tmp_k6)', \
|
|
'k7 = alignment_timestamp("year", tmp_k7)', \
|
|
'k8 = strftime("%Y-%m-%d %H:%M:%S", tmp_k8)', \
|
|
'v2 = strftime("%Y-%m-%d %H:%M:%S", tmp_v2)', \
|
|
'v3 = alignment_timestamp("year", tmp_v3)', \
|
|
'v4 = strftime("%Y-%m-%d %H:%M:%S", tmp_v4)', \
|
|
'v5 = alignment_timestamp("month", tmp_v5)', \
|
|
'v6 = alignment_timestamp("day", tmp_v6)', \
|
|
'v7 = strftime("%Y-%m-%d %H:%M:%S", tmp_v7)', \
|
|
'v8 = alignment_timestamp("hour", tmp_v8)', \
|
|
'v9 = strftime("%Y-%m-%d %H:%M:%S", tmp_v9)']
|
|
|
|
column_name_list = [\
|
|
'tmp_k1', 'k2', 'tmp_k3', 'tmp_k4', 'tmp_k5', \
|
|
'tmp_k6', 'tmp_k7', 'tmp_k8', \
|
|
'v1', 'tmp_v2', 'tmp_v3', 'tmp_v4', 'tmp_v5', \
|
|
'tmp_v6', 'tmp_v7', 'tmp_v8', 'tmp_v9']
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_6, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list)
|
|
assert client.batch_load(util.get_label(), data_desc_list, \
|
|
max_filter_ratio='1', is_wait=True, broker=broker_info)
|
|
assert client.verify(DATA_TS.expected_data_file_list_3, table_name)
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_to_default_value_column():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_to_default_value_column",
|
|
"describe": "函数的返回值列为schema中指定默认值的列",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
函数的返回值列为schema中指定默认值的列
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \
|
|
'2099-12-21 23:59:59', 'MAXVALUE']
|
|
partition_info = palo_client.PartitionInfo('k1', \
|
|
partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
assert client.create_table(table_name, DATA_TS.schema_4, \
|
|
partition_info, distribution_info)
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
set_list = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \
|
|
'k3 = alignment_timestamp("hour", tmp_k3)', \
|
|
'k4 = alignment_timestamp("day", tmp_k4)', \
|
|
'k5 = strftime("%Y-%m-%d %H:%M:%S", tmp_k5)', \
|
|
'k6 = alignment_timestamp("month", tmp_k6)', \
|
|
'k7 = alignment_timestamp("year", tmp_k7)', \
|
|
'k8 = strftime("%Y-%m-%d %H:%M:%S", tmp_k8)', \
|
|
'v2 = strftime("%Y-%m-%d %H:%M:%S", tmp_v2)', \
|
|
'v3 = alignment_timestamp("year", tmp_v3)', \
|
|
'v4 = strftime("%Y-%m-%d %H:%M:%S", tmp_v4)', \
|
|
'v5 = alignment_timestamp("month", tmp_v5)', \
|
|
'v6 = alignment_timestamp("day", tmp_v6)', \
|
|
'v7 = strftime("%Y-%m-%d %H:%M:%S", tmp_v7)', \
|
|
'v8 = alignment_timestamp("hour", tmp_v8)', \
|
|
'v9 = strftime("%Y-%m-%d %H:%M:%S", tmp_v9)']
|
|
|
|
column_name_list = [\
|
|
'tmp_k1', 'k2', 'tmp_k3', 'tmp_k4', 'tmp_k5', \
|
|
'tmp_k6', 'tmp_k7', 'tmp_k8', \
|
|
'v1', 'tmp_v2', 'tmp_v3', 'tmp_v4', 'tmp_v5', \
|
|
'tmp_v6', 'tmp_v7', 'tmp_v8', 'tmp_v9']
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_6, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list)
|
|
assert client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info,
|
|
max_filter_ratio=0.1)
|
|
assert client.verify(DATA_TS.expected_data_file_list_3, table_name)
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_datetime_timestamp_invalid():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_datetime_timestamp_invalid",
|
|
"describe": "数据文件中时间戳列含有无效字符串、八进制、十六进制字符串、算术表达式、小数",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
数据文件中时间戳列含有无效字符串、八进制、十六进制字符串、算术表达式、小数
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \
|
|
'2099-12-21 23:59:59', 'MAXVALUE']
|
|
partition_info = palo_client.PartitionInfo('k1', \
|
|
partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
assert client.create_table(table_name, DATA_TS.schema_3, \
|
|
partition_info, distribution_info)
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
set_list = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \
|
|
'k3 = alignment_timestamp("hour", tmp_k3)', \
|
|
'k4 = alignment_timestamp("day", tmp_k4)', \
|
|
'k5 = strftime("%Y-%m-%d %H:%M:%S", tmp_k5)', \
|
|
'k6 = alignment_timestamp("month", tmp_k6)', \
|
|
'k7 = alignment_timestamp("year", tmp_k7)', \
|
|
'k8 = strftime("%Y-%m-%d %H:%M:%S", tmp_k8)', \
|
|
'v2 = strftime("%Y-%m-%d %H:%M:%S", tmp_v2)', \
|
|
'v3 = alignment_timestamp("year", tmp_v3)', \
|
|
'v4 = strftime("%Y-%m-%d %H:%M:%S", tmp_v4)', \
|
|
'v5 = alignment_timestamp("month", tmp_v5)', \
|
|
'v6 = alignment_timestamp("day", tmp_v6)', \
|
|
'v7 = strftime("%Y-%m-%d %H:%M:%S", tmp_v7)', \
|
|
'v8 = alignment_timestamp("hour", tmp_v8)', \
|
|
'v9 = strftime("%Y-%m-%d %H:%M:%S", tmp_v9)']
|
|
|
|
column_name_list = [\
|
|
'tmp_k1', 'k2', 'tmp_k3', 'tmp_k4', 'tmp_k5', \
|
|
'tmp_k6', 'tmp_k7', 'tmp_k8', \
|
|
'v1', 'tmp_v2', 'tmp_v3', 'tmp_v4', 'tmp_v5', \
|
|
'tmp_v6', 'tmp_v7', 'tmp_v8', 'tmp_v9']
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_7, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list)
|
|
assert client.batch_load(util.get_label(), data_desc_list, \
|
|
max_filter_ratio='1', is_wait=True, broker=broker_info)
|
|
assert client.verify(DATA_TS.expected_data_file_list_3, table_name)
|
|
client.clean(database_name)
|
|
|
|
|
|
def timestamp_special_data():
|
|
"""
|
|
特殊数据
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \
|
|
'2099-12-21 23:59:59', 'MAXVALUE']
|
|
partition_info = palo_client.PartitionInfo('k1', \
|
|
partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
assert client.create_table(table_name, DATA_TS.schema_5, \
|
|
partition_info, distribution_info)
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
set_list = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \
|
|
'k2 = alignment_timestamp("month", tmp_k2)', \
|
|
'k3 = alignment_timestamp("year", tmp_k3)']
|
|
|
|
column_name_list = ['tmp_k1', 'tmp_k2', 'tmp_k3']
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list)
|
|
assert client.batch_load(util.get_label(), data_desc_list, \
|
|
max_filter_ratio='1', is_wait=True, broker=broker_info)
|
|
assert client.verify(DATA_TS.expected_data_file_list_4, table_name)
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_invalid_parameter_part_1():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_invalid_parameter_part_1",
|
|
"describe": "函数缺少第1或2个参数，或同时缺少两个参数",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
函数缺少第1或2个参数，或同时缺少两个参数
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \
|
|
'2099-12-21 23:59:59', 'MAXVALUE']
|
|
partition_info = palo_client.PartitionInfo('k1', \
|
|
partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
assert client.create_table(table_name, DATA_TS.schema_5, \
|
|
partition_info, distribution_info)
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
set_list_1 = ['k1 = strftime("%Y-%m-%d %H:%M:%S")', \
|
|
'k2 = alignment_timestamp("month", tmp_k2)', \
|
|
'k3 = alignment_timestamp("year", tmp_k3)']
|
|
|
|
set_list_2 = ['k1 = strftime(tmp_k1)', \
|
|
'k2 = alignment_timestamp("month", tmp_k2)', \
|
|
'k3 = alignment_timestamp("year", tmp_k3)']
|
|
|
|
set_list_3 = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \
|
|
'k2 = alignment_timestamp(tmp_k2)', \
|
|
'k3 = alignment_timestamp("year", tmp_k3)']
|
|
|
|
ret = False
|
|
column_name_list = ['tmp_k1', 'tmp_k2', 'tmp_k3']
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list_1)
|
|
try:
|
|
ret = client.batch_load(util.get_label(), data_desc_list, \
|
|
max_filter_ratio='1', is_wait=True, broker=broker_info)
|
|
except:
|
|
pass
|
|
assert not ret
|
|
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list_2)
|
|
try:
|
|
ret = client.batch_load(util.get_label(), data_desc_list, \
|
|
max_filter_ratio='1', is_wait=True, broker=broker_info)
|
|
except:
|
|
pass
|
|
assert not ret
|
|
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list_3)
|
|
try:
|
|
ret = client.batch_load(util.get_label(), data_desc_list, \
|
|
max_filter_ratio='1', is_wait=True, broker=broker_info)
|
|
except:
|
|
pass
|
|
assert not ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_invalid_parameter_part_2():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_invalid_parameter_part_2",
|
|
"describe": "函数缺少第1或2个参数，或同时缺少两个参数",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
函数缺少第1或2个参数，或同时缺少两个参数
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \
|
|
'2099-12-21 23:59:59', 'MAXVALUE']
|
|
partition_info = palo_client.PartitionInfo('k1', \
|
|
partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
assert client.create_table(table_name, DATA_TS.schema_5, \
|
|
partition_info, distribution_info)
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
set_list_4 = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \
|
|
'k2 = alignment_timestamp("month", tmp_k2)', \
|
|
'k3 = alignment_timestamp("year")']
|
|
|
|
set_list_5 = ['k1 = strftime()', \
|
|
'k2 = alignment_timestamp("month", tmp_k2)', \
|
|
'k3 = alignment_timestamp("year", tmp_k3)']
|
|
|
|
set_list_6 = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \
|
|
'k2 = alignment_timestamp("month", tmp_k2)', \
|
|
'k3 = alignment_timestamp()']
|
|
|
|
ret = False
|
|
column_name_list = ['tmp_k1', 'tmp_k2', 'tmp_k3']
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list_4)
|
|
try:
|
|
ret = client.batch_load(util.get_label(), data_desc_list, \
|
|
max_filter_ratio='1', is_wait=True, broker=broker_info)
|
|
except:
|
|
pass
|
|
assert not ret
|
|
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list_5)
|
|
try:
|
|
ret = client.batch_load(util.get_label(), data_desc_list, \
|
|
max_filter_ratio='1', is_wait=True, broker=broker_info)
|
|
except:
|
|
pass
|
|
assert not ret
|
|
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list_6)
|
|
try:
|
|
ret = client.batch_load(util.get_label(), data_desc_list, \
|
|
max_filter_ratio='1', is_wait=True, broker=broker_info)
|
|
except:
|
|
pass
|
|
assert not ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_invalid_parameter_part_3():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_invalid_parameter_part_3",
|
|
"describe": "函数传递空串""或"NULL"",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
函数传递空串""或"NULL"
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \
|
|
'2099-12-21 23:59:59', 'MAXVALUE']
|
|
partition_info = palo_client.PartitionInfo('k1', \
|
|
partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
assert client.create_table(table_name, DATA_TS.schema_5, \
|
|
partition_info, distribution_info)
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
set_list_1 = ['k1 = strftime("", tmp_k1)', \
|
|
'k2 = alignment_timestamp("month", tmp_k2)', \
|
|
'k3 = alignment_timestamp("year", tmp_k3)']
|
|
set_list_2 = ['k1 = strftime("%Y-%m-%d %H:%M:%S", "")', \
|
|
'k2 = alignment_timestamp("month", tmp_k2)', \
|
|
'k3 = alignment_timestamp("year", tmp_k3)']
|
|
set_list_3 = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \
|
|
'k2 = alignment_timestamp("NULL", tmp_k2)', \
|
|
'k3 = alignment_timestamp("year", tmp_k3)']
|
|
|
|
ret = False
|
|
column_name_list = ['tmp_k1', 'tmp_k2', 'tmp_k3']
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list_1)
|
|
try:
|
|
ret = client.batch_load(util.get_label(), data_desc_list, \
|
|
max_filter_ratio='1', is_wait=True, broker=broker_info)
|
|
except:
|
|
pass
|
|
assert not ret
|
|
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list_2)
|
|
try:
|
|
ret = client.batch_load(util.get_label(), data_desc_list, \
|
|
max_filter_ratio='1', is_wait=True, broker=broker_info)
|
|
except:
|
|
pass
|
|
assert not ret
|
|
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list_3)
|
|
try:
|
|
ret = client.batch_load(util.get_label(), data_desc_list, \
|
|
max_filter_ratio='1', is_wait=True, broker=broker_info)
|
|
except:
|
|
pass
|
|
assert not ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_invalid_parameter_part_4():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_invalid_parameter_part_4",
|
|
"describe": "函数传递空串""或"NULL", 或无效的时间格式字符串参数",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
函数传递空串""或"NULL", 或无效的时间格式字符串参数
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \
|
|
'2099-12-21 23:59:59', 'MAXVALUE']
|
|
partition_info = palo_client.PartitionInfo('k1', \
|
|
partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
assert client.create_table(table_name, DATA_TS.schema_5, \
|
|
partition_info, distribution_info)
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
set_list_4 = ['k1 = strftime(None, tmp_k1)', \
|
|
'k2 = alignment_timestamp("month", tmp_k2)', \
|
|
'k3 = alignment_timestamp("year", tmp_k3)']
|
|
set_list_5 = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \
|
|
'k2 = alignment_timestamp("month", tmp_k2)', \
|
|
'k3 = alignment_timestamp(None, tmp_k3)']
|
|
set_list_6 = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \
|
|
'k2 = alignment_timestamp("month", tmp_k2)', \
|
|
'k3 = alignment_timestamp("year", None)']
|
|
|
|
set_list_7 = ['k1 = strftime("%k-%m-%d %H:%M:%S", tmp_k1)', \
|
|
'k2 = alignment_timestamp("month", tmp_k2)', \
|
|
'k3 = alignment_timestamp("year", tmp_k3)']
|
|
|
|
ret = False
|
|
column_name_list = ['tmp_k1', 'tmp_k2', 'tmp_k3']
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list_4)
|
|
try:
|
|
ret = client.batch_load(util.get_label(), data_desc_list, \
|
|
max_filter_ratio='1', is_wait=True, broker=broker_info)
|
|
except:
|
|
pass
|
|
assert not ret
|
|
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list_5)
|
|
try:
|
|
ret = client.batch_load(util.get_label(), data_desc_list, \
|
|
max_filter_ratio='1', is_wait=True, broker=broker_info)
|
|
except:
|
|
pass
|
|
assert not ret
|
|
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list_6)
|
|
try:
|
|
ret = client.batch_load(util.get_label(), data_desc_list, \
|
|
max_filter_ratio='1', is_wait=True, broker=broker_info)
|
|
except:
|
|
pass
|
|
assert not ret
|
|
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list_7)
|
|
try:
|
|
ret = client.batch_load(util.get_label(), data_desc_list, \
|
|
max_filter_ratio='1', is_wait=True, broker=broker_info)
|
|
except:
|
|
pass
|
|
assert not ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_invalid_parameter_part_5():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_invalid_parameter_part_5",
|
|
"describe": "不指定返回值或返回值列不存在",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
不指定返回值或返回值列不存在
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \
|
|
'2099-12-21 23:59:59', 'MAXVALUE']
|
|
partition_info = palo_client.PartitionInfo('k1', \
|
|
partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
assert client.create_table(table_name, DATA_TS.schema_5, \
|
|
partition_info, distribution_info)
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
set_list_1 = ['strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \
|
|
'k2 = alignment_timestamp("month", tmp_k2)', \
|
|
'k3 = alignment_timestamp("year", tmp_k3)']
|
|
|
|
set_list_2 = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \
|
|
'k2 = alignment_timestamp("month", tmp_k2)', \
|
|
'k5 = alignment_timestamp("year", tmp_k3)']
|
|
ret = False
|
|
column_name_list = ['tmp_k1', 'tmp_k2', 'tmp_k3']
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list_1)
|
|
try:
|
|
ret = client.batch_load(util.get_label(), data_desc_list, \
|
|
max_filter_ratio='1', is_wait=True, broker=broker_info)
|
|
except:
|
|
pass
|
|
assert not ret
|
|
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list_2)
|
|
try:
|
|
ret = client.batch_load(util.get_label(), data_desc_list, \
|
|
max_filter_ratio='1', is_wait=True, broker=broker_info)
|
|
except:
|
|
pass
|
|
assert not ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_timestamp_invalid_format():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_timestamp_invalid_format",
|
|
"describe": "函数返回值列为datetime类型，但是不是Palo支持的格式",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
函数返回值列为datetime类型，但是不是Palo支持的格式
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \
|
|
'2099-12-21 23:59:59', 'MAXVALUE']
|
|
partition_info = palo_client.PartitionInfo('k1', \
|
|
partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
assert client.create_table(table_name, DATA_TS.schema_5, \
|
|
partition_info, distribution_info)
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
set_list = ['k1 = strftime("%Y:%m:%d %H:%M:%S", tmp_k1)', \
|
|
'k2 = alignment_timestamp("month", tmp_k2)', \
|
|
'k3 = alignment_timestamp("year", tmp_k3)']
|
|
|
|
column_name_list = ['tmp_k1', 'tmp_k2', 'tmp_k3']
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list)
|
|
assert not client.batch_load(util.get_label(), data_desc_list, \
|
|
max_filter_ratio='0', is_wait=True, broker=broker_info)
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_timestamp_upper_lower():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_timestamp_upper_lower",
|
|
"describe": "year|month|day|hour大小写",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
year|month|day|hour大小写
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \
|
|
'2099-12-21 23:59:59', 'MAXVALUE']
|
|
partition_info = palo_client.PartitionInfo('k1', \
|
|
partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
assert client.create_table(table_name, DATA_TS.schema_5, \
|
|
partition_info, distribution_info)
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
set_list = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \
|
|
'k2 = alignment_timestamp("MONTH", tmp_k2)', \
|
|
'k3 = alignment_timestamp("YEAR", tmp_k3)']
|
|
|
|
column_name_list = ['tmp_k1', 'tmp_k2', 'tmp_k3']
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list)
|
|
assert not client.batch_load(util.get_label(), data_desc_list, \
|
|
max_filter_ratio='0', is_wait=True, broker=broker_info)
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_no_column_list():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_no_column_list",
|
|
"describe": "不指定column_list",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
不指定column_list
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \
|
|
'2099-12-21 23:59:59', 'MAXVALUE']
|
|
partition_info = palo_client.PartitionInfo('k1', \
|
|
partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
assert client.create_table(table_name, DATA_TS.schema_3, \
|
|
partition_info, distribution_info)
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
set_list = ['k1 = strftime("%Y-%m-%d %H:%M:%S", k1)', \
|
|
'k3 = alignment_timestamp("hour", k3)', \
|
|
'k4 = alignment_timestamp("day", k4)', \
|
|
'k5 = strftime("%Y-%m-%d %H:%M:%S", k5)', \
|
|
'k6 = alignment_timestamp("month", k6)', \
|
|
'k7 = alignment_timestamp("year", k7)', \
|
|
'k8 = strftime("%Y-%m-%d %H:%M:%S", k8)', \
|
|
'v2 = strftime("%Y-%m-%d %H:%M:%S", v2)', \
|
|
'v3 = alignment_timestamp("year", v3)', \
|
|
'v4 = strftime("%Y-%m-%d %H:%M:%S", v4)', \
|
|
'v5 = alignment_timestamp("month", v5)', \
|
|
'v6 = alignment_timestamp("day", v6)', \
|
|
'v7 = strftime("%Y-%m-%d %H:%M:%S", v7)', \
|
|
'v8 = alignment_timestamp("hour", v8)', \
|
|
'v9 = strftime("%Y-%m-%d %H:%M:%S", v9)']
|
|
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_6, \
|
|
table_name, set_list=set_list)
|
|
assert client.batch_load(util.get_label(), data_desc_list, \
|
|
max_filter_ratio='1', is_wait=True, broker=broker_info)
|
|
assert client.verify(DATA_TS.expected_data_file_list_3, table_name)
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_to_varchar():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_to_varchar",
|
|
"describe": "时间戳转换成字符串类型",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
时间戳转换成字符串类型
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
assert client.create_table(table_name, DATA_TS.schema_6)
|
|
assert client.show_tables(table_name)
|
|
|
|
set_list = ['k1 = strftime("%Y-%m-%d %H:%M:%S", k1)', \
|
|
'k3 = alignment_timestamp("hour", k3)', \
|
|
'k4 = alignment_timestamp("day", k4)', \
|
|
'k5 = strftime("%Y-%m-%d %H:%M:%S", k5)', \
|
|
'k6 = alignment_timestamp("month", k6)', \
|
|
'k7 = alignment_timestamp("year", k7)', \
|
|
'k8 = strftime("%Y-%m-%d %H:%M:%S", k8)', \
|
|
'v2 = strftime("%Y-%m-%d %H:%M:%S", v2)', \
|
|
'v3 = alignment_timestamp("year", v3)', \
|
|
'v4 = strftime("%Y-%m-%d %H:%M:%S", v4)', \
|
|
'v5 = alignment_timestamp("month", v5)', \
|
|
'v6 = alignment_timestamp("day", v6)', \
|
|
'v7 = strftime("%Y-%m-%d %H:%M:%S", v7)', \
|
|
'v8 = alignment_timestamp("hour", v8)', \
|
|
'v9 = strftime("%Y-%m-%d %H:%M:%S", v9)']
|
|
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_6, \
|
|
table_name, set_list=set_list)
|
|
assert client.batch_load(util.get_label(), data_desc_list, \
|
|
max_filter_ratio='1', is_wait=True, broker=broker_info)
|
|
assert client.verify(DATA_TS.expected_data_file_list_3, table_name)
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_one_to_one():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_one_to_one",
|
|
"describe": "同一列多次转换或对齐都对应到表中的同一列",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
同一列多次转换或对齐都对应到表中的同一列
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['-300000', '0', '300000', 'MAXVALUE']
|
|
partition_info = palo_client.PartitionInfo('k1', \
|
|
partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
assert client.create_table(table_name, DATA_TS.schema_2, \
|
|
partition_info, distribution_info)
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
set_list = ['k1 = alignment_timestamp("hour", tmp_k1)', \
|
|
'k3 = alignment_timestamp("day", tmp_k3)', \
|
|
'k4 = alignment_timestamp("month", tmp_k4)', \
|
|
'k5 = alignment_timestamp("year", tmp_k5)', \
|
|
'k5 = alignment_timestamp("year", tmp_k5)', \
|
|
'v2 = alignment_timestamp("year", tmp_v2)', \
|
|
'v4 = alignment_timestamp("month", tmp_v4)', \
|
|
'v6 = alignment_timestamp("day", tmp_v6)', \
|
|
'v7 = alignment_timestamp("hour", tmp_v7)']
|
|
|
|
column_name_list = ['tmp_k1', 'k2', 'tmp_k3', 'tmp_k4', 'tmp_k5', \
|
|
'v1', 'tmp_v2', 'v3', 'tmp_v4', 'v5', 'tmp_v6', 'tmp_v7']
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_4, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list)
|
|
ret = False
|
|
try:
|
|
ret = client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info)
|
|
except:
|
|
pass
|
|
assert not ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_one_to_many():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_load.test_one_to_many",
|
|
"describe": "同一列转换后对应到表中的多列，及多个函数具有相同的参数列，不同的返回值列",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
同一列转换后对应到表中的多列，及多个函数具有相同的参数列，不同的返回值列
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['-300000', '0', '300000', 'MAXVALUE']
|
|
partition_info = palo_client.PartitionInfo('k1', \
|
|
partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
assert client.create_table(table_name, DATA_TS.schema_7, \
|
|
partition_info, distribution_info)
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
set_list = ['k1 = alignment_timestamp("hour", tmp_k1)', \
|
|
'k3 = alignment_timestamp("day", tmp_k3)', \
|
|
'k4 = alignment_timestamp("month", tmp_k4)', \
|
|
'k5 = alignment_timestamp("year", tmp_k5)', \
|
|
'v2 = alignment_timestamp("year", tmp_v2)', \
|
|
'v4 = alignment_timestamp("month", tmp_v4)', \
|
|
'v6 = alignment_timestamp("day", tmp_v6)', \
|
|
'v7 = alignment_timestamp("day", tmp_v6)']
|
|
|
|
column_name_list = ['tmp_k1', 'k2', 'tmp_k3', 'tmp_k4', 'tmp_k5', \
|
|
'v1', 'tmp_v2', 'v3', 'tmp_v4', 'v5', 'tmp_v6', 'tmp_v7']
|
|
data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_4, \
|
|
table_name, column_name_list=column_name_list, set_list=set_list)
|
|
assert client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info, max_filter_ratio=0.05)
|
|
assert client.verify(DATA_TS.expected_data_file_list_5, table_name)
|
|
client.clean(database_name)
|
|
|
|
|
|
def teardown_module():
|
|
"""
|
|
tearDown
|
|
"""
|
|
pass
|
|
|
|
|
|
if __name__ == '__main__':
|
|
import pdb
|
|
pdb.set_trace()
|
|
setup_module()
|
|
|