1042 lines
35 KiB
Python
1042 lines
35 KiB
Python
#!/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
############################################################################
|
|
#
|
|
# @file test_sys_partition_basic.py
|
|
# @date 2015/05/13 15:26:21
|
|
# @brief This file is a test file for palo data loading and verifying.
|
|
#
|
|
#############################################################################
|
|
|
|
"""
|
|
按照所有支持分区的数据类型进行分区建表，对查询结果进行正确性校验
|
|
"""
|
|
|
|
from data import partition as DATA
|
|
from lib import palo_config
|
|
from lib import palo_client
|
|
from lib import util
|
|
import random
|
|
|
|
LOG = palo_client.LOG
|
|
L = palo_client.L
|
|
config = palo_config.config
|
|
broker_info = palo_config.broker_info
|
|
|
|
|
|
def setup_module():
|
|
"""
|
|
setUp
|
|
"""
|
|
global client
|
|
client = palo_client.get_client(config.fe_host, config.fe_query_port, user=config.fe_user,
|
|
password=config.fe_password, http_port=config.fe_http_port)
|
|
|
|
|
|
def check_partition_list(table_name, partition_name_list):
|
|
"""
|
|
验证分区是否创建成功
|
|
"""
|
|
for partition_name in partition_name_list:
|
|
assert client.get_partition(table_name, partition_name)
|
|
|
|
|
|
def check_load_and_verify(table_name, partition_name_list):
|
|
"""
|
|
验证表是否创建成功，分区是否创建成功，导入数据，校验
|
|
"""
|
|
data_desc_list = palo_client.LoadDataInfo(DATA.file_path_1, table_name)
|
|
assert client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info)
|
|
assert client.verify(DATA.expected_data_file_list_1, table_name)
|
|
assert client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info)
|
|
assert client.verify(list(DATA.expected_data_file_list_1) * 2, table_name)
|
|
|
|
|
|
def partition_check(table_name, column_name, partition_name_list, \
|
|
partition_value_list, distribution_type, bucket_num, storage_type):
|
|
"""
|
|
检查，验证
|
|
"""
|
|
partition_info = palo_client.PartitionInfo(column_name, \
|
|
partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo(distribution_type, bucket_num)
|
|
client.create_table(table_name, DATA.schema_1, \
|
|
partition_info, distribution_info)
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
check_load_and_verify(table_name, partition_name_list)
|
|
|
|
|
|
def test_partition_by_datetime_1():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_basic_b.test_partition_by_datetime_1",
|
|
"describe": "datetime分区",
|
|
"tag": "system,p1"
|
|
}
|
|
"""
|
|
"""
|
|
datetime分区
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['2017-01-01 00:00:00', '2027-01-01 00:00:00', \
|
|
'2039-01-01 00:00:00', 'MAXVALUE']
|
|
|
|
random_bucket_num = random.randrange(1, 300)
|
|
LOG.info(L('', random_bucket_num=random_bucket_num))
|
|
partition_check(table_name, 'k5', partition_name_list, \
|
|
partition_value_list, 'RANDOM', random_bucket_num, 'row')
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_partition_by_datetime_2():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_basic_b.test_partition_by_datetime_2",
|
|
"describe": "datetime分区，两个相同的分区值",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
datetime分区，两个相同的分区值
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['2017-01-01 00:00:00', '2017-01-01 00:00:00', \
|
|
'2039-01-01 00:00:00', 'MAXVALUE']
|
|
|
|
partition_info = palo_client.PartitionInfo('k5', partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
|
|
ret = False
|
|
try:
|
|
ret = client.create_table(table_name, DATA.schema_1, \
|
|
partition_info, distribution_info)
|
|
except:
|
|
pass
|
|
|
|
assert not ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_partition_by_datetime_3():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_basic_b.test_partition_by_datetime_3",
|
|
"describe": "datetime分区，边界值",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
datetime分区，边界值
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['1900-01-01 00:00:00', '2027-01-01 23:59:59', \
|
|
'2039-01-01 00:00:00', 'MAXVALUE']
|
|
|
|
partition_info = palo_client.PartitionInfo('k5', partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
|
|
ret = False
|
|
try:
|
|
ret = client.create_table(table_name, DATA.schema_1, \
|
|
partition_info, distribution_info)
|
|
except:
|
|
pass
|
|
|
|
assert ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_partition_by_datetime_4():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_basic_b.test_partition_by_datetime_4",
|
|
"describe": "datetime分区，越界",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
datetime分区，越界
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['1899-12-31 23:59:59', '2027-01-01 00:00:00', \
|
|
'2039-01-01 00:00:00', 'MAXVALUE']
|
|
|
|
partition_info = palo_client.PartitionInfo('k5', partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
|
|
ret = False
|
|
try:
|
|
ret = client.create_table(table_name, DATA.schema_1, \
|
|
partition_info, distribution_info)
|
|
except:
|
|
pass
|
|
assert ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_partition_by_datetime_5():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_basic_b.test_partition_by_datetime_5",
|
|
"describe": "datetime分区，HASH(分区列)",
|
|
"tag": "system,p1"
|
|
}
|
|
"""
|
|
"""
|
|
datetime分区，HASH(分区列)
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['1999-12-31 23:59:59', '2027-01-01 00:00:00', \
|
|
'2039-01-01 00:00:00', 'MAXVALUE']
|
|
|
|
random_bucket_num = random.randrange(1, 300)
|
|
LOG.info(L('', random_bucket_num=random_bucket_num))
|
|
partition_check(table_name, 'k5', partition_name_list, \
|
|
partition_value_list, 'HASH(k5)', random_bucket_num, 'column')
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_partition_by_datetime_6():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_basic_b.test_partition_by_datetime_6",
|
|
"describe": "datetime分区，HASH(非分区列)",
|
|
"tag": "system,p1"
|
|
}
|
|
"""
|
|
"""
|
|
datetime分区，HASH(非分区列)
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['1999-12-31 23:59:59', '2027-01-01 00:00:00', \
|
|
'2039-01-01 00:00:00', 'MAXVALUE']
|
|
|
|
random_bucket_num = random.randrange(1, 300)
|
|
LOG.info(L('', random_bucket_num=random_bucket_num))
|
|
partition_check(table_name, 'k5', partition_name_list, \
|
|
partition_value_list, 'HASH(k4, k1, k2)', random_bucket_num, 'column')
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_partition_by_datetime_7():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_basic_b.test_partition_by_datetime_7",
|
|
"describe": "datetime分区，一个分区",
|
|
"tag": "system,p1"
|
|
}
|
|
"""
|
|
"""
|
|
datetime分区，一个分区
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a']
|
|
partition_value_list = ['MAXVALUE']
|
|
|
|
random_bucket_num = random.randrange(1, 300)
|
|
LOG.info(L('', random_bucket_num=random_bucket_num))
|
|
partition_check(table_name, 'k5', partition_name_list, \
|
|
partition_value_list, 'HASH(k5)', random_bucket_num, 'column')
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_illegal_two_same_partition_value():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_basic_b.test_illegal_two_same_partition_value",
|
|
"describe": "非法输入：指定两个相同的range分区值",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
非法输入：指定两个相同的range分区值
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['1000', '1000', '3000', 'MAXVALUE']
|
|
partition_info = palo_client.PartitionInfo('k4', partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
|
|
ret = False
|
|
try:
|
|
ret = client.create_table(table_name, DATA.schema_1, \
|
|
partition_info, distribution_info)
|
|
except:
|
|
pass
|
|
|
|
assert not ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_illegal_partition_value_bigger_first():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_basic_b.test_illegal_partition_value_bigger_first",
|
|
"describe": "非法输入：分区值先大后小",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
非法输入：分区值先大后小
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['2000', '1000', '3000', 'MAXVALUE']
|
|
partition_info = palo_client.PartitionInfo('k4', partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
|
|
ret = False
|
|
try:
|
|
ret = client.create_table(table_name, DATA.schema_1, \
|
|
partition_info, distribution_info)
|
|
except:
|
|
pass
|
|
|
|
assert not ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_illegal_partition_value_out_range_1():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_basic_b.test_illegal_partition_value_out_range_1",
|
|
"describe": "非法输入：分区值超过范围",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
非法输入：分区值超过范围
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['20', '100', '101', '3000']
|
|
partition_info = palo_client.PartitionInfo('k1', partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
|
|
ret = False
|
|
try:
|
|
ret = client.create_table(table_name, DATA.schema_1, \
|
|
partition_info, distribution_info)
|
|
except:
|
|
pass
|
|
|
|
assert not ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_illegal_partition_value_out_range_2():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_basic_b.test_illegal_partition_value_out_range_2",
|
|
"describe": "非法输入：分区值超过范围",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
非法输入：分区值超过范围
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['-150', '20', '100', '101']
|
|
partition_info = palo_client.PartitionInfo('k1', partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
|
|
ret = False
|
|
try:
|
|
ret = client.create_table(table_name, DATA.schema_1, \
|
|
partition_info, distribution_info)
|
|
except:
|
|
pass
|
|
|
|
assert not ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_illegal_partition_value_not_int():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_basic_b.test_illegal_partition_value_not_int",
|
|
"describe": "非法输入：分区值小数",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
非法输入：分区值小数
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['-50.1', '20.5', '100.0', '101']
|
|
partition_info = palo_client.PartitionInfo('k1', partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
|
|
ret = False
|
|
try:
|
|
ret = client.create_table(table_name, DATA.schema_1, \
|
|
partition_info, distribution_info)
|
|
except:
|
|
pass
|
|
|
|
assert not ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_add_partition_1():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_basic_b.test_add_partition_1",
|
|
"describe": "增加分区",
|
|
"tag": "system,p1"
|
|
}
|
|
"""
|
|
"""
|
|
增加分区
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['50', '200', '1000', '65535000']
|
|
partition_info = palo_client.PartitionInfo('k3', partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
client.create_table(table_name, DATA.schema_1, \
|
|
partition_info, distribution_info)
|
|
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
client.add_partition(table_name, 'add_1', '65537000')
|
|
data_desc_list = palo_client.LoadDataInfo(DATA.file_path_1, table_name)
|
|
ret = client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info)
|
|
assert ret
|
|
ret = client.verify(DATA.expected_data_file_list_1, table_name)
|
|
assert ret
|
|
|
|
client.add_partition(table_name, 'add_2', '65538000')
|
|
ret = client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info)
|
|
assert ret
|
|
ret = client.verify(list(DATA.expected_data_file_list_1) * 2, table_name)
|
|
assert ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_add_partition_2():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_basic_b.test_add_partition_2",
|
|
"describe": "非法输入：增加分区, 分区值位于中间和最前",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
非法输入：增加分区, 分区值位于中间和最前
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['50', '200', '1000', '65535000']
|
|
partition_info = palo_client.PartitionInfo('k3', partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
client.create_table(table_name, DATA.schema_1, \
|
|
partition_info, distribution_info)
|
|
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
ret = False
|
|
try:
|
|
ret = client.add_partition(table_name, 'add_1', '100')
|
|
except:
|
|
pass
|
|
assert not ret
|
|
|
|
try:
|
|
ret = client.add_partition(table_name, 'add_1', '50')
|
|
except:
|
|
pass
|
|
|
|
assert not ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_add_partition_3():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_basic_b.test_add_partition_3",
|
|
"describe": "增加分区, 修改桶数",
|
|
"tag": "system,p1"
|
|
}
|
|
"""
|
|
"""
|
|
增加分区, 修改桶数
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['50', '200', '1000', '65535000']
|
|
partition_info = palo_client.PartitionInfo('k3', partition_name_list, partition_value_list)
|
|
distribution_type = 'RANDOM'
|
|
distribution_info = palo_client.DistributionInfo(distribution_type, 13)
|
|
client.create_table(table_name, DATA.schema_1, \
|
|
partition_info, distribution_info)
|
|
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
client.add_partition(table_name, 'add_1', '65537000', distribution_type, 20)
|
|
data_desc_list = palo_client.LoadDataInfo(DATA.file_path_1, table_name)
|
|
ret = client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info)
|
|
assert ret
|
|
ret = client.verify(DATA.expected_data_file_list_1, table_name)
|
|
assert ret
|
|
|
|
client.add_partition(table_name, 'add_2', '65538000', distribution_type, 37)
|
|
ret = client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info)
|
|
assert ret
|
|
ret = client.verify(list(DATA.expected_data_file_list_1) * 2, table_name)
|
|
assert ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_add_partition_4():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_basic_b.test_add_partition_4",
|
|
"describe": "增加分区, 修改桶数, 变小",
|
|
"tag": "system,p1"
|
|
}
|
|
"""
|
|
"""
|
|
增加分区, 修改桶数, 变小
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['50', '200', '1000', '65535000']
|
|
partition_info = palo_client.PartitionInfo('k3', partition_name_list, partition_value_list)
|
|
distribution_type = 'RANDOM'
|
|
distribution_info = palo_client.DistributionInfo(distribution_type, 13)
|
|
client.create_table(table_name, DATA.schema_1, \
|
|
partition_info, distribution_info)
|
|
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
client.add_partition(table_name, 'add_1', '65537000', distribution_type, 20)
|
|
data_desc_list = palo_client.LoadDataInfo(DATA.file_path_1, table_name)
|
|
ret = client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info)
|
|
assert ret
|
|
ret = client.verify(DATA.expected_data_file_list_1, table_name)
|
|
assert ret
|
|
|
|
client.add_partition(table_name, 'add_2', str(2 ** 31 - 1), distribution_type, 7)
|
|
ret = client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info)
|
|
assert ret
|
|
ret = client.verify(list(DATA.expected_data_file_list_1) * 2, table_name)
|
|
assert ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_add_partition_5():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_basic_b.test_add_partition_5",
|
|
"describe": "增加分区, 修改distribution方式",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
增加分区, 修改distribution方式
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['50', '200', '1000', '65535000']
|
|
partition_info = palo_client.PartitionInfo('k3', partition_name_list, partition_value_list)
|
|
distribution_type = 'RANDOM'
|
|
distribution_info = palo_client.DistributionInfo(distribution_type, 13)
|
|
client.create_table(table_name, DATA.schema_1, \
|
|
partition_info, distribution_info)
|
|
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
ret = False
|
|
try:
|
|
ret = client.add_partition(table_name, 'add_1', \
|
|
str(2 ** 31 - 1), 'HASH(k1, k2)', 13)
|
|
except:
|
|
pass
|
|
assert not ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_add_partition_6():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_basic_b.test_add_partition_6",
|
|
"describe": "单分区表，增加分区",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
单分区表，增加分区
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
distribution_type = 'RANDOM'
|
|
distribution_info = palo_client.DistributionInfo(distribution_type, 13)
|
|
client.create_table(table_name, \
|
|
DATA.schema_1, distribution_info=distribution_info)
|
|
|
|
assert client.show_tables(table_name)
|
|
|
|
ret = False
|
|
try:
|
|
ret = client.add_partition(table_name, 'add_1', \
|
|
str(2 ** 31 - 1), 'HASH(k1, k2)', 13)
|
|
except:
|
|
pass
|
|
assert not ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_drop_partition_1():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_basic_b.test_drop_partition_1",
|
|
"describe": "删除不存在的分区",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
删除不存在的分区
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['50', '200', '1000', '65535000']
|
|
partition_info = palo_client.PartitionInfo('k3', partition_name_list, partition_value_list)
|
|
distribution_type = 'RANDOM'
|
|
distribution_info = palo_client.DistributionInfo(distribution_type, 13)
|
|
client.create_table(table_name, DATA.schema_1, \
|
|
partition_info, distribution_info)
|
|
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
ret = False
|
|
try:
|
|
ret = client.drop_partition(table_name, 'add_1')
|
|
except:
|
|
pass
|
|
assert not ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_drop_partition_2():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_basic_b.test_drop_partition_2",
|
|
"describe": "删除分区到一个分区也不留",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
删除分区到只剩一个分区
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['50', '200', '1000', '65535000']
|
|
partition_info = palo_client.PartitionInfo('k3', partition_name_list, partition_value_list)
|
|
distribution_type = 'RANDOM'
|
|
distribution_info = palo_client.DistributionInfo(distribution_type, 13)
|
|
client.create_table(table_name, DATA.schema_1, \
|
|
partition_info, distribution_info)
|
|
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
assert client.drop_partition(table_name, partition_name_list[0])
|
|
assert client.drop_partition(table_name, partition_name_list[1])
|
|
assert client.drop_partition(table_name, partition_name_list[2])
|
|
assert client.drop_partition(table_name, partition_name_list[3])
|
|
ret = False
|
|
try:
|
|
ret = client.drop_partition(table_name, partition_name_list[3])
|
|
except:
|
|
pass
|
|
assert not ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_drop_partition_3():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_basic_b.test_drop_partition_3",
|
|
"describe": "删除，导入，数据校验",
|
|
"tag": "system,p1"
|
|
}
|
|
"""
|
|
"""
|
|
删除，导入，数据校验
|
|
"""
|
|
# TODO 删分区，加分区，导入过滤, 导入到指定分区
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', \
|
|
'partition_d', 'partition_e', 'partition_f', 'partition_g']
|
|
partition_value_list = ['300', '500', '900', '2200', '2300', '2800', '4000']
|
|
partition_info = palo_client.PartitionInfo('k3', partition_name_list, partition_value_list)
|
|
distribution_type = 'RANDOM'
|
|
distribution_info = palo_client.DistributionInfo(distribution_type, 13)
|
|
client.create_table(table_name, DATA.schema_1, \
|
|
partition_info, distribution_info)
|
|
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
data_desc_list = palo_client.LoadDataInfo(DATA.file_path_1, table_name)
|
|
#导入
|
|
assert client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info)
|
|
#校验
|
|
assert client.verify(DATA.expected_data_file_list_1, table_name)
|
|
#删分区
|
|
client.drop_partition(table_name, 'partition_a')
|
|
#加分区
|
|
assert client.add_partition(table_name, 'add_1', '5000')
|
|
#校验
|
|
assert client.verify([DATA.expected_file_1], table_name)
|
|
#加分区
|
|
assert client.add_partition(table_name, 'add_2', '6000')
|
|
#删分区
|
|
client.drop_partition(table_name, 'partition_c')
|
|
#校验
|
|
assert client.verify([DATA.expected_file_2], table_name)
|
|
#删分区
|
|
client.drop_partition(table_name, 'partition_e')
|
|
#校验
|
|
assert client.verify([DATA.expected_file_3], table_name)
|
|
#加分区
|
|
assert client.add_partition(table_name, 'add_3', 'MAXVALUE')
|
|
#删分区
|
|
client.drop_partition(table_name, 'partition_g')
|
|
#校验
|
|
assert client.verify([DATA.expected_file_4], table_name)
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_loop_add_drop():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_basic_b.test_loop_add_drop",
|
|
"describe": "增删分区反复",
|
|
"tag": "system,p1"
|
|
}
|
|
"""
|
|
"""
|
|
增删分区反复
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', \
|
|
'partition_d', 'partition_e', 'partition_f', 'partition_g']
|
|
partition_value_list = ['300', '500', '900', '2200', '2300', '3800', '4000']
|
|
partition_info = palo_client.PartitionInfo('k3', partition_name_list, partition_value_list)
|
|
distribution_type = 'RANDOM'
|
|
distribution_info = palo_client.DistributionInfo(distribution_type, 13)
|
|
client.create_table(table_name, DATA.schema_1, \
|
|
partition_info, distribution_info)
|
|
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
data_desc_list = palo_client.LoadDataInfo(DATA.file_path_1, table_name)
|
|
#导入
|
|
assert client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info)
|
|
#校验
|
|
assert client.verify(DATA.expected_data_file_list_1, table_name)
|
|
count = 0
|
|
#while count < 500:
|
|
#ext 32000
|
|
while count < 10:
|
|
#删分区
|
|
client.drop_partition(table_name, 'partition_g')
|
|
#加分区
|
|
assert client.add_partition(table_name, 'partition_g', '4000', \
|
|
'RANDOM', random.randrange(1, 300))
|
|
count = count + 1
|
|
#校验
|
|
assert client.verify(DATA.expected_data_file_list_1, table_name)
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_add_partition_times_then_drop_times():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_basic_b.test_add_partition_times_then_drop_times",
|
|
"describe": "连续增加分区，连续删除分区",
|
|
"tag": "system,p1"
|
|
}
|
|
"""
|
|
"""
|
|
连续增加分区，连续删除分区
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', \
|
|
'partition_d', 'partition_e', 'partition_f', 'partition_g']
|
|
partition_value_list = ['300', '500', '900', '2200', '2300', '3800', '4000']
|
|
partition_info = palo_client.PartitionInfo('k3', partition_name_list, partition_value_list)
|
|
distribution_type = 'RANDOM'
|
|
distribution_info = palo_client.DistributionInfo(distribution_type, 13)
|
|
client.create_table(table_name, DATA.schema_1, \
|
|
partition_info, distribution_info)
|
|
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
data_desc_list = palo_client.LoadDataInfo(DATA.file_path_1, table_name)
|
|
#导入
|
|
assert client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info)
|
|
#校验
|
|
assert client.verify(DATA.expected_data_file_list_1, table_name)
|
|
count = 1
|
|
while count < 30:
|
|
#加分区
|
|
assert client.add_partition(table_name, \
|
|
'partition_add_%d' % (count), str(4000 + count * 10), \
|
|
'RANDOM', random.randrange(1, 300))
|
|
count = count + 1
|
|
#校验
|
|
assert client.verify(DATA.expected_data_file_list_1, table_name)
|
|
|
|
count = 1
|
|
while count < 30:
|
|
#删分区
|
|
assert client.drop_partition(table_name, 'partition_add_%d' % (count))
|
|
count = count + 1
|
|
#校验
|
|
assert client.verify(DATA.expected_data_file_list_1, table_name)
|
|
#导入
|
|
assert client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info)
|
|
#校验
|
|
assert client.verify(list(DATA.expected_data_file_list_1) * 2, table_name)
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_drop_partition_key():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_basic_b.test_drop_partition_key",
|
|
"describe": "删除分区列",
|
|
"tag": "system,p1,fuzz"
|
|
}
|
|
"""
|
|
"""
|
|
删除分区列
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['5', '20', '30', '50']
|
|
|
|
partition_info = palo_client.PartitionInfo('k1', \
|
|
partition_name_list, partition_value_list)
|
|
distribution_info = palo_client.DistributionInfo('RANDOM', 13)
|
|
client.create_table(table_name, DATA.schema_1, \
|
|
partition_info, distribution_info)
|
|
assert client.show_tables(table_name)
|
|
check_partition_list(table_name, partition_name_list)
|
|
|
|
data_desc_list = palo_client.LoadDataInfo(DATA.file_path_1, table_name)
|
|
assert client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info)
|
|
assert client.verify(DATA.expected_data_file_list_1, table_name)
|
|
|
|
column_name_list = ['k1',]
|
|
ret = False
|
|
try:
|
|
ret = client.schema_change_drop_column(table_name, column_name_list, \
|
|
is_wait_job=True, is_wait_delete_old_schema=True)
|
|
except:
|
|
pass
|
|
|
|
assert not ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def test_partition_by_value():
|
|
"""
|
|
{
|
|
"title": "test_sys_partition_basic_b.test_partition_by_value",
|
|
"describe": "用value列分区",
|
|
"tag": "system,p1"
|
|
}
|
|
"""
|
|
"""
|
|
用value列分区
|
|
"""
|
|
database_name, table_name, index_name = util.gen_num_format_name_list()
|
|
LOG.info(L('', database_name=database_name, \
|
|
table_name=table_name, index_name=index_name))
|
|
client.clean(database_name)
|
|
client.create_database(database_name)
|
|
|
|
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
|
|
partition_value_list = ['5', '20', '30', 'MAXVALUE']
|
|
|
|
partition_info = palo_client.PartitionInfo('v1', \
|
|
partition_name_list, partition_value_list)
|
|
ret = False
|
|
try:
|
|
ret = client.create_table(table_name, DATA.schema_1, partition_info)
|
|
except:
|
|
pass
|
|
assert not ret
|
|
client.clean(database_name)
|
|
|
|
|
|
def teardown_module():
|
|
"""
|
|
tearDown
|
|
"""
|
|
pass
|
|
|
|
|
|
if __name__ == '__main__':
|
|
import pdb
|
|
pdb.set_trace()
|
|
setup_module()
|
|
|