441 lines
18 KiB
Python
441 lines
18 KiB
Python
#!/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
"""palo client verify"""
|
|
import petl
|
|
import math
|
|
from decimal import Decimal
|
|
from collections import OrderedDict
|
|
from datetime import datetime
|
|
|
|
import palo_logger
|
|
LOG = palo_logger.Logger.getLogger()
|
|
L = palo_logger.StructedLogMessage
|
|
|
|
|
|
class VerifyFile(object):
|
|
"""
|
|
VerifyFile
|
|
"""
|
|
|
|
def __init__(self, file_name, delimiter='\t'):
|
|
self.file_name = file_name
|
|
self.delimiter = delimiter
|
|
|
|
def get_file_name(self):
|
|
"""
|
|
get file name
|
|
"""
|
|
return self.file_name
|
|
|
|
def get_delimiter(self):
|
|
"""
|
|
get delimiter
|
|
"""
|
|
return self.delimiter
|
|
|
|
def __str__(self):
|
|
return str(self.file_name)
|
|
|
|
|
|
class Verify(object):
|
|
"""verify class"""
|
|
def __init__(self, expected_file_list, datas, schema, table_name, database_name, encoding=None):
|
|
"""
|
|
file：校验文件，可以是str，['file1', 'file2'] or VerifyFile
|
|
sql_ret：sql执行的结果
|
|
schema：verify校验使用的表的desc结果做schema
|
|
verify_by_sql使用的4元组，sql查询结果的schema, 由四元组(name, type, agg_type, default_value)组成的list
|
|
table_name, database_name: 生成默认的校验文件名称用
|
|
"""
|
|
self.expected_file_list = expected_file_list
|
|
self.table_name = table_name
|
|
self.database_name = database_name
|
|
self.schema = schema
|
|
self.datas = datas
|
|
self.encoding = encoding
|
|
|
|
@staticmethod
|
|
def __get_type_convert_handler(field_type):
|
|
""""""
|
|
def __int_type(min, max):
|
|
"""Return a function that will attempt to parse the value as a number,
|
|
"""
|
|
|
|
def f(v):
|
|
"""check and return type
|
|
"""
|
|
try:
|
|
value = int(v)
|
|
except (ValueError, TypeError) as e:
|
|
raise e
|
|
if min <= value <= max:
|
|
return int(v)
|
|
else:
|
|
return None
|
|
|
|
return f
|
|
|
|
def __char_type():
|
|
"""regurn a function"""
|
|
|
|
def f(v):
|
|
"""check v if null"""
|
|
if v == "None":
|
|
v = None
|
|
return v
|
|
|
|
return f
|
|
|
|
tinyint = __int_type(-2 ** 7, 2 ** 7 - 1)
|
|
smallint = __int_type(-2 ** 15, 2 ** 15 - 1)
|
|
paloint = __int_type(-2 ** 31, 2 ** 31 - 1)
|
|
bigint = __int_type(-2 ** 63, 2 ** 63 - 1)
|
|
largeint = __int_type(-2 ** 127, 2 ** 127 - 1)
|
|
datetime = petl.datetimeparser('%Y-%m-%d %H:%M:%S')
|
|
date = petl.dateparser('%Y-%m-%d')
|
|
char = __char_type()
|
|
varchar = __char_type()
|
|
|
|
field_type = field_type.lower().split('<')[0]
|
|
field_type = field_type.lower().split('(')[0]
|
|
field_type_handler_dict = {'char': char, 'varchar': varchar, 'decimal': Decimal,
|
|
'tinyint': tinyint, 'smallint': smallint, 'int': paloint,
|
|
'bigint': bigint, 'largeint': largeint, 'text': varchar,
|
|
'float': float, 'double': float, 'datetime': datetime, 'date': date,
|
|
'boolean': tinyint,
|
|
'array': varchar, 'decimalv3': Decimal}
|
|
return field_type_handler_dict[field_type]
|
|
|
|
def __get_convert_dict(self):
|
|
"""get column type from schema, and get convert func"""
|
|
convert_dict = {field[0]: self.__get_type_convert_handler(field[1]) for field in self.schema}
|
|
return convert_dict
|
|
|
|
def __get_field_list(self):
|
|
"""get column name from schema"""
|
|
field_list = [field[0] for field in self.schema]
|
|
return field_list
|
|
|
|
def __get_key_list(self):
|
|
"""get key column from schema"""
|
|
key_list = [field[0] for field in self.schema if field[3] == 'true']
|
|
return tuple(key_list)
|
|
|
|
def __get_type_list(self):
|
|
"""get column type from schema"""
|
|
type_list = [field[1] for field in self.schema]
|
|
return type_list
|
|
|
|
@staticmethod
|
|
def __get_aggregate_key(key_list):
|
|
"""get key"""
|
|
if len(key_list) == 1:
|
|
return key_list[0]
|
|
else:
|
|
return key_list
|
|
|
|
def __get_aggregation_ordereddict(self):
|
|
"""aggregation table value agg func"""
|
|
def _sum(l):
|
|
items = []
|
|
for i in l:
|
|
if i is not None:
|
|
items.append(i)
|
|
if len(items) == 0:
|
|
return None
|
|
else:
|
|
return sum(items)
|
|
|
|
def __agg_replace(l):
|
|
items = []
|
|
for i in l:
|
|
items.append(i)
|
|
return items[-1]
|
|
|
|
def __agg_replace_if_not_null(l):
|
|
""" replace if not null """
|
|
items = []
|
|
for i in l:
|
|
if i is not None:
|
|
items.append(i)
|
|
if len(items) == 0:
|
|
return None
|
|
else:
|
|
return items[-1]
|
|
|
|
agg_function_dict = {'max': max, 'min': min, 'sum': _sum, 'replace': __agg_replace,
|
|
'replace_if_not_null': __agg_replace_if_not_null}
|
|
|
|
aggregation = OrderedDict()
|
|
|
|
aggtype_list = [(field[0], field[5]) for field in self.schema if field[5] != '']
|
|
|
|
for item in aggtype_list:
|
|
aggregation[item[0]] = item[0], agg_function_dict[item[1].lower()]
|
|
|
|
return aggregation
|
|
|
|
def __write_data_to_file(self, data_from_database, data_from_file, save_verifyfile_list):
|
|
"""将文件中的数据写入tmp文件中"""
|
|
if self.encoding is not None:
|
|
if save_verifyfile_list[0] is not None:
|
|
petl.tocsv(data_from_database, save_verifyfile_list[0].get_file_name(),
|
|
encoding=self.encoding, delimiter=save_verifyfile_list[0].get_delimiter())
|
|
if save_verifyfile_list[1] is not None:
|
|
petl.tocsv(data_from_file, save_verifyfile_list[1].get_file_name(),
|
|
encoding=self.encoding, delimiter=save_verifyfile_list[1].get_delimiter())
|
|
else:
|
|
if save_verifyfile_list[0] is not None:
|
|
petl.tocsv(data_from_database, save_verifyfile_list[0].get_file_name(),
|
|
delimiter=save_verifyfile_list[0].get_delimiter())
|
|
if save_verifyfile_list[1] is not None:
|
|
petl.tocsv(data_from_file, save_verifyfile_list[1].get_file_name(),
|
|
delimiter=save_verifyfile_list[1].get_delimiter())
|
|
|
|
@staticmethod
|
|
def __check_float(field_of_database, field_of_file, type):
|
|
def __adjust_data(num):
|
|
if num is None:
|
|
return None
|
|
else:
|
|
num = float(num)
|
|
if num == 0.0:
|
|
return 0.0
|
|
else:
|
|
return num / 10 ** (math.floor(math.log10(abs(num))) + 1)
|
|
data_of_database = __adjust_data(field_of_database)
|
|
data_of_file = __adjust_data(field_of_file)
|
|
# 最后一个有效数字可以相差 1，比如： 0.123456001 == 0.123456999 => True
|
|
# 0.123456001 == 0.123457999 => True 0.123456001 == 0.123458999 => False
|
|
# 0.123456001 == 0.123455999 => True 0.123456001 == 0.123454999 => False
|
|
precision = None
|
|
if type.lower() == 'float':
|
|
precision = 2e-6
|
|
elif type.lower() == 'double':
|
|
precision = 2e-15
|
|
if math.fabs(data_of_database - data_of_file) < precision or \
|
|
math.fabs(data_of_database - data_of_file) / data_of_file < 2e-3:
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
def __check_data(self, data_from_database, data_from_file):
|
|
rows_number_of_database = petl.nrows(data_from_database)
|
|
rows_number_of_file = petl.nrows(data_from_file)
|
|
|
|
if rows_number_of_database != rows_number_of_file:
|
|
LOG.warning(L("verify data error", lines_of_database=rows_number_of_database,
|
|
lines_of_file=rows_number_of_file))
|
|
return False
|
|
result_of_database = petl.records(data_from_database)
|
|
result_of_file = petl.records(data_from_file)
|
|
type_list = self.__get_type_list()
|
|
|
|
for record_of_database, record_of_file in zip(result_of_database, result_of_file):
|
|
for field_of_database, field_of_file, field_type in \
|
|
zip(record_of_database, record_of_file, type_list):
|
|
if field_of_database is None and field_of_file is None:
|
|
continue
|
|
else:
|
|
if field_of_database is None or field_of_file is None:
|
|
return False
|
|
if field_type.lower() == 'float' or field_type.lower() == 'double':
|
|
if not self.__check_float(field_of_database, field_of_file,
|
|
type=field_type.lower()):
|
|
LOG.error(L("FLOAT VERIFY FAIL", field_of_database=field_of_database,
|
|
field_of_file=field_of_file, record_of_database=record_of_database,
|
|
record_of_file=record_of_file))
|
|
return False
|
|
elif field_of_database != field_of_file:
|
|
LOG.error(L("VERIFY FAIL", field_of_database=field_of_database,
|
|
field_of_file=field_of_file, record_of_database=record_of_database,
|
|
record_of_file=record_of_file))
|
|
return False
|
|
return True
|
|
|
|
def __get_data_from_database(self):
|
|
"""
|
|
处理数据库中的数据，datas是client.execute(sql)的结果
|
|
"""
|
|
key_list = self.__get_key_list()
|
|
header = self.__get_field_list()
|
|
field_list = self.__get_field_list()
|
|
convert_dict = {}
|
|
for field in self.schema:
|
|
if field[1].lower().startswith('largeint'):
|
|
convert_dict[field[0]] = self.__get_type_convert_handler(field[1])
|
|
dict_list = []
|
|
for row in self.datas:
|
|
field_value_dict = {}
|
|
for field, value in zip(header, row):
|
|
field_value_dict[field] = value
|
|
dict_list.append(field_value_dict)
|
|
table_database_from = petl.fromdicts(dict_list, header)
|
|
table_database_convert = petl.convert(table_database_from, convert_dict)
|
|
table_database_sort = petl.sort(table_database_convert, field_list)
|
|
table_database_merge_sort = petl.mergesort(table_database_sort,
|
|
key=field_list, presorted=False)
|
|
return table_database_merge_sort
|
|
|
|
def __get_data_from_file(self):
|
|
"""
|
|
从文件中获取数据，排序，按照表的聚合模型处理数据
|
|
"""
|
|
# 为了兼容以前的代码
|
|
if type(self.expected_file_list) is str:
|
|
from_verifyfile_list = [VerifyFile(self.expected_file_list, '\t')]
|
|
elif type(self.expected_file_list) is list and type(self.expected_file_list[0]) is str:
|
|
from_verifyfile_list = [VerifyFile(file, '\t') for file in self.expected_file_list]
|
|
elif type(self.expected_file_list) is VerifyFile:
|
|
from_verifyfile_list = [self.expected_file_list]
|
|
else:
|
|
from_verifyfile_list = None
|
|
header = self.__get_field_list()
|
|
key_list = self.__get_key_list()
|
|
field_list = self.__get_field_list()
|
|
convert_dict = self.__get_convert_dict()
|
|
dup = False
|
|
for col in self.schema:
|
|
if col[5] == 'NONE':
|
|
dup = True
|
|
|
|
table_file_to_merge_list = []
|
|
for etl_file in from_verifyfile_list:
|
|
# 读取csv文件数据
|
|
table_file_from = petl.fromcsv(etl_file.get_file_name(),
|
|
encoding='utf8', delimiter=etl_file.get_delimiter())
|
|
# 给数据增加表头
|
|
table_file_push = petl.pushheader(table_file_from, header)
|
|
# 给数据加类型
|
|
table_file_convert = petl.convert(table_file_push, convert_dict)
|
|
table_file_to_merge_list.append(table_file_convert)
|
|
if not dup:
|
|
table_file_merge_sort = petl.mergesort(*table_file_to_merge_list,
|
|
key=key_list, presorted=False)
|
|
aggregation = self.__get_aggregation_ordereddict()
|
|
aggregate_key = self.__get_aggregate_key(key_list)
|
|
# 聚合表，按照聚合方式聚合
|
|
table_file_aggregate = petl.aggregate(table_file_merge_sort,
|
|
key=aggregate_key, aggregation=aggregation,
|
|
presorted=True)
|
|
table_file_merge_sort = petl.mergesort(table_file_aggregate,
|
|
key=key_list, presorted=True)
|
|
return table_file_merge_sort
|
|
else:
|
|
table_file_merge_sort = petl.mergesort(*table_file_to_merge_list,
|
|
key=field_list, presorted=False)
|
|
return table_file_merge_sort
|
|
|
|
def __generate_dafault_save_verifyfile_list(self):
|
|
"""根据库名，表名生成校验文件的名称"""
|
|
name_prefix = ".%s.%s" % (self.database_name, self.table_name)
|
|
name_for_database = "%s.%s" % (name_prefix, 'DB')
|
|
name_for_file = "%s.%s" % (name_prefix, 'FILE')
|
|
return [VerifyFile(name_for_database), VerifyFile(name_for_file)]
|
|
|
|
def verify(self, save_file_list=None):
|
|
"""
|
|
崭新的校验函数
|
|
"""
|
|
LOG.info(L("check file:", file=self.expected_file_list))
|
|
self.__adjust_schema_for_verify()
|
|
# 获取db中的数据
|
|
data_from_database = self.__get_data_from_database()
|
|
# 获取file中的文件
|
|
data_from_file = self.__get_data_from_file()
|
|
if save_file_list is None:
|
|
save_file_list = self.__generate_dafault_save_verifyfile_list()
|
|
# 分别写入数据
|
|
self.__write_data_to_file(data_from_database, data_from_file, save_file_list)
|
|
# 返回check结果, true / false
|
|
return self.__check_data(data_from_database, data_from_file)
|
|
|
|
def __adjust_schema_for_verify(self):
|
|
adjust_schema = []
|
|
for field in self.schema:
|
|
adjust_field = list(field)
|
|
if adjust_field[3] == 'false':
|
|
if adjust_field[5] is not None:
|
|
adjust_field[5] = adjust_field[5].split(',')[0]
|
|
else:
|
|
adjust_field[5] = ''
|
|
adjust_schema.append(tuple(adjust_field))
|
|
self.schema = tuple(adjust_schema)
|
|
return self.schema
|
|
|
|
def __adjust_schema_for_self_defined_sql(self):
|
|
# TODO
|
|
# 这个函数可能有问题，以后修改
|
|
adjust_schema = []
|
|
for column in self.schema:
|
|
adjust_column = []
|
|
adjust_column.append(column[0])
|
|
adjust_column.append(column[1])
|
|
adjust_column.append('No')
|
|
if len(column) > 2 and column[2] is not None:
|
|
adjust_column.append('false')
|
|
else:
|
|
adjust_column.append('true')
|
|
adjust_column.append('N/A')
|
|
if len(column) > 2 and column[2] is not None:
|
|
adjust_column.append(column[2])
|
|
else:
|
|
adjust_column.append('')
|
|
adjust_schema.append(tuple(adjust_column))
|
|
self.schema = adjust_schema
|
|
return self.schema
|
|
|
|
def verify_by_sql(self, save_file_list=None):
|
|
"""
|
|
校验自定义的SQL语句的查询结果
|
|
expected_file_list: VerifyFile对象的list
|
|
sql: SQL语句字符串
|
|
schema: 查询结果的schema, 由四元组(name, type, agg_type, default_value)组成的list
|
|
四元组中后两项可省略, 需要注意的是key列指定默认值是agg_type设置为None
|
|
Example -> [("k1", "INT"), ("k2", "CHAR", None, ""), ("v", "DATE", "REPLACE")]
|
|
save_file_list: VerifyFile对象的list
|
|
"""
|
|
self.__adjust_schema_for_self_defined_sql()
|
|
data_from_database = self.__get_data_from_database()
|
|
data_from_file = self.__get_data_from_file()
|
|
if save_file_list is not None:
|
|
self.__write_data_to_file(data_from_database, data_from_file, save_file_list)
|
|
return self.__check_data(data_from_database, data_from_file)
|
|
|
|
|
|
def verify(file, sql_ret, schema, table_name, database_name, encoding, save_file_list):
|
|
"""
|
|
verify, schema为palo desc结果
|
|
适用于
|
|
1. 多个文件的时候，会对文件进行拼接，排序读取
|
|
2. 适用于原始文件，palo对原始文件进行过滤、聚合等处理时，无需额外保存校验文件，直接使用原始文件进行处理生成校验文件
|
|
"""
|
|
verifier = Verify(file, sql_ret, schema, table_name, database_name, encoding)
|
|
return verifier.verify(save_file_list)
|
|
|
|
|
|
def verify_by_sql(file, sql_ret, schema, table_name, database_name, encoding, save_file_list):
|
|
"""
|
|
verify by sql
|
|
指定四元组为schema
|
|
"""
|
|
verifier = Verify(file, sql_ret, schema, table_name, database_name, encoding)
|
|
return verifier.verify_by_sql(save_file_list)
|
|
|