243 lines
7.3 KiB
Python
243 lines
7.3 KiB
Python
#!/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
"""
|
|
for palo query test
|
|
Date: 2018-07-24 16:16:33
|
|
"""
|
|
import sys
|
|
import time
|
|
import optparse
|
|
import subprocess as commands
|
|
import json
|
|
import functools
|
|
sys.path.append("../..")
|
|
|
|
from lib import palo_client
|
|
from lib import palo_config
|
|
from lib import palo_logger
|
|
|
|
|
|
LOG = palo_logger.Logger.getLogger()
|
|
L = palo_logger.StructedLogMessage
|
|
config = palo_config.config
|
|
|
|
|
|
def run_check(test_file, result_file):
|
|
"""
|
|
base test
|
|
"""
|
|
print("\t---------------------------------------------------------\t")
|
|
print("input file:%s, result file:%s" % (test_file, result_file))
|
|
LOG.info(L('input file: %s, result file: %s' % (test_file, result_file)))
|
|
actual = _execute(test_file)
|
|
if not actual:
|
|
return False
|
|
f_result = open(result_file, "r")
|
|
lines = f_result.readlines()
|
|
k = 0
|
|
j = 0
|
|
ret = True
|
|
total_line = len(lines)
|
|
for res in lines:
|
|
res = res.strip()
|
|
j += 1
|
|
if actual[k].upper().endswith("#IGNORE CHECK"):
|
|
if k >= total_line - 1:
|
|
break
|
|
k += 1
|
|
continue
|
|
if res.upper().endswith("#IGNORE CHECK") or res.startswith("--"):
|
|
continue
|
|
if actual[k] != res:
|
|
if actual[k].startswith("False") and res.startswith("False"):
|
|
print('-----------------diff info--------------------------------')
|
|
print("different false info in result %d line" % j)
|
|
print('expected: %s\nactual: %s' % (res, actual[k]))
|
|
LOG.warning(L("different false info in result %d line" % j))
|
|
LOG.warning(L('expected: %s\nactual: %s' % (res, actual[k])))
|
|
else:
|
|
ret = False
|
|
print('****************diff result*******************************')
|
|
print("different in result %d line" % j)
|
|
LOG.error(L("different in result %d line" % j))
|
|
if k > 0:
|
|
print(actual[k - 1])
|
|
LOG.error(L('this is wrong sql', sql=actual[k - 1]))
|
|
print("expected: %s\nactual: %s" % (res, actual[k]))
|
|
LOG.error(L("expected: %s but actual: %s" % (res, actual[k])))
|
|
|
|
k += 1
|
|
LOG.info(L('test %s result is %s' % (test_file, ret)))
|
|
return ret
|
|
|
|
|
|
def gen_result(test_file, result_file):
|
|
"""
|
|
gen result
|
|
"""
|
|
result = _execute(test_file)
|
|
f_result = open(result_file, "w")
|
|
for r in result:
|
|
f_result.write(r + "\n")
|
|
|
|
f_result.close()
|
|
return True
|
|
|
|
|
|
def compare(a, b):
|
|
"""compare data to None"""
|
|
assert isinstance(a, (tuple, list))
|
|
assert isinstance(b, (tuple, list))
|
|
for i in range(0, len(a)):
|
|
if a[i] is not None and b[i] is not None:
|
|
if a[i] > b[i]:
|
|
return 1
|
|
elif a[i] < b[i]:
|
|
return -1
|
|
else:
|
|
continue
|
|
elif a[i] is None and b[i] is None:
|
|
continue
|
|
elif a[i] is None:
|
|
return -1
|
|
else:
|
|
return 1
|
|
return 0
|
|
|
|
|
|
def _execute(test_file):
|
|
client = palo_client.get_client(config.fe_host, config.fe_query_port,
|
|
user=config.fe_user,
|
|
password=config.fe_password)
|
|
f_sql = open(test_file, "r")
|
|
result = list()
|
|
for sql in f_sql:
|
|
IGNORE_CHECK = False
|
|
sql = sql.strip()
|
|
# 注释,忽略
|
|
if not sql or sql.startswith("--"):
|
|
continue
|
|
|
|
# ignore this sql's result
|
|
# select * from a#IGNORE CHECK ==> select * from a
|
|
# () ==> ()#IGNORE CHECK
|
|
if sql.upper().endswith("#IGNORE CHECK"):
|
|
sql = sql[:len(sql) - 13].strip()
|
|
IGNORE_CHECK = True
|
|
print(sql)
|
|
LOG.info(L('execute', sql=sql))
|
|
result.append(sql)
|
|
|
|
# 对于use和导入语句,只记录SQL,不记录结果
|
|
if sql.upper().startswith("USE"):
|
|
selected_db = sql[4:].strip()
|
|
client.use(selected_db)
|
|
continue
|
|
# 异步导入
|
|
if sql.upper().startswith("LOAD") or sql.upper().startswith("INSERT"):
|
|
try:
|
|
client.execute(sql)
|
|
client.wait_table_load_job()
|
|
except Exception as e:
|
|
print(str(e))
|
|
continue
|
|
# curl multi load & stream load
|
|
if sql.upper().startswith("CURL"):
|
|
try:
|
|
sql = sql.format(FE_HOST=config.fe_host, HTTP_PORT=config.fe_http_port, FE_PASSWORD=config.fe_password)
|
|
print(sql)
|
|
ret = commands.getoutput(sql)
|
|
print(ret)
|
|
client.wait_table_load_job()
|
|
except Exception as e:
|
|
print(str(e))
|
|
break
|
|
continue
|
|
|
|
# 不支持异常操作，execute的SQL应该是支持的
|
|
r = client.execute(sql)
|
|
# 排序
|
|
if r:
|
|
r = list(r)
|
|
try:
|
|
r.sort()
|
|
except Exception as e:
|
|
if isinstance(e, TypeError):
|
|
r.sort(key=functools.cmp_to_key(compare))
|
|
# 字符串格式化
|
|
r = str_format(r)
|
|
actual = ("%s" % r).strip()
|
|
|
|
if IGNORE_CHECK:
|
|
actual += "#IGNORE CHECK"
|
|
print(actual)
|
|
result.append(actual)
|
|
f_sql.close()
|
|
return result
|
|
|
|
|
|
def str_format(result):
|
|
"""fromat result and return str"""
|
|
ret = list()
|
|
for line in result:
|
|
record = list()
|
|
for col in line:
|
|
record.append(str(col))
|
|
ret.append(record)
|
|
return str(tuple(ret))
|
|
|
|
|
|
def check_cluster():
|
|
"""check be if enough"""
|
|
client = palo_client.get_client(config.fe_host, config.fe_query_port,
|
|
user=config.fe_user,
|
|
password=config.fe_password)
|
|
alive_be = client.get_alive_backend_list()
|
|
if len(alive_be) >= 3:
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
|
|
def main():
|
|
"""
|
|
main
|
|
"""
|
|
usage = "Usage: python implement.py test_file result_file"
|
|
parser = optparse.OptionParser(usage)
|
|
parser.add_option("-g", "--gen", dest="gen", default=False, action='store_true',
|
|
help="generate result and write into file")
|
|
options, args = parser.parse_args()
|
|
if len(args) != 2:
|
|
parser.print_help()
|
|
return
|
|
#s = run_check("atomic.sql", "atomic.result")
|
|
if options.gen:
|
|
s = gen_result(args[0], args[1])
|
|
return
|
|
s = run_check(args[0], args[1])
|
|
print("--------------------------------------------------------------------------------------")
|
|
print("Check Result: %s" % s)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|