#!/bin/env python
# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

"""
/***************************************************************************
  *
  * @file test_sys_pull_load_external.py
  * @brief Test for create external table
  *
  **************************************************************************/
"""

import pytest
from data import support_null as DATA2
from data import schema as DATA1
from data import pull_load_apache as DATA
from lib import palo_client
from lib import util
from lib import palo_config
from lib import common

LOG = palo_client.LOG
L = palo_client.L
config = palo_config.config
broker_name = config.broker_name
broker_info = palo_config.broker_info


def setup_module():
    """
    setUp
    """
    global client
    client = palo_client.get_client(config.fe_host, config.fe_query_port, user=config.fe_user,
                                    password=config.fe_password, http_port=config.fe_http_port)


def test_mysql_external_table():
    """
    {
    "title": "test_sys_pull_load_external.test_mysql_external_table",
    "describe": "create mysql external table and select",
    "tag": "system,p1"
    }
    """
    """create mysql external table and select"""
    database_name, table_name, index_name = util.gen_name_list()
    LOG.info(L('', database_name=database_name,
               table_name=table_name,
               index_name=index_name))
    client.clean(database_name)
    client.create_database(database_name)
    client.use(database_name)
    property = {"host": config.mysql_host, 
                "port": config.mysql_port, 
                "user": config.mysql_user, 
                "password": config.mysql_password, 
                "database": config.mysql_db, 
                "table": "baseall"}
    ret = client.create_external_table(table_name, DATA1.baseall_column_list, engine='mysql',
                                 property=property)
    assert ret
    sql1 = 'select * from %s.%s order by k1' % (database_name, table_name)
    sql2 = 'select * from %s.baseall order by k1' % config.palo_db
    ret1 = client.execute(sql1)
    ret2 = client.execute(sql2)
    util.check(ret1, ret2)
    client.clean(database_name)


def test_mysql_external_issue_5066():
    """
    {
    "title": "test_sys_pull_load_external.test_mysql_external_table_select",
    "describe": "create mysql external table and select",
    "tag": "system,p1"
    }
    """
    """create mysql external table and select"""
    database_name, table_name, index_name = util.gen_name_list()
    LOG.info(L('', database_name=database_name,
               table_name=table_name,
               index_name=index_name))
    client.clean(database_name)
    client.create_database(database_name)
    client.use(database_name)
    property = {"host": config.mysql_host,
                "port": config.mysql_port,
                "user": config.mysql_user,
                "password": config.mysql_password,
                "database": config.mysql_db,
                "table": "baseall"}
    tb1 = table_name + '_1'
    tb2 = table_name + '_2'
    tb3 = table_name + '_3'
    assert client.create_external_table(tb1, DATA1.baseall_column_list, engine='mysql',
                                        property=property)
    assert client.create_external_table(tb2, DATA1.baseall_column_list, engine='mysql',
                                        property=property)
    assert client.create_external_table(tb3, DATA1.baseall_column_list, engine='mysql',
                                        property=property)
    sql = 'create view v1 as select * from %s union all select * from %s union all select * from %s' % (tb1, tb2, tb3)
    assert client.execute(sql) == ()
    sql1 = 'select count(*) from v1'
    sql2 = 'select 45'
    common.check2(client, sql1=sql1, sql2=sql2)
    client.clean(database_name)


def test_mysql_external_alter_engine():
    """
    {
    "title": "test_sys_pull_load_external.test_mysql_external_table",
    "describe": "create mysql external table and alter external engine/driver, issue-6993",
    "tag": "system,p1"
    }
    """
    """create mysql external table and select"""
    database_name, table_name, index_name = util.gen_name_list()
    LOG.info(L('', database_name=database_name,
               table_name=table_name,
               index_name=index_name))
    client.clean(database_name)
    client.create_database(database_name)
    client.use(database_name)
    property = {"host": config.mysql_host,
                "port": config.mysql_port,
                "user": config.mysql_user,
                "password": config.mysql_password,
                "database": config.mysql_db,
                "table": "baseall"}
    ret = client.create_external_table(table_name, DATA1.baseall_column_list, engine='mysql',
                                 property=property)
    assert ret, 'create table failed'
    sql = 'select engine from information_schema.tables where ' \
          'table_schema="%s" and table_name="%s"' % (database_name, table_name)
    ret = client.execute(sql)
    assert ret[0][0].upper() == "MYSQL", 'expect mysql engine, but %s' % ret[0][0]
    sql = 'alter table %s modify engine to odbc properties("driver"="odbc")' % table_name
    ret = client.execute(sql)
    sql = 'select engine from information_schema.tables where ' \
          'table_schema="%s" and table_name="%s"' % (database_name, table_name)
    ret = client.execute(sql)
    assert ret[0][0].upper() == "ODBC", 'expect odbc engine, but %s' % ret[0][0]
    client.clean(database_name)


def test_hdfs_external_table_disable():
    """
    {
    "title": "test_sys_pull_load_external.test_hdfs_external_table_disable",
    "describe": "test hdfs csv file external table",
    "tag": "system,p1"
    }
    """
    """test hdfs csv file external table"""
    database_name, table_name, index_name = util.gen_name_list()
    LOG.info(L('', database_name=database_name,
               table_name=table_name,
               index_name=index_name))
    client.clean(database_name)
    client.create_database(database_name)
    client.use(database_name)
    property = {"broker_name": config.broker_name,
                "path": DATA.data_1,
                "column_separator": "\t",
                "format": "csv"}
    ret = client.create_external_table(table_name, DATA.schema_1, engine='broker',
                                       property=property, broker_property=config.broker_property)
    assert ret
    msg = 'Broker external table is not supported, try to use table function please'
    util.assert_return(False, msg, client.select_all, table_name)
    client.clean(database_name)


def test_hdfs_external_table_csv():
    """
    {
    "title": "test_sys_pull_load_external.test_hdfs_external_table_csv",
    "describe": "test hdfs csv file external table",
    "tag": "system,p1"
    }
    """
    """test hdfs csv file external table"""
    database_name, table_name, index_name = util.gen_name_list()
    LOG.info(L('', database_name=database_name,
               table_name=table_name,
               index_name=index_name))
    client.clean(database_name)
    client.create_database(database_name)
    client.use(database_name)
    property = {
                "uri": DATA.data_1,
                "fs.defaultFS": config.defaultFS, 
                "hadoop.username": config.hdfs_username,
                "column_separator": "\t",
                "format": "csv"}
    sql = 'select * from hdfs %s' % util.convert_dict2property(property)
    common.check_by_file(DATA.verify_1[0], sql=sql, client=client)
    client.clean(database_name)


def test_hdfs_external_table_gz():
    """
    {
    "title": "test_sys_pull_load_external.test_hdfs_external_table_gz",
    "describe": "test hdfs gz file external file",
    "tag": "system,p1"
    }
    """
    """test hdfs gz file external file"""
    database_name, table_name, index_name = util.gen_name_list()
    LOG.info(L('', database_name=database_name,
               table_name=table_name,
               index_name=index_name))
    client.clean(database_name)
    client.create_database(database_name)
    client.use(database_name)
    property = {
                "uri": DATA.data_1_gz,
                "fs.defaultFS": config.defaultFS,
                "hadoop.username": config.hdfs_username,
                "column_separator": "\t"
               }

    sql = 'select * from hdfs %s' % util.convert_dict2property(property)
    msg = 'format: is not supported.'
    util.assert_return(False, msg, client.execute, sql)
    # common.check_by_file(DATA.verify_1[0], sql=sql, client=client)
    client.clean(database_name)


@pytest.mark.skip()
def test_hdfs_external_table_bz2():
    """
    {
    "title": "test_sys_pull_load_external.test_hdfs_external_table_bz2",
    "describe": "test hdfs bz2 external table",
    "tag": "system,p1"
    }
    """
    """test hdfs bz2 external table"""
    database_name, table_name, index_name = util.gen_name_list()
    LOG.info(L('', database_name=database_name,
               table_name=table_name,
               index_name=index_name))
    client.clean(database_name)
    client.create_database(database_name)
    client.use(database_name)
    property = {"broker_name": config.broker_name,
                "path": DATA.data_1_bz2,
               }
    ret = client.create_external_table(table_name, DATA.schema_1, engine='broker',
                                       property=property, broker_property=broker_info)
    assert ret
    assert client.verify(DATA.verify_1, table_name)
    client.clean(database_name)


@pytest.mark.skip()
def test_hdfs_external_table_lzo():
    """
    {
    "title": "test_sys_pull_load_external.test_hdfs_external_table_lzo",
    "describe": "test hdfs lzo external table",
    "tag": "system,p1"
    }
    """
    """test hdfs lzo external table"""
    database_name, table_name, index_name = util.gen_name_list()
    LOG.info(L('', database_name=database_name,
               table_name=table_name,
               index_name=index_name))
    client.clean(database_name)
    client.create_database(database_name)
    client.use(database_name)
    property = {"broker_name": config.broker_name,
                "path": DATA.data_1_lzo,
               }
    ret = client.create_external_table(table_name, DATA.schema_1, engine='broker',
                                       property=property, broker_property=broker_info)
    assert ret
    assert client.verify(DATA.verify_1, table_name)
    client.clean(database_name)


@pytest.mark.skip()
def test_hdfs_external_table_lz4():
    """
    {
    "title": "test_sys_pull_load_external.test_hdfs_external_table_lz4",
    "describe": "test hdfs lz4 file external table",
    "tag": "system,p1"
    }
    """
    """test hdfs lz4 file external table"""
    database_name, table_name, index_name = util.gen_name_list()
    LOG.info(L('', database_name=database_name,
               table_name=table_name,
               index_name=index_name))
    client.clean(database_name)
    client.create_database(database_name)
    client.use(database_name)
    property = {"broker_name": config.broker_name,
                "path": DATA.data_1_lz4,
               }
    ret = client.create_external_table(table_name, DATA.schema_1, engine='broker',
                                       property=property, broker_property=broker_info)
    assert ret
    assert client.verify(DATA.verify_1, table_name)
    client.clean(database_name)


def test_hdfs_external_table_parquet():
    """
    {
    "title": "test_sys_pull_load_external.test_hdfs_external_table_parquet",
    "describe": "test hdfs parquet file external table",
    "tag": "system,p1"
    }
    """
    database_name, table_name, index_name = util.gen_name_list()
    LOG.info(L('', database_name=database_name,
               table_name=table_name,
               index_name=index_name))
    client.clean(database_name)
    client.create_database(database_name)
    client.use(database_name)
    property = {
                "uri": DATA.data_1_parquet,
                "fs.defaultFS": config.defaultFS,
                "hadoop.username": config.hdfs_username,
                "column_separator": "\t",
                "format": "parquet"
               }

    sql = 'select tinyint_key, smallint_key, int_key, bigint_key, largeint_key, char_key, ' \
          'varchar_key, cast(decimal_key as decimal(27, 9)), date_key, datetime_key, ' \
          'tinyint_value_max, smallint_value_min, int_value_sum, bigint_value_sum, ' \
          'largeint_value_sum, largeint_value_replace, char_value_replace, ' \
          'varchar_value_replace, cast(decimal_value_replace as decimal(27, 9)), ' \
          'date_value_replace, datetime_value_replace, float_value_sum, double_value_sum ' \
          'from hdfs %s' % util.convert_dict2property(property)
    common.check_by_file(DATA.verify_6[0], sql=sql, client=client)
    client.clean(database_name)


def test_external_table_null():
    """
    {
    "title": "test_sys_pull_load_external.test_external_table_null",
    "describe": "test external table column null",
    "tag": "system,p1"
    }
    """
    """test external table column null"""
    database_name, table_name, index_name = util.gen_num_format_name_list()
    LOG.info(L('', database_name=database_name,
               table_name=table_name, index_name=index_name))
    client.clean(database_name)
    client.create_database(database_name)
    client.use(database_name)

    property = {
                "uri": DATA2.hdfs_file_5,
                "fs.defaultFS": config.defaultFS,
                "hadoop.username": config.hdfs_username,
                "column_separator": "\t",
                "format": "csv"
               }
    sql = 'select c1,c2,c3,c4,c5,c6,c7,c8,c9,cast(c10 as decimal(9,3)), ' \
          'c11,c12,c13,c14,c15,c16,c17,c18,c19,cast(c20 as decimal(27,9)) ' \
          'from hdfs%s' % util.convert_dict2property(property)
    common.check_by_file(DATA2.expected_file_external_1, sql=sql, client=client)

    sql = 'SELECT COUNT(*) FROM hdfs%s' % util.convert_dict2property(property)
    ret = client.execute(sql)
    assert int(ret[0][0]) == 12
    sql = 'SELECT COUNT(c1) FROM hdfs%s' % util.convert_dict2property(property)
    ret = client.execute(sql)
    assert int(ret[0][0]) == 10
    client.clean(database_name)
    

def test_external_table_not_null():
    """
    {
    "title": "test_sys_pull_load_external.test_external_table_not_null",
    "describe": "test external table column not null, filtered null value",
    "tag": "system,p1"
    }
    """
    """test external table column not null, filtered null value"""
    database_name, table_name, index_name = util.gen_num_format_name_list()
    LOG.info(L('', database_name=database_name,
               table_name=table_name, index_name=index_name))
    
    client.clean(database_name)
    client.create_database(database_name)
    client.use(database_name)
    property = {"broker_name": config.broker_name,
                "path": DATA2.hdfs_file_5,
                "column_separator": "\t",
                "format": "csv"}
    ret = client.create_external_table(table_name, DATA2.schema_2, engine='broker',
                                       property=property, broker_property=broker_info, 
                                       set_null=False)
    assert ret
    
    sql = 'SELECT * FROM %s.%s ' \
          'ORDER BY k2, k1, k3, k4, k5, k6, k7, k8, k9, k10' \
          % (database_name, table_name)
    assert common.check_by_file(DATA2.expected_file_external_2, sql=sql, client=client)
    client.clean(database_name)
    

if __name__ == '__main__':
    setup_module()
    # test_mysql_external_table()
    # test_hdfs_external_table_gz()
    # test_hdfs_external_table_bz2()
    # test_hdfs_external_table_lz4()
    # test_hdfs_external_table_lzo()
    test_hdfs_external_table_csv()
    # test_hdfs_external_table_parquet()
    test_external_table_not_null()
    # test_external_table_null()