249 lines
7.9 KiB
Python
249 lines
7.9 KiB
Python
#!/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
"""
|
|
/***************************************************************************
|
|
*
|
|
* @file palo_config.py
|
|
* @date 2015/01/30 09:38:51
|
|
* @brief this file is config about palo for test
|
|
*
|
|
**************************************************************************/
|
|
"""
|
|
|
|
# 测试环境相关的信息记在这里
|
|
# 比如hadoop, mysql
|
|
import sys
|
|
sys.path.append('../deploy')
|
|
import env_config
|
|
import util
|
|
|
|
|
|
class PaloConfig(object):
|
|
"""
|
|
PaloConfig类
|
|
"""
|
|
def __init__(self):
|
|
"""
|
|
PaloConfig__init__方法
|
|
"""
|
|
import os
|
|
self.fe_host = env_config.master
|
|
self.fe_query_port = env_config.fe_query_port
|
|
self.fe_http_port = env_config.fe_query_port - 1000
|
|
self.fe_user = "root"
|
|
self.fe_password = ""
|
|
self.palo_db = "test_query_qa"
|
|
|
|
self.host_username = env_config.host_username
|
|
self.host_password = env_config.host_password
|
|
self.java_home = env_config.JAVA_HOME
|
|
self.fe_path = env_config.fe_path
|
|
self.be_path = env_config.be_path
|
|
self.be_data_path = env_config.be_data_path_list
|
|
self.dpp_config_str = env_config.dpp_config_str
|
|
self.fe_observer_list = env_config.observer_list
|
|
self.fe_follower_list = env_config.follower_list
|
|
|
|
self.mysql_host = "127.0.0.1"
|
|
self.mysql_port = 61001
|
|
self.mysql_user = "root"
|
|
self.mysql_password = ""
|
|
self.mysql_db = "test"
|
|
|
|
self.canal_user = 'canal'
|
|
self.canal_password = 'canal'
|
|
self.canal_ip = '127.0.0.1'
|
|
|
|
if "FE_HOST" in os.environ.keys():
|
|
self.fe_host = os.environ["FE_HOST"]
|
|
if "FE_QUERY_PORT" in os.environ.keys():
|
|
self.fe_query_port = int(os.environ["FE_QUERY_PORT"])
|
|
if "FE_USER" in os.environ.keys():
|
|
self.fe_user = os.environ["FE_USER"]
|
|
if "FE_PASSWORD" in os.environ.keys():
|
|
self.fe_password = os.environ["FE_PASSWORD"]
|
|
if "FE_WEB_PORT" in os.environ.keys():
|
|
self.fe_http_port = int(os.environ["FE_WEB_PORT"])
|
|
if "FE_DB" in os.environ.keys():
|
|
self.palo_db = os.environ["FE_DB"]
|
|
|
|
if "MYSQL_HOST" in os.environ.keys():
|
|
self.mysql_host = os.environ["MYSQL_HOST"]
|
|
if "MYSQL_PORT" in os.environ.keys():
|
|
self.mysql_port = int(os.environ["MYSQL_PORT"])
|
|
if "MYSQL_USER" in os.environ.keys():
|
|
self.mysql_user = os.environ["MYSQL_USER"]
|
|
if "MYSQL_PASSWORD" in os.environ.keys():
|
|
self.mysql_password = os.environ["MYSQL_PASSWORD"]
|
|
if "MYSQL_DB" in os.environ.keys():
|
|
self.mysql_db = os.environ["MYSQL_DB"]
|
|
|
|
self.bos_accesskey = "xxxxxxxx"
|
|
self.bos_secret_accesskey = "xxxxx"
|
|
self.bos_endpoint = "http://bj.bcebos.com"
|
|
self.bos_region = "bj"
|
|
self.s3_endpoint = "https://s3.bj.bcebos.com"
|
|
# kafka例行导入信息
|
|
self.kafka_broker_list = 'xxxx'
|
|
self.kafka_zookeeper = 'xxxxx'
|
|
# 备份恢复的仓库地址
|
|
self.defaultFS = 'xxx'
|
|
self.hdfs_location = 'xxxx'
|
|
self.repo_location = 'xxxx'
|
|
self.hdfs_username = 'root'
|
|
self.hdfs_passwd = ''
|
|
|
|
# apache hdfs
|
|
self.broker_name = 'hdfs'
|
|
self.broker_property = {"username": self.hdfs_username, "password": self.hdfs_passwd}
|
|
|
|
|
|
config = PaloConfig()
|
|
|
|
|
|
def gen_hdfs_file_path(file_path):
|
|
"""
|
|
hadoop上的数据文件的路径，这个路径以"data"目录结束
|
|
hadoop/
|
|
data/
|
|
dir1/
|
|
dir2/...
|
|
file1
|
|
file2...
|
|
这个目录对应于hdfs/data目录
|
|
hdfs/
|
|
data/
|
|
dir1/
|
|
dir2/...
|
|
file1
|
|
file2...
|
|
更改hadoop地址时，只需要将hdfs下的的data目录上传到新的hadoop
|
|
并修改这个配置即可，所有的case就可以正常运行
|
|
放到代码库便于数据文件的组织和维护
|
|
case中只需定义hdfs_path之后的路径
|
|
"""
|
|
hdfs_path = 'hdfs://xxxxxxx/data/'
|
|
entire_path = "%s/%s" % (hdfs_path, file_path)
|
|
return entire_path.replace(r"//", r"/").replace(r":/", r"://")
|
|
|
|
|
|
def gen_apache_hdfs_file_path(file_path):
|
|
"""
|
|
apache
|
|
"""
|
|
hdfs_path = 'hdfs://xxxxxxxx/data'
|
|
entire_path = "%s/%s" % (hdfs_path, file_path)
|
|
return entire_path.replace(r"//", r"/").replace(r":/", r"://")
|
|
|
|
|
|
def gen_bos_file_path(file_path):
|
|
"""
|
|
bos
|
|
"""
|
|
hdfs_path = 'bos://xxxxx/data/'
|
|
entire_path = "%s/%s" % (hdfs_path, file_path)
|
|
return entire_path.replace(r"//", r"/").replace(r":/", r"://")
|
|
|
|
|
|
def gen_remote_file_path(file_path):
|
|
"""生成远端数据的文件地址"""
|
|
return gen_apache_hdfs_file_path(file_path)
|
|
# return gen_hdfs_file_path(file_path)
|
|
|
|
|
|
def gen_s3_file_path(file_path):
|
|
"""s3 storage path"""
|
|
s3_path = 's3://xxxx/data/'
|
|
entire_path = "%s/%s" % (s3_path, file_path)
|
|
return entire_path.replace(r"//", r"/").replace(r":/", r"://")
|
|
|
|
|
|
class HDFSInfo(object):
|
|
"""
|
|
HDFSInfo
|
|
"""
|
|
def __init__(self, properties):
|
|
self.properties = properties
|
|
|
|
def get_property(self):
|
|
"""get propert str"""
|
|
if isinstance(self.properties, str):
|
|
property = '(%s)' % self.properties
|
|
elif isinstance(self.properties, dict):
|
|
property = util.convert_dict2property(self.properties)
|
|
return property
|
|
|
|
def __str__(self):
|
|
s = 'WITH HDFS %s' % (self.get_property())
|
|
return s
|
|
|
|
|
|
class S3Info(HDFSInfo):
|
|
"""
|
|
S3Info
|
|
"""
|
|
def __str__(self):
|
|
s = 'WITH S3 %s' % (self.get_property())
|
|
return s
|
|
|
|
|
|
class BrokerInfo(HDFSInfo):
|
|
"""
|
|
BrokerInfo
|
|
"""
|
|
def __init__(self, broker_name, properties):
|
|
super(BrokerInfo, self).__init__(properties)
|
|
self.broker_name = broker_name
|
|
|
|
def __str__(self):
|
|
s = 'WITH BROKER "%s" %s' % (self.broker_name, self.get_property())
|
|
return s
|
|
|
|
def to_broker_property_dict(self):
|
|
"""make broker property to dict"""
|
|
if isinstance(self.properties, str):
|
|
str_p = '{%s}' % self.properties
|
|
property = eval(str_p.replace('=', ':'))
|
|
elif isinstance(self.properties, dict):
|
|
property = self.properties
|
|
return property
|
|
|
|
def to_select_into_broker_property_str(self):
|
|
"""generate broker property str for select into """
|
|
property_dict = self.to_broker_property_dict()
|
|
select_into_broker_property_list = list()
|
|
select_into_broker_property_list.append('"broker.name"="%s"' % self.broker_name)
|
|
for k, v in property_dict.items():
|
|
select_into_broker_property_list.append('"broker.%s"="%s"' % (k, v))
|
|
return ','.join(select_into_broker_property_list)
|
|
|
|
|
|
broker_info = BrokerInfo(config.broker_name, config.broker_property)
|
|
s3_info = S3Info({"AWS_ENDPOINT": config.s3_endpoint,
|
|
"AWS_ACCESS_KEY": config.bos_accesskey,
|
|
"AWS_SECRET_KEY": config.bos_secret_accesskey,
|
|
"AWS_REGION": config.bos_region})
|
|
hdfs_info = HDFSInfo(config.broker_property)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
print(config.broker_property['username'])
|
|
|