98 lines
4.2 KiB
Python
98 lines
4.2 KiB
Python
#!/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
"""
|
|
palo broker load info
|
|
"""
|
|
|
|
|
|
class LoadDataInfo(object):
|
|
"""
|
|
load data description
|
|
Attributes:
|
|
file_path_list: 可以为string表示只有一个导入文件，或一个list表示多个导入文件
|
|
column_name_list: 列名组成的list
|
|
column_terminator: 数据文件列分隔符
|
|
is_negative: 是否为数据恢复
|
|
format_as: str，导入文件的格式，如csv，parquet
|
|
columns_from_path: list，从路径中解析列的数据，如['k1', 'k2', 'city']
|
|
where_clause: string, k1 > k2
|
|
"""
|
|
def __init__(self, file_path_list, table_name, partition_list=None, temp_partition_list=None,
|
|
column_name_list=None, column_terminator=None, set_list=None, format_as=None,
|
|
is_negative=False, columns_from_path=None, where_clause=None, merge_type=None,
|
|
delete_on_predicates=None, order_by_list=None):
|
|
if isinstance(file_path_list, list):
|
|
self.file_path_list = file_path_list
|
|
elif isinstance(file_path_list, str):
|
|
self.file_path_list = [file_path_list]
|
|
else:
|
|
raise Exception("File path list should be list or string", file_path_list=file_path_list)
|
|
self.table_name = table_name
|
|
self.partition_list = partition_list
|
|
self.temp_partition_list = temp_partition_list
|
|
self.column_name_list = column_name_list
|
|
self.column_terminator = column_terminator
|
|
self.is_negative = is_negative
|
|
self.set_list = set_list
|
|
self.format_as = format_as
|
|
self.columns_from_path = columns_from_path
|
|
self.where_clause = where_clause
|
|
self.merge_type = merge_type
|
|
self.delete_on_predicates = delete_on_predicates
|
|
self.order_by_list = order_by_list
|
|
|
|
def __str__(self):
|
|
if self.merge_type is None:
|
|
self.merge_type = ''
|
|
sql = '%s DATA INFILE("%s")' % (self.merge_type, '", "'.join(self.file_path_list))
|
|
if self.is_negative:
|
|
sql = '%s NEGATIVE' % sql
|
|
sql = '%s INTO TABLE `%s`' % (sql, self.table_name)
|
|
if self.partition_list is not None:
|
|
sql = '%s PARTITION (%s)' % (sql, ', '.join(self.partition_list))
|
|
if self.temp_partition_list is not None:
|
|
sql = '%s TEMPORARY PARTITION (%s)' % (sql, ', '.join(self.temp_partition_list))
|
|
if self.column_terminator is not None:
|
|
sql = '%s COLUMNS TERMINATED BY "%s"' % (sql, self.column_terminator)
|
|
if self.format_as is not None:
|
|
sql = '%s FORMAT AS "%s"' % (sql, self.format_as)
|
|
if self.column_name_list is not None:
|
|
sql = '%s (`%s`)' % (sql, '`, `'.join(self.column_name_list))
|
|
if self.columns_from_path is not None:
|
|
sql = '%s COLUMNS FROM PATH AS (%s)' % (sql, ','.join(self.columns_from_path))
|
|
if self.set_list is not None:
|
|
sql = '%s SET(%s)' % (sql, ', '.join(self.set_list))
|
|
if self.where_clause is not None:
|
|
sql = '%s WHERE %s' % (sql, self.where_clause)
|
|
if self.delete_on_predicates is not None:
|
|
sql = '%s DELETE ON %s' % (sql, self.delete_on_predicates)
|
|
if self.order_by_list is not None:
|
|
sql = '%s ORDER BY %s' % (sql, ','.join(self.order_by_list))
|
|
return sql
|
|
|
|
def set(self, k, v):
|
|
"""设置属性"""
|
|
if hasattr(self, k):
|
|
setattr(self, k, v)
|
|
else:
|
|
raise Exception('data descriptor set error: %s' % k)
|
|
|
|
|