217 lines
7.8 KiB
Python
Executable File
217 lines
7.8 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# encoding: utf-8
|
|
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
"""
|
|
This module is doris builtin functions
|
|
"""
|
|
|
|
import sys
|
|
import os
|
|
import errno
|
|
from string import Template
|
|
import doris_builtins_functions
|
|
|
|
java_registry_preamble = '\
|
|
// Licensed to the Apache Software Foundation (ASF) under one \n\
|
|
// or more contributor license agreements. See the NOTICE file \n\
|
|
// distributed with this work for additional information \n\
|
|
// regarding copyright ownership. The ASF licenses this file \n\
|
|
// to you under the Apache License, Version 2.0 (the \n\
|
|
// "License"); you may not use this file except in compliance \n\
|
|
// with the License. You may obtain a copy of the License at \n\
|
|
// \n\
|
|
// http://www.apache.org/licenses/LICENSE-2.0\n\
|
|
// \n\
|
|
// Unless required by applicable law or agreed to in writing, software\n\
|
|
// distributed under the License is distributed on an "AS IS" BASIS,\n\
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n\
|
|
// See the License for the specific language governing permissions and\n\
|
|
// limitations under the License.\n\
|
|
\n\
|
|
// This is a generated file, DO NOT EDIT.\n\
|
|
// To add new functions, see the generator at\n\
|
|
// common/function-registry/gen_builtins_catalog.py or the function list at\n\
|
|
// common/function-registry/doris_builtins_functions.py.\n\
|
|
\n\
|
|
package org.apache.doris.builtins;\n\
|
|
\n\
|
|
import org.apache.doris.catalog.ArrayType;\n\
|
|
import org.apache.doris.catalog.Type;\n\
|
|
import org.apache.doris.catalog.Function;\n\
|
|
import org.apache.doris.catalog.FunctionSet;\n\
|
|
import com.google.common.collect.Sets;\n\
|
|
import java.util.Set;\n\
|
|
\n\
|
|
public class ScalarBuiltins { \n\
|
|
public static void initBuiltins(FunctionSet functionSet) { \
|
|
\n'
|
|
|
|
java_registry_epilogue = '\
|
|
}\n\
|
|
}\n'
|
|
|
|
FE_PATH = "../../../fe/fe-core/target/generated-sources/build/org/apache/doris/builtins/"
|
|
print(FE_PATH)
|
|
|
|
# This contains all the metadata to describe all the builtins.
|
|
# Each meta data entry is itself a map to store all the meta data
|
|
# - fn_name, ret_type, args, symbol, sql_names
|
|
meta_data_entries = []
|
|
|
|
# Read in the function and add it to the meta_data_entries map
|
|
def add_function(fn_meta_data, user_visible):
|
|
"""add function
|
|
"""
|
|
assert len(fn_meta_data) == 8, \
|
|
"Invalid function entry in doris_builtins_functions.py:\n\t" + repr(fn_meta_data)
|
|
entry = {}
|
|
entry["sql_names"] = fn_meta_data[0]
|
|
entry["ret_type"] = fn_meta_data[1]
|
|
entry["args"] = fn_meta_data[2]
|
|
entry["symbol"] = fn_meta_data[3]
|
|
if fn_meta_data[4] != '':
|
|
entry["prepare"] = fn_meta_data[4]
|
|
if fn_meta_data[5] != '':
|
|
entry["close"] = fn_meta_data[5]
|
|
if fn_meta_data[6] != '':
|
|
entry['vec'] = True
|
|
if fn_meta_data[7] != '':
|
|
entry['nullable_mode'] = fn_meta_data[7]
|
|
else:
|
|
entry['nullable_mode'] = 'DEPEND_ON_ARGUMENT'
|
|
|
|
entry["user_visible"] = user_visible
|
|
meta_data_entries.append(entry)
|
|
|
|
|
|
"""
|
|
generate fe data type, support nested ARRAY type.
|
|
for example:
|
|
in[TINYINT] --> out[Type.TINYINT]
|
|
in[INT] --> out[Type.INT]
|
|
in[ARRAY_INT] --> out[new ArrayType(Type.INT)]
|
|
"""
|
|
def generate_fe_datatype(str_type):
|
|
if str_type.startswith("ARRAY_"):
|
|
vec_type = str_type.split('_', 1);
|
|
if len(vec_type) > 1 and vec_type[0] == "ARRAY":
|
|
return "new ArrayType(" + generate_fe_datatype(vec_type[1]) + ")"
|
|
if str_type == "DECIMALV2":
|
|
return "Type.MAX_DECIMALV2_TYPE"
|
|
if str_type == "DECIMAL32":
|
|
return "Type.DECIMAL32"
|
|
if str_type == "DECIMAL64":
|
|
return "Type.DECIMAL64"
|
|
if str_type == "DECIMAL128":
|
|
return "Type.DECIMAL128"
|
|
return "Type." + str_type
|
|
|
|
"""
|
|
Order of params:
|
|
name, symbol, user_visible, prepare, close, nullable_mode, ret_type, has_var_args, args
|
|
"""
|
|
def generate_fe_entry(entry, name):
|
|
"""add function
|
|
"""
|
|
java_output = ""
|
|
java_output += "\"" + name + "\""
|
|
java_output += ", \"" + entry["symbol"] + "\""
|
|
if entry["user_visible"]:
|
|
java_output += ", true"
|
|
else:
|
|
java_output += ", false"
|
|
if 'prepare' in entry:
|
|
java_output += ', "%s"' % entry["prepare"]
|
|
else:
|
|
java_output += ', null'
|
|
if 'close' in entry:
|
|
java_output += ', "%s"' % entry["close"]
|
|
else:
|
|
java_output += ', null'
|
|
|
|
java_output += ", Function.NullableMode." + entry["nullable_mode"]
|
|
java_output += ", " + generate_fe_datatype(entry["ret_type"])
|
|
|
|
# Check the last entry for varargs indicator.
|
|
if entry["args"] and entry["args"][-1] == "...":
|
|
entry["args"].pop()
|
|
java_output += ", true"
|
|
else:
|
|
java_output += ", false"
|
|
for arg in entry["args"]:
|
|
java_output += ", " + generate_fe_datatype(arg)
|
|
return java_output
|
|
|
|
# Generates the FE builtins init file that registers all the builtins.
|
|
def generate_fe_registry_init(filename):
|
|
"""add function
|
|
"""
|
|
java_registry_file = open(filename, "w")
|
|
java_registry_file.write(java_registry_preamble)
|
|
|
|
for entry in meta_data_entries:
|
|
for name in entry["sql_names"]:
|
|
java_output = generate_fe_entry(entry, name)
|
|
if ("vec" not in entry):
|
|
java_registry_file.write(" functionSet.addScalarBuiltin(%s);\n" % java_output)
|
|
else:
|
|
java_registry_file.write(" functionSet.addScalarAndVectorizedBuiltin(%s);\n" % java_output)
|
|
|
|
|
|
java_registry_file.write("\n")
|
|
|
|
# add non_null_result_with_null_param_functions
|
|
java_registry_file.write(" Set<String> funcNames = Sets.newHashSet();\n")
|
|
for entry in doris_builtins_functions.null_result_with_one_null_param_functions:
|
|
java_registry_file.write(" funcNames.add(\"%s\");\n" % entry)
|
|
java_registry_file.write(" functionSet.buildNullResultWithOneNullParamFunction(funcNames);\n");
|
|
|
|
# add nondeterministic functions
|
|
java_registry_file.write(" Set<String> nondeterministicFuncNames = Sets.newHashSet();\n")
|
|
for entry in doris_builtins_functions.nondeterministic_functions:
|
|
java_registry_file.write(" nondeterministicFuncNames.add(\"%s\");\n" % entry)
|
|
java_registry_file.write(" functionSet.buildNondeterministicFunctions(nondeterministicFuncNames);\n");
|
|
|
|
java_registry_file.write(" funcNames = Sets.newHashSet();\n")
|
|
for entry in doris_builtins_functions.null_result_with_one_null_param_functions:
|
|
java_registry_file.write(" funcNames.add(\"%s\");\n" % entry)
|
|
java_registry_file.write(" functionSet.buildNullResultWithOneNullParamFunction(funcNames);\n");
|
|
|
|
java_registry_file.write(java_registry_epilogue)
|
|
java_registry_file.close()
|
|
|
|
if __name__ == "__main__":
|
|
|
|
try:
|
|
os.makedirs(FE_PATH)
|
|
except OSError as e:
|
|
if e.errno == errno.EEXIST:
|
|
pass
|
|
else:
|
|
raise
|
|
|
|
# Read the function metadata inputs
|
|
for function in doris_builtins_functions.visible_functions:
|
|
add_function(function, True)
|
|
for function in doris_builtins_functions.invisible_functions:
|
|
add_function(function, False)
|
|
|
|
generate_fe_registry_init(FE_PATH + "ScalarBuiltins.java")
|