From c6090965e21ec9cc05166df8044735e5a66712bb Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Mon, 10 Feb 2020 09:45:01 +0800 Subject: [PATCH] [Unused Code] Remove unused file in gensrc/scripts (#2863) gensrc/script/doris_functions.py gensrc/script/gen_opcodes.py These 2 files are useless. Remove them. --- gensrc/script/Makefile | 16 +- gensrc/script/doris_functions.py | 328 ------------------------- gensrc/script/gen_opcodes.py | 396 ------------------------------- 3 files changed, 2 insertions(+), 738 deletions(-) delete mode 100755 gensrc/script/doris_functions.py delete mode 100755 gensrc/script/gen_opcodes.py diff --git a/gensrc/script/Makefile b/gensrc/script/Makefile index 11725bbfb1..75af53bdd2 100644 --- a/gensrc/script/Makefile +++ b/gensrc/script/Makefile @@ -20,7 +20,7 @@ BUILD_DIR = ${CURDIR}/../build/ # Prerequisites on the right side of '|' is only order -all: gen_opcode gen_version gen_builtins +all: gen_builtins gen_version .PHONY: all ${BUILD_DIR}/python: @@ -44,27 +44,15 @@ ${GEN_VEC_FUNC_OUTPUT}: gen_vector_functions.py | ${BUILD_DIR}/python gen_vec_func: ${GEN_VEC_FUNC_OUTPUT} .PHONY: gen_vec_func -# generate opcode -GEN_OPCODE_OUTPUT = ${BUILD_DIR}/thrift/Opcodes.thrift \ - ${BUILD_DIR}/gen_cpp/opcode/opcode-registry-init.cc \ - ${BUILD_DIR}/java/org/apache/doris/opcode/FunctionRegistry.java \ - ${BUILD_DIR}/java/org/apache/doris/opcode/FunctionOperator.java - -${GEN_OPCODE_OUTPUT}: doris_functions.py ${GEN_FUNC_OUTPUT} ${GEN_VEC_FUNC_OUTPUT} | ${BUILD_DIR}/python -gen_opcode: ${GEN_OPCODE_OUTPUT} -.PHONY: gen_opcode - # generate GEN_BUILTINS_OUTPUT = ${BUILD_DIR}/java/org/apache/doris/builtins/ScalarBuiltins.java -${GEN_BUILTINS_OUTPUT}: doris_builtins_functions.py gen_builtins_functions.py +${GEN_BUILTINS_OUTPUT}: doris_builtins_functions.py gen_builtins_functions.py ${GEN_FUNC_OUTPUT} ${GEN_VEC_FUNC_OUTPUT} cd ${BUILD_DIR}/python && ${PYTHON} ${CURDIR}/gen_builtins_functions.py gen_builtins: ${GEN_BUILTINS_OUTPUT} .PHONY: gen_builtins # generate version info -VERSION_OUTPUT = ${BUILD_DIR}/java/org/apache/doris/common/Version.java - gen_version: ${CURDIR}/gen_build_version.sh .PHONY: gen_version diff --git a/gensrc/script/doris_functions.py b/gensrc/script/doris_functions.py deleted file mode 100755 index a651747bc4..0000000000 --- a/gensrc/script/doris_functions.py +++ /dev/null @@ -1,328 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" -# This is a list of all the functions that are not auto-generated. -# It contains all the meta data that describes the function. The format is: -# , , [], , [] -# -# 'function name' is the base of what the opcode enum will be generated from. It does not -# have to be unique, the script will mangle the name with the signature if necessary. -# -# 'sql aliases' are the function names that can be used from sql. They are -# optional and there can be multiple aliases for a function. -# -# This is combined with the list in generated_functions to code-gen the opcode -# registry in the FE and BE. -""" - -functions = [ - ['Compound_And', 'BOOLEAN', ['BOOLEAN', 'BOOLEAN'], \ - 'CompoundPredicate::not_compute_fn', []], - ['Compound_Or', 'BOOLEAN', ['BOOLEAN', 'BOOLEAN'], \ - 'CompoundPredicate::not_compute_fn', []], - ['Compound_Not', 'BOOLEAN', ['BOOLEAN', 'BOOLEAN'], \ - 'CompoundPredicate::not_compute_fn', []], - - ['Constant_Regex', 'BOOLEAN', ['BOOLEAN', 'BOOLEAN'], \ - 'LikePredicate::constant_regex_fn', []], - ['Constant_Substring', 'BOOLEAN', ['BOOLEAN', 'BOOLEAN'], \ - 'LikePredicate::constant_substring_fn', []], - ['Like', 'BOOLEAN', ['VARCHAR', 'VARCHAR'], 'LikePredicate::like_fn', []], - ['Regex', 'BOOLEAN', ['VARCHAR', 'VARCHAR'], 'LikePredicate::regex_fn', []], - - ['Math_Pi', 'DOUBLE', [], 'MathFunctions::pi', ['pi']], - ['Math_E', 'DOUBLE', [], 'MathFunctions::e', ['e']], - ['Math_Abs', 'DOUBLE', ['DOUBLE'], 'MathFunctions::abs', ['abs']], - ['Math_Sign', 'FLOAT', ['DOUBLE'], 'MathFunctions::sign', ['sign']], - ['Math_Sin', 'DOUBLE', ['DOUBLE'], 'MathFunctions::sin', ['sin']], - ['Math_Asin', 'DOUBLE', ['DOUBLE'], 'MathFunctions::asin', ['asin']], - ['Math_Cos', 'DOUBLE', ['DOUBLE'], 'MathFunctions::cos', ['cos']], - ['Math_Acos', 'DOUBLE', ['DOUBLE'], 'MathFunctions::acos', ['acos']], - ['Math_Tan', 'DOUBLE', ['DOUBLE'], 'MathFunctions::tan', ['tan']], - ['Math_Atan', 'DOUBLE', ['DOUBLE'], 'MathFunctions::atan', ['atan']], - ['Math_Radians', 'DOUBLE', ['DOUBLE'], 'MathFunctions::radians', ['radians']], - ['Math_Degrees', 'DOUBLE', ['DOUBLE'], 'MathFunctions::degrees', ['degrees']], - ['Math_Ceil', 'BIGINT', ['DOUBLE'], 'MathFunctions::ceil', ['ceil', 'ceiling']], - ['Math_Floor', 'BIGINT', ['DOUBLE'], 'MathFunctions::floor', ['floor']], - ['Math_Round', 'BIGINT', ['DOUBLE'], 'MathFunctions::round', ['round']], - ['Math_Round', 'DOUBLE', ['DOUBLE', 'INT'], 'MathFunctions::round_up_to', ['round']], - ['Math_Truncate', 'BIGINT', ['DOUBLE'], 'MathFunctions::truncate', ['truncate']], - ['Math_Truncate', 'DOUBLE', ['DOUBLE', 'INT'], 'MathFunctions::truncate_to', ['truncate']], - ['Math_Exp', 'DOUBLE', ['DOUBLE'], 'MathFunctions::exp', ['exp']], - ['Math_Ln', 'DOUBLE', ['DOUBLE'], 'MathFunctions::ln', ['ln']], - ['Math_Log10', 'DOUBLE', ['DOUBLE'], 'MathFunctions::log10', ['log10']], - ['Math_Log2', 'DOUBLE', ['DOUBLE'], 'MathFunctions::log2', ['log2']], - ['Math_Log', 'DOUBLE', ['DOUBLE', 'DOUBLE'], 'MathFunctions::log', ['log']], - ['Math_Pow', 'DOUBLE', ['DOUBLE', 'DOUBLE'], 'MathFunctions::pow', ['pow', 'power']], - ['Math_Sqrt', 'DOUBLE', ['DOUBLE'], 'MathFunctions::sqrt', ['sqrt']], - ['Math_Rand', 'DOUBLE', [], 'MathFunctions::rand', ['rand']], - ['Math_Rand', 'DOUBLE', ['INT'], 'MathFunctions::rand_seed', ['rand']], - ['Math_Bin', 'VARCHAR', ['BIGINT'], 'MathFunctions::bin', ['bin']], - ['Math_Hex', 'VARCHAR', ['BIGINT'], 'MathFunctions::hex_int', ['hex']], - ['Math_Hex', 'VARCHAR', ['VARCHAR'], 'MathFunctions::hex_string', ['hex']], - ['Math_Unhex', 'VARCHAR', ['VARCHAR'], 'MathFunctions::unhex', ['unhex']], - ['Math_Conv', 'VARCHAR', ['BIGINT', 'TINYINT', 'TINYINT'], \ - 'MathFunctions::conv_int', ['conv']], - ['Math_Conv', 'VARCHAR', ['VARCHAR', 'TINYINT', 'TINYINT'], \ - 'MathFunctions::conv_string', ['conv']], - ['Math_Pmod', 'BIGINT', ['BIGINT', 'BIGINT'], 'MathFunctions::pmod_bigint', ['pmod']], - ['Math_Pmod', 'DOUBLE', ['DOUBLE', 'DOUBLE'], 'MathFunctions::pmod_double', ['pmod']], - ['Math_Positive', 'BIGINT', ['BIGINT'], 'MathFunctions::positive_bigint', ['positive']], - ['Math_Positive', 'DOUBLE', ['DOUBLE'], 'MathFunctions::positive_double', ['positive']], - ['Math_Negative', 'DOUBLE', ['DOUBLE'], 'MathFunctions::negative_double', ['negative']], - ['Math_Negative', 'BIGINT', ['BIGINT'], 'MathFunctions::negative_bigint', ['negative']], - ['Math_Greatest', 'BIGINT', ['BIGINT', '...'], 'MathFunctions::greatest_bigint', ['greatest']], - ['Math_Greatest', 'DOUBLE', ['DOUBLE', '...'], 'MathFunctions::greatest_double', ['greatest']], - ['Math_Greatest', 'DECIMAL', ['DECIMAL', '...'], 'MathFunctions::greatest_decimal', ['greatest']], - ['Math_Greatest', 'DECIMALV2', ['DECIMALV2', '...'], 'MathFunctions::greatest_decimal', ['greatest']], - ['Math_Greatest', 'VARCHAR', ['VARCHAR', '...'], 'MathFunctions::greatest_string', ['greatest']], - ['Math_Greatest', 'DATETIME', ['DATETIME', '...'], \ - 'MathFunctions::greatest_timestamp', ['greatest']], - ['Math_Least', 'BIGINT', ['BIGINT', '...'], 'MathFunctions::least_bigint', ['least']], - ['Math_Least', 'DOUBLE', ['DOUBLE', '...'], 'MathFunctions::least_double', ['least']], - ['Math_Least', 'DECIMAL', ['DECIMAL', '...'], 'MathFunctions::least_decimal', ['least']], - ['Math_Least', 'DECIMALV2', ['DECIMALV2', '...'], 'MathFunctions::least_decimalv2', ['least']], - ['Math_Least', 'VARCHAR', ['VARCHAR', '...'], 'MathFunctions::least_string', ['least']], - ['Math_Least', 'DATETIME', ['DATETIME', '...'], 'MathFunctions::least_timestamp', ['least']], - -# left and right are key words, leave them out for now. - ['String_Left', 'VARCHAR', ['VARCHAR', 'INT'], 'StringFunctions::left', ['strleft', 'left']], - ['String_Right', 'VARCHAR', ['VARCHAR', 'INT'], 'StringFunctions::right', ['strright', 'right']], - ['String_Ends_With', 'BOOLEAN', ['VARCHAR', 'VARCHAR'], 'StringFunctions::ends_with', ['ends_with']], - ['String_Starts_With', 'BOOLEAN', ['VARCHAR', 'VARCHAR'], 'StringFunctions::starts_with', ['starts_with']], - ['String_Length', 'INT', ['VARCHAR'], 'StringFunctions::length', ['length']], - ['String_Lower', 'VARCHAR', ['VARCHAR'], 'StringFunctions::lower', ['lower', 'lcase']], - ['String_Upper', 'VARCHAR', ['VARCHAR'], 'StringFunctions::upper', ['upper', 'ucase']], - ['String_Reverse', 'VARCHAR', ['VARCHAR'], 'StringFunctions::reverse', ['reverse']], - ['String_Trim', 'VARCHAR', ['VARCHAR'], 'StringFunctions::trim', ['trim']], - ['String_Ltrim', 'VARCHAR', ['VARCHAR'], 'StringFunctions::ltrim', ['ltrim']], - ['String_Rtrim', 'VARCHAR', ['VARCHAR'], 'StringFunctions::rtrim', ['rtrim']], - ['String_Space', 'VARCHAR', ['INT'], 'StringFunctions::space', ['space']], - ['String_Repeat', 'VARCHAR', ['VARCHAR', 'INT'], 'StringFunctions::repeat', ['repeat']], - ['String_Ascii', 'INT', ['VARCHAR'], 'StringFunctions::ascii', ['ascii']], - ['String_Lpad', 'VARCHAR', ['VARCHAR', 'INT', 'VARCHAR'], \ - 'StringFunctions::lpad', ['lpad']], - ['String_Rpad', 'VARCHAR', ['VARCHAR', 'INT', 'VARCHAR'], \ - 'StringFunctions::rpad', ['rpad']], - ['String_Instr', 'INT', ['VARCHAR', 'VARCHAR'], 'StringFunctions::instr', ['instr']], - ['String_Locate', 'INT', ['VARCHAR', 'VARCHAR'], 'StringFunctions::locate', ['locate']], - ['String_Locate', 'INT', ['VARCHAR', 'VARCHAR', 'INT'], \ - 'StringFunctions::locate_pos', ['locate']], - ['String_Regexp_Extract', 'VARCHAR', ['VARCHAR', 'VARCHAR', 'INT'], \ - 'StringFunctions::regexp_extract', ['regexp_extract']], - ['String_Regexp_Replace', 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR'], \ - 'StringFunctions::regexp_replace', ['regexp_replace']], - ['String_Concat', 'VARCHAR', ['VARCHAR', '...'], 'StringFunctions::concat', ['concat']], - ['String_Concat_Ws', 'VARCHAR', ['VARCHAR', 'VARCHAR', '...'], \ - 'StringFunctions::concat_ws', ['concat_ws']], - ['String_Find_In_Set', 'INT', ['VARCHAR', 'VARCHAR'], \ - 'StringFunctions::find_in_set', ['find_in_set']], - ['String_Parse_Url', 'VARCHAR', ['VARCHAR', 'VARCHAR'], \ - 'StringFunctions::parse_url', ['parse_url']], - ['String_Parse_Url', 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR'], \ - 'StringFunctions::parse_url_key', ['parse_url']], - ['Utility_Sleep', 'BOOLEAN', ['INT'], 'UtilityFunctions::sleep', ['sleep']], - ['Utility_Version', 'VARCHAR', [], 'UtilityFunctions::version', ['version']], - -#JsonFunction - ['Get_json_int', 'INT', ['VARCHAR', 'VARCHAR'], \ - 'JsonFunctions::get_json_int_value', ['get_json_int']], - ['Get_json_string', 'VARCHAR', ['VARCHAR', 'VARCHAR'], \ - 'JsonFunctions::get_json_string_value', ['get_json_string']], - ['Get_json_double', 'DOUBLE', ['VARCHAR', 'VARCHAR'], \ - 'JsonFunctions::get_json_double_value', ['get_json_double']], - -#hll function - ['Hll_cardinality', 'VARCHAR', ['VARCHAR'], \ - 'HllHashFunction::hll_cardinality', ['hll_cardinality']], - ['Hll_hash_string', 'VARCHAR', ['VARCHAR'], \ - 'HllHashFunction::hll_hash_string', ['hll_hash_string']], - -# Timestamp Functions -# ['Unix_Timestamp', 'INT', [], \ -# 'TimestampFunctions::to_unix', ['unix_timestamp']], -# ['Unix_Timestamp', 'INT', ['DATETIME'], \ -# 'TimestampFunctions::to_unix', ['unix_timestamp']], -# ['Unix_Timestamp', 'INT', ['DATETIME', 'VARCHAR'], \ -# 'TimestampFunctions::to_unix', ['unix_timestamp']], -# ['From_UnixTime', 'VARCHAR', ['INT'], \ -# 'TimestampFunctions::from_unix', ['from_unixtime']], -# ['From_UnixTime', 'VARCHAR', ['INT', 'VARCHAR'], \ -# 'TimestampFunctions::from_unix', ['from_unixtime']], - ['Timestamp_year', 'INT', ['DATETIME'], 'TimestampFunctions::year', ['year']], - ['Timestamp_quarter', 'INT', ['DATETIME'], 'TimestampFunctions::quarter', ['quarter']], - ['Timestamp_month', 'INT', ['DATETIME'], 'TimestampFunctions::month', ['month']], - ['Timestamp_dayofmonth', 'INT', ['DATETIME'], \ - 'TimestampFunctions::day_of_month', ['day', 'dayofmonth']], - ['Timestamp_dayofyear', 'INT', ['DATETIME'], - 'TimestampFunctions::day_of_year', ['dayofyear']], - ['Timestamp_weekofyear', 'INT', ['DATETIME'], \ - 'TimestampFunctions::week_of_year', ['weekofyear']], - ['Timestamp_hour', 'INT', ['DATETIME'], 'TimestampFunctions::hour', ['hour']], - ['Timestamp_minute', 'INT', ['DATETIME'], 'TimestampFunctions::minute', ['minute']], - ['Timestamp_second', 'INT', ['DATETIME'], 'TimestampFunctions::second', ['second']], - ['Timestamp_now', 'DATETIME', [], \ - 'TimestampFunctions::now', ['now', 'current_timestamp']], - ['Time_now', 'DATETIME', [], \ - 'TimestampFunctions::curtime', ['curtime', 'utc_time']], - ['Timestamp_to_date', 'DATE', ['DATETIME'], \ - 'TimestampFunctions::to_date', ['date', 'to_date']], - ['Timestamp_years_add', 'DATETIME', ['DATETIME', 'INT'], \ - 'TimestampFunctions::years_add', ['years_add']], - ['Timestamp_years_sub', 'DATETIME', ['DATETIME', 'INT'], \ - 'TimestampFunctions::years_sub', ['years_sub']], - ['Timestamp_months_add', 'DATETIME', ['DATETIME', 'INT'], \ - 'TimestampFunctions::months_add', ['months_add']], - ['Timestamp_months_sub', 'DATETIME', ['DATETIME', 'INT'], \ - 'TimestampFunctions::months_sub', ['months_sub']], - ['Timestamp_weeks_add', 'DATETIME', ['DATETIME', 'INT'], \ - 'TimestampFunctions::weeks_add', ['weeks_add']], - ['Timestamp_weeks_sub', 'DATETIME', ['DATETIME', 'INT'], \ - 'TimestampFunctions::weeks_sub', ['weeks_sub']], - ['Timestamp_days_add', 'DATETIME', ['DATETIME', 'INT'], \ - 'TimestampFunctions::days_add', ['days_add', 'date_add', 'adddate']], - ['Timestamp_days_sub', 'DATETIME', ['DATETIME', 'INT'], \ - 'TimestampFunctions::days_sub', ['days_sub', 'date_sub', 'subdate']], - ['Timestamp_hours_add', 'DATETIME', ['DATETIME', 'INT'], \ - 'TimestampFunctions::hours_add', ['hours_add']], - ['Timestamp_hours_sub', 'DATETIME', ['DATETIME', 'INT'], \ - 'TimestampFunctions::hours_sub', ['hours_sub']], - ['Timestamp_minutes_add', 'DATETIME', ['DATETIME', 'INT'], \ - 'TimestampFunctions::minutes_add', ['minutes_add']], - ['Timestamp_minutes_sub', 'DATETIME', ['DATETIME', 'INT'], \ - 'TimestampFunctions::minutes_sub', ['minutes_sub']], - ['Timestamp_seconds_add', 'DATETIME', ['DATETIME', 'INT'], \ - 'TimestampFunctions::seconds_add', ['seconds_add']], - ['Timestamp_seconds_sub', 'DATETIME', ['DATETIME', 'INT'], \ - 'TimestampFunctions::seconds_sub', ['seconds_sub']], - ['Timestamp_microseconds_add', 'DATETIME', ['DATETIME', 'INT'], \ - 'TimestampFunctions::micros_add', ['microseconds_add']], - ['Timestamp_microseconds_sub', 'DATETIME', ['DATETIME', 'INT'], \ - 'TimestampFunctions::micros_sub', ['microseconds_sub']], - ['Timestamp_diff', 'INT', ['DATETIME', 'DATETIME'], \ - 'TimestampFunctions::date_diff', ['datediff']], - ['Time_diff', 'DATETIME', ['DATETIME', 'DATETIME'], \ - 'TimestampFunctions::time_diff', ['timediff']], - ['From_utc_timestamp', 'DATETIME', ['DATETIME', 'VARCHAR'], \ - 'TimestampFunctions::from_utc', ['from_utc_timestamp']], - ['To_utc_timestamp', 'DATETIME', ['DATETIME', 'VARCHAR'], \ - 'TimestampFunctions::to_utc', ['to_utc_timestamp']], - ['Timestamp_date_format', 'VARCHAR', ['DATETIME', 'VARCHAR'], \ - 'TimestampFunctions::date_format', ['date_format']], - ['Timestamp_from_days', 'DATE', ['INT'], \ - 'TimestampFunctions::from_days', ['from_days']], - ['Timestamp_to_days', 'INT', ['DATE'], \ - 'TimestampFunctions::to_days', ['to_days']], - ['Timestamp_str_to_date', 'DATETIME', ['VARCHAR', 'VARCHAR'], \ - 'TimestampFunctions::str_to_date', ['str_to_date']], - ['Timestamp_day_name', 'VARCHAR', ['DATE'], \ - 'TimestampFunctions::day_name', ['dayname']], - ['Timestamp_mont_name', 'VARCHAR', ['DATE'], \ - 'TimestampFunctions::month_name', ['monthname']], - ['Timestamp_timestamp', 'DATETIME', ['DATETIME'], \ - 'TimestampFunctions::timestamp', ['timestamp']], - -# Conditional Functions -# ['Conditional_NullIf', 'BOOLEAN', ['BOOLEAN', 'BOOLEAN'], \ -# 'ConditionalFunctions::null_if_bool', ['nullif']], -# ['Conditional_NullIf', 'TINYINT', ['TINYINT', 'TINYINT'], \ -# 'ConditionalFunctions::null_if_tinyint', ['nullif']], -# ['Conditional_NullIf', 'SMALLINT', ['SMALLINT', 'SMALLINT'], \ -# 'ConditionalFunctions::null_if_smallint', ['nullif']], -# ['Conditional_NullIf', 'INT', ['INT', 'INT'], \ -# 'ConditionalFunctions::null_if_int', ['nullif']], -# ['Conditional_NullIf', 'BIGINT', ['BIGINT', 'BIGINT'], \ -# 'ConditionalFunctions::null_if_bigint', ['nullif']], -# ['Conditional_NullIf', 'FLOAT', ['FLOAT', 'FLOAT'], \ -# 'ConditionalFunctions::null_if_float', ['nullif']], -# ['Conditional_NullIf', 'DOUBLE', ['DOUBLE', 'DOUBLE'], \ -# 'ConditionalFunctions::null_if_double', ['nullif']], -# ['Conditional_NullIf', 'VARCHAR', ['VARCHAR', 'VARCHAR'], \ -# 'ConditionalFunctions::null_if_string', ['nullif']], -# ['Conditional_NullIf', 'DATETIME', ['DATETIME', 'DATETIME'], \ -# 'ConditionalFunctions::null_if_timestamp', ['nullif']], -# ['Conditional_Ifnull', 'BOOLEAN', ['BOOLEAN', 'BOOLEAN'], \ -# 'ConditionalFunctions::if_null_bool', ['ifnull']], -# ['Conditional_Ifnull', 'TINYINT', ['TINYINT', 'TINYINT'], \ -# 'ConditionalFunctions::if_null_tinyint', ['ifnull']], -# ['Conditional_Ifnull', 'SMALLINT', ['SMALLINT', 'SMALLINT'], \ -# 'ConditionalFunctions::if_null_smallint', ['ifnull']], -# ['Conditional_Ifnull', 'INT', ['INT', 'INT'], \ -# 'ConditionalFunctions::if_null_int', ['ifnull']], -# ['Conditional_Ifnull', 'BIGINT', ['BIGINT', 'BIGINT'], \ -# 'ConditionalFunctions::if_null_bigint', ['ifnull']], -# ['Conditional_Ifnull', 'FLOAT', ['FLOAT', 'FLOAT'], \ -# 'ConditionalFunctions::if_null_float', ['ifnull']], -# ['Conditional_Ifnull', 'DOUBLE', ['DOUBLE', 'DOUBLE'], \ -# 'ConditionalFunctions::if_null_double', ['ifnull']], -# ['Conditional_Ifnull', 'VARCHAR', ['VARCHAR', 'VARCHAR'], \ -# 'ConditionalFunctions::if_null_string', ['ifnull']], -# ['Conditional_Ifnull', 'DATETIME', ['DATETIME', 'DATETIME'], \ -# 'ConditionalFunctions::if_null_timestamp', ['ifnull']], -# ['Conditional_If', 'BOOLEAN', ['BOOLEAN', 'BOOLEAN', 'BOOLEAN'], \ -# 'ConditionalFunctions::if_bool', ['if']], -# ['Conditional_If', 'TINYINT', ['BOOLEAN', 'TINYINT', 'TINYINT'], \ -# 'ConditionalFunctions::if_tinyint', ['if']], -# ['Conditional_If', 'SMALLINT', ['BOOLEAN', 'SMALLINT', 'SMALLINT'], \ -# 'ConditionalFunctions::if_smallint', ['if']], -# ['Conditional_If', 'INT', ['BOOLEAN', 'INT', 'INT'], \ -# 'ConditionalFunctions::if_int', ['if']], -# ['Conditional_If', 'BIGINT', ['BOOLEAN', 'BIGINT', 'BIGINT'], \ -# 'ConditionalFunctions::if_bigint', ['if']], -# ['Conditional_If', 'FLOAT', ['BOOLEAN', 'FLOAT', 'FLOAT'], \ -# 'ConditionalFunctions::if_float', ['if']], -# ['Conditional_If', 'DOUBLE', ['BOOLEAN', 'DOUBLE', 'DOUBLE'], \ -# 'ConditionalFunctions::if_double', ['if']], -# ['Conditional_If', 'VARCHAR', ['BOOLEAN', 'VARCHAR', 'VARCHAR'], \ -# 'ConditionalFunctions::if_string', ['if']], -# ['Conditional_If', 'DATETIME', ['BOOLEAN', 'DATETIME', 'DATETIME'], \ -# 'ConditionalFunctions::if_timestamp', ['if']], -# ['Conditional_Coalesce', 'BOOLEAN', ['BOOLEAN', '...'], \ -# 'ConditionalFunctions::coalesce_bool', ['coalesce']], -# ['Conditional_Coalesce', 'BIGINT', ['BIGINT', '...'], \ -# 'ConditionalFunctions::coalesce_int', ['coalesce']], -# ['Conditional_Coalesce', 'DOUBLE', ['DOUBLE', '...'], \ -# 'ConditionalFunctions::coalesce_float', ['coalesce']], -# ['Conditional_Coalesce', 'VARCHAR', ['VARCHAR', '...'], \ -# 'ConditionalFunctions::coalesce_string', ['coalesce']], -# ['Conditional_Coalesce', 'DATETIME', ['DATETIME', '...'], \ -# 'ConditionalFunctions::coalesce_timestamp', ['coalesce']], -] - -# These functions are implemented against the UDF interface. -# TODO: this list should subsume the one above when all builtins are migrated. -udf_functions = [ - ['Udf_Math_Abs', 'DECIMAL', ['DECIMAL'], 'UdfBuiltins::decimal_abs', ['udf_abs'], - ''], - ['Udf_Math_Abs', 'DECIMALV2', ['DECIMALV2'], 'UdfBuiltins::decimal_abs', ['udf_abs'], - ''], - ['Udf_Sub_String', 'VARCHAR', ['VARCHAR', 'INT', 'INT'], - ['Udf_Sub_String', 'VARCHAR', ['VARCHAR', 'INT', 'INT'], - 'UdfBuiltins::sub_string', ['udf_substring'], ''], - ['Udf_Add_Two_Number', 'BIGINT', ['BIGINT', 'BIGINT'], - 'UdfBuiltins::add_two_number', ['udf_add_two_number'], ''], - ['Udf_Math_Pi', 'DOUBLE', [], 'UdfBuiltins::pi', ['udf_pi'], - '_ZN6impala11UdfBuiltins2PiEPN10impala_udf15FunctionContextE'], - ['Udf_Math_Abs', 'DOUBLE', ['DOUBLE'], 'UdfBuiltins::abs', ['udf_abs'], - '_ZN6impala11UdfBuiltins3AbsEPN10impala_udf15FunctionContextERKNS1_9DoubleValE'], - ['Udf_String_Lower', 'VARCHAR', ['VARCHAR'], 'UdfBuiltins::lower', ['udf_lower'], - '_ZN6impala11UdfBuiltins5LowerEPN10impala_udf15FunctionContextERKNS1_9StringValE'], -] diff --git a/gensrc/script/gen_opcodes.py b/gensrc/script/gen_opcodes.py deleted file mode 100755 index 94ce47ec9a..0000000000 --- a/gensrc/script/gen_opcodes.py +++ /dev/null @@ -1,396 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" -# This script generates the necessary files to coordinate function calls between the FE -# and BE. In the FE, this creates a mapping between function signature (Operation & -# Arguments) to an opcode. The opcode is a thrift enum which is passed to the backend. -# The backend has all the information from just the opcode and does not need to worry -# about type checking. -# -# This scripts pulls function metadata input from -# - src/common/function/doris_functions.py (manually maintained) -# - src/common/function/generated_functions.py (auto-generated metadata) -# -# This script will generate 4 outputs -# 1. Thrift enum for all the opcodes -# - impala/fe/src/thrift/Opcodes.thrift -# 2. FE java operators (one per function, ignoring overloading) -# - impala/fe/generated-sources/gen-java/com/cloudera/impala/opcode/FunctionOperater.java -# 3 Java registry setup (registering all the functions with signatures) -# - impala/fe/generated-sources/gen-java/com/cloudera/impala/opcode/FunctionRegistry.java -# 4. BE registry setup (mapping opcodes to ComputeFunctions) -# - impala/be/generated-sources/opcode/opcode-registry-init.cc -# -# TODO: version the registry on the FE and BE so we can identify if they are out of sync -""" - - -import sys -import os -import string -sys.path.append(os.getcwd()) -import doris_functions -import generated_functions -import generated_vector_functions - -native_types = { - 'BOOLEAN': 'bool', - 'TINYINT': 'char', - 'SMALLINT': 'short', - 'INT': 'int', - 'BIGINT': 'long', - 'LARGEINT': 'LargeIntValue', - 'FLOAT': 'float', - 'DOUBLE': 'double', - 'VARCHAR': 'StringValue', - 'DATE': 'Date', - 'DATETIME': 'DateTime', - 'DECIMAL': 'DecimalValue', - 'DECIMALV2': 'DecimalV2Value', - 'TIME': 'double' -} - -thrift_preamble = '\ -//\n\ -namespace java org.apache.doris.thrift\n\ -\n\ -enum TExprOpcode {\n' - -thrift_epilogue = '\ -}\n\ -\n' - -cc_registry_preamble = '\ -// Licensed to the Apache Software Foundation (ASF) under one \n\ -// or more contributor license agreements. See the NOTICE file \n\ -// distributed with this work for additional information \n\ -// regarding copyright ownership. The ASF licenses this file \n\ -// to you under the Apache License, Version 2.0 (the \n\ -// "License"); you may not use this file except in compliance \n\ -// with the License. You may obtain a copy of the License at \n\ -// \n\ -// http://www.apache.org/licenses/LICENSE-2.0\n\ -// \n\ -// Unless required by applicable law or agreed to in writing, software\n\ -// distributed under the License is distributed on an "AS IS" BASIS,\n\ -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n\ -// See the License for the specific language governing permissions and\n\ -// limitations under the License.\n\ -// limitations under the License.\n\ -\n\ -// This is a generated file, DO NOT EDIT.\n\ -// To add new functions, see impala/common/function-registry/gen_opcodes.py\n\ -\n\ -#include "exprs/opcode_registry.h"\n\ -#include "exprs/expr.h"\n\ -#include "exprs/compound_predicate.h"\n\ -#include "exprs/like_predicate.h"\n\ -#include "exprs/math_functions.h"\n\ -#include "exprs/string_functions.h"\n\ -#include "exprs/timestamp_functions.h"\n\ -#include "exprs/conditional_functions.h"\n\ -#include "exprs/udf_builtins.h"\n\ -#include "exprs/utility_functions.h"\n\ -#include "gen_cpp/opcode/functions.h"\n\ -#include "gen_cpp/opcode/vector-functions.h"\n\ -#include "exprs/json_functions.h"\n\ -#include "exprs/encryption_functions.h"\n\ -#include "exprs/es_functions.h"\n\ -#include "exprs/hll_hash_function.h"\n\ -\n\ -using namespace boost::posix_time;\n\ -using namespace boost::gregorian;\n\ -\n\ -namespace doris {\n\ -\n\ -void OpcodeRegistry::init() {\n' - -cc_registry_epilogue = '\ -}\n\ -\n\ -}\n' - -operator_file_preamble = '\ -// This is a generated file, DO NOT EDIT.\n\ -// To add new functions, see impala/common/function-registry/gen_opcodes.py\n\ -\n\ -package org.apache.doris.opcode;\n\ -\n\ -public enum FunctionOperator {\n' - -operator_file_epilogue = '\ -}\n' - -java_registry_preamble = '\ -// This is a generated file, DO NOT EDIT.\n\ -// To add new functions, see impala/common/function-registry/gen_opcodes.py\n\ -\n\ -package org.apache.doris.opcode;\n\ -\n\ -import org.apache.doris.analysis.OpcodeRegistry;\n\ -import org.apache.doris.catalog.PrimitiveType;\n\ -import org.apache.doris.thrift.TExprOpcode;\n\ -import com.google.common.base.Preconditions;\n\ -\n\ -public class FunctionRegistry { \n\ - public static void InitFunctions(OpcodeRegistry registry) { \n\ - boolean result = true;\n\ -\n' - -java_registry_epilogue = '\ - Preconditions.checkState(result); \n\ - }\n\ -}\n' - - -def initialize_sub(op, return_type, arg_types): - """ - initialize_sub - """ - sub = {} - java_args = "PrimitiveType." + return_type - sub["fn_class"] = "GetValueFunctions" - sub["fn_signature"] = op - sub["num_args"] = len(arg_types) - for idx in range(0, len(arg_types)): - arg = arg_types[idx] - sub["fn_signature"] += "_" + native_types[arg] - sub["native_type" + repr(idx + 1)] = native_types[arg] - java_args += ", PrimitiveType." + arg - sub["thrift_enum"] = sub["fn_signature"].upper() - sub["java_output"] = "FunctionOperator." + op.upper() + ", TExprOpcode." + sub["thrift_enum"] - sub["java_output"] += ", " + java_args - return sub - -FE_PATH = "../java/org.apache.doris/opcode/" -BE_PATH = "../gen_cpp/opcode/" -THRIFT_PATH = "../thrift/" - -# This contains a list of all the opcodes that are built base on the -# function name from the input. Inputs can have multiple signatures -# with the same function name and the opcode is mangled using the -# arg types. -opcodes = [] - -# This contains a list of all the function names (no overloading/mangling) -operators = [] - -# This is a mapping of operators to a list of function meta data entries -# Each meta data entry is itself a map to store all the meta data -# - fn_name, ret_type, args, be_fn, sql_names -meta_data_entries = {} - -def add_function(fn_meta_data, udf_interface, is_vector_function=False): - """ - Read in the function and add it to the meta_data_entries map - """ - fn_name = fn_meta_data[0] - ret_type = fn_meta_data[1] - args = fn_meta_data[2] - be_fn = fn_meta_data[3] - - entry = {} - entry["fn_name"] = fn_name - entry["ret_type"] = fn_meta_data[1] - entry["args"] = fn_meta_data[2] - entry["be_fn"] = fn_meta_data[3] - entry["sql_names"] = fn_meta_data[4] - entry["is_vector_function"] = is_vector_function - if udf_interface: - entry["symbol"] = fn_meta_data[5] - else: - entry["symbol"] = "" - entry["udf_interface"] = udf_interface - - if fn_name in meta_data_entries: - meta_data_entries[fn_name].append(entry) - else: - fn_list = [entry] - meta_data_entries[fn_name] = fn_list - operators.append(fn_name.upper()) - - -def generate_opcodes(): - """ - Iterate over entries in the meta_data_entries map and generate opcodes. Some - entries will have the same name at this stage, quality the name withe the - signature to generate unique enums. - Resulting opcode list is sorted with INVALID_OPCODE at beginning and LAST_OPCODE - at end. - """ - for fn in meta_data_entries: - entries = meta_data_entries[fn] - if len(entries) > 1: - for entry in entries: - opcode = fn.upper() - for arg in entry["args"]: - if arg == "...": - opcode += "_" + 'VARARGS' - else: - opcode += "_" + native_types[arg].upper() - opcodes.append(opcode) - entry["opcode"] = opcode - else: - opcodes.append(fn.upper()) - entries[0]["opcode"] = fn.upper() - opcodes.sort() - opcodes.insert(0, 'INVALID_OPCODE') - opcodes.append('LAST_OPCODE') - - -def generate_be_registry_init(filename): - """ - Generates the BE registry init file that will add all the compute functions - to the registry. Outputs the generated-file to 'filename' - """ - cc_registry_file = open(filename, "w") - cc_registry_file.write(cc_registry_preamble) - - for fn in meta_data_entries: - entries = meta_data_entries[fn] - for entry in entries: - opcode = entry["opcode"] - be_fn = entry["be_fn"] - symbol = entry["symbol"] - # We generate two casts to work around GCC Bug 11407 - if entry["is_vector_function"]: - cc_output = 'TExprOpcode::%s, (void*)(Expr::VectorComputeFn)%s, "%s"' \ - % (opcode, be_fn, symbol) - else: - cc_output = 'TExprOpcode::%s, (void*)(Expr::ComputeFn)%s, "%s"' \ - % (opcode, be_fn, symbol) - cc_registry_file.write(" this->add(%s);\n" % (cc_output)) - - cc_registry_file.write(cc_registry_epilogue) - cc_registry_file.close() - - -def generate_fe_registry_init(filename): - """ - Generates the FE registry init file that registers all the functions. This file - contains all the opcode->function signature mappings and all of the string->operator - mappings for sql functions - """ - java_registry_file = open(filename, "w") - java_registry_file.write(java_registry_preamble) - - for fn in meta_data_entries: - entries = meta_data_entries[fn] - for entry in entries: - java_output = "" - if entry["udf_interface"]: - java_output += "true" - else: - java_output += "false" - if entry["is_vector_function"]: - java_output += ", true" - else: - java_output += ", false" - java_output += ", FunctionOperator." + fn.upper() - java_output += ", TExprOpcode." + entry["opcode"] - # Check the last entry for varargs indicator. - if entry["args"] and entry["args"][-1] == "...": - entry["args"].pop() - java_output += ", true" - else: - java_output += ", false" - java_output += ", PrimitiveType." + entry["ret_type"] - for arg in entry["args"]: - java_output += ", PrimitiveType." + arg - java_registry_file.write(" result &= registry.add(%s);\n" % java_output) - java_registry_file.write("\n") - - mappings = {} - - for fn in meta_data_entries: - entries = meta_data_entries[fn] - for entry in entries: - for name in entry["sql_names"]: - if name in mappings: - if mappings[name] != fn.upper(): - print "Invalid mapping \"%s\" -> FunctionOperator.%s." \ - % (name, mappings[name]) - print "There is already a mapping \"%s\" -> FunctionOperator.%s.\n" \ - % (name, fn.upper()) - sys.exit(1) - continue - mappings[name] = fn.upper() - java_output = "\"%s\", FunctionOperator.%s" % (name, fn.upper()) - java_registry_file.write(" result &= registry.addFunctionMapping(%s);\n" \ - % java_output) - java_registry_file.write("\n") - - java_registry_file.write(java_registry_epilogue) - java_registry_file.close() - -# Read the function metadata inputs -for function in doris_functions.functions: - if len(function) != 5: - print "Invalid function entry in doris_functions.py:\n\t" + repr(function) - sys.exit(1) - add_function(function, False) - -for function in doris_functions.udf_functions: - assert len(function) == 6, \ - "Invalid function entry in doris_functions.py:\n\t" + repr(function) - add_function(function, True) - -for function in generated_functions.functions: - if len(function) != 5: - print "Invalid function entry in generated_functions.py:\n\t" + repr(function) - sys.exit(1) - add_function(function, False) - -for function in generated_vector_functions.functions: - if len(function) != 5: - print "Invalid function entry in generated_functions.py:\n\t" + repr(function) - sys.exit(1) - add_function(function, False, True) - -generate_opcodes() - -if not os.path.exists(FE_PATH): - os.makedirs(FE_PATH) -if not os.path.exists(BE_PATH): - os.makedirs(BE_PATH) -if not os.path.exists(THRIFT_PATH): - os.makedirs(THRIFT_PATH) - -generate_be_registry_init(BE_PATH + "opcode-registry-init.cc") -generate_fe_registry_init(FE_PATH + "FunctionRegistry.java") - -# Output the opcodes to thrift -thrift_file = open(THRIFT_PATH + "Opcodes.thrift", "w") -thrift_file.write(thrift_preamble) -for opcode in opcodes: - thrift_file.write(" %s,\n" % opcode) -thrift_file.write(thrift_epilogue) -thrift_file.close() - -# Output the operators to java -operators.sort() -operators.insert(0, "INVALID_OPERATOR") -operator_java_file = open(FE_PATH + "FunctionOperator.java", "w") -operator_java_file.write(operator_file_preamble) -for op in operators: - operator_java_file.write(" %s,\n" % op) -operator_java_file.write(operator_file_epilogue) -operator_java_file.close()