hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[FEAT MERGE] 开源裁减颗粒度优化

Browse Source 
Co-authored-by: nroskill <nroskill@gmail.com>
Co-authored-by: akaError <lzg020616@163.com>
Co-authored-by: yinyj17 <yinyijun92@gmail.com>
This commit is contained in:
wenxingsen
2023-08-15 02:40:25 +00:00
committed by ob-robot
parent a31e422133
commit 9b31f8aa03
378 changed files with 26718 additions and 2379 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
src/sql/parser/CMakeLists.txt
View File

@ -1,5 +1,7 @@
# charset objects used for proxy_parser
if (OB_BUILD_OPENSOURCE)
set(ob_sql_parser_charset_object_list
ob_ctype_bin_os.cc
ob_ctype_gb18030_os.cc
@ -12,10 +14,34 @@ set(ob_sql_parser_charset_object_list
ob_ctype_utf8_os.cc
ob_dtoa_os.cc
)
endif()
list(TRANSFORM ob_sql_parser_charset_object_list
PREPEND ${PROJECT_SOURCE_DIR}/deps/oblib/src/lib/charset/)
if(OB_BUILD_FULL_CHARSET)
set(ob_sql_parser_full_charset_object_list
ob_ctype_bin.cc
ob_ctype.cc
ob_ctype_gbk.cc
ob_ctype_latin1.cc
ob_ctype_mb.cc
ob_ctype_simple.cc
ob_ctype_utf8.cc
ob_dtoa.cc
ob_ctype_utf16.cc
ob_ctype_uca.cc
ob_ctype_gb18030.cc
uca900_ja_tbls.cc
uca900_zh_tbls.cc
uca900_zh2_tbls.cc
uca900_zh3_tbls.cc
)
endif()
list(TRANSFORM ob_sql_parser_full_charset_object_list
PREPEND ${PROJECT_SOURCE_DIR}/close_modules/charset/deps/oblib/src/lib/charset/)
# hash objects used for proxy parser
set(ob_sql_parser_hash_object_list
murmur_hash.h
@ -28,10 +54,42 @@ list(TRANSFORM ob_sql_parser_hash_object_list
add_library(ob_sql_parser_base INTERFACE)
target_include_directories(ob_sql_parser_base INTERFACE
${PROJECT_SOURCE_DIR}/deps/oblib/src/common/sql_mode/
${PROJECT_SOURCE_DIR}/src/objit/include/objit/common)
${PROJECT_SOURCE_DIR}/src/objit/include/objit/common
${CMAKE_SOURCE_DIR}/deps/oblib/src/lib/charset/
)
# parser objects for proxy parser
if (OB_BUILD_ORACLE_PARSER)
set(ob_inner_sql_parser_object_list
sql_parser_oracle_utf8_mode_lex.c
sql_parser_oracle_utf8_mode_lex.h
sql_parser_oracle_utf8_mode_tab.c
sql_parser_oracle_utf8_mode_tab.h
sql_parser_oracle_gbk_mode_lex.c
sql_parser_oracle_gbk_mode_lex.h
sql_parser_oracle_gbk_mode_tab.c
sql_parser_oracle_gbk_mode_tab.h
sql_parser_oracle_latin1_mode_lex.c
sql_parser_oracle_latin1_mode_lex.h
sql_parser_oracle_latin1_mode_tab.c
sql_parser_oracle_latin1_mode_tab.h
)
set(ob_inner_sql_parser_object_list
${ob_inner_sql_parser_object_list}
non_reserved_keywords_oracle_utf8_mode.c
non_reserved_keywords_oracle_gbk_mode.c
non_reserved_keywords_oracle_latin1_mode.c
)
endif()
# common
set(ob_inner_sql_parser_object_list
${ob_inner_sql_parser_object_list}
sql_parser_mysql_mode_lex.c
sql_parser_mysql_mode_lex.h
sql_parser_mysql_mode_tab.c
sql_parser_mysql_mode_tab.h
non_reserved_keywords_mysql_mode.c
ob_item_type_str.h
ob_non_reserved_keywords.c
@ -51,10 +109,6 @@ set(ob_inner_sql_parser_object_list
sql_parser_base.c
sql_parser_base.h
sql_parser_base.h
sql_parser_mysql_mode_lex.c
sql_parser_mysql_mode_lex.h
sql_parser_mysql_mode_tab.c
sql_parser_mysql_mode_tab.h
parser_utility.cpp
parser_utility.h
type_name.c
@ -70,11 +124,19 @@ set(ob_extra_sql_parser_object_list
)
# ob_sql_proxy_parser_objects is the static library for proxy, it does not link against observer
add_library(ob_sql_proxy_parser_objects OBJECT
${ob_inner_sql_parser_object_list}
${ob_sql_parser_charset_object_list}
${ob_sql_parser_hash_object_list}
)
if (OB_BUILD_FULL_CHARSET)
add_library(ob_sql_proxy_parser_objects OBJECT
${ob_inner_sql_parser_object_list}
${ob_sql_parser_hash_object_list}
${ob_sql_parser_full_charset_object_list}
)
else()
add_library(ob_sql_proxy_parser_objects OBJECT
${ob_inner_sql_parser_object_list}
${ob_sql_parser_hash_object_list}
${ob_sql_parser_charset_object_list}
)
endif()
# ob_sql_server_parser_object is the static library for observer
add_library(ob_sql_server_parser_objects OBJECT

165
src/sql/parser/gen_parser.sh
View File

@ -42,5 +42,170 @@ sed "/Setup the input buffer state to scan the given bytes/,/}/{/for ( i = 0; i
sed "/Setup the input buffer state to scan the given bytes/,/}/{s/\tbuf\[i\] = yybytes\[i\]/memcpy(buf, yybytes, _yybytes_len)/g}" -i ../../../src/sql/parser/sql_parser_mysql_mode_lex.c
if [ -d "../../../close_modules/oracle_parser/sql/parser" ]; then
ln -sf ../../../close_modules/oracle_parser/sql/parser/sql_parser_oracle_mode.y ../../../src/sql/parser/sql_parser_oracle_mode.y
ln -sf ../../../close_modules/oracle_parser/sql/parser/sql_parser_oracle_mode.l ../../../src/sql/parser/sql_parser_oracle_mode.l
# generate oracle latin1 sql_parser(do not support multi_byte_space、multi_byte_comma、multi_byte_left_parenthesis、multi_byte_right_parenthesis)
##1.copy lex and yacc files
cat ../../../src/sql/parser/sql_parser_oracle_mode.y > ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y
cat ../../../src/sql/parser/sql_parser_oracle_mode.l > ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l
##2.replace name
sed "s/obsql_oracle_yy/obsql_oracle_latin1_yy/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y
sed "s/obsql_oracle_yy/obsql_oracle_latin1_yy/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l
sed "s/sql_parser_oracle_mode/sql_parser_oracle_latin1_mode/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y
sed "s/sql_parser_oracle_mode/sql_parser_oracle_latin1_mode/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l
sed "s/obsql_oracle_parser_fatal_error/obsql_oracle_latin1_parser_fatal_error/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y
sed "s/obsql_oracle_parser_fatal_error/obsql_oracle_latin1_parser_fatal_error/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l
sed "s/obsql_oracle_fast_parse/obsql_oracle_latin1_fast_parse/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y
sed "s/obsql_oracle_multi_fast_parse/obsql_oracle_latin1_multi_fast_parse/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y
sed "s/obsql_oracle_multi_values_parse/obsql_oracle_latin1_multi_values_parse/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y
##3.do not need to replace multi_byte_space、multi_byte_comma、multi_byte_left_parenthesis、multi_byte_right_parenthesis code
sed "s/multi_byte_space \[\\\u3000\]/multi_byte_space \[\\\x20]/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l
sed "s/multi_byte_comma \[\\\uff0c\]/multi_byte_comma \[\\\x2c]/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l
sed "s/multi_byte_left_parenthesis \[\\\uff08\]/multi_byte_left_parenthesis \[\\\x28]/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l
sed "s/multi_byte_right_parenthesis \[\\\uff09\]/multi_byte_right_parenthesis \[\\\x29]/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l
echo "LATIN1_CHAR [\x80-\xFF]" > ../../../src/sql/parser/latin1.txt
sed '/following character status will be rewrite by gen_parse.sh according to connection character/d' -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l
sed '/multi_byte_connect_char \/\*According to connection character to set by gen_parse.sh\*\//r ../../../src/sql/parser/latin1.txt' -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l
sed '/multi_byte_connect_char \/\*According to connection character to set by gen_parse.sh\*\//d' -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l
sed 's/multi_byte_connect_char/LATIN1_CHAR/g' -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l
sed -i '/<hint>{multi_byte_space}/,+5d' sql_parser_oracle_latin1_mode.l
sed -i '/<hint>{multi_byte_comma}/,+35d' sql_parser_oracle_latin1_mode.l
sed -i '/{multi_byte_comma}/,+23d' sql_parser_oracle_latin1_mode.l
sed -i '/{multi_byte_space}/,+4d' sql_parser_oracle_latin1_mode.l
##4.generate oracle latin1 parser files
bison_parser ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y ../../../src/sql/parser/sql_parser_oracle_latin1_mode_tab.c
flex -o ../../../src/sql/parser/sql_parser_oracle_latin1_mode_lex.c ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l ../../../src/sql/parser/sql_parser_oracle_latin1_mode_tab.h
##5.replace other info
sed "/Setup the input buffer state to scan the given bytes/,/}/{/int i/d}" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode_lex.c
sed "/Setup the input buffer state to scan the given bytes/,/}/{/for ( i = 0; i < _yybytes_len; ++i )/d}" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode_lex.c
sed "/Setup the input buffer state to scan the given bytes/,/}/{s/\tbuf\[i\] = yybytes\[i\]/memcpy(buf, yybytes, _yybytes_len)/g}" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode_lex.c
cat ../../../src/sql/parser/non_reserved_keywords_oracle_mode.c > ../../../src/sql/parser/non_reserved_keywords_oracle_latin1_mode.c
sed '/#include "ob_non_reserved_keywords.h"/a\#include "sql/parser/sql_parser_oracle_latin1_mode_tab.h\"' -i ../../../src/sql/parser/non_reserved_keywords_oracle_latin1_mode.c
sed "s/non_reserved_keywords_oracle_mode.c is for …/non_reserved_keywords_oracle_latin1_mode.c is auto generated by gen_parser.sh/g" -i ../../../src/sql/parser/non_reserved_keywords_oracle_latin1_mode.c
##6.clean useless files
rm -f ../../../src/sql/parser/latin1.txt
rm -f ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l
rm -f ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y
# generate oracle utf8 sql_parser(support multi_byte_space、multi_byte_comma、multi_byte_left_parenthesis、multi_byte_right_parenthesis)
##1.copy lex and yacc files
cat ../../../src/sql/parser/sql_parser_oracle_mode.y > ../../../src/sql/parser/sql_parser_oracle_utf8_mode.y
cat ../../../src/sql/parser/sql_parser_oracle_mode.l > ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l
##2.replace name
sed "s/obsql_oracle_yy/obsql_oracle_utf8_yy/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.y
sed "s/obsql_oracle_yy/obsql_oracle_utf8_yy/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l
sed "s/sql_parser_oracle_mode/sql_parser_oracle_utf8_mode/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.y
sed "s/sql_parser_oracle_mode/sql_parser_oracle_utf8_mode/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l
sed "s/obsql_oracle_parser_fatal_error/obsql_oracle_utf8_parser_fatal_error/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.y
sed "s/obsql_oracle_parser_fatal_error/obsql_oracle_utf8_parser_fatal_error/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l
sed "s/obsql_oracle_fast_parse/obsql_oracle_utf8_fast_parse/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.y
sed "s/obsql_oracle_multi_fast_parse/obsql_oracle_utf8_multi_fast_parse/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.y
sed "s/obsql_oracle_multi_values_parse/obsql_oracle_utf8_multi_values_parse/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.y
##3.add multi_byte_space、multi_byte_comma、multi_byte_left_parenthesis、multi_byte_right_parenthesis code.
sed "s/multi_byte_space \[\\\u3000\]/multi_byte_space ([\\\xe3\][\\\x80\][\\\x80])/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l
sed "s/multi_byte_comma \[\\\uff0c\]/multi_byte_comma ([\\\xef\][\\\xbc\][\\\x8c])/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l
sed "s/multi_byte_left_parenthesis \[\\\uff08\]/multi_byte_left_parenthesis ([\\\xef\][\\\xbc\][\\\x88])/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l
sed "s/multi_byte_right_parenthesis \[\\\uff09\]/multi_byte_right_parenthesis ([\\\xef\][\\\xbc\][\\\x89])/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l
echo "U [\x80-\xbf]
U_1_1 [\x80]
U_1_2 [\x81-\xbf]
U_1_3 [\x80-\xbb]
U_1_4 [\xbc]
U_1_5 [\xbd-\xbf]
U_1_6 [\x80-\x87]
U_1_7 [\x8a-\x8b]
U_1_8 [\x8d-\xbf]
U_2 [\xc2-\xdf]
U_3 [\xe0-\xe2]
U_3_1 [\xe3]
U_3_2 [\xe4-\xee]
U_3_3 [\xef]
U_4 [\xf0-\xf4]
u_except_space ({U_3_1}{U_1_2}{U}|{U_3_1}{U_1_1}{U_1_2})
u_except_comma_parenthesis ({U_3_3}{U_1_3}{U}|{U_3_3}{U_1_4}{U_1_6}|{U_3_3}{U_1_4}{U_1_7}|{U_3_3}{U_1_4}{U_1_8}|{U_3_3}{U_1_5}{U})
UTF8_CHAR ({U_2}{U}|{U_3}{U}{U}|{u_except_space}|{U_3_2}{U}{U}|{u_except_comma_parenthesis}|{U_4}{U}{U}{U})" > ../../../src/sql/parser/utf8.txt
sed '/following character status will be rewrite by gen_parse.sh according to connection character/d' -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l
sed '/multi_byte_connect_char \/\*According to connection character to set by gen_parse.sh\*\//r ../../../src/sql/parser/utf8.txt' -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l
sed '/multi_byte_connect_char \/\*According to connection character to set by gen_parse.sh\*\//d' -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l
sed 's/space \[ \\t\\n\\r\\f\]/space (\[ \\t\\n\\r\\f\]|{multi_byte_space})/g' -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l
sed 's/multi_byte_connect_char/UTF8_CHAR/g' -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l
##4.generate oracle utf8 parser files
bison_parser ../../../src/sql/parser/sql_parser_oracle_utf8_mode.y ../../../src/sql/parser/sql_parser_oracle_utf8_mode_tab.c
flex -o ../../../src/sql/parser/sql_parser_oracle_utf8_mode_lex.c ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l ../../../src/sql/parser/sql_parser_oracle_utf8_mode_tab.h
##5.replace other info
sed "/Setup the input buffer state to scan the given bytes/,/}/{/int i/d}" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode_lex.c
sed "/Setup the input buffer state to scan the given bytes/,/}/{/for ( i = 0; i < _yybytes_len; ++i )/d}" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode_lex.c
sed "/Setup the input buffer state to scan the given bytes/,/}/{s/\tbuf\[i\] = yybytes\[i\]/memcpy(buf, yybytes, _yybytes_len)/g}" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode_lex.c
cat ../../../src/sql/parser/non_reserved_keywords_oracle_mode.c > ../../../src/sql/parser/non_reserved_keywords_oracle_utf8_mode.c
sed '/#include "ob_non_reserved_keywords.h"/a\#include "sql/parser/sql_parser_oracle_utf8_mode_tab.h\"' -i ../../../src/sql/parser/non_reserved_keywords_oracle_utf8_mode.c
sed "s/non_reserved_keywords_oracle_mode.c is for …/non_reserved_keywords_oracle_utf8_mode.c is auto generated by gen_parser.sh/g" -i ../../../src/sql/parser/non_reserved_keywords_oracle_utf8_mode.c
##6.clean useless files
rm -f ../../../src/sql/parser/utf8.txt
rm -f ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l
rm -f ../../../src/sql/parser/sql_parser_oracle_utf8_mode.y
# generate oracle gbk sql_parser(support multi_byte_space、multi_byte_comma、multi_byte_left_parenthesis、multi_byte_right_parenthesis)
##1.copy lex and yacc files
cat ../../../src/sql/parser/sql_parser_oracle_mode.y > ../../../src/sql/parser/sql_parser_oracle_gbk_mode.y
cat ../../../src/sql/parser/sql_parser_oracle_mode.l > ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l
##2.replace name
sed "s/obsql_oracle_yy/obsql_oracle_gbk_yy/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.y
sed "s/obsql_oracle_yy/obsql_oracle_gbk_yy/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l
sed "s/sql_parser_oracle_mode/sql_parser_oracle_gbk_mode/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.y
sed "s/sql_parser_oracle_mode/sql_parser_oracle_gbk_mode/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l
sed "s/obsql_oracle_parser_fatal_error/obsql_oracle_gbk_parser_fatal_error/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.y
sed "s/obsql_oracle_parser_fatal_error/obsql_oracle_gbk_parser_fatal_error/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l
sed "s/obsql_oracle_fast_parse/obsql_oracle_gbk_fast_parse/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.y
sed "s/obsql_oracle_multi_fast_parse/obsql_oracle_gbk_multi_fast_parse/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.y
sed "s/obsql_oracle_multi_values_parse/obsql_oracle_gbk_multi_values_parse/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.y
##3.add multi_byte_space、multi_byte_comma、multi_byte_left_parenthesis、multi_byte_right_parenthesis code.
sed "s/multi_byte_space \[\\\u3000\]/multi_byte_space ([\\\xa1][\\\xa1])/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l
sed "s/multi_byte_comma \[\\\uff0c\]/multi_byte_comma ([\\\xa3][\\\xac])/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l
sed "s/multi_byte_left_parenthesis \[\\\uff08\]/multi_byte_left_parenthesis ([\\\xa3][\\\xa8])/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l
sed "s/multi_byte_right_parenthesis \[\\\uff09\]/multi_byte_right_parenthesis ([\\\xa3][\\\xa9])/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l
echo "GB_1 [\x81-\xfe]
GB_1_1 [\x81-\xa0]
GB_1_2 [\xa1]
GB_1_3 [\xa2]
GB_1_4 [\xa3]
GB_1_5 [\xa4-\xfe]
GB_2 [\x40-\xfe]
GB_2_1 [\x40-\xa0]
GB_2_2 [\xa2-\xfe]
GB_2_3 [\x40-\xa7]
GB_2_4 [\xaa-\xab]
GB_2_5 [\xad-\xfe]
GB_3 [\x30-\x39]
g_except_space ({GB_1_2}{GB_2_1}|{GB_1_2}{GB_2_2})
g_except_comma_parenthesis ({GB_1_4}{GB_2_3}|{GB_1_4}{GB_2_4}|{GB_1_4}{GB_2_5})
GB_CHAR ({GB_1_1}{GB_2}|{g_except_space}|{GB_1_3}{GB_2}|{g_except_comma_parenthesis}|{GB_1_5}{GB_2}|{GB_1}{GB_3}{GB_1}{GB_3})" > ../../../src/sql/parser/gbk.txt
sed '/following character status will be rewrite by gen_parse.sh according to connection character/d' -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l
sed '/multi_byte_connect_char \/\*According to connection character to set by gen_parse.sh\*\//r ../../../src/sql/parser/gbk.txt' -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l
sed '/multi_byte_connect_char \/\*According to connection character to set by gen_parse.sh\*\//d' -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l
sed 's/space \[ \\t\\n\\r\\f\]/space (\[ \\t\\n\\r\\f\]|{multi_byte_space})/g' -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l
sed 's/multi_byte_connect_char/GB_CHAR/g' -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l
##4.generate oracle gbk parser files
bison_parser ../../../src/sql/parser/sql_parser_oracle_gbk_mode.y ../../../src/sql/parser/sql_parser_oracle_gbk_mode_tab.c
flex -o ../../../src/sql/parser/sql_parser_oracle_gbk_mode_lex.c ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l ../../../src/sql/parser/sql_parser_oracle_gbk_mode_tab.h
##5.replace other info
sed "/Setup the input buffer state to scan the given bytes/,/}/{/int i/d}" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode_lex.c
sed "/Setup the input buffer state to scan the given bytes/,/}/{/for ( i = 0; i < _yybytes_len; ++i )/d}" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode_lex.c
sed "/Setup the input buffer state to scan the given bytes/,/}/{s/\tbuf\[i\] = yybytes\[i\]/memcpy(buf, yybytes, _yybytes_len)/g}" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode_lex.c
cat ../../../src/sql/parser/non_reserved_keywords_oracle_mode.c > ../../../src/sql/parser/non_reserved_keywords_oracle_gbk_mode.c
sed '/#include "ob_non_reserved_keywords.h"/a\#include "sql/parser/sql_parser_oracle_gbk_mode_tab.h\"' -i ../../../src/sql/parser/non_reserved_keywords_oracle_gbk_mode.c
sed "s/non_reserved_keywords_oracle_mode.c is for …/non_reserved_keywords_oracle_gbk_mode.c is auto generated by gen_parser.sh/g" -i ../../../src/sql/parser/non_reserved_keywords_oracle_gbk_mode.c
##6.clean useless files
rm -f ../../../src/sql/parser/gbk.txt
rm -f ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l
rm -f ../../../src/sql/parser/sql_parser_oracle_gbk_mode.y
rm -rf ../../../src/sql/parser/sql_parser_oracle_mode.y
rm -rf ../../../src/sql/parser/sql_parser_oracle_mode.l
fi
# generate type name
./gen_type_name.sh ../../../src/objit/include/objit/common/ob_item_type.h > type_name.c

1436
src/sql/parser/non_reserved_keywords_oracle_mode.c
View File

File diff suppressed because it is too large Load Diff

219
src/sql/parser/sql_parser_base.c
View File

@ -16,6 +16,8 @@
#define yyconst const
typedef void* yyscan_t;
typedef struct yy_buffer_state *YY_BUFFER_STATE;
#define IS_ORACLE_MODE(mode) (0 != (mode & SMO_ORACLE))
#define IS_ORACLE_COMPATIBLE (IS_ORACLE_MODE(p->sql_mode_))
extern int obsql_mysql_yylex_init_extra(YY_EXTRA_TYPE yy_user_defined,yyscan_t* ptr_yy_globals );
extern int obsql_mysql_yyparse(ParseResult *result);
extern int obsql_mysql_multi_fast_parse(ParseResult *p);
@ -25,6 +27,35 @@ extern int obsql_mysql_yylex_destroy (yyscan_t yyscanner );
extern YY_BUFFER_STATE obsql_mysql_yy_scan_bytes (yyconst char *bytes,int len ,yyscan_t yyscanner );
extern void obsql_mysql_yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner );
extern void obsql_mysql_yy_delete_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner );
#ifdef OB_BUILD_ORACLE_PARSER
extern int obsql_oracle_latin1_yylex_init_extra(YY_EXTRA_TYPE yy_user_defined,yyscan_t* ptr_yy_globals );
extern int obsql_oracle_latin1_yyparse(ParseResult *result);
extern int obsql_oracle_latin1_multi_fast_parse(ParseResult *p);
extern int obsql_oracle_latin1_multi_values_parse(ParseResult *p);
extern int obsql_oracle_latin1_fast_parse(ParseResult *p);
extern int obsql_oracle_latin1_yylex_destroy (yyscan_t yyscanner );
extern YY_BUFFER_STATE obsql_oracle_latin1_yy_scan_bytes (yyconst char *bytes,int len ,yyscan_t yyscanner );
extern void obsql_oracle_latin1_yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner );
extern void obsql_oracle_latin1_yy_delete_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner );
extern int obsql_oracle_utf8_yylex_init_extra(YY_EXTRA_TYPE yy_user_defined,yyscan_t* ptr_yy_globals );
extern int obsql_oracle_utf8_yyparse(ParseResult *result);
extern int obsql_oracle_utf8_multi_fast_parse(ParseResult *p);
extern int obsql_oracle_utf8_multi_values_parse(ParseResult *p);
extern int obsql_oracle_utf8_fast_parse(ParseResult *p);
extern int obsql_oracle_utf8_yylex_destroy (yyscan_t yyscanner );
extern YY_BUFFER_STATE obsql_oracle_utf8_yy_scan_bytes (yyconst char *bytes,int len ,yyscan_t yyscanner );
extern void obsql_oracle_utf8_yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner );
extern void obsql_oracle_utf8_yy_delete_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner );
extern int obsql_oracle_gbk_yylex_init_extra(YY_EXTRA_TYPE yy_user_defined,yyscan_t* ptr_yy_globals );
extern int obsql_oracle_gbk_yyparse(ParseResult *result);
extern int obsql_oracle_gbk_multi_fast_parse(ParseResult *p);
extern int obsql_oracle_gbk_multi_values_parse(ParseResult *p);
extern int obsql_oracle_gbk_fast_parse(ParseResult *p);
extern int obsql_oracle_gbk_yylex_destroy (yyscan_t yyscanner );
extern YY_BUFFER_STATE obsql_oracle_gbk_yy_scan_bytes (yyconst char *bytes,int len ,yyscan_t yyscanner );
extern void obsql_oracle_gbk_yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner );
extern void obsql_oracle_gbk_yy_delete_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner );
#endif
int parse_init(ParseResult *p)
{
int ret = 0; // can not include C++ file "ob_define.h"
@ -38,7 +69,45 @@ int parse_init(ParseResult *p)
}
if (OB_LIKELY( 0 == ret)) {
#ifdef OB_BUILD_ORACLE_PARSER
if (IS_ORACLE_COMPATIBLE) {
switch (p->connection_collation_) {
case 28/*CS_TYPE_GBK_CHINESE_CI*/:
case 87/*CS_TYPE_GBK_BIN*/:
case 216/*CS_TYPE_GB18030_2022_BIN*/:
case 217/*CS_TYPE_GB18030_2022_PINYIN_CI*/:
case 218/*CS_TYPE_GB18030_2022_PINYIN_CS*/:
case 219/*CS_TYPE_GB18030_2022_RADICAL_CI*/:
case 220/*CS_TYPE_GB18030_2022_RADICAL_CS*/:
case 221/*CS_TYPE_GB18030_2022_STROKE_CI*/:
case 222/*CS_TYPE_GB18030_2022_STROKE_CS*/:
case 248/*CS_TYPE_GB18030_CHINESE_CI*/:
case 249/*CS_TYPE_GB18030_BIN*/:
ret = obsql_oracle_gbk_yylex_init_extra(p, &(p->yyscan_info_));
break;
case 45/*CS_TYPE_UTF8MB4_GENERAL_CI*/:
case 46/*CS_TYPE_UTF8MB4_BIN*/:
case 63/*CS_TYPE_BINARY*/:
case 224/*CS_TYPE_UTF8MB4_UNICODE_CI*/:
ret = obsql_oracle_utf8_yylex_init_extra(p, &(p->yyscan_info_));
break;
case 8/*CS_TYPE_LATIN1_SWEDISH_CI*/:
case 47/*CS_TYPE_LATIN1_BIN*/:
ret = obsql_oracle_latin1_yylex_init_extra(p, &(p->yyscan_info_));
break;
default: {
ret = -1;
(void)snprintf(p->error_msg_, MAX_ERROR_MSG, "get not support connection collation: %u",
p->connection_collation_);
break;
}
}
} else {
#endif
ret = obsql_mysql_yylex_init_extra(p, &(p->yyscan_info_));
#ifdef OB_BUILD_ORACLE_PARSER
}
#endif
}
return ret;
}
@ -47,7 +116,45 @@ int parse_terminate(ParseResult *p)
{
int ret = 0;
if (OB_LIKELY(NULL != p->yyscan_info_)) {
#ifdef OB_BUILD_ORACLE_PARSER
if (IS_ORACLE_COMPATIBLE) {
switch (p->connection_collation_) {
case 28/*CS_TYPE_GBK_CHINESE_CI*/:
case 87/*CS_TYPE_GBK_BIN*/:
case 216/*CS_TYPE_GB18030_2022_BIN*/:
case 217/*CS_TYPE_GB18030_2022_PINYIN_CI*/:
case 218/*CS_TYPE_GB18030_2022_PINYIN_CS*/:
case 219/*CS_TYPE_GB18030_2022_RADICAL_CI*/:
case 220/*CS_TYPE_GB18030_2022_RADICAL_CS*/:
case 221/*CS_TYPE_GB18030_2022_STROKE_CI*/:
case 222/*CS_TYPE_GB18030_2022_STROKE_CS*/:
case 248/*CS_TYPE_GB18030_CHINESE_CI*/:
case 249/*CS_TYPE_GB18030_BIN*/:
ret = obsql_oracle_gbk_yylex_destroy(p->yyscan_info_);
break;
case 45/*CS_TYPE_UTF8MB4_GENERAL_CI*/:
case 46/*CS_TYPE_UTF8MB4_BIN*/:
case 63/*CS_TYPE_BINARY*/:
case 224/*CS_TYPE_UTF8MB4_UNICODE_CI*/:
ret = obsql_oracle_utf8_yylex_destroy(p->yyscan_info_);
break;
case 8/*CS_TYPE_LATIN1_SWEDISH_CI*/:
case 47/*CS_TYPE_LATIN1_BIN*/:
ret = obsql_oracle_latin1_yylex_destroy(p->yyscan_info_);
break;
default: {
ret = -1;
(void)snprintf(p->error_msg_, MAX_ERROR_MSG, "get not support connection collation: %u",
p->connection_collation_);
break;
}
}
} else {
#endif
ret = obsql_mysql_yylex_destroy(p->yyscan_info_);
#ifdef OB_BUILD_ORACLE_PARSER
}
#endif
}
return ret;
}
@ -105,6 +212,115 @@ int parse_sql(ParseResult *p, const char *buf, size_t input_len)
ret = OB_PARSER_ERR_NO_MEMORY;
#endif
} else {
#ifdef OB_BUILD_ORACLE_PARSER
if (IS_ORACLE_COMPATIBLE) {
switch (p->connection_collation_) {
case 28/*CS_TYPE_GBK_CHINESE_CI*/:
case 87/*CS_TYPE_GBK_BIN*/:
case 216/*CS_TYPE_GB18030_2022_BIN*/:
case 217/*CS_TYPE_GB18030_2022_PINYIN_CI*/:
case 218/*CS_TYPE_GB18030_2022_PINYIN_CS*/:
case 219/*CS_TYPE_GB18030_2022_RADICAL_CI*/:
case 220/*CS_TYPE_GB18030_2022_RADICAL_CS*/:
case 221/*CS_TYPE_GB18030_2022_STROKE_CI*/:
case 222/*CS_TYPE_GB18030_2022_STROKE_CS*/:
case 248/*CS_TYPE_GB18030_CHINESE_CI*/:
case 249/*CS_TYPE_GB18030_BIN*/: {
YY_BUFFER_STATE bp = obsql_oracle_gbk_yy_scan_bytes(buf, len, p->yyscan_info_);
obsql_oracle_gbk_yy_switch_to_buffer(bp, p->yyscan_info_);
int tmp_ret = -1;
if (p->is_fp_) {
tmp_ret = obsql_oracle_gbk_fast_parse(p);
} else if (p->is_multi_query_) {
tmp_ret = obsql_oracle_gbk_multi_fast_parse(p);
} else if (p->is_multi_values_parser_) {
tmp_ret = obsql_oracle_gbk_multi_values_parse(p);
} else {
tmp_ret = obsql_oracle_gbk_yyparse(p);
}
if (0 == tmp_ret) {
ret = OB_PARSER_SUCCESS;
} else if (2 == tmp_ret) {
ret = OB_PARSER_ERR_NO_MEMORY;
} else {
if (0 != p->extra_errno_) {
ret = p->extra_errno_;
} else {
ret = OB_PARSER_ERR_PARSE_SQL;
}
}
obsql_oracle_gbk_yy_delete_buffer(bp, p->yyscan_info_);
break;
}
case 45/*CS_TYPE_UTF8MB4_GENERAL_CI*/:
case 46/*CS_TYPE_UTF8MB4_BIN*/:
case 63/*CS_TYPE_BINARY*/:
case 224/*CS_TYPE_UTF8MB4_UNICODE_CI*/:{
YY_BUFFER_STATE bp = obsql_oracle_utf8_yy_scan_bytes(buf, len, p->yyscan_info_);
obsql_oracle_utf8_yy_switch_to_buffer(bp, p->yyscan_info_);
int tmp_ret = -1;
if (p->is_fp_) {
tmp_ret = obsql_oracle_utf8_fast_parse(p);
} else if (p->is_multi_query_) {
tmp_ret = obsql_oracle_utf8_multi_fast_parse(p);
} else if (p->is_multi_values_parser_) {
tmp_ret = obsql_oracle_utf8_multi_values_parse(p);
} else {
tmp_ret = obsql_oracle_utf8_yyparse(p);
}
if (0 == tmp_ret) {
ret = OB_PARSER_SUCCESS;
} else if (2 == tmp_ret) {
ret = OB_PARSER_ERR_NO_MEMORY;
} else {
if (0 != p->extra_errno_) {
ret = p->extra_errno_;
} else {
ret = OB_PARSER_ERR_PARSE_SQL;
}
}
obsql_oracle_utf8_yy_delete_buffer(bp, p->yyscan_info_);
break;
}
case 8/*CS_TYPE_LATIN1_SWEDISH_CI*/:
case 47/*CS_TYPE_LATIN1_BIN*/:{
YY_BUFFER_STATE bp = obsql_oracle_latin1_yy_scan_bytes(buf, len, p->yyscan_info_);
obsql_oracle_latin1_yy_switch_to_buffer(bp, p->yyscan_info_);
int tmp_ret = -1;
if (p->is_fp_) {
tmp_ret = obsql_oracle_latin1_fast_parse(p);
} else if (p->is_multi_query_) {
tmp_ret = obsql_oracle_latin1_multi_fast_parse(p);
} else if (p->is_multi_values_parser_) {
tmp_ret = obsql_oracle_latin1_multi_values_parse(p);
} else {
tmp_ret = obsql_oracle_latin1_yyparse(p);
}
if (0 == tmp_ret) {
ret = OB_PARSER_SUCCESS;
} else if (2 == tmp_ret) {
ret = OB_PARSER_ERR_NO_MEMORY;
} else {
if (0 != p->extra_errno_) {
ret = p->extra_errno_;
} else {
ret = OB_PARSER_ERR_PARSE_SQL;
}
}
obsql_oracle_latin1_yy_delete_buffer(bp, p->yyscan_info_);
break;
}
default: {
ret = OB_PARSER_ERR_UNEXPECTED;
(void)snprintf(p->error_msg_, MAX_ERROR_MSG, "get not support conn collation: %u",
p->connection_collation_);
break;
}
}
#endif
#ifdef OB_BUILD_ORACLE_PARSER
} else {
#endif
YY_BUFFER_STATE bp = obsql_mysql_yy_scan_bytes(buf, len, p->yyscan_info_);
obsql_mysql_yy_switch_to_buffer(bp, p->yyscan_info_);
int tmp_ret = -1;
@ -129,6 +345,9 @@ int parse_sql(ParseResult *p, const char *buf, size_t input_len)
}
}
obsql_mysql_yy_delete_buffer(bp, p->yyscan_info_);
#ifdef OB_BUILD_ORACLE_PARSER
}
#endif
}
}
}

70
src/sql/parser/sql_parser_base.h
View File

@ -240,6 +240,50 @@ do {
setup_token_pos_info(node, word_start - 1, word_end - word_start + 1); \
} while (0)
//oralce下生成非保留关键字结点请使用该宏,区别于mysql的是做了大写的转换
#define get_oracle_non_reserved_node(node, malloc_pool, expr_start, expr_end) \
do { \
malloc_terminal_node(node, malloc_pool, T_IDENT); \
if (OB_UNLIKELY(NULL == node || NULL == result || NULL == result->input_sql_)) {\
yyerror(NULL, result, "invalid argument, node:%p, result:%p or input_sql is NULL", node, result);\
YYABORT_UNEXPECTED; \
} else if (OB_UNLIKELY(expr_start < 0 || expr_end < 0 || expr_start > expr_end)) { \
yyerror(NULL, result, "invalid argument, expr_start:%d, expr_end:%d", (int32_t)expr_start, (int32_t)expr_end);\
YYABORT_UNEXPECTED; \
} else { \
int start = expr_start; \
char * upper_value = NULL; \
char * raw_str = NULL; \
node->str_value_ = NULL; \
node->str_len_ = 0; \
node->raw_text_ = NULL; \
node->text_len_ = 0; \
while (start <= expr_end && ISSPACE(result->input_sql_[start - 1])) {\
start++; \
} \
if ('"' == result->input_sql_[start - 1]) { \
start++; \
expr_end--; \
} \
if (start >= expr_start \
&& (OB_UNLIKELY((NULL == (upper_value = copy_expr_string(result, start, expr_end)))))) { \
yyerror(NULL, result, "No more space for copying expression string"); \
YYABORT_NO_MEMORY; \
} else { \
if (start >= expr_start \
&& (OB_UNLIKELY((NULL == (raw_str = copy_expr_string(result, start, expr_end)))))) { \
yyerror(NULL, result, "No more space for copying expression string"); \
YYABORT_NO_MEMORY; \
} else { \
node->raw_text_ = raw_str; \
node->text_len_ = expr_end - start + 1; \
node->str_value_ = str_toupper(upper_value, expr_end - start + 1); \
node->str_len_ = expr_end - start + 1; \
setup_token_pos_info(node, expr_start - 1, expr_end - expr_start + 1); \
} \
} \
} \
} while(0)
#define make_name_node(node, malloc_pool, name) \
do { \
@ -863,6 +907,32 @@ for (int32_t _i = 0; _i < _yyleng; ++_i) {
} \
} while (0);
#define PARSE_INT_STR_ORACLE(param_node, malloc_pool, errno) \
do { \
if ('-' == param_node->str_value_[0]) { \
char *copied_str = parse_strndup(param_node->str_value_, param_node->str_len_, malloc_pool); \
if (OB_ISNULL(copied_str)) { \
((ParseResult *)yyextra)->extra_errno_ = OB_PARSER_ERR_NO_MEMORY;\
yyerror(NULL, yyextra, "No more space for mallocing"); \
return ERROR; \
} else { \
int pos = 1; \
for (; pos < param_node->str_len_ && ISSPACE(copied_str[pos]); pos++) ; \
copied_str[--pos] = '-'; \
param_node->value_ = ob_strntoll(copied_str + pos, param_node->str_len_ - pos, 10, NULL, &errno); \
if (ERANGE == errno) { \
param_node->type_ = T_NUMBER; \
token_ret = DECIMAL_VAL; \
} \
} \
} else { \
param_node->value_ = ob_strntoll(param_node->str_value_, param_node->str_len_, 10, NULL, &errno); \
if (ERANGE == errno) { \
param_node->type_ = T_NUMBER; \
token_ret = DECIMAL_VAL; \
} \
} \
} while (0);
#define RM_MULTI_STMT_END_P(p) \
do \