hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

add cache when generate parser in cmake process

Browse Source
This commit is contained in:
LINxiansheng 2024-12-05 08:45:10 +00:00 committed by ob-robot
parent b3984bd166
commit 443063ac91
7 changed files with 83 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -164,6 +164,7 @@ src/sql/parser/sql_parser_oracle_single_byte_mode_tab.h
src/sql/parser/non_reserved_keywords_oracle_single_byte_mode.c
src/sql/parser/_gen_parser.output
src/sql/parser/_gen_parser.error
src/sql/parser/_MD5
src/pl/parser/pl_parser_mysql_mode_lex.c
src/pl/parser/pl_parser_mysql_mode_lex.h
src/pl/parser/pl_parser_mysql_mode_tab.c
@ -174,6 +175,7 @@ src/pl/parser/pl_parser_oracle_mode_tab.c
src/pl/parser/pl_parser_oracle_mode_tab.h
src/pl/parser/_gen_parser.error
src/pl/parser/_gen_pl_parser.output
src/pl/parser/_MD5
src/pl/parser/pl_parser_mysql_mode.output
src/pl/parser/pl_parser_oracle_mode.output
src/share/inner_table/sys_package/system_package.cpp

1
cmake/Env.cmake
View File

@ -27,6 +27,7 @@ ob_define(OB_BUILD_CCLS OFF)
ob_define(LTO_JOBS all)
ob_define(LTO_CACHE_DIR "${CMAKE_BINARY_DIR}/cache")
ob_define(LTO_CACHE_POLICY cache_size=100%:cache_size_bytes=0k:cache_size_files=0:prune_after=0s:prune_interval=72h)
ob_define(NEED_PARSER_CACHE ON)
# get compiler from build.sh
ob_define(OB_CC "")
ob_define(OB_CXX "")

18
src/objit/src/ob_llvm_di_helper.cpp
View File

@ -40,9 +40,9 @@ uint64_t ObLLVMDIType::get_align_bits()
{
return OB_ISNULL(v_) ? 0 : v_->getAlignInBits();
}
int ObLLVMDIHelper::init(core::JitContext *jc)
{
int ObLLVMDIHelper::init(core::JitContext *jc)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(jc)) {
ret = OB_ERR_UNEXPECTED;
@ -155,9 +155,9 @@ int ObLLVMDIHelper::create_local_variable(const ObString &name, uint32_t arg_no,
ObDISubprogram *sp = NULL;
ObDILocalVariable *var_ptr;
if (name.empty() || OB_ISNULL(jc_) || OB_ISNULL(file = jc_->file_)
|| OB_ISNULL(sp = jc_->sp_) || OB_ISNULL(type.get_v())) {
ret = OB_NOT_INIT;
LOG_WARN("jc or file or sp or type is NULL", K(name), K(ret));
|| OB_ISNULL(sp = jc_->sp_) || OB_ISNULL(type.get_v())) {
ret = OB_NOT_INIT;
LOG_WARN("jc or file or sp or type is NULL", K(name), K(ret));
} else {
var_ptr = (arg_no > 0) ?
jc_->dbuilder_.createParameterVariable(sp, StringRef(name.ptr(), name.length()),
@ -227,9 +227,9 @@ int ObLLVMDIHelper::create_pointer_type(ObLLVMDIType &pointee_type,
} else if (OB_ISNULL(ptr_type = jc_->dbuilder_.createPointerType(pte_type, 64, 64))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to create pointer type", K(ret));
} else {
pointer_type.set_v(ptr_type);
}
} else {
pointer_type.set_v(ptr_type);
}
return ret;
}

2
src/pl/CMakeLists.txt
View File

@ -1,5 +1,5 @@
execute_process(
COMMAND bash gen_parser.sh
COMMAND env NEED_PARSER_CACHE=${NEED_PARSER_CACHE} bash gen_parser.sh
RESULT_VARIABLE RET
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src/pl/parser
OUTPUT_FILE _gen_pl_parser.output

48
src/pl/parser/gen_parser.sh
View File

@ -9,6 +9,8 @@ CURDIR="$(dirname $(readlink -f "$0"))"
#export PATH=/usr/local/bin:$PATH
export PATH=$CURDIR/../../../deps/3rd/usr/local/oceanbase/devtools/bin/:$PATH
export BISON_PKGDATADIR=$CURDIR/../../../deps/3rd/usr/local/oceanbase/devtools/share/bison
CACHE_MD5_FILE=$CURDIR/_MD5
TEMP_FILE=$(mktemp)
BISON_VERSION=`bison -V| grep 'bison (GNU Bison)'|awk '{ print $4;}'`
NEED_VERSION='2.4.1'
@ -34,17 +36,43 @@ then
fi
}
# generate pl_parser
bison_parser ../../../src/pl/parser/pl_parser_mysql_mode.y ../../../src/pl/parser/pl_parser_mysql_mode_tab.c
flex -o ../../../src/pl/parser/pl_parser_mysql_mode_lex.c ../../../src/pl/parser/pl_parser_mysql_mode.l ../../../src/pl/parser/pl_parser_mysql_mode_tab.h
# TODO delete the following line at 9.30
rm -rf ../../../src/pl/parser/pl_parser_oracle_mode_lex.c ../../../src/pl/parser/pl_parser_oracle_mode_tab.c ../../../src/pl/parser/pl_parser_oracle_mode_tab.h
cat ../../../src/pl/parser/pl_parser_mysql_mode.y >> $TEMP_FILE
cat ../../../src/pl/parser/pl_parser_mysql_mode.l >> $TEMP_FILE
if [ -d "../../../close_modules/oracle_pl/pl/parser/" ]; then
bison_parser ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode.y ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode_tab.c
flex -o ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode_lex.c ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode.l ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode_tab.h
cat ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode.y >> $TEMP_FILE
cat ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode.l >> $TEMP_FILE
fi
#./gen_type_name.sh ob_item_type.h >type_name.c
md5sum_value=$(md5sum "$TEMP_FILE" | awk '{ print $1 }')
function generate_parser {
# generate pl_parser
bison_parser ../../../src/pl/parser/pl_parser_mysql_mode.y ../../../src/pl/parser/pl_parser_mysql_mode_tab.c
flex -o ../../../src/pl/parser/pl_parser_mysql_mode_lex.c ../../../src/pl/parser/pl_parser_mysql_mode.l ../../../src/pl/parser/pl_parser_mysql_mode_tab.h
# TODO delete the following line at 9.30
rm -rf ../../../src/pl/parser/pl_parser_oracle_mode_lex.c ../../../src/pl/parser/pl_parser_oracle_mode_tab.c ../../../src/pl/parser/pl_parser_oracle_mode_tab.h
if [ -d "../../../close_modules/oracle_pl/pl/parser/" ]; then
bison_parser ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode.y ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode_tab.c
flex -o ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode_lex.c ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode.l ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode_tab.h
fi
#./gen_type_name.sh ob_item_type.h >type_name.c
echo "$md5sum_value" > $CACHE_MD5_FILE
}
if [[ -n "$NEED_PARSER_CACHE" && "$NEED_PARSER_CACHE" == "ON" ]]; then
echo "generate pl parser with cache"
origin_md5sum_value=$(<$CACHE_MD5_FILE)
if [[ "$md5sum_value" == "$origin_md5sum_value" ]]; then
echo "hit the md5 cache"
else
generate_parser
fi
else
echo "generate pl parser without cache"
generate_parser
fi
rm -rf $TEMP_FILE

2
src/sql/CMakeLists.txt
View File

@ -1536,7 +1536,7 @@ target_link_libraries(ob_sql_static
endif()
execute_process(
COMMAND bash gen_parser.sh
COMMAND env NEED_PARSER_CACHE=${NEED_PARSER_CACHE} bash gen_parser.sh
RESULT_VARIABLE RET
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/src/sql/parser
OUTPUT_FILE _gen_parser.output

31
src/sql/parser/gen_parser.sh
View File

@ -9,6 +9,9 @@ CURDIR="$(dirname $(readlink -f "$0"))"
#export PATH=/usr/local/bin:$PATH
export PATH=$CURDIR/../../../deps/3rd/usr/local/oceanbase/devtools/bin/:$PATH
export BISON_PKGDATADIR=$CURDIR/../../../deps/3rd/usr/local/oceanbase/devtools/share/bison
CACHE_MD5_FILE=$CURDIR/_MD5
TEMP_FILE=$(mktemp)
BISON_VERSION=`bison -V| grep 'bison (GNU Bison)'|awk '{ print $4;}'`
NEED_VERSION='2.4.1'
@ -17,6 +20,15 @@ if [ "$BISON_VERSION" != "$NEED_VERSION" ]; then
exit 1
fi
cat ../../../src/sql/parser/sql_parser_mysql_mode.y >> $TEMP_FILE
cat ../../../src/sql/parser/sql_parser_mysql_mode.l >> $TEMP_FILE
if [ -d "../../../close_modules/oracle_pl/pl/parser/" ]; then
cat ../../../close_modules/oracle_parser/sql/parser/sql_parser_oracle_mode.y >> $TEMP_FILE
cat ../../../close_modules/oracle_parser/sql/parser/sql_parser_oracle_mode.l >> $TEMP_FILE
fi
md5sum_value=$(md5sum "$TEMP_FILE" | awk '{ print $1 }')
bison_parser() {
BISON_OUTPUT="$(bison -v -Werror -d $1 -o $2 2>&1)"
BISON_RETURN="$?"
@ -33,6 +45,7 @@ then
fi
}
function generate_parser {
# generate mysql sql_parser
bison_parser ../../../src/sql/parser/sql_parser_mysql_mode.y ../../../src/sql/parser/sql_parser_mysql_mode_tab.c
flex -Cfa -B -8 -o ../../../src/sql/parser/sql_parser_mysql_mode_lex.c ../../../src/sql/parser/sql_parser_mysql_mode.l ../../../src/sql/parser/sql_parser_mysql_mode_tab.h
@ -264,3 +277,21 @@ fi
# generate type name
./gen_type_name.sh ../../../src/objit/include/objit/common/ob_item_type.h > type_name.c
echo "$md5sum_value" > $CACHE_MD5_FILE
}
if [[ -n "$NEED_PARSER_CACHE" && "$NEED_PARSER_CACHE" == "ON" ]]; then
echo "generate pl parser with cache"
origin_md5sum_value=$(<$CACHE_MD5_FILE)
if [[ "$md5sum_value" == "$origin_md5sum_value" ]]; then
echo "hit the md5 cache"
else
generate_parser
fi
else
echo "generate pl parser without cache"
generate_parser
fi
rm -rf $TEMP_FILE