From 6677ad6fb72c82c78214ef6c9b105dd343be7b79 Mon Sep 17 00:00:00 2001 From: obdev Date: Thu, 3 Nov 2022 10:38:39 +0000 Subject: [PATCH] Add meta info for DECIMAL VARCHAR --- deps/init/oceanbase.el7.aarch64.deps | 2 +- deps/init/oceanbase.el7.x86_64.deps | 2 +- deps/init/oceanbase.el8.aarch64.deps | 2 +- deps/init/oceanbase.el8.x86_64.deps | 2 +- .../libobcdc/src/ob_log_meta_manager.cpp | 17 ++++++- .../tests/ob_binlog_record_printer.cpp | 44 ++++++++++++++----- 6 files changed, 51 insertions(+), 18 deletions(-) diff --git a/deps/init/oceanbase.el7.aarch64.deps b/deps/init/oceanbase.el7.aarch64.deps index bca2d3c2b..d3f7708bf 100644 --- a/deps/init/oceanbase.el7.aarch64.deps +++ b/deps/init/oceanbase.el7.aarch64.deps @@ -40,7 +40,7 @@ obdevtools-gcc9-9.3.0-52022092914.el7.aarch64.rpm obdevtools-llvm-11.0.1-312022092921.el7.aarch64.rpm [tools-deps] -devdeps-oblogmsg-1.0-32022100420.el7.aarch64.rpm +devdeps-oblogmsg-1.0-42022101823.el7.aarch64.rpm devdeps-rocksdb-6.22.1.1-52022100420.el7.aarch64.rpm [test-utils] diff --git a/deps/init/oceanbase.el7.x86_64.deps b/deps/init/oceanbase.el7.x86_64.deps index eb53bde1c..1405f301f 100644 --- a/deps/init/oceanbase.el7.x86_64.deps +++ b/deps/init/oceanbase.el7.x86_64.deps @@ -36,7 +36,7 @@ obdevtools-gcc9-9.3.0-52022092914.el7.x86_64.rpm obdevtools-llvm-11.0.1-312022092921.el7.x86_64.rpm [tools-deps] -devdeps-oblogmsg-1.0-32022100420.el7.x86_64.rpm +devdeps-oblogmsg-1.0-42022101823.el7.x86_64.rpm devdeps-rocksdb-6.22.1.1-52022100420.el7.x86_64.rpm [test-utils] diff --git a/deps/init/oceanbase.el8.aarch64.deps b/deps/init/oceanbase.el8.aarch64.deps index f3f3ccc2e..499f98b72 100644 --- a/deps/init/oceanbase.el8.aarch64.deps +++ b/deps/init/oceanbase.el8.aarch64.deps @@ -35,7 +35,7 @@ obdevtools-gcc9-9.3.0-52022092914.el8.aarch64.rpm obdevtools-llvm-11.0.1-312022092921.el8.aarch64.rpm [tools-deps] -devdeps-oblogmsg-1.0-32022100420.el8.aarch64.rpm +devdeps-oblogmsg-1.0-42022101823.el8.aarch64.rpm devdeps-rocksdb-6.22.1.1-52022100420.el8.aarch64.rpm [test-utils] diff --git a/deps/init/oceanbase.el8.x86_64.deps b/deps/init/oceanbase.el8.x86_64.deps index b0e4891c3..a374e3d01 100644 --- a/deps/init/oceanbase.el8.x86_64.deps +++ b/deps/init/oceanbase.el8.x86_64.deps @@ -36,7 +36,7 @@ obdevtools-gcc9-9.3.0-52022092914.el8.x86_64.rpm obdevtools-llvm-11.0.1-312022092921.el8.x86_64.rpm [tools-deps] -devdeps-oblogmsg-1.0-32022100420.el8.x86_64.rpm +devdeps-oblogmsg-1.0-42022101823.el8.x86_64.rpm devdeps-rocksdb-6.22.1.1-52022100420.el8.x86_64.rpm [test-utils] diff --git a/src/logservice/libobcdc/src/ob_log_meta_manager.cpp b/src/logservice/libobcdc/src/ob_log_meta_manager.cpp index be526f982..03d74fc1e 100644 --- a/src/logservice/libobcdc/src/ob_log_meta_manager.cpp +++ b/src/logservice/libobcdc/src/ob_log_meta_manager.cpp @@ -768,15 +768,28 @@ int ObLogMetaManager::set_column_meta_(IColMeta *col_meta, col_meta->setValuesOfEnumSet(extended_type_info_vec); //mysql treat it as MYSQL_TYPE_STRING, it is not suitable for libobcdc - if (ObEnumType == column_schema.get_data_type()) { + if (ObEnumType == col_type) { mysql_type = obmysql::MYSQL_TYPE_ENUM; - } else if (ObSetType == column_schema.get_data_type()) { + } else if (ObSetType == col_type) { mysql_type = obmysql::MYSQL_TYPE_SET; } + } else if (ObNumberType == col_type || ObUNumberType == col_type) { + col_meta->setScale(column_schema.get_data_scale()); + col_meta->setPrecision(column_schema.get_data_precision()); } bool signed_flag = ((type_flag & UNSIGNED_FLAG) == 0); + if (ObBitType == col_type) { + // the length of BIT type is required, + // the "length" of the BIT type is store in precision + col_meta->setLength(column_schema.get_data_precision()); + } else { + // for types with valid length(string\enumset\rowid\json\raw\lob\geo), + // get_data_length returns the valid length, returns 0 for other types. + col_meta->setLength(column_schema.get_data_length()); + } + col_meta->setName(column_schema.get_column_name()); col_meta->setType(static_cast(mysql_type)); col_meta->setSigned(signed_flag); diff --git a/src/logservice/libobcdc/tests/ob_binlog_record_printer.cpp b/src/logservice/libobcdc/tests/ob_binlog_record_printer.cpp index c92a0b82e..0f642e09d 100644 --- a/src/logservice/libobcdc/tests/ob_binlog_record_printer.cpp +++ b/src/logservice/libobcdc/tests/ob_binlog_record_printer.cpp @@ -520,6 +520,9 @@ int ObBinlogRecordPrinter::output_data_file_column_data(IBinlogRecord *br, const char *is_not_null = col_meta ? (col_meta->isNotNull() ? "true" : "false") : "NULL"; // const char *default_val = col_meta ? col_meta->getDefault() : "NULL"; const char *is_signed = col_meta ? (col_meta->isSigned() ? "true" : "false") : "NULL"; + const long scale = col_meta ? col_meta->getScale(): 0; + const long precision = col_meta ? col_meta->getPrecision(): 0; + const long col_data_length = col_meta ? col_meta->getLength(): 0; bool is_generated_column = col_meta ? col_meta->isGenerated() : false; bool is_hidden_row_key_column = col_meta ? col_meta->isHiddenRowKey() : false; bool is_lob = is_lob_type(ctype); @@ -533,23 +536,40 @@ int ObBinlogRecordPrinter::output_data_file_column_data(IBinlogRecord *br, ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_is_signed:%s", column_index, is_signed); ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_encoding:%s", column_index, encoding); ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_is_not_null:%s", column_index, is_not_null); - if (enable_print_detail && is_hidden_row_key_column) { - ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_is_hidden_rowkey:%d", column_index, is_hidden_row_key_column); + if (enable_print_detail) { + if (is_hidden_row_key_column) { + ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_is_hidden_rowkey:%d", column_index, is_hidden_row_key_column); + } + // print the length of varchar only in print detail mode, + // because there have been many test cases with varchar type before the varchar length info is added into column meta + if (oceanbase::obmysql::MYSQL_TYPE_VAR_STRING == ctype || oceanbase::obmysql::MYSQL_TYPE_BIT == ctype) { + ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_define_length:%ld", column_index, col_data_length); + } + else if ((oceanbase::obmysql::MYSQL_TYPE_ENUM == ctype) || (oceanbase::obmysql::MYSQL_TYPE_SET == ctype)) { + const std::string delim = ","; + for (int i = 0; i < values_of_enum_set->size(); i++) { + enum_set_values_str += (*values_of_enum_set)[i]; + if (i != values_of_enum_set->size() - 1) { + enum_set_values_str += delim; + } + } + ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_extend_info:%s", column_index, enum_set_values_str.c_str()); + } + // print precision & scale only in print detail mode, becacuse INT in oracle mode is also a kind of NUMBER(DECIMAL) + // whose precision is 38 and scale is 0, more importantly, the default precision(-1, PRECISION_UNKNOWN_YET) + // and scale(-85, ORA_NUMBER_SCALE_UNKNOWN_YET) of NUMBER in oracle mode is confusing, so we decide not to + // modify test results for oracle mode temporarily for convenience and efficiency. + // TODO + else if ((oceanbase::obmysql::MYSQL_TYPE_DECIMAL == ctype) || (oceanbase::obmysql::MYSQL_TYPE_NEWDECIMAL == ctype)) { + // Not sure if MYSQL_TYPE_DECIMAL is deprecated, DECIMAL in mysql & oracle mode should be MYSQL_TYPE_NEWDECIMAL + ROW_PRINTF(ptr, size, pos , ri, "[C%ld] column_precision:%ld", column_index, precision); + ROW_PRINTF(ptr, size, pos , ri, "[C%ld] column_scale:%ld", column_index, scale); + } else { } } if (is_generated_column) { ROW_PRINTF(ptr, size, pos, ri, "[C%ld] is_generated_column:%d", column_index, is_generated_column); } - if ((oceanbase::obmysql::MYSQL_TYPE_ENUM == ctype) || (oceanbase::obmysql::MYSQL_TYPE_SET == ctype)) { - const std::string delim = ","; - for (int i = 0; i < values_of_enum_set->size(); i++) { - enum_set_values_str += (*values_of_enum_set)[i]; - if (i != values_of_enum_set->size() - 1) { - enum_set_values_str += delim; - } - } - ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_extend_info:%s", column_index, enum_set_values_str.c_str()); - } // FIXME: does not check the value of the field until the length of the default value can be obtained // ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_default_value:%s", column_index, default_val);