diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index af254038bc..1735c3debc 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -627,8 +627,14 @@ Status SchemaChangeForInvertedIndex::process(RowsetReaderSharedPtr rowset_reader DCHECK_EQ(inverted_index.columns.size(), 1); auto index_id = inverted_index.index_id; auto column_name = inverted_index.columns[0]; - auto column = _tablet_schema->column(column_name); auto column_idx = _tablet_schema->field_index(column_name); + if (column_idx < 0) { + LOG(WARNING) << "referenced column was missing. " + << "[column=" << column_name << " referenced_column=" << column_idx + << "]"; + return Status::Error(); + } + auto column = _tablet_schema->column(column_idx); return_columns.emplace_back(column_idx); _olap_data_convertor->add_column_data_convertor(column); @@ -782,7 +788,13 @@ Status SchemaChangeForInvertedIndex::_write_inverted_index(int32_t segment_idx, int idx = 0; for (auto& inverted_index : _alter_inverted_indexs) { auto column_name = inverted_index.columns[0]; - auto column = _tablet_schema->column(column_name); + auto column_idx = _tablet_schema->field_index(column_name); + if (column_idx < 0) { + LOG(WARNING) << "referenced column was missing. " + << "[column=" << column_name << " referenced_column=" << column_idx << "]"; + return Status::Error(); + } + auto column = _tablet_schema->column(column_idx); auto index_id = inverted_index.index_id; auto converted_result = _olap_data_convertor->convert_column_data(idx++); @@ -1275,6 +1287,11 @@ Status SchemaChangeHandler::_get_rowset_readers(TabletSharedPtr tablet, DCHECK_EQ(inverted_index.columns.size(), 1); auto column_name = inverted_index.columns[0]; auto idx = tablet_schema->field_index(column_name); + if (idx < 0) { + LOG(WARNING) << "referenced column was missing. " + << "[column=" << column_name << " referenced_column=" << idx << "]"; + return Status::Error(); + } return_columns.emplace_back(idx); } @@ -1382,7 +1399,14 @@ Status SchemaChangeHandler::_drop_inverted_index(std::vectorrowset_id(), i); for (auto& inverted_index : alter_inverted_indexs) { auto column_name = inverted_index.columns[0]; - auto column = tablet_schema->column(column_name); + auto column_idx = tablet_schema->field_index(column_name); + if (column_idx < 0) { + LOG(WARNING) << "referenced column was missing. " + << "[column=" << column_name << " referenced_column=" << column_idx + << "]"; + return Status::Error(); + } + auto column = tablet_schema->column(column_idx); auto index_id = inverted_index.index_id; std::string inverted_index_file = diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index 1c77ac3234..dcfac6ef7e 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -503,7 +503,12 @@ void TabletIndex::init_from_thrift(const TOlapTableIndex& index, // get column unique id by name std::vector col_unique_ids(index.columns.size()); for (size_t i = 0; i < index.columns.size(); i++) { - col_unique_ids[i] = tablet_schema.column(index.columns[i]).unique_id(); + auto column_idx = tablet_schema.field_index(index.columns[i]); + if (column_idx >= 0) { + col_unique_ids[i] = tablet_schema.column(column_idx).unique_id(); + } else { + col_unique_ids[i] = -1; + } } _col_unique_ids = std::move(col_unique_ids); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 3bfee997fd..11fb77c3ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -2128,7 +2128,8 @@ public class SchemaChangeHandler extends AlterHandler { */ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTable, List newIndexes) throws UserException { - if (alterClause.getIndex() == null) { + Index alterIndex = alterClause.getIndex(); + if (alterIndex == null) { return false; } @@ -2166,7 +2167,12 @@ public class SchemaChangeHandler extends AlterHandler { } } - newIndexes.add(alterClause.getIndex()); + // the column name in CreateIndexClause is not check case sensitivity, + // when send index description to BE, there maybe cannot find column by name, + // so here update column name in CreateIndexClause after checkColumn for indexDef, + // there will use the column name in olapTable insead of the column name in CreateIndexClause. + alterIndex.setColumns(indexDef.getColumns()); + newIndexes.add(alterIndex); return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java index d1c21b5d37..711a83496b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java @@ -24,6 +24,7 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.AnalysisException; import com.google.common.base.Strings; +import com.google.common.collect.Lists; import java.util.HashMap; import java.util.List; @@ -34,6 +35,9 @@ public class IndexDef { private String indexName; private boolean ifNotExists; private List columns; + // add the column name of olapTable column into caseSensitivityColumns + // instead of the column name which from sql_parser analyze + private List caseSensitivityColumns = Lists.newArrayList(); private IndexType indexType; private String comment; private Map properties; @@ -142,6 +146,9 @@ public class IndexDef { } public List getColumns() { + if (caseSensitivityColumns.size() > 0) { + return caseSensitivityColumns; + } return columns; } @@ -176,6 +183,7 @@ public class IndexDef { if (indexType == IndexType.BITMAP || indexType == IndexType.INVERTED || indexType == IndexType.BLOOMFILTER || indexType == IndexType.NGRAM_BF) { String indexColName = column.getName(); + caseSensitivityColumns.add(indexColName); PrimitiveType colType = column.getDataType(); if (indexType == IndexType.INVERTED && colType.isArrayType()) { colType = ((ArrayType) column.getType()).getItemType().getPrimitiveType(); diff --git a/regression-test/suites/inverted_index_p0/test_add_drop_index_ignore_case_column.groovy b/regression-test/suites/inverted_index_p0/test_add_drop_index_ignore_case_column.groovy new file mode 100644 index 0000000000..ca8e390f67 --- /dev/null +++ b/regression-test/suites/inverted_index_p0/test_add_drop_index_ignore_case_column.groovy @@ -0,0 +1,194 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +suite("test_add_drop_index_with_ignore_case_column", "inverted_index"){ + // prepare test table + def timeout = 60000 + def delta_time = 1000 + def alter_res = "null" + def useTime = 0 + def wait_for_latest_op_on_table_finish = { table_name, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${table_name}" ORDER BY CreateTime DESC LIMIT 1;""" + alter_res = alter_res.toString() + if(alter_res.contains("FINISHED")) { + break + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout) + } + + def indexTbName1 = "test_add_drop_inverted_index4" + + sql "DROP TABLE IF EXISTS ${indexTbName1}" + // create 1 replica table + sql """ + CREATE TABLE IF NOT EXISTS ${indexTbName1} ( + `id` int(11) NULL, + `Name` text NULL, + `Description` text NULL, + INDEX idx_id (`id`) USING INVERTED COMMENT '', + INDEX idx_name (`name`) USING INVERTED PROPERTIES("parser"="none") COMMENT '' + ) + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 10 + properties("replication_num" = "1"); + """ + + // set enable_vectorized_engine=true + sql """ SET enable_vectorized_engine=true; """ + def var_result = sql "show variables" + logger.info("show variales result: " + var_result ) + + // show index of create table + def show_result = sql "show index from ${indexTbName1}" + logger.info("show index from " + indexTbName1 + " result: " + show_result) + assertEquals(show_result[0][2], "idx_id") + assertEquals(show_result[1][2], "idx_name") + + // insert data + sql "insert into ${indexTbName1} values (1, 'name1', 'desc 1'), (2, 'name2', 'desc 2')" + + // query all rows + def select_result = sql "select * from ${indexTbName1} order by id" + assertEquals(select_result.size(), 2) + assertEquals(select_result[0][0], 1) + assertEquals(select_result[0][1], "name1") + assertEquals(select_result[0][2], "desc 1") + assertEquals(select_result[1][0], 2) + assertEquals(select_result[1][1], "name2") + assertEquals(select_result[1][2], "desc 2") + + // query rows where name='name1' + select_result = sql "select * from ${indexTbName1} where name='name1'" + assertEquals(select_result.size(), 1) + assertEquals(select_result[0][0], 1) + assertEquals(select_result[0][1], "name1") + assertEquals(select_result[0][2], "desc 1") + + // query rows where name match 'name2' + select_result = sql "select * from ${indexTbName1} where name match 'name2'" + assertEquals(select_result.size(), 1) + assertEquals(select_result[0][0], 2) + assertEquals(select_result[0][1], "name2") + assertEquals(select_result[0][2], "desc 2") + + // query rows where description match 'desc', should fail without index + def success = false + try { + sql "select * from ${indexTbName1} where description match 'desc'" + success = true + } catch(Exception ex) { + logger.info("sql exception: " + ex) + } + assertEquals(success, false) + + // add index on column description + sql "create index idx_desc on ${indexTbName1}(description) USING INVERTED PROPERTIES(\"parser\"=\"standard\");" + wait_for_latest_op_on_table_finish(indexTbName1, timeout) + + // show index after add index + show_result = sql "show index from ${indexTbName1}" + logger.info("show index from " + indexTbName1 + " result: " + show_result) + assertEquals(show_result[0][2], "idx_id") + assertEquals(show_result[1][2], "idx_name") + assertEquals(show_result[2][2], "idx_desc") + + // query rows where description match 'desc' + select_result = sql "select * from ${indexTbName1} where description match 'desc' order by id" + assertEquals(select_result.size(), 2) + assertEquals(select_result[0][0], 1) + assertEquals(select_result[0][1], "name1") + assertEquals(select_result[0][2], "desc 1") + assertEquals(select_result[1][0], 2) + assertEquals(select_result[1][1], "name2") + assertEquals(select_result[1][2], "desc 2") + + // query rows where description match_all 'desc 1' + select_result = sql "select * from ${indexTbName1} where description match_all 'desc 1'" + assertEquals(select_result.size(), 1) + assertEquals(select_result[0][0], 1) + assertEquals(select_result[0][1], "name1") + assertEquals(select_result[0][2], "desc 1") + + // query rows where description match_all 'desc 2' + select_result = sql "select * from ${indexTbName1} where description match_all 'desc 2'" + assertEquals(select_result.size(), 1) + assertEquals(select_result[0][0], 2) + assertEquals(select_result[0][1], "name2") + assertEquals(select_result[0][2], "desc 2") + + // drop index + // add index on column description + sql "drop index idx_desc on ${indexTbName1}" + wait_for_latest_op_on_table_finish(indexTbName1, timeout) + + // query rows where description match 'desc', should fail without index + success = false + try { + sql "select * from ${indexTbName1} where description match 'desc'" + success = true + } catch(Exception ex) { + logger.info("sql exception: " + ex) + } + assertEquals(success, false) + + // query rows where name='name1' + select_result = sql "select * from ${indexTbName1} where name='name1'" + assertEquals(select_result.size(), 1) + assertEquals(select_result[0][0], 1) + assertEquals(select_result[0][1], "name1") + assertEquals(select_result[0][2], "desc 1") + + // query rows where name='name2' + select_result = sql "select * from ${indexTbName1} where name='name2'" + assertEquals(select_result.size(), 1) + assertEquals(select_result[0][0], 2) + assertEquals(select_result[0][1], "name2") + assertEquals(select_result[0][2], "desc 2") + + // add index on column description + sql "create index idx_desc on ${indexTbName1}(DESCRIPTION) USING INVERTED PROPERTIES(\"parser\"=\"standard\");" + wait_for_latest_op_on_table_finish(indexTbName1, timeout) + + // query rows where description match 'desc' + select_result = sql "select * from ${indexTbName1} where description match 'desc' order by id" + assertEquals(select_result.size(), 2) + assertEquals(select_result[0][0], 1) + assertEquals(select_result[0][1], "name1") + assertEquals(select_result[0][2], "desc 1") + assertEquals(select_result[1][0], 2) + assertEquals(select_result[1][1], "name2") + assertEquals(select_result[1][2], "desc 2") + + // query rows where description match_all 'desc 1' + select_result = sql "select * from ${indexTbName1} where description match_all 'desc 1'" + assertEquals(select_result.size(), 1) + assertEquals(select_result[0][0], 1) + assertEquals(select_result[0][1], "name1") + assertEquals(select_result[0][2], "desc 1") + + // query rows where description match_all 'desc 2' + select_result = sql "select * from ${indexTbName1} where description match_all 'desc 2'" + assertEquals(select_result.size(), 1) + assertEquals(select_result[0][0], 2) + assertEquals(select_result[0][1], "name2") + assertEquals(select_result[0][2], "desc 2") +} \ No newline at end of file