diff --git a/be/src/olap/inverted_index_parser.h b/be/src/olap/inverted_index_parser.h index 5bdfba6a7e..87ea726723 100644 --- a/be/src/olap/inverted_index_parser.h +++ b/be/src/olap/inverted_index_parser.h @@ -101,12 +101,11 @@ std::string get_parser_ignore_above_value_from_properties( template std::string get_parser_lowercase_from_properties( const std::map& properties) { + DBUG_EXECUTE_IF("inverted_index_parser.get_parser_lowercase_from_properties", { return ""; }) + if (properties.find(INVERTED_INDEX_PARSER_LOWERCASE_KEY) != properties.end()) { return properties.at(INVERTED_INDEX_PARSER_LOWERCASE_KEY); } else { - DBUG_EXECUTE_IF("inverted_index_parser.get_parser_lowercase_from_properties", - { return ""; }) - if constexpr (ReturnTrue) { return INVERTED_INDEX_PARSER_TRUE; } else { diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index 99bdf5a897..0418f4c633 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -798,6 +798,11 @@ void TabletIndex::to_schema_pb(TabletIndexPB* index) const { } index->set_index_type(_index_type); for (const auto& kv : _properties) { + DBUG_EXECUTE_IF("tablet_schema.to_schema_pb", { + if (kv.first == INVERTED_INDEX_PARSER_LOWERCASE_KEY) { + continue; + } + }) (*index->mutable_properties())[kv.first] = kv.second; } index->set_index_suffix_name(_escaped_index_suffix_path); @@ -805,9 +810,11 @@ void TabletIndex::to_schema_pb(TabletIndexPB* index) const { DBUG_EXECUTE_IF("tablet_schema.to_schema_pb", { return; }) // lowercase by default - if (!_properties.contains(INVERTED_INDEX_PARSER_LOWERCASE_KEY)) { - (*index->mutable_properties())[INVERTED_INDEX_PARSER_LOWERCASE_KEY] = - INVERTED_INDEX_PARSER_TRUE; + if (!_properties.empty()) { + if (!_properties.contains(INVERTED_INDEX_PARSER_LOWERCASE_KEY)) { + (*index->mutable_properties())[INVERTED_INDEX_PARSER_LOWERCASE_KEY] = + INVERTED_INDEX_PARSER_TRUE; + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java index e2235868a1..24abe49095 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java @@ -69,6 +69,14 @@ public class Index implements Writable { this.indexType = indexType; this.properties = properties; this.comment = comment; + if (indexType == IndexDef.IndexType.INVERTED) { + if (this.properties != null && !this.properties.isEmpty()) { + String key = InvertedIndexUtil.INVERTED_INDEX_PARSER_LOWERCASE_KEY; + if (!properties.containsKey(key)) { + this.properties.put(key, "true"); + } + } + } } public Index() { diff --git a/regression-test/suites/backup_restore/test_backup_restore_inverted_index.groovy b/regression-test/suites/backup_restore/test_backup_restore_inverted_index.groovy index 09b9f0b31e..7c4f04cbd2 100644 --- a/regression-test/suites/backup_restore/test_backup_restore_inverted_index.groovy +++ b/regression-test/suites/backup_restore/test_backup_restore_inverted_index.groovy @@ -74,7 +74,7 @@ suite("test_backup_restore_inverted_index", "backup_restore") { def restore_index_comment = sql "SHOW CREATE TABLE ${dbName}.${tableName}" - assertTrue(restore_index_comment[0][1].contains("USING INVERTED PROPERTIES(\"parser\" = \"english\")")) + assertTrue(restore_index_comment[0][1].contains("USING INVERTED PROPERTIES(\"parser\" = \"english\", \"lower_case\" = \"true\")")) result = sql "SELECT id, TOKENIZE(comment,'\"parser\"=\"english\"') as token FROM ${dbName}.${tableName}" assertEquals(result.size(), 2) diff --git a/regression-test/suites/fault_injection_p0/test_index_lowercase_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_index_lowercase_fault_injection.groovy index 4408660047..e18060a7d6 100644 --- a/regression-test/suites/fault_injection_p0/test_index_lowercase_fault_injection.groovy +++ b/regression-test/suites/fault_injection_p0/test_index_lowercase_fault_injection.groovy @@ -35,7 +35,11 @@ suite("test_index_lowercase_fault_injection", "nonConcurrent") { COMMENT "OLAP" DISTRIBUTED BY HASH(`@timestamp`) BUCKETS 1 PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" + "replication_allocation" = "tag.location.default: 1", + "compaction_policy" = "time_series", + "time_series_compaction_goal_size_mbytes" = "1024", + "time_series_compaction_file_count_threshold" = "20", + "time_series_compaction_time_threshold_seconds" = "3600" ); """ } @@ -55,14 +59,14 @@ suite("test_index_lowercase_fault_injection", "nonConcurrent") { sql """ INSERT INTO ${testTable} VALUES (893964653, '232.0.0.0', 'GET /images/hm_bg.jpg HTTP/1.0', 200, 3781); """ sql 'sync' + + qt_sql """ select count() from ${testTable} where (request match 'HTTP'); """ + qt_sql """ select count() from ${testTable} where (request match 'http'); """ } finally { GetDebugPoint().disableDebugPointForAllBEs("inverted_index_parser.get_parser_lowercase_from_properties") GetDebugPoint().disableDebugPointForAllBEs("tablet_schema.to_schema_pb") } - qt_sql """ select count() from ${testTable} where (request match 'HTTP'); """ - qt_sql """ select count() from ${testTable} where (request match 'http'); """ - sql """ INSERT INTO ${testTable} VALUES (893964672, '26.1.0.0', 'GET /images/hm_bg.jpg HTTP/1.0', 304, 0); """ sql """ INSERT INTO ${testTable} VALUES (893964672, '26.1.0.0', 'GET /images/hm_bg.jpg HTTP/1.0', 304, 0); """ sql """ INSERT INTO ${testTable} VALUES (893964653, '232.0.0.0', 'GET /images/hm_bg.jpg HTTP/1.0', 200, 3781); """ diff --git a/regression-test/suites/index_p0/test_index_meta.groovy b/regression-test/suites/index_p0/test_index_meta.groovy index 75ca902649..e1d26f825f 100644 --- a/regression-test/suites/index_p0/test_index_meta.groovy +++ b/regression-test/suites/index_p0/test_index_meta.groovy @@ -71,7 +71,7 @@ suite("index_meta", "p0") { assertEquals(show_result[1][4], "name") assertEquals(show_result[1][10], "INVERTED") assertEquals(show_result[1][11], "'index for name'") - assertEquals(show_result[1][12], "(\"parser\" = \"none\")") + assertEquals(show_result[1][12], "(\"parser\" = \"none\", \"lower_case\" = \"true\")") // add index on column description sql "create index idx_desc on ${tableName}(description) USING INVERTED PROPERTIES(\"parser\"=\"standard\") COMMENT 'index for description';" @@ -90,12 +90,12 @@ suite("index_meta", "p0") { assertEquals(show_result[1][4], "name") assertEquals(show_result[1][10], "INVERTED") assertEquals(show_result[1][11], "'index for name'") - assertEquals(show_result[1][12], "(\"parser\" = \"none\")") + assertEquals(show_result[1][12], "(\"parser\" = \"none\", \"lower_case\" = \"true\")") assertEquals(show_result[2][2], "idx_desc") assertEquals(show_result[2][4], "description") assertEquals(show_result[2][10], "INVERTED") assertEquals(show_result[2][11], "index for description") - assertEquals(show_result[2][12], "(\"parser\" = \"standard\")") + assertEquals(show_result[2][12], "(\"parser\" = \"standard\", \"lower_case\" = \"true\")") // drop index // add index on column description @@ -114,7 +114,7 @@ suite("index_meta", "p0") { assertEquals(show_result[1][4], "description") assertEquals(show_result[1][10], "INVERTED") assertEquals(show_result[1][11], "index for description") - assertEquals(show_result[1][12], "(\"parser\" = \"standard\")") + assertEquals(show_result[1][12], "(\"parser\" = \"standard\", \"lower_case\" = \"true\")") // add index on column description sql "create index idx_name on ${tableName}(name) USING INVERTED COMMENT 'new index for name';" @@ -133,7 +133,7 @@ suite("index_meta", "p0") { assertEquals(show_result[1][4], "description") assertEquals(show_result[1][10], "INVERTED") assertEquals(show_result[1][11], "index for description") - assertEquals(show_result[1][12], "(\"parser\" = \"standard\")") + assertEquals(show_result[1][12], "(\"parser\" = \"standard\", \"lower_case\" = \"true\")") assertEquals(show_result[2][2], "idx_name") assertEquals(show_result[2][4], "name") assertEquals(show_result[2][10], "INVERTED")