From d194014914b1e254ce8ae562f7bac01dfbc1e205 Mon Sep 17 00:00:00 2001 From: amory Date: Fri, 20 Jun 2025 19:51:38 +0800 Subject: [PATCH] [fix](ip) fix ip type in IndexDef support (#50637) ### What problem does this PR solve? only in branch-2.1 we do not support old indexDef to support Ip type and fix array_contains && arrays_overlap for with type array which backport: https://github.com/apache/doris/pull/50637 Issue Number: close #xxx --- .../functions/array/function_array_index.h | 10 ++ .../functions/array/function_arrays_overlap.h | 6 + .../org/apache/doris/analysis/IndexDef.java | 2 +- .../apache/doris/analysis/IndexDefTest.java | 63 ++++++++ .../test_array_ip_with_inverted_index.out | 148 ++++++++++++++++++ .../test_array_ip_with_inverted_index.groovy | 97 ++++++++++++ 6 files changed, 325 insertions(+), 1 deletion(-) create mode 100644 regression-test/data/inverted_index_p0/test_array_ip_with_inverted_index.out create mode 100644 regression-test/suites/inverted_index_p0/test_array_ip_with_inverted_index.groovy diff --git a/be/src/vec/functions/array/function_array_index.h b/be/src/vec/functions/array/function_array_index.h index b313c8385d..3f65d523f6 100644 --- a/be/src/vec/functions/array/function_array_index.h +++ b/be/src/vec/functions/array/function_array_index.h @@ -458,6 +458,16 @@ private: offsets, nested_null_map, *nested_column, *right_column, right_nested_null_map, array_null_map); } + } else if (is_ip(right_type) && is_ip(left_element_type)) { + if (left_which_type.is_ipv4()) { + return_column = _execute_number_expanded( + offsets, nested_null_map, *nested_column, *right_column, + right_nested_null_map, array_null_map); + } else if (left_which_type.is_ipv6()) { + return_column = _execute_number_expanded( + offsets, nested_null_map, *nested_column, *right_column, + right_nested_null_map, array_null_map); + } } if (return_column) { diff --git a/be/src/vec/functions/array/function_arrays_overlap.h b/be/src/vec/functions/array/function_arrays_overlap.h index d641b78e65..8e319c8709 100644 --- a/be/src/vec/functions/array/function_arrays_overlap.h +++ b/be/src/vec/functions/array/function_arrays_overlap.h @@ -341,6 +341,12 @@ public: ret = _execute_internal(left_exec_data, right_exec_data, dst_null_map_data, dst_nested_col->get_data().data()); + } else if (left_which_type.is_ipv4()) { + ret = _execute_internal(left_exec_data, right_exec_data, dst_null_map_data, + dst_nested_col->get_data().data()); + } else if (left_which_type.is_ipv6()) { + ret = _execute_internal(left_exec_data, right_exec_data, dst_null_map_data, + dst_nested_col->get_data().data()); } if (ret.ok()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java index 9d2233690e..145bccb543 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java @@ -225,7 +225,7 @@ public class IndexDef { } if (!(colType.isDateType() || colType.isDecimalV2Type() || colType.isDecimalV3Type() || colType.isFixedPointType() || colType.isStringType() || colType == PrimitiveType.BOOLEAN - || colType.isVariantType()) || colType.isIPType()) { + || colType.isVariantType() || colType.isIPType())) { throw new AnalysisException(colType + " is not supported in " + indexType.toString() + " index. " + "invalid index: " + indexName); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java index 56c78e2430..eb30163b7c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java @@ -17,9 +17,13 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.MapType; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.StructType; import org.apache.doris.common.AnalysisException; import com.google.common.collect.Lists; @@ -75,6 +79,65 @@ public class IndexDefTest { } } + @Test + public void testIndexType() { + // support types + PrimitiveType[] supportedTypes = new PrimitiveType[] { + PrimitiveType.DATE, PrimitiveType.DATETIME, PrimitiveType.DATEV2, PrimitiveType.DATETIMEV2, + PrimitiveType.DECIMALV2, PrimitiveType.DECIMAL32, PrimitiveType.DECIMAL64, PrimitiveType.DECIMAL128, PrimitiveType.DECIMAL256, + PrimitiveType.TINYINT, PrimitiveType.SMALLINT, PrimitiveType.INT, PrimitiveType.BIGINT, PrimitiveType.LARGEINT, + PrimitiveType.VARCHAR, PrimitiveType.CHAR, PrimitiveType.STRING, + PrimitiveType.BOOLEAN, + PrimitiveType.IPV4, PrimitiveType.IPV6 + }; + for (PrimitiveType type : supportedTypes) { + try { + IndexDef def = new IndexDef("idx", false, Lists.newArrayList("col1"), IndexDef.IndexType.INVERTED, null, ""); + Column col = new Column("col1", type); + def.checkColumn(col, KeysType.DUP_KEYS, true, true); + // check in array + def.checkColumn(new Column("col1", ArrayType.create(ScalarType.createType(type), true)), KeysType.DUP_KEYS, true, true); + } catch (AnalysisException e) { + Assert.fail("Should support type: " + type + ", but got: " + e.getMessage()); + } + } + // not support types + PrimitiveType[] unsupportedTypes = new PrimitiveType[] { + PrimitiveType.FLOAT, PrimitiveType.DOUBLE, + PrimitiveType.INVALID_TYPE, PrimitiveType.NULL_TYPE, + PrimitiveType.HLL, PrimitiveType.BITMAP, PrimitiveType.QUANTILE_STATE, + PrimitiveType.TIME, PrimitiveType.TIMEV2, PrimitiveType.JSONB, + PrimitiveType.LAMBDA_FUNCTION, PrimitiveType.ALL + }; + for (PrimitiveType type : unsupportedTypes) { + try { + IndexDef def = new IndexDef("idx", false, Lists.newArrayList("col1"), IndexDef.IndexType.INVERTED, null, ""); + Column col = new Column("col1", type); + def.checkColumn(col, KeysType.DUP_KEYS, true, true); + Assert.fail("Should not support type: " + type); + } catch (AnalysisException e) { + Assert.assertTrue(e.getMessage().contains("is not supported")); + } + } + // check in map/struct + try { + IndexDef def = new IndexDef("idx", false, Lists.newArrayList("col1"), IndexDef.IndexType.INVERTED, null, ""); + Column col = new Column("col1", new MapType(ScalarType.createType(PrimitiveType.INT), ScalarType.createType(PrimitiveType.STRING))); + def.checkColumn(col, KeysType.DUP_KEYS, true, true); + Assert.fail("Should not support map type"); + } catch (AnalysisException e) { + Assert.assertTrue(e.getMessage().contains("is not supported")); + } + try { + IndexDef def = new IndexDef("idx", false, Lists.newArrayList("col1"), IndexDef.IndexType.INVERTED, null, ""); + Column col = new Column("col1", new StructType(ScalarType.createType(PrimitiveType.INT), ScalarType.createType(PrimitiveType.STRING)), true); + def.checkColumn(col, KeysType.DUP_KEYS, true, true); + Assert.fail("Should not support map type"); + } catch (AnalysisException e) { + Assert.assertTrue(e.getMessage().contains("is not supported")); + } + } + @Test public void toSql() { Assert.assertEquals("INDEX `index1` (`col1`) USING INVERTED COMMENT 'balabala'", def.toSql()); diff --git a/regression-test/data/inverted_index_p0/test_array_ip_with_inverted_index.out b/regression-test/data/inverted_index_p0/test_array_ip_with_inverted_index.out new file mode 100644 index 0000000000..ccc09bee8a --- /dev/null +++ b/regression-test/data/inverted_index_p0/test_array_ip_with_inverted_index.out @@ -0,0 +1,148 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 ["192.168.0.1", "10.0.0.1"] ["2001:db8::1", "fe80::1"] 192.168.0.1 10.0.0.1 2001:db8::1 fe80::1 + +-- !sql -- +2 ["172.16.0.1"] ["::1"] 172.16.0.1 1.1.1.1 ::1 ::2 +4 ["1.1.1.1"] ["2001:db8::2"] 1.1.1.1 \N 2001:db8::2 \N + +-- !sql -- +1 ["192.168.0.1", "10.0.0.1"] ["2001:db8::1", "fe80::1"] 192.168.0.1 10.0.0.1 2001:db8::1 fe80::1 + +-- !sql -- +2 ["172.16.0.1"] ["::1"] 172.16.0.1 1.1.1.1 ::1 ::2 +4 ["1.1.1.1"] ["2001:db8::2"] 1.1.1.1 \N 2001:db8::2 \N + +-- !sql -- +4 ["1.1.1.1"] ["2001:db8::2"] 1.1.1.1 \N 2001:db8::2 \N + +-- !sql -- +1 ["192.168.0.1", "10.0.0.1"] ["2001:db8::1", "fe80::1"] 192.168.0.1 10.0.0.1 2001:db8::1 fe80::1 +2 ["172.16.0.1"] ["::1"] 172.16.0.1 1.1.1.1 ::1 ::2 + +-- !sql -- + +-- !sql -- +1 ["192.168.0.1", "10.0.0.1"] ["2001:db8::1", "fe80::1"] 192.168.0.1 10.0.0.1 2001:db8::1 fe80::1 +2 ["172.16.0.1"] ["::1"] 172.16.0.1 1.1.1.1 ::1 ::2 +4 ["1.1.1.1"] ["2001:db8::2"] 1.1.1.1 \N 2001:db8::2 \N + +-- !sql -- +1 ["192.168.0.1", "10.0.0.1"] ["2001:db8::1", "fe80::1"] 192.168.0.1 10.0.0.1 2001:db8::1 fe80::1 + +-- !sql -- +2 ["172.16.0.1"] ["::1"] 172.16.0.1 1.1.1.1 ::1 ::2 +4 ["1.1.1.1"] ["2001:db8::2"] 1.1.1.1 \N 2001:db8::2 \N + +-- !sql -- +1 ["192.168.0.1", "10.0.0.1"] ["2001:db8::1", "fe80::1"] 192.168.0.1 10.0.0.1 2001:db8::1 fe80::1 + +-- !sql -- +2 ["172.16.0.1"] ["::1"] 172.16.0.1 1.1.1.1 ::1 ::2 +4 ["1.1.1.1"] ["2001:db8::2"] 1.1.1.1 \N 2001:db8::2 \N + +-- !sql -- +2 ["172.16.0.1"] ["::1"] 172.16.0.1 1.1.1.1 ::1 ::2 + +-- !sql -- +1 ["192.168.0.1", "10.0.0.1"] ["2001:db8::1", "fe80::1"] 192.168.0.1 10.0.0.1 2001:db8::1 fe80::1 +4 ["1.1.1.1"] ["2001:db8::2"] 1.1.1.1 \N 2001:db8::2 \N + +-- !sql -- +4 ["1.1.1.1"] ["2001:db8::2"] 1.1.1.1 \N 2001:db8::2 \N + +-- !sql -- +1 ["192.168.0.1", "10.0.0.1"] ["2001:db8::1", "fe80::1"] 192.168.0.1 10.0.0.1 2001:db8::1 fe80::1 +2 ["172.16.0.1"] ["::1"] 172.16.0.1 1.1.1.1 ::1 ::2 + +-- !sql -- + +-- !sql -- +1 ["192.168.0.1", "10.0.0.1"] ["2001:db8::1", "fe80::1"] 192.168.0.1 10.0.0.1 2001:db8::1 fe80::1 +2 ["172.16.0.1"] ["::1"] 172.16.0.1 1.1.1.1 ::1 ::2 +4 ["1.1.1.1"] ["2001:db8::2"] 1.1.1.1 \N 2001:db8::2 \N + +-- !sql -- +1 ["192.168.0.1", "10.0.0.1"] ["2001:db8::1", "fe80::1"] 192.168.0.1 10.0.0.1 2001:db8::1 fe80::1 + +-- !sql -- +2 ["172.16.0.1"] ["::1"] 172.16.0.1 1.1.1.1 ::1 ::2 +4 ["1.1.1.1"] ["2001:db8::2"] 1.1.1.1 \N 2001:db8::2 \N + +-- !sql -- +4 ["1.1.1.1"] ["2001:db8::2"] 1.1.1.1 \N 2001:db8::2 \N + +-- !sql -- +1 ["192.168.0.1", "10.0.0.1"] ["2001:db8::1", "fe80::1"] 192.168.0.1 10.0.0.1 2001:db8::1 fe80::1 +2 ["172.16.0.1"] ["::1"] 172.16.0.1 1.1.1.1 ::1 ::2 + +-- !sql -- + +-- !sql -- +1 ["192.168.0.1", "10.0.0.1"] ["2001:db8::1", "fe80::1"] 192.168.0.1 10.0.0.1 2001:db8::1 fe80::1 +2 ["172.16.0.1"] ["::1"] 172.16.0.1 1.1.1.1 ::1 ::2 +4 ["1.1.1.1"] ["2001:db8::2"] 1.1.1.1 \N 2001:db8::2 \N + +-- !sql -- + +-- !sql -- + +-- !sql -- +1 ["192.168.0.1", "10.0.0.1"] ["2001:db8::1", "fe80::1"] 192.168.0.1 10.0.0.1 2001:db8::1 fe80::1 + +-- !sql -- +2 ["172.16.0.1"] ["::1"] 172.16.0.1 1.1.1.1 ::1 ::2 +4 ["1.1.1.1"] ["2001:db8::2"] 1.1.1.1 \N 2001:db8::2 \N + +-- !sql -- +2 ["172.16.0.1"] ["::1"] 172.16.0.1 1.1.1.1 ::1 ::2 + +-- !sql -- +1 ["192.168.0.1", "10.0.0.1"] ["2001:db8::1", "fe80::1"] 192.168.0.1 10.0.0.1 2001:db8::1 fe80::1 +4 ["1.1.1.1"] ["2001:db8::2"] 1.1.1.1 \N 2001:db8::2 \N + +-- !sql -- + +-- !sql -- +1 ["192.168.0.1", "10.0.0.1"] ["2001:db8::1", "fe80::1"] 192.168.0.1 10.0.0.1 2001:db8::1 fe80::1 +2 ["172.16.0.1"] ["::1"] 172.16.0.1 1.1.1.1 ::1 ::2 +4 ["1.1.1.1"] ["2001:db8::2"] 1.1.1.1 \N 2001:db8::2 \N + +-- !sql -- + +-- !sql -- + +-- !sql -- +1 ["192.168.0.1", "10.0.0.1"] ["2001:db8::1", "fe80::1"] 192.168.0.1 10.0.0.1 2001:db8::1 fe80::1 + +-- !sql -- +1 ["192.168.0.1", "10.0.0.1"] ["2001:db8::1", "fe80::1"] 192.168.0.1 10.0.0.1 2001:db8::1 fe80::1 + +-- !sql -- +1 ["192.168.0.1", "10.0.0.1"] ["2001:db8::1", "fe80::1"] 192.168.0.1 10.0.0.1 2001:db8::1 fe80::1 + +-- !sql -- +1 ["192.168.0.1", "10.0.0.1"] ["2001:db8::1", "fe80::1"] 192.168.0.1 10.0.0.1 2001:db8::1 fe80::1 + +-- !sql -- +1 ["192.168.0.1", "10.0.0.1"] ["2001:db8::1", "fe80::1"] 192.168.0.1 10.0.0.1 2001:db8::1 fe80::1 +4 ["1.1.1.1"] ["2001:db8::2"] 1.1.1.1 \N 2001:db8::2 \N + +-- !sql -- +3 \N \N \N \N \N \N +4 ["1.1.1.1"] ["2001:db8::2"] 1.1.1.1 \N 2001:db8::2 \N + +-- !sql -- +1 ["192.168.0.1", "10.0.0.1"] ["2001:db8::1", "fe80::1"] 192.168.0.1 10.0.0.1 2001:db8::1 fe80::1 + +-- !sql -- +1 ["192.168.0.1", "10.0.0.1"] ["2001:db8::1", "fe80::1"] 192.168.0.1 10.0.0.1 2001:db8::1 fe80::1 +2 ["172.16.0.1"] ["::1"] 172.16.0.1 1.1.1.1 ::1 ::2 + +-- !sql -- +1 ["192.168.0.1", "10.0.0.1"] ["2001:db8::1", "fe80::1"] 192.168.0.1 10.0.0.1 2001:db8::1 fe80::1 + +-- !sql -- +3 \N \N \N \N \N \N +4 ["1.1.1.1"] ["2001:db8::2"] 1.1.1.1 \N 2001:db8::2 \N + diff --git a/regression-test/suites/inverted_index_p0/test_array_ip_with_inverted_index.groovy b/regression-test/suites/inverted_index_p0/test_array_ip_with_inverted_index.groovy new file mode 100644 index 0000000000..5de728d85c --- /dev/null +++ b/regression-test/suites/inverted_index_p0/test_array_ip_with_inverted_index.groovy @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_array_ip_with_inverted_index") { + def tbl = "tai_ip_test" + sql "DROP TABLE IF EXISTS ${tbl}" + + sql """ + CREATE TABLE IF NOT EXISTS ${tbl} ( + id int, + ips_v4 array NULL, + ips_v6 array NULL, + ipv4_1 ipv4 NULL, + ipv4_2 ipv4 NULL, + ipv6_1 ipv6 NULL, + ipv6_2 ipv6 NULL, + INDEX idx_ips_v4(ips_v4) USING INVERTED COMMENT '', + INDEX idx_ips_v6(ips_v6) USING INVERTED COMMENT '', + INDEX idx_ipv4_1(ipv4_1) USING INVERTED COMMENT '', + INDEX idx_ipv6_1(ipv6_1) USING INVERTED COMMENT '' + ) ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + // insert some data + sql """ INSERT INTO ${tbl} VALUES (1, ['192.168.0.1', '10.0.0.1'], ['2001:db8::1', 'fe80::1'], '192.168.0.1', '10.0.0.1', '2001:db8::1', 'fe80::1'); """ + sql """ INSERT INTO ${tbl} VALUES (2, ['172.16.0.1'], ['::1'], '172.16.0.1', '1.1.1.1', '::1', '::2'); """ + sql """ INSERT INTO ${tbl} VALUES (3, NULL, NULL, NULL, NULL, NULL, NULL); """ + sql """ INSERT INTO ${tbl} VALUES (4, ['1.1.1.1'], ['2001:db8::2'], '1.1.1.1', NULL, '2001:db8::2', NULL); """ + + // for array 和 array query array_contains/arrays_overlap + def ipv4_test_values = ["'192.168.0.1'", "'10.0.0.1'", "'1.1.1.1'", null] + for (def p : ipv4_test_values) { + order_qt_sql """ select * from ${tbl} where array_contains(ips_v4, ${p}) order by id; """ + order_qt_sql """ select * from ${tbl} where !array_contains(ips_v4, ${p}) order by id; """ + } + def ipv6_test_values = ["'2001:db8::1'", "'fe80::1'", "'::1'", "'2001:db8::2'", null] + for (def p : ipv6_test_values) { + order_qt_sql """ select * from ${tbl} where array_contains(ips_v6, ${p}) order by id; """ + order_qt_sql """ select * from ${tbl} where !array_contains(ips_v6, ${p}) order by id; """ + } + def ipv4_overlap = ["['192.168.0.1','10.0.0.1']", "['1.1.1.1']", "[]", null] + for (def p : ipv4_overlap) { + order_qt_sql """ select * from ${tbl} where arrays_overlap(ips_v4, ${p}) order by id; """ + order_qt_sql """ select * from ${tbl} where !arrays_overlap(ips_v4, ${p}) order by id; """ + } + def ipv6_overlap = ["['2001:db8::1','fe80::1']", "['::1']", "[]", null] + for (def p : ipv6_overlap) { + order_qt_sql """ select * from ${tbl} where arrays_overlap(ips_v6, ${p}) order by id; """ + order_qt_sql """ select * from ${tbl} where !arrays_overlap(ips_v6, ${p}) order by id; """ + } + + // for ipv4_1/ipv4_2/ipv6_1/ipv6_2 query =、IN、IS NULL + order_qt_sql """ select * from ${tbl} where ipv4_1 = '192.168.0.1' order by id; """ + order_qt_sql """ select * from ${tbl} where ipv4_2 = '10.0.0.1' order by id; """ + order_qt_sql """ select * from ${tbl} where ipv6_1 = '2001:db8::1' order by id; """ + order_qt_sql """ select * from ${tbl} where ipv6_2 = 'fe80::1' order by id; """ + order_qt_sql """ select * from ${tbl} where ipv4_1 in ('192.168.0.1', '1.1.1.1') order by id; """ + order_qt_sql """ select * from ${tbl} where ipv6_2 is null order by id; """ + + // alter for inverted index + sql """ ALTER TABLE ${tbl} ADD INDEX idx_ipv4_2(ipv4_2) USING INVERTED COMMENT ''; """ + waitForSchemaChangeDone { + sql """ SHOW ALTER TABLE COLUMN WHERE TableName='${tbl}' ORDER BY createtime DESC LIMIT 1 """ + time 600 + } + sql """ ALTER TABLE ${tbl} ADD INDEX idx_ipv6_2(ipv6_2) USING INVERTED COMMENT ''; """ + waitForSchemaChangeDone { + sql """ SHOW ALTER TABLE COLUMN WHERE TableName='${tbl}' ORDER BY createtime DESC LIMIT 1 """ + time 600 + } + + + order_qt_sql """ select * from ${tbl} where ipv4_2 = '10.0.0.1' order by id; """ + order_qt_sql """ select * from ${tbl} where ipv4_2 in ('10.0.0.1', '1.1.1.1') order by id; """ + order_qt_sql """ select * from ${tbl} where ipv6_2 = 'fe80::1' order by id; """ + order_qt_sql """ select * from ${tbl} where ipv6_2 is null order by id; """ +} +