diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapIntersect.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapIntersect.java index 7c9b33d2ad..956f585f0a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapIntersect.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapIntersect.java @@ -17,13 +17,16 @@ package org.apache.doris.nereids.trees.expressions.functions.agg; +import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable; import org.apache.doris.nereids.trees.expressions.functions.BitmapIntersectFunction; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.BitmapType; import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import java.util.List; @@ -33,6 +36,10 @@ import java.util.List; public class OrthogonalBitmapIntersect extends AggregateFunction implements AlwaysNotNullable, OrthogonalBitmapFunction, BitmapIntersectFunction { + static final List FUNCTION_SIGNATURES = SUPPORTED_TYPES.stream() + .map(type -> FunctionSignature.ret(BitmapType.INSTANCE).varArgs(BitmapType.INSTANCE, type, type)) + .collect(ImmutableList.toImmutableList()); + /** * constructor with 3 or more arguments. */ @@ -49,6 +56,11 @@ public class OrthogonalBitmapIntersect extends AggregateFunction ExpressionUtils.mergeArguments(arg0, arg1, arg2, varArgs)); } + @Override + public List getSignatures() { + return FUNCTION_SIGNATURES; + } + /** * withDistinctAndChildren. */ diff --git a/regression-test/data/nereids_syntax_p0/test_bitmap_function_nereids.out b/regression-test/data/nereids_syntax_p0/test_bitmap_function_nereids.out new file mode 100644 index 0000000000..ed771ea325 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/test_bitmap_function_nereids.out @@ -0,0 +1,273 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +0 + +-- !sql -- +1 + +-- !sql -- +1 + +-- !sql -- +1,2 + +-- !sql -- + + +-- !sql -- +\N + +-- !sql -- +false + +-- !sql -- +true + +-- !sql -- +0 + +-- !sql -- + + +-- !sql -- +0,1,2 + +-- !sql -- +\N + +-- !sql -- +false + +-- !sql -- +true + +-- !sql -- +true + +-- !sql -- +false + +-- !sql_bitmap_hash1 -- +1 + +-- !sql_bitmap_hash2 -- +1 + +-- !sql_bitmap_hash3 -- +0 + +-- !sql_bitmap_hash64_1 -- +1 + +-- !sql_bitmap_hash64_2 -- +1 + +-- !sql_bitmap_hash64_3 -- +0 + +-- !sql -- +2 + +-- !sql -- +1 + +-- !sql -- +1,2 + +-- !sql -- +\N + +-- !sql -- +0,1,2,10 + +-- !sql -- +1,2,3,4,5,10 + +-- !sql -- +0 + +-- !sql -- +3 + +-- !sql -- +1 + +-- !sql -- +2 + +-- !sql -- +0 + +-- !sql -- +\N + +-- !sql -- +3 + +-- !sql -- +3 + +-- !sql -- +5 + +-- !sql -- +6 + +-- !sql -- +\N + +-- !sql -- +2 + +-- !sql -- +1,4 + +-- !sql -- +1,3,5 + +-- !sql -- +1,3,5 + +-- !sql -- +\N + +-- !sql -- +4 + +-- !sql -- +0 + +-- !sql -- +6 + +-- !sql -- +3 + +-- !sql -- +3 + +-- !sql -- +\N + +-- !sql -- +0 + +-- !sql -- +5 + +-- !sql -- +2 + +-- !sql -- +2 + +-- !sql -- +1,2,3,4,5 + +-- !sql -- +2 + +-- !sql -- +1,2,3 + +-- !sql -- +4,5 + +-- !sql -- +0,1,2 + +-- !sql -- +2,3 + +-- !sql -- +2,3,5 + +-- !sql -- +\N + +-- !sql -- + + +-- !sql -- +1 + +-- !sql -- +1,2 + +-- !sql -- +1 \N +2 \N + +-- !sql -- +1 3 +2 2 + +-- !sql -- +1 3 +2 2 + +-- !sql -- +2 + +-- !sql -- +1,4 + +-- !sql -- +1,3,5 + +-- !sql -- +1,3,5 + +-- !sql -- +\N + +-- !sql -- +1 + +-- !sql -- + + +-- !sql -- +\N + +-- !sql -- +9999999999 + +-- !sql -- +4 1,2,3 +3 1,2,3,4,5 + +-- !sql -- +3 + +-- !sql -- +\N + +-- !sql -- +0 + +-- !sql -- +0 + +-- !sql -- +[1, 2, 3] +[1, 2, 3, 4, 5] + +-- !sql -- +[] + +-- !sql -- +[3, 4, 100, 200] + +-- !sql -- +1,2,3 + +-- !sql -- +1 + +-- !sql -- +100 + +-- !sql -- +20221103 + diff --git a/regression-test/suites/nereids_syntax_p0/test_bitmap_function_nereids.groovy b/regression-test/suites/nereids_syntax_p0/test_bitmap_function_nereids.groovy new file mode 100644 index 0000000000..cb00d6cd66 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/test_bitmap_function_nereids.groovy @@ -0,0 +1,194 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_bitmap_function_nereids") { + sql 'set enable_vectorized_engine=true;' + sql 'set enable_fallback_to_original_planner=false;' + sql 'set enable_nereids_planner=true;' + // BITMAP_AND + qt_sql """ select bitmap_count(bitmap_and(to_bitmap(1), to_bitmap(2))) cnt """ + qt_sql """ select bitmap_count(bitmap_and(to_bitmap(1), to_bitmap(1))) cnt """ + qt_sql """ select bitmap_to_string(bitmap_and(to_bitmap(1), to_bitmap(1))) """ + qt_sql """ select bitmap_to_string(bitmap_and(bitmap_from_string('1,2,3'), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5'))) """ + qt_sql """ select bitmap_to_string(bitmap_and(bitmap_from_string('1,2,3'), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5'),bitmap_empty())) """ + qt_sql """ select bitmap_to_string(bitmap_and(bitmap_from_string('1,2,3'), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5'),NULL)) """ + + // BITMAP_CONTAINS + qt_sql """ select bitmap_contains(to_bitmap(1),2) cnt """ + qt_sql """ select bitmap_contains(to_bitmap(1),1) cnt """ + + // BITMAP_EMPTY + qt_sql """ select bitmap_count(bitmap_empty()) """ + + // BITMAP_FROM_STRING + qt_sql """ select bitmap_to_string(bitmap_empty()) """ + qt_sql """ select bitmap_to_string(bitmap_from_string("0, 1, 2")) """ + qt_sql """ select bitmap_from_string("-1, 0, 1, 2") """ + + // BITMAP_HAS_ANY + qt_sql """ select bitmap_has_any(to_bitmap(1),to_bitmap(2)) cnt """ + qt_sql """ select bitmap_has_any(to_bitmap(1),to_bitmap(1)) cnt """ + + // BITMAP_HAS_ALL + qt_sql """ select bitmap_has_all(bitmap_from_string("0, 1, 2"), bitmap_from_string("1, 2")) cnt """ + qt_sql """ select bitmap_has_all(bitmap_empty(), bitmap_from_string("1, 2")) cnt """ + + // BITMAP_HASH + qt_sql_bitmap_hash1 """ select bitmap_count(bitmap_hash('hello')) """ + qt_sql_bitmap_hash2 """ select bitmap_count(bitmap_hash('')) """ + qt_sql_bitmap_hash3 """ select bitmap_count(bitmap_hash(null)) """ + + // BITMAP_HASH64 + qt_sql_bitmap_hash64_1 """ select bitmap_count(bitmap_hash64('hello')) """ + qt_sql_bitmap_hash64_2 """ select bitmap_count(bitmap_hash64('')) """ + qt_sql_bitmap_hash64_3 """ select bitmap_count(bitmap_hash64(null)) """ + + // BITMAP_OR + qt_sql """ select bitmap_count(bitmap_or(to_bitmap(1), to_bitmap(2))) cnt """ + qt_sql """ select bitmap_count(bitmap_or(to_bitmap(1), to_bitmap(1))) cnt """ + qt_sql """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2))) """ + qt_sql """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2), to_bitmap(10), to_bitmap(0), NULL)) """ + qt_sql """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2), to_bitmap(10), to_bitmap(0), bitmap_empty())) """ + qt_sql """ select bitmap_to_string(bitmap_or(to_bitmap(10), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5'))) """ + + // bitmap_and_count + qt_sql """ select bitmap_and_count(bitmap_from_string('1,2,3'),bitmap_empty()) """ + qt_sql """ select bitmap_and_count(bitmap_from_string('1,2,3'),bitmap_from_string('1,2,3')) """ + qt_sql """ select bitmap_and_count(bitmap_from_string('1,2,3'),bitmap_from_string('3,4,5')) """ + qt_sql """ select bitmap_and_count(bitmap_from_string('1,2,3'), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5')) """ + qt_sql """ select bitmap_and_count(bitmap_from_string('1,2,3'), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5'),bitmap_empty()) """ + qt_sql """ select bitmap_and_count(bitmap_from_string('1,2,3'), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5'), NULL) """ + + // bitmap_or_count + qt_sql """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_empty()) """ + qt_sql """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_from_string('1,2,3'))""" + qt_sql """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_from_string('3,4,5')) """ + qt_sql """ select bitmap_or_count(bitmap_from_string('1,2,3'), bitmap_from_string('3,4,5'), to_bitmap(100), bitmap_empty()) """ + qt_sql """ select bitmap_or_count(bitmap_from_string('1,2,3'), bitmap_from_string('3,4,5'), to_bitmap(100), NULL) """ + + // BITMAP_XOR + qt_sql """ select bitmap_count(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'))) cnt """ + qt_sql """ select bitmap_to_string(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'))) """ + qt_sql """ select bitmap_to_string(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'))) """ + qt_sql """ select bitmap_to_string(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'),bitmap_empty())) """ + qt_sql """ select bitmap_to_string(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'),NULL)) """ + + // BITMAP_XOR_COUNT + qt_sql """ select bitmap_xor_count(bitmap_from_string('1,2,3'),bitmap_from_string('3,4,5')) """ + qt_sql """ select bitmap_xor_count(bitmap_from_string('1,2,3'),bitmap_from_string('1,2,3')) """ + qt_sql """ select bitmap_xor_count(bitmap_from_string('1,2,3'),bitmap_from_string('4,5,6')) """ + qt_sql """ select (bitmap_xor_count(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'))) """ + qt_sql """ select (bitmap_xor_count(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'),bitmap_empty())) """ + qt_sql """ select (bitmap_xor_count(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'),NULL)) """ + + // BITMAP_NOT + qt_sql """ select bitmap_count(bitmap_not(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'))) cnt """ + qt_sql """ select bitmap_to_string(bitmap_not(bitmap_from_string('2,3,5'),bitmap_from_string('1,2,3,4'))) """ + + // BITMAP_AND_NOT + qt_sql """ select bitmap_count(bitmap_and_not(bitmap_from_string('1,2,3'),bitmap_from_string('3,4,5'))) cnt """ + + // BITMAP_AND_NOT_COUNT + qt_sql """ select bitmap_and_not_count(bitmap_from_string('1,2,3'),bitmap_from_string('3,4,5')) cnt """ + + // BITMAP_SUBSET_IN_RANGE + qt_sql """ select bitmap_to_string(bitmap_subset_in_range(bitmap_from_string('1,2,3,4,5'), 0, 9)) value """ + qt_sql """ select bitmap_to_string(bitmap_subset_in_range(bitmap_from_string('1,2,3,4,5'), 2, 3)) value """ + + // BITMAP_SUBSET_LIMIT + qt_sql """ select bitmap_to_string(bitmap_subset_limit(bitmap_from_string('1,2,3,4,5'), 0, 3)) value """ + qt_sql """ select bitmap_to_string(bitmap_subset_limit(bitmap_from_string('1,2,3,4,5'), 4, 3)) value """ + + // SUB_BITMAP + qt_sql """ select bitmap_to_string(sub_bitmap(bitmap_from_string('1,0,1,2,3,1,5'), 0, 3)) value """ + qt_sql """ select bitmap_to_string(sub_bitmap(bitmap_from_string('1,0,1,2,3,1,5'), -3, 2)) value """ + qt_sql """ select bitmap_to_string(sub_bitmap(bitmap_from_string('1,0,1,2,3,1,5'), 2, 100)) value """ + + // BITMAP_TO_STRING + qt_sql """ select bitmap_to_string(null) """ + qt_sql """ select bitmap_to_string(bitmap_empty()) """ + qt_sql """ select bitmap_to_string(to_bitmap(1)) """ + qt_sql """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2))) """ + + // BITMAP_UNION + def bitmapUnionTable = "test_bitmap_union" + sql """ DROP TABLE IF EXISTS ${bitmapUnionTable} """ + sql """ create table if not exists ${bitmapUnionTable} (page_id int,user_id bitmap bitmap_union) aggregate key (page_id) distributed by hash (page_id) PROPERTIES("replication_num" = "1") """ + + sql """ insert into ${bitmapUnionTable} values(1, to_bitmap(1)); """ + sql """ insert into ${bitmapUnionTable} values(1, to_bitmap(2)); """ + sql """ insert into ${bitmapUnionTable} values(1, to_bitmap(3)); """ + sql """ insert into ${bitmapUnionTable} values(2, to_bitmap(1)); """ + sql """ insert into ${bitmapUnionTable} values(2, to_bitmap(2)); """ + + qt_sql """ select page_id, bitmap_union(user_id) from ${bitmapUnionTable} group by page_id order by page_id """ + qt_sql """ select page_id, bitmap_count(bitmap_union(user_id)) from ${bitmapUnionTable} group by page_id order by page_id """ + qt_sql """ select page_id, count(distinct user_id) from ${bitmapUnionTable} group by page_id order by page_id """ + + sql """ drop table ${bitmapUnionTable} """ + + // BITMAP_XOR + qt_sql """ select bitmap_count(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'))) cnt; """ + qt_sql """ select bitmap_to_string(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'))); """ + qt_sql """ select bitmap_to_string(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'))); """ + qt_sql """ select bitmap_to_string(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'),bitmap_empty())); """ + qt_sql """ select bitmap_to_string(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'),NULL)); """ + + // TO_BITMAP + qt_sql """ select bitmap_count(to_bitmap(10)) """ + qt_sql """ select bitmap_to_string(to_bitmap(-1)) """ + + // BITMAP_MAX + qt_sql """ select bitmap_max(bitmap_from_string('')) value; """ + qt_sql """ select bitmap_max(bitmap_from_string('1,9999999999')) value """ + + // INTERSECT_COUNT + def intersectCountTable = "test_intersect_count" + sql """ DROP TABLE IF EXISTS ${intersectCountTable} """ + sql """ create table if not exists ${intersectCountTable} (dt int (11),page varchar (10),user_id bitmap BITMAP_UNION ) DISTRIBUTED BY HASH(dt) BUCKETS 2 PROPERTIES("replication_num" = "1") """ + + + sql """ insert into ${intersectCountTable} values(3,"110001", to_bitmap(1)); """ + sql """ insert into ${intersectCountTable} values(3,"110001", to_bitmap(2)); """ + sql """ insert into ${intersectCountTable} values(3,"110001", to_bitmap(3)); """ + sql """ insert into ${intersectCountTable} values(3,"110001", to_bitmap(4)); """ + sql """ insert into ${intersectCountTable} values(3,"110001", to_bitmap(5)); """ + sql """ insert into ${intersectCountTable} values(4,"110001", to_bitmap(1)); """ + sql """ insert into ${intersectCountTable} values(4,"110001", to_bitmap(2)); """ + sql """ insert into ${intersectCountTable} values(4,"110001", to_bitmap(3)); """ + + qt_sql """ select dt,bitmap_to_string(user_id) from ${intersectCountTable} where dt in (3,4) order by dt desc; """ + qt_sql """ select intersect_count(user_id,dt,3,4) from ${intersectCountTable}; """ + + // ARTHOGONAL_BITMAP_**** + def arthogonalBitmapTable = "test_arthogonal_bitmap" + sql """ DROP TABLE IF EXISTS ${arthogonalBitmapTable} """ + sql """ CREATE TABLE IF NOT EXISTS ${arthogonalBitmapTable} ( tag_group bigint(20) NULL COMMENT "标签组", tag_value_id varchar(64) NULL COMMENT "标签值", tag_range int(11) NOT NULL DEFAULT "0" COMMENT "", partition_sign varchar(32) NOT NULL COMMENT "分区标识", bucket int(11) NOT NULL COMMENT "分桶字段", confidence tinyint(4) NULL DEFAULT "100" COMMENT "置信度", members bitmap BITMAP_UNION NULL COMMENT "人群") ENGINE=OLAP AGGREGATE KEY(tag_group, tag_value_id, tag_range, partition_sign, bucket, confidence) COMMENT "dmp_tag_map" PARTITION BY LIST(partition_sign) (PARTITION p202203231 VALUES IN ("2022-03-23-1"), PARTITION p202203251 VALUES IN ("2022-03-25-1"), PARTITION p202203261 VALUES IN ("2022-03-26-1"), PARTITION p202203271 VALUES IN ("2022-03-27-1"), PARTITION p202203281 VALUES IN ("2022-03-28-1"), PARTITION p202203291 VALUES IN ("2022-03-29-1"), PARTITION p202203301 VALUES IN ("2022-03-30-1"), PARTITION p202203311 VALUES IN ("2022-03-31-1"), PARTITION p202204011 VALUES IN ("2022-04-01-1"), PARTITION crowd VALUES IN ("crowd"), PARTITION crowd_tmp VALUES IN ("crowd_tmp"), PARTITION extend_crowd VALUES IN ("extend_crowd"), PARTITION partition_sign VALUES IN ("online_crowd")) DISTRIBUTED BY HASH(bucket) BUCKETS 64 PROPERTIES ("replication_allocation" = "tag.location.default: 1", "in_memory" = "false", "storage_format" = "V2");""" + + qt_sql """ select orthogonal_bitmap_intersect(members, tag_group, 1150000, 1150001, 390006) from ${arthogonalBitmapTable} where tag_group in ( 1150000, 1150001, 390006); """ + qt_sql """ select orthogonal_bitmap_intersect_count(members, tag_group, 1150000, 1150001, 390006) from ${arthogonalBitmapTable} where tag_group in ( 1150000, 1150001, 390006); """ + qt_sql """ select orthogonal_bitmap_union_count(members) from ${arthogonalBitmapTable} where tag_group in ( 1150000, 1150001, 390006); """ + + qt_sql """ select bitmap_to_array(user_id) from ${intersectCountTable} order by dt desc; """ + qt_sql """ select bitmap_to_array(bitmap_empty()); """ + qt_sql """ select bitmap_to_array(bitmap_from_string('100,200,3,4')); """ + + qt_sql """ select bitmap_to_string(sub_bitmap(bitmap_from_string('1,2,3,4,5'), 0, 3)) value; """ + qt_sql """ select bitmap_to_string(sub_bitmap(bitmap_from_string('1'), 0, 3)) value; """ + qt_sql """ select bitmap_to_string(bitmap_subset_limit(bitmap_from_string('100'), 0, 3)) value; """ + qt_sql """ select bitmap_to_string(bitmap_subset_in_range(bitmap_from_string('20221103'), 0, 20221104)) date_list_bitmap; """ +}