diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index cf208f08dc..eece72cf5a 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -178,10 +178,8 @@ Status SegmentWriter::init(const std::vector& col_ids, bool has_key) { init_column_meta(opts.meta, cid, column, _tablet_schema); // now we create zone map for key columns in AGG_KEYS or all column in UNIQUE_KEYS or DUP_KEYS - // and not support zone map for array type and jsonb type. - opts.need_zone_map = - (column.is_key() || _tablet_schema->keys_type() != KeysType::AGG_KEYS) && - column.type() != FieldType::OLAP_FIELD_TYPE_OBJECT; + // except for columns whose type don't support zone map. + opts.need_zone_map = column.is_key() || _tablet_schema->keys_type() != KeysType::AGG_KEYS; opts.need_bloom_filter = column.is_bf_column(); auto* tablet_index = _tablet_schema->get_ngram_bf_index(column.unique_id()); if (tablet_index) { @@ -234,6 +232,9 @@ Status SegmentWriter::init(const std::vector& col_ids, bool has_key) { CHECK_FIELD_TYPE(AGG_STATE, "agg_state") CHECK_FIELD_TYPE(MAP, "map") CHECK_FIELD_TYPE(VARIANT, "variant") + CHECK_FIELD_TYPE(OBJECT, "object") + CHECK_FIELD_TYPE(HLL, "hll") + CHECK_FIELD_TYPE(QUANTILE_STATE, "quantile_state") #undef CHECK_FIELD_TYPE diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp index 766c3e47a7..54a0e9ad2d 100644 --- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp @@ -131,9 +131,8 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo _init_column_meta(opts.meta, cid, column); // now we create zone map for key columns in AGG_KEYS or all column in UNIQUE_KEYS or DUP_KEYS - // and not support zone map for array type and jsonb type. - opts.need_zone_map = (column.is_key() || _tablet_schema->keys_type() != KeysType::AGG_KEYS) && - column.type() != FieldType::OLAP_FIELD_TYPE_OBJECT; + // except for columns whose type don't support zone map. + opts.need_zone_map = column.is_key() || _tablet_schema->keys_type() != KeysType::AGG_KEYS; opts.need_bloom_filter = column.is_bf_column(); auto* tablet_index = _tablet_schema->get_ngram_bf_index(column.unique_id()); if (tablet_index) { @@ -186,6 +185,9 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo CHECK_FIELD_TYPE(AGG_STATE, "agg_state") CHECK_FIELD_TYPE(MAP, "map") CHECK_FIELD_TYPE(VARIANT, "variant") + CHECK_FIELD_TYPE(OBJECT, "object") + CHECK_FIELD_TYPE(HLL, "hll") + CHECK_FIELD_TYPE(QUANTILE_STATE, "quantile_state") #undef CHECK_FIELD_TYPE diff --git a/be/src/vec/functions/function.h b/be/src/vec/functions/function.h index cb8ff34cdb..44a629bf24 100644 --- a/be/src/vec/functions/function.h +++ b/be/src/vec/functions/function.h @@ -716,7 +716,8 @@ ColumnPtr wrap_in_nullable(const ColumnPtr& src, const Block& block, const Colum M(Struct, ColumnStruct) \ M(VARIANT, ColumnObject) \ M(BitMap, ColumnBitmap) \ - M(HLL, ColumnHLL) + M(HLL, ColumnHLL) \ + M(QuantileState, ColumnQuantileState) #define TYPE_TO_BASIC_COLUMN_TYPE(M) \ NUMERIC_TYPE_TO_COLUMN_TYPE(M) \ diff --git a/be/src/vec/functions/function_case.h b/be/src/vec/functions/function_case.h index 1360e47aa6..2ecc6bd186 100644 --- a/be/src/vec/functions/function_case.h +++ b/be/src/vec/functions/function_case.h @@ -261,6 +261,7 @@ public: std::is_same_v || std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v || std::is_same_v) { // result_column and all then_column is not nullable. diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Types/HLL.md b/docs/en/docs/sql-manual/sql-reference/Data-Types/HLL.md index 65e452f546..9cfe4d5e0e 100644 --- a/docs/en/docs/sql-manual/sql-reference/Data-Types/HLL.md +++ b/docs/en/docs/sql-manual/sql-reference/Data-Types/HLL.md @@ -28,7 +28,7 @@ under the License. ### Description HLL -HLL cannot be used as a key column, and the aggregation type is HLL_UNION when create table. +HLL cannot be used as a key column. The columns of the HLL type can be used in Aggregate tables, Duplicate tables and Unique tables. When used in an Aggregate table, the aggregation type is HLL_UNION when building table. The user does not need to specify the length and default value. The length is controlled within the system according to the degree of data aggregation. And HLL columns can only be queried or used through the matching hll_union_agg, hll_raw_agg, hll_cardinality, and hll_hash. diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Types/QUANTILE_STATE.md b/docs/en/docs/sql-manual/sql-reference/Data-Types/QUANTILE_STATE.md index 8dfe546317..e53c63ce5f 100644 --- a/docs/en/docs/sql-manual/sql-reference/Data-Types/QUANTILE_STATE.md +++ b/docs/en/docs/sql-manual/sql-reference/Data-Types/QUANTILE_STATE.md @@ -31,10 +31,11 @@ QUANTILE_STATE **In 2.0, we support the [agg_state](AGG_STATE.md) function, and it is recommended to use agg_state quantile_union(quantile_state not null) instead of this type.** - QUANTILE_STATE cannot be used as a key column, and the aggregation type is QUANTILE_UNION when building the table. - The user does not need to specify the length and default value. The length is controlled within the system according to the degree of data aggregation. - And the QUANTILE_STATE column can only be queried or used through the supporting QUANTILE_PERCENT, QUANTILE_UNION and TO_QUANTILE_STATE functions. - QUANTILE_STATE is a type for calculating the approximate value of quantiles. Different values with the same key are pre-aggregated during loading process. When the number of aggregated values does not exceed 2048, all data are recorded in detail. When the number of aggregated values is greater than 2048, [TDigest] is used. (https://github.com/tdunning/t-digest/blob/main/docs/t-digest-paper/histo.pdf) algorithm to aggregate (cluster) the data and save the centroid points after clustering. +QUANTILE_STATE cannot be used as a key column. The columns of the HLL type can be used in Aggregate tables, Duplicate tables and Unique tables. When used in an Aggregate table, the aggregation type is HLL_UNION when building table. + +The user does not need to specify the length and default value. The length is controlled within the system according to the degree of data aggregation. +And the QUANTILE_STATE column can only be queried or used through the supporting QUANTILE_PERCENT, QUANTILE_UNION and TO_QUANTILE_STATE functions. +QUANTILE_STATE is a type for calculating the approximate value of quantiles. Different values with the same key are pre-aggregated during loading process. When the number of aggregated values does not exceed 2048, all data are recorded in detail. When the number of aggregated values is greater than 2048, [TDigest] is used. (https://github.com/tdunning/t-digest/blob/main/docs/t-digest-paper/histo.pdf) algorithm to aggregate (cluster) the data and save the centroid points after clustering. related functions: diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Types/HLL.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Types/HLL.md index 0290eb82b8..dfc4472344 100644 --- a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Types/HLL.md +++ b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Types/HLL.md @@ -26,13 +26,13 @@ under the License. ## HLL(HyperLogLog) ### description - HLL - HLL不能作为key列使用，建表时配合聚合类型为HLL_UNION。 - 用户不需要指定长度和默认值。长度根据数据的聚合程度系统内控制。 - 并且HLL列只能通过配套的hll_union_agg、hll_raw_agg、hll_cardinality、hll_hash进行查询或使用。 - - HLL是模糊去重，在数据量大的情况性能优于Count Distinct。 - HLL的误差通常在1%左右，有时会达到2%。 +HLL +HLL不能作为key列使用，支持在Aggregate模型、Duplicate模型和Unique模型的表中使用。在Aggregate模型表中使用时，建表时配合的聚合类型为HLL_UNION。 +用户不需要指定长度和默认值。长度根据数据的聚合程度系统内控制。 +并且HLL列只能通过配套的hll_union_agg、hll_raw_agg、hll_cardinality、hll_hash进行查询或使用。 + +HLL是模糊去重，在数据量大的情况性能优于Count Distinct。 +HLL的误差通常在1%左右，有时会达到2%。 ### example diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Types/QUANTILE_STATE.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Types/QUANTILE_STATE.md index b249dbe5e4..c91befee89 100644 --- a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Types/QUANTILE_STATE.md +++ b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Types/QUANTILE_STATE.md @@ -28,27 +28,27 @@ under the License. ### description QUANTILE_STATE - **在2.0中我们支持了[agg_state](AGG_STATE.md)功能，推荐使用agg_state quantile_union(quantile_state not null)来代替本类型。** +**在2.0中我们支持了[agg_state](AGG_STATE.md)功能，推荐使用agg_state quantile_union(quantile_state not null)来代替本类型。** - QUANTILE_STATE不能作为key列使用，建表时配合聚合类型为QUANTILE_UNION。 - 用户不需要指定长度和默认值。长度根据数据的聚合程度系统内控制。 - 并且QUANTILE_STATE列只能通过配套的QUANTILE_PERCENT、QUANTILE_UNION、TO_QUANTILE_STATE等函数进行查询或使用。 - - QUANTILE_STATE 是一种计算分位数近似值的类型，在导入时会对相同的key，不同 value 进行预聚合，当value数量不超过2048时采用明细记录所有数据，当 value 数量大于2048时采用 [TDigest](https://github.com/tdunning/t-digest/blob/main/docs/t-digest-paper/histo.pdf) 算法，对数据进行聚合（聚类）保存聚类后的质心点。 +QUANTILE_STATE不能作为key列使用，支持在Aggregate模型、Duplicate模型和Unique模型的表中使用。在Aggregate模型表中使用时，建表时配合的聚合类型为QUANTILE_UNION。 +用户不需要指定长度和默认值。长度根据数据的聚合程度系统内控制。 +并且QUANTILE_STATE列只能通过配套的QUANTILE_PERCENT、QUANTILE_UNION、TO_QUANTILE_STATE等函数进行查询或使用。 - 相关函数: - - QUANTILE_UNION(QUANTILE_STATE): - 此函数为聚合函数，用于将不同的分位数计算中间结果进行聚合操作。此函数返回的结果仍是QUANTILE_STATE - - - TO_QUANTILE_STATE(DOUBLE raw_data [,FLOAT compression]): - 此函数将数值类型转化成QUANTILE_STATE类型 - compression参数是可选项，可设置范围是[2048, 10000]，值越大，后续分位数近似计算的精度越高，内存消耗越大，计算耗时越长。 - compression参数未指定或设置的值在[2048, 10000]范围外，以2048的默认值运行 +QUANTILE_STATE 是一种计算分位数近似值的类型，在导入时会对相同的key，不同 value 进行预聚合，当value数量不超过2048时采用明细记录所有数据，当 value 数量大于2048时采用 [TDigest](https://github.com/tdunning/t-digest/blob/main/docs/t-digest-paper/histo.pdf) 算法，对数据进行聚合（聚类）保存聚类后的质心点。 - QUANTILE_PERCENT(QUANTILE_STATE, percent): - 此函数将分位数计算的中间结果变量（QUANTILE_STATE）转化为具体的分位数数值 +相关函数: + + QUANTILE_UNION(QUANTILE_STATE): + 此函数为聚合函数，用于将不同的分位数计算中间结果进行聚合操作。此函数返回的结果仍是QUANTILE_STATE + + + TO_QUANTILE_STATE(DOUBLE raw_data [,FLOAT compression]): + 此函数将数值类型转化成QUANTILE_STATE类型 + compression参数是可选项，可设置范围是[2048, 10000]，值越大，后续分位数近似计算的精度越高，内存消耗越大，计算耗时越长。 + compression参数未指定或设置的值在[2048, 10000]范围外，以2048的默认值运行 + + QUANTILE_PERCENT(QUANTILE_STATE, percent): + 此函数将分位数计算的中间结果变量（QUANTILE_STATE）转化为具体的分位数数值 diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java index 8ecf686805..3bd12b64ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java @@ -521,14 +521,6 @@ public class CreateTableStmt extends DdlStmt { throw new AnalysisException("Time type is not supported for olap table"); } - if (columnDef.getType().isObjectStored()) { - if (!columnDef.getType().isBitmapType()) { - if (keysDesc.getKeysType() != KeysType.AGG_KEYS) { - throw new AnalysisException("column:" + columnDef.getName() + " must be used in AGG_KEYS."); - } - } - } - if (!columnSet.add(columnDef.getName())) { ErrorReport.reportAnalysisException(ErrorCode.ERR_DUP_FIELDNAME, columnDef.getName()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateType.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateType.java index f86ea7855e..47ee91c0b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateType.java @@ -115,11 +115,8 @@ public enum AggregateType { compatibilityMap.put(MAX, EnumSet.copyOf(primitiveTypeList)); primitiveTypeList.clear(); - // all types except object stored column type, such as bitmap hll - // quantile_state. + // all types except agg_state. EnumSet excObjectStored = EnumSet.allOf(PrimitiveType.class); - excObjectStored.remove(PrimitiveType.HLL); - excObjectStored.remove(PrimitiveType.QUANTILE_STATE); excObjectStored.remove(PrimitiveType.AGG_STATE); compatibilityMap.put(REPLACE, EnumSet.copyOf(excObjectStored)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index 86d48556f8..2736d71a50 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -191,13 +191,7 @@ public class ColumnDefinition { throw new AnalysisException("Type exceeds the maximum nesting depth of 9"); } } - if (type.isHllType() || type.isQuantileStateType()) { - if (aggType == null) { - throw new AnalysisException("column: " + name + " must be used in AGG_KEYS."); - } - isNullable = false; - } - if (type.isBitmapType()) { + if (type.isHllType() || type.isQuantileStateType() || type.isBitmapType()) { if (aggType != null) { isNullable = false; } @@ -349,14 +343,6 @@ public class ColumnDefinition { if (type.isTimeLikeType()) { throw new AnalysisException("Time type is not supported for olap table"); } - - if (type.isObjectType()) { - if (!type.isBitmapType()) { - if (keysType != KeysType.AGG_KEYS) { - throw new AnalysisException("column:" + name + " must be used in AGG_KEYS."); - } - } - } } /** diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableStmtTest.java index 539c42dbd7..30da403397 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableStmtTest.java @@ -355,19 +355,6 @@ public class CreateTableStmtTest { stmt.analyze(analyzer); } - @Test - public void testHllNoAggTab() throws Exception { - ColumnDef hll = new ColumnDef("col3", new TypeDef(ScalarType.createType(PrimitiveType.HLL))); - cols.add(hll); - CreateTableStmt stmt = new CreateTableStmt(false, false, tblNameNoDb, cols, "olap", - new KeysDesc(KeysType.DUP_KEYS, colsName), null, new RandomDistributionDesc(10), - null, null, ""); - expectedEx.expect(AnalysisException.class); - expectedEx.expectMessage( - "Aggregate type `col3` HLL NONE NOT NULL COMMENT \"\" is not compatible with primitive type HLL"); - stmt.analyze(analyzer); - } - @Test public void testBmpHllIncAgg() throws Exception { ColumnDef bitmap = new ColumnDef("col3", new TypeDef(ScalarType.createType(PrimitiveType.BITMAP))); diff --git a/regression-test/data/data_model_p0/duplicate/storage/test_duplicate_hll.out b/regression-test/data/data_model_p0/duplicate/storage/test_duplicate_hll.out new file mode 100644 index 0000000000..deeceba298 --- /dev/null +++ b/regression-test/data/data_model_p0/duplicate/storage/test_duplicate_hll.out @@ -0,0 +1,113 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 0 +2 6 +3 2 + +-- !sql -- +7 + +-- !from_agg -- +1 0 +2 6 +3 2 + +-- !from_agg -- +7 + +-- !from_values -- +1 0 +1 1 +2 1 +2 6 +3 2 +4 1 + +-- !from_values -- +1 1 +2 6 +3 2 +4 1 + +-- !from_values -- +8 + +-- !from_dup -- +1 0 +1 0 +1 1 +1 1 +2 1 +2 1 +2 6 +2 6 +3 2 +3 2 +4 1 +4 1 + +-- !from_dup -- +1 1 +2 6 +3 2 +4 1 + +-- !from_dup -- +8 + +-- !sql -- +1 0 +2 6 +3 2 + +-- !sql -- +7 + +-- !from_agg -- +1 0 +2 6 +3 2 + +-- !from_agg -- +7 + +-- !from_values -- +1 0 +1 1 +2 1 +2 6 +3 2 +4 1 + +-- !from_values -- +1 1 +2 6 +3 2 +4 1 + +-- !from_values -- +8 + +-- !from_dup -- +1 0 +1 0 +1 1 +1 1 +2 1 +2 1 +2 6 +2 6 +3 2 +3 2 +4 1 +4 1 + +-- !from_dup -- +1 1 +2 6 +3 2 +4 1 + +-- !from_dup -- +8 + diff --git a/regression-test/data/data_model_p0/duplicate/storage/test_duplicate_quantile_state.out b/regression-test/data/data_model_p0/duplicate/storage/test_duplicate_quantile_state.out new file mode 100644 index 0000000000..d75ad2aa66 --- /dev/null +++ b/regression-test/data/data_model_p0/duplicate/storage/test_duplicate_quantile_state.out @@ -0,0 +1,85 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 -1.0 -1.0 -1.0 +2 0.0 0.5 1.0 +3 0.0 1.0 2.0 + +-- !from_agg -- +1 -1.0 -1.0 -1.0 +2 0.0 0.5 1.0 +3 0.0 1.0 2.0 + +-- !from_values -- +1 -2.0 -2.0 -2.0 +1 -1.0 -1.0 -1.0 +1 0.0 0.0 0.0 +2 -100.0 -100.0 -100.0 +2 0.0 0.5 1.0 +3 0.0 1.0 2.0 + +-- !from_values -- +1 -2.0 -1.0 0.0 +2 -100.0 0.0 1.0 +3 0.0 1.0 2.0 + +-- !from_dup -- +1 -2.0 -2.0 -2.0 +1 -2.0 -2.0 -2.0 +1 -1.0 -1.0 -1.0 +1 -1.0 -1.0 -1.0 +1 0.0 0.0 0.0 +1 0.0 0.0 0.0 +2 -100.0 -100.0 -100.0 +2 -100.0 -100.0 -100.0 +2 0.0 0.5 1.0 +2 0.0 0.5 1.0 +3 0.0 1.0 2.0 +3 0.0 1.0 2.0 + +-- !from_dup -- +1 -2.0 -1.0 0.0 +2 -100.0 0.0 1.0 +3 0.0 1.0 2.0 + +-- !sql -- +1 -1.0 -1.0 -1.0 +2 0.0 0.5 1.0 +3 0.0 1.0 2.0 + +-- !from_agg -- +1 -1.0 -1.0 -1.0 +2 0.0 0.5 1.0 +3 0.0 1.0 2.0 + +-- !from_values -- +1 -2.0 -2.0 -2.0 +1 -1.0 -1.0 -1.0 +1 0.0 0.0 0.0 +2 -100.0 -100.0 -100.0 +2 0.0 0.5 1.0 +3 0.0 1.0 2.0 + +-- !from_values -- +1 -2.0 -1.0 0.0 +2 -100.0 0.0 1.0 +3 0.0 1.0 2.0 + +-- !from_dup -- +1 -2.0 -2.0 -2.0 +1 -2.0 -2.0 -2.0 +1 -1.0 -1.0 -1.0 +1 -1.0 -1.0 -1.0 +1 0.0 0.0 0.0 +1 0.0 0.0 0.0 +2 -100.0 -100.0 -100.0 +2 -100.0 -100.0 -100.0 +2 0.0 0.5 1.0 +2 0.0 0.5 1.0 +3 0.0 1.0 2.0 +3 0.0 1.0 2.0 + +-- !from_dup -- +1 -2.0 -1.0 0.0 +2 -100.0 0.0 1.0 +3 0.0 1.0 2.0 + diff --git a/regression-test/data/data_model_p0/unique/test_unique_bitmap.out b/regression-test/data/data_model_p0/unique/test_unique_bitmap.out index 85f559e238..2240e4201a 100644 --- a/regression-test/data/data_model_p0/unique/test_unique_bitmap.out +++ b/regression-test/data/data_model_p0/unique/test_unique_bitmap.out @@ -19,3 +19,23 @@ 2 2 3,1000 3 8 0,1,2,3,5,99,876,2445 +-- !sql -- +1 1 1 +2 2 3,1000 +3 3 999,1000,888888 + +-- !sql -- +1 4 5,90,876,1000 +2 2 3,1000 +3 8 0,1,2,3,5,99,876,2445 + +-- !sql -- +1 1 1 +2 2 3,1000 +3 3 999,1000,888888 + +-- !sql -- +1 4 5,90,876,1000 +2 2 3,1000 +3 8 0,1,2,3,5,99,876,2445 + diff --git a/regression-test/data/data_model_p0/unique/test_unique_hll.out b/regression-test/data/data_model_p0/unique/test_unique_hll.out new file mode 100644 index 0000000000..c6bc798fa3 --- /dev/null +++ b/regression-test/data/data_model_p0/unique/test_unique_hll.out @@ -0,0 +1,185 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 0 +2 6 +3 2 + +-- !sql -- +7 + +-- !from_agg -- +1 0 +2 6 +3 2 + +-- !from_agg -- +7 + +-- !from_values -- +1 1 +2 1 +3 2 +4 1 + +-- !from_values -- +1 1 +2 1 +3 2 +4 1 + +-- !from_values -- +4 + +-- !from_uniq -- +1 1 +2 1 +3 2 +4 1 + +-- !from_uniq -- +1 1 +2 1 +3 2 +4 1 + +-- !from_uniq -- +4 + +-- !sql -- +1 0 +2 6 +3 2 + +-- !sql -- +7 + +-- !from_agg -- +1 0 +2 6 +3 2 + +-- !from_agg -- +7 + +-- !from_values -- +1 1 +2 1 +3 2 +4 1 + +-- !from_values -- +1 1 +2 1 +3 2 +4 1 + +-- !from_values -- +4 + +-- !from_uniq -- +1 1 +2 1 +3 2 +4 1 + +-- !from_uniq -- +1 1 +2 1 +3 2 +4 1 + +-- !from_uniq -- +4 + +-- !sql -- +1 0 +2 6 +3 2 + +-- !sql -- +7 + +-- !from_agg -- +1 0 +2 6 +3 2 + +-- !from_agg -- +7 + +-- !from_values -- +1 1 +2 1 +3 2 +4 1 + +-- !from_values -- +1 1 +2 1 +3 2 +4 1 + +-- !from_values -- +4 + +-- !from_uniq -- +1 1 +2 1 +3 2 +4 1 + +-- !from_uniq -- +1 1 +2 1 +3 2 +4 1 + +-- !from_uniq -- +4 + +-- !sql -- +1 0 +2 6 +3 2 + +-- !sql -- +7 + +-- !from_agg -- +1 0 +2 6 +3 2 + +-- !from_agg -- +7 + +-- !from_values -- +1 1 +2 1 +3 2 +4 1 + +-- !from_values -- +1 1 +2 1 +3 2 +4 1 + +-- !from_values -- +4 + +-- !from_uniq -- +1 1 +2 1 +3 2 +4 1 + +-- !from_uniq -- +1 1 +2 1 +3 2 +4 1 + +-- !from_uniq -- +4 + diff --git a/regression-test/data/data_model_p0/unique/test_unique_quantile_state.out b/regression-test/data/data_model_p0/unique/test_unique_quantile_state.out new file mode 100644 index 0000000000..280ea0a2cc --- /dev/null +++ b/regression-test/data/data_model_p0/unique/test_unique_quantile_state.out @@ -0,0 +1,121 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 -1.0 -1.0 -1.0 +2 0.0 0.5 1.0 +3 0.0 1.0 2.0 + +-- !from_agg -- +1 -1.0 -1.0 -1.0 +2 0.0 0.5 1.0 +3 0.0 1.0 2.0 + +-- !from_values -- +1 0.0 0.0 0.0 +2 -100.0 -100.0 -100.0 +3 0.0 1.0 2.0 + +-- !from_values -- +1 0.0 0.0 0.0 +2 -100.0 -100.0 -100.0 +3 0.0 1.0 2.0 + +-- !from_uniq -- +1 0.0 0.0 0.0 +2 -100.0 -100.0 -100.0 +3 0.0 1.0 2.0 + +-- !from_uniq -- +1 0.0 0.0 0.0 +2 -100.0 -100.0 -100.0 +3 0.0 1.0 2.0 + +-- !sql -- +1 -1.0 -1.0 -1.0 +2 0.0 0.5 1.0 +3 0.0 1.0 2.0 + +-- !from_agg -- +1 -1.0 -1.0 -1.0 +2 0.0 0.5 1.0 +3 0.0 1.0 2.0 + +-- !from_values -- +1 0.0 0.0 0.0 +2 -100.0 -100.0 -100.0 +3 0.0 1.0 2.0 + +-- !from_values -- +1 0.0 0.0 0.0 +2 -100.0 -100.0 -100.0 +3 0.0 1.0 2.0 + +-- !from_uniq -- +1 0.0 0.0 0.0 +2 -100.0 -100.0 -100.0 +3 0.0 1.0 2.0 + +-- !from_uniq -- +1 0.0 0.0 0.0 +2 -100.0 -100.0 -100.0 +3 0.0 1.0 2.0 + +-- !sql -- +1 -1.0 -1.0 -1.0 +2 0.0 0.5 1.0 +3 0.0 1.0 2.0 + +-- !from_agg -- +1 -1.0 -1.0 -1.0 +2 0.0 0.5 1.0 +3 0.0 1.0 2.0 + +-- !from_values -- +1 0.0 0.0 0.0 +2 -100.0 -100.0 -100.0 +3 0.0 1.0 2.0 + +-- !from_values -- +1 0.0 0.0 0.0 +2 -100.0 -100.0 -100.0 +3 0.0 1.0 2.0 + +-- !from_uniq -- +1 0.0 0.0 0.0 +2 -100.0 -100.0 -100.0 +3 0.0 1.0 2.0 + +-- !from_uniq -- +1 0.0 0.0 0.0 +2 -100.0 -100.0 -100.0 +3 0.0 1.0 2.0 + +-- !sql -- +1 -1.0 -1.0 -1.0 +2 0.0 0.5 1.0 +3 0.0 1.0 2.0 + +-- !from_agg -- +1 -1.0 -1.0 -1.0 +2 0.0 0.5 1.0 +3 0.0 1.0 2.0 + +-- !from_values -- +1 0.0 0.0 0.0 +2 -100.0 -100.0 -100.0 +3 0.0 1.0 2.0 + +-- !from_values -- +1 0.0 0.0 0.0 +2 -100.0 -100.0 -100.0 +3 0.0 1.0 2.0 + +-- !from_uniq -- +1 0.0 0.0 0.0 +2 -100.0 -100.0 -100.0 +3 0.0 1.0 2.0 + +-- !from_uniq -- +1 0.0 0.0 0.0 +2 -100.0 -100.0 -100.0 +3 0.0 1.0 2.0 + diff --git a/regression-test/suites/data_model_p0/duplicate/storage/test_duplicate_bitmap.groovy b/regression-test/suites/data_model_p0/duplicate/storage/test_duplicate_bitmap.groovy index 9dc9052a48..36586b7dc3 100644 --- a/regression-test/suites/data_model_p0/duplicate/storage/test_duplicate_bitmap.groovy +++ b/regression-test/suites/data_model_p0/duplicate/storage/test_duplicate_bitmap.groovy @@ -17,73 +17,52 @@ suite("test_duplicate_table_bitmap") { - // 1. nerieds planner - sql "set enable_nereids_planner=true" - sql "set enable_fallback_to_original_planner=false" - sql "set enable_nereids_dml=true;" - sql "sync;" - def tbName = "test_duplicate_bitmap1" - sql "DROP TABLE IF EXISTS ${tbName}" - sql """ CREATE TABLE IF NOT EXISTS ${tbName} ( - k int, - id_bitmap bitmap - ) DUPLICATE KEY(k) - DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1"); """ - sql "insert into ${tbName} values(1,to_bitmap(1));" - sql "insert into ${tbName} values(2,bitmap_or(to_bitmap(3),to_bitmap(1000)));" - sql "insert into ${tbName} values(3,bitmap_or(to_bitmap(999),to_bitmap(1000),to_bitmap(888888)));" - qt_sql "select k,bitmap_count(id_bitmap),bitmap_to_string(id_bitmap) from ${tbName} order by k, bitmap_count(id_bitmap);" - sql "insert into ${tbName} values(3,bitmap_from_string('1,0,1,2,3,1,5,99,876,2445'));" - sql "insert into ${tbName} values(1,bitmap_or(bitmap_from_string('90,5,876'),to_bitmap(1000)));" - qt_sql "select k,bitmap_count(id_bitmap),bitmap_to_string(id_bitmap) from ${tbName} order by k, bitmap_count(id_bitmap);" - sql "insert into ${tbName} select * from ${tbName};" - qt_sql "select k,bitmap_count(id_bitmap),bitmap_to_string(id_bitmap) from ${tbName} order by k, bitmap_count(id_bitmap);" - sql "DROP TABLE IF EXISTS ${tbName};" + for (def use_nereids : [true, false]) { + if (use_nereids) { + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_nereids_dml=true;" + } else { + sql "set enable_nereids_planner=false" + sql "set enable_nereids_dml=false;" + } + sql "sync;" - tbName = "test_duplicate_bitmap2" - sql "DROP TABLE IF EXISTS ${tbName}" - test { + def tbName = "test_duplicate_bitmap1" + sql "DROP TABLE IF EXISTS ${tbName}" sql """ CREATE TABLE IF NOT EXISTS ${tbName} ( - k bitmap, - v int - ) DUPLICATE KEY(k) - DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1"); """ - exception "Key column can not set complex type:k" + k int, + id_bitmap bitmap + ) DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1"); """ + + def result = sql "show create table ${tbName}" + logger.info("${result}") + assertTrue(result.toString().containsIgnoreCase('`id_bitmap` BITMAP NOT NULL')) + + sql "insert into ${tbName} values(1,to_bitmap(1));" + sql "insert into ${tbName} values(2,bitmap_or(to_bitmap(3),to_bitmap(1000)));" + sql "insert into ${tbName} values(3,bitmap_or(to_bitmap(999),to_bitmap(1000),to_bitmap(888888)));" + qt_sql "select k,bitmap_count(id_bitmap),bitmap_to_string(id_bitmap) from ${tbName} order by k, bitmap_count(id_bitmap);" + + sql "insert into ${tbName} values(3,bitmap_from_string('1,0,1,2,3,1,5,99,876,2445'));" + sql "insert into ${tbName} values(1,bitmap_or(bitmap_from_string('90,5,876'),to_bitmap(1000)));" + qt_sql "select k,bitmap_count(id_bitmap),bitmap_to_string(id_bitmap) from ${tbName} order by k, bitmap_count(id_bitmap);" + + sql "insert into ${tbName} select * from ${tbName};" + qt_sql "select k,bitmap_count(id_bitmap),bitmap_to_string(id_bitmap) from ${tbName} order by k, bitmap_count(id_bitmap);" + + sql "DROP TABLE IF EXISTS ${tbName};" + + tbName = "test_duplicate_bitmap2" + sql "DROP TABLE IF EXISTS ${tbName}" + test { + sql """ CREATE TABLE IF NOT EXISTS ${tbName} ( + k bitmap, + v int + ) DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1"); """ + exception "Key column can not set complex type:k" + } } - - - // 2. origin planner - sql "set enable_nereids_planner=false" - sql "set enable_nereids_dml=false;" - sql "sync;" - tbName = "test_duplicate_bitmap3" - sql "DROP TABLE IF EXISTS ${tbName}" - sql """ CREATE TABLE IF NOT EXISTS ${tbName} ( - k int, - id_bitmap bitmap - ) DUPLICATE KEY(k) - DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1"); """ - sql "insert into ${tbName} values(1,to_bitmap(1));" - sql "insert into ${tbName} values(2,bitmap_or(to_bitmap(3),to_bitmap(1000)));" - sql "insert into ${tbName} values(3,bitmap_or(to_bitmap(999),to_bitmap(1000),to_bitmap(888888)));" - qt_sql "select k,bitmap_count(id_bitmap),bitmap_to_string(id_bitmap) from ${tbName} order by k, bitmap_count(id_bitmap);" - sql "insert into ${tbName} values(3,bitmap_from_string('1,0,1,2,3,1,5,99,876,2445'));" - sql "insert into ${tbName} values(1,bitmap_or(bitmap_from_string('90,5,876'),to_bitmap(1000)));" - qt_sql "select k,bitmap_count(id_bitmap),bitmap_to_string(id_bitmap) from ${tbName} order by k, bitmap_count(id_bitmap);" - sql "insert into ${tbName} select * from ${tbName};" - qt_sql "select k,bitmap_count(id_bitmap),bitmap_to_string(id_bitmap) from ${tbName} order by k, bitmap_count(id_bitmap);" - sql "DROP TABLE IF EXISTS ${tbName};" - - - tbName = "test_duplicate_bitmap4" - sql "DROP TABLE IF EXISTS ${tbName}" - test { - sql """ CREATE TABLE IF NOT EXISTS ${tbName} ( - k bitmap, - v int - ) DUPLICATE KEY(k) - DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1"); """ - exception "Key column can not set complex type:k" - } - } diff --git a/regression-test/suites/data_model_p0/duplicate/storage/test_duplicate_hll.groovy b/regression-test/suites/data_model_p0/duplicate/storage/test_duplicate_hll.groovy new file mode 100644 index 0000000000..5185d5c93c --- /dev/null +++ b/regression-test/suites/data_model_p0/duplicate/storage/test_duplicate_hll.groovy @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_duplicate_table_hll") { + + for (def use_nereids : [true, false]) { + if (use_nereids) { + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_nereids_dml=true;" + } else { + sql "set enable_nereids_planner=false" + sql "set enable_nereids_dml=false;" + } + sql "sync;" + + def tbName = "test_duplicate_hll1" + sql "DROP TABLE IF EXISTS ${tbName}" + sql """ CREATE TABLE IF NOT EXISTS ${tbName} ( k int, v hll ) DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1"); """ + + def result = sql "show create table ${tbName}" + logger.info("${result}") + assertTrue(result.toString().containsIgnoreCase('`v` HLL NOT NULL')) + + def tbNameAgg = "test_duplicate_hll_agg1" + sql "DROP TABLE IF EXISTS ${tbNameAgg}" + sql """ CREATE TABLE IF NOT EXISTS ${tbNameAgg} ( k int, v hll hll_union ) AGGREGATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1"); """ + + sql """ insert into ${tbNameAgg} values + (1,hll_empty()),(2, hll_hash(100)), + (2,hll_hash(0)),(2, hll_hash(4875)), + (2,hll_hash(9234)),(2, hll_hash(45)), + (2,hll_hash(0)),(2,hll_hash(100000)), + (3,hll_hash(0)),(3,hll_hash(1)); """ + qt_sql "select k, hll_cardinality(v) from ${tbNameAgg} order by k;" + qt_sql "select HLL_UNION_AGG(v) from ${tbNameAgg};" + + // 1. insert from aggregate table + sql "insert into ${tbName} select * from ${tbNameAgg};" + qt_from_agg "select k, hll_cardinality(v) from ${tbName} order by k, hll_cardinality(v);" + qt_from_agg "select HLL_UNION_AGG(v) from ${tbName};" + // 2. insert into values + sql """ insert into ${tbName} values (4, hll_hash(100)), (1, hll_hash(999)), (2, hll_hash(0));""" + qt_from_values "select k, hll_cardinality(v) from ${tbName} order by k, hll_cardinality(v);" + qt_from_values "select k, hll_cardinality(hll_union(v)) from ${tbName} group by k order by k, hll_cardinality(hll_union(v));" + qt_from_values "select HLL_UNION_AGG(v) from ${tbName};" + // 3. insert from duplicate table + sql "insert into ${tbName} select * from ${tbName};" + qt_from_dup "select k, hll_cardinality(v) from ${tbName} order by k, hll_cardinality(v);" + qt_from_dup "select k, hll_cardinality(hll_union(v)) from ${tbName} group by k order by k, hll_cardinality(hll_union(v));" + qt_from_dup "select HLL_UNION_AGG(v) from ${tbName};" + + sql "DROP TABLE IF EXISTS ${tbName};" + sql "DROP TABLE IF EXISTS ${tbNameAgg};" + + + tbName = "test_duplicate_hll3" + sql "DROP TABLE IF EXISTS ${tbName}" + test { + sql """ CREATE TABLE IF NOT EXISTS ${tbName} (k hll, v int) DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1"); """ + exception "Key column can not set complex type:k" + } + } +} diff --git a/regression-test/suites/data_model_p0/duplicate/storage/test_duplicate_quantile_state.groovy b/regression-test/suites/data_model_p0/duplicate/storage/test_duplicate_quantile_state.groovy new file mode 100644 index 0000000000..b83c37c181 --- /dev/null +++ b/regression-test/suites/data_model_p0/duplicate/storage/test_duplicate_quantile_state.groovy @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_duplicate_table_quantile_state") { + + for (def use_nereids : [true, false]) { + if (use_nereids) { + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_nereids_dml=true;" + } else { + sql "set enable_nereids_planner=false" + sql "set enable_nereids_dml=false;" + } + sql "sync;" + + def tbName = "test_duplicate_quantile_state1" + sql "DROP TABLE IF EXISTS ${tbName}" + sql """ CREATE TABLE IF NOT EXISTS ${tbName} ( k int, v QUANTILE_STATE ) DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1"); """ + + def result = sql "show create table ${tbName}" + logger.info("${result}") + assertTrue(result.toString().containsIgnoreCase('`v` QUANTILE_STATE NOT NULL')) + + def tbNameAgg = "test_duplicate_quantile_state_agg1" + sql "DROP TABLE IF EXISTS ${tbNameAgg}" + sql """ CREATE TABLE IF NOT EXISTS ${tbNameAgg} ( k int, v QUANTILE_STATE QUANTILE_UNION NOT NULL ) AGGREGATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1"); """ + + sql """ insert into ${tbNameAgg} values + (1,to_quantile_state(-1, 2048)),(2,to_quantile_state(0, 2048)), + (2,to_quantile_state(1, 2048)),(3,to_quantile_state(0, 2048)), + (3,to_quantile_state(1, 2048)),(3,to_quantile_state(2, 2048));""" + qt_sql "select k, quantile_percent(v, 0), quantile_percent(v, 0.5), quantile_percent(v, 1) from ${tbNameAgg} order by k;" + + // 1. insert from aggregate table + sql "insert into ${tbName} select * from ${tbNameAgg};" + qt_from_agg "select k, quantile_percent(v, 0) c1, quantile_percent(v, 0.5) c2, quantile_percent(v, 1) c3 from ${tbName} order by k, c1, c2, c3;" + // 2. insert into values + sql """ insert into ${tbName} values (1, to_quantile_state(-2, 2048)), (1, to_quantile_state(0, 2048)), (2, to_quantile_state(-100, 2048));""" + qt_from_values "select k, quantile_percent(v, 0) c1, quantile_percent(v, 0.5) c2, quantile_percent(v, 1) c3 from ${tbName} order by k, c1, c2, c3;" + qt_from_values """ select k, quantile_percent(QUANTILE_UNION(v), 0) c1, quantile_percent(QUANTILE_UNION(v), 0.5) c2, quantile_percent(QUANTILE_UNION(v), 1) c3 + from ${tbName} group by k order by k, c1, c2, c3; """ + // 3. insert from duplicate table + sql "insert into ${tbName} select * from ${tbName};" + qt_from_dup "select k, quantile_percent(v, 0) c1, quantile_percent(v, 0.5) c2, quantile_percent(v, 1) c3 from ${tbName} order by k, c1, c2, c3;" + qt_from_dup """ select k, quantile_percent(QUANTILE_UNION(v), 0) c1, quantile_percent(QUANTILE_UNION(v), 0.5) c2, quantile_percent(QUANTILE_UNION(v), 1) c3 + from ${tbName} group by k order by k, c1, c2, c3; """ + + sql "DROP TABLE IF EXISTS ${tbName};" + sql "DROP TABLE IF EXISTS ${tbNameAgg};" + + + tbName = "test_duplicate_quantile_state3" + sql "DROP TABLE IF EXISTS ${tbName}" + test { + sql """ CREATE TABLE IF NOT EXISTS ${tbName} (k QUANTILE_STATE, v int) DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1"); """ + exception "Key column can not set complex type:k" + } + } +} diff --git a/regression-test/suites/data_model_p0/unique/test_unique_bitmap.groovy b/regression-test/suites/data_model_p0/unique/test_unique_bitmap.groovy index 64cf809eea..a0fec954cb 100644 --- a/regression-test/suites/data_model_p0/unique/test_unique_bitmap.groovy +++ b/regression-test/suites/data_model_p0/unique/test_unique_bitmap.groovy @@ -16,40 +16,43 @@ // under the License. suite("test_unique_table_bitmap") { - def tbName = "test_uniq_table_bitmap1" - sql "DROP TABLE IF EXISTS ${tbName}" - sql """ - CREATE TABLE IF NOT EXISTS ${tbName} ( - k int, - id_bitmap bitmap - ) UNIQUE KEY(k) - DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1"); - """ - sql "insert into ${tbName} values(1,to_bitmap(1));" - sql "insert into ${tbName} values(2,bitmap_or(to_bitmap(3),to_bitmap(1000)));" - sql "insert into ${tbName} values(3,bitmap_or(to_bitmap(999),to_bitmap(1000),to_bitmap(888888)));" - qt_sql "select k,bitmap_count(id_bitmap),bitmap_to_string(id_bitmap) from ${tbName} order by k;" - sql "insert into ${tbName} values(3,bitmap_from_string('1,0,1,2,3,1,5,99,876,2445'));" - sql "insert into ${tbName} values(1,bitmap_or(bitmap_from_string('90,5,876'),to_bitmap(1000)));" - qt_sql "select k,bitmap_count(id_bitmap),bitmap_to_string(id_bitmap) from ${tbName} order by k;" - sql "DROP TABLE ${tbName};" - def tbName2 = "test_uniq_table_bitmap2" - sql "DROP TABLE IF EXISTS ${tbName2}" - sql """ - CREATE TABLE IF NOT EXISTS ${tbName2} ( - k int, - id_bitmap bitmap - ) UNIQUE KEY(k) - DISTRIBUTED BY HASH(k) BUCKETS 1 - properties("replication_num" = "1", "enable_unique_key_merge_on_write" = "true"); - """ - sql "insert into ${tbName2} values(1,to_bitmap(1));" - sql "insert into ${tbName2} values(2,bitmap_or(to_bitmap(3),to_bitmap(1000)));" - sql "insert into ${tbName2} values(3,bitmap_or(to_bitmap(999),to_bitmap(1000),to_bitmap(888888)));" - qt_sql "select k,bitmap_count(id_bitmap),bitmap_to_string(id_bitmap) from ${tbName2} order by k;" - sql "insert into ${tbName2} values(3,bitmap_from_string('1,0,1,2,3,1,5,99,876,2445'));" - sql "insert into ${tbName2} values(1,bitmap_or(bitmap_from_string('90,5,876'),to_bitmap(1000)));" - qt_sql "select k,bitmap_count(id_bitmap),bitmap_to_string(id_bitmap) from ${tbName2} order by k;" - sql "DROP TABLE ${tbName2};" + for (def enable_mow : [true, false]) { + for (def use_nereids : [true, false]) { + if (use_nereids) { + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_nereids_dml=true;" + } else { + sql "set enable_nereids_planner=false" + sql "set enable_nereids_dml=false;" + } + sql "sync;" + + def tbName = "test_uniq_table_bitmap" + sql "DROP TABLE IF EXISTS ${tbName}" + sql """ + CREATE TABLE IF NOT EXISTS ${tbName} ( + k int, + id_bitmap bitmap + ) UNIQUE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1", "enable_unique_key_merge_on_write" = "${enable_mow}"); + """ + + def result = sql "show create table ${tbName}" + logger.info("${result}") + assertTrue(result.toString().containsIgnoreCase('`id_bitmap` BITMAP NOT NULL')) + + sql "insert into ${tbName} values(1,to_bitmap(1));" + sql "insert into ${tbName} values(2,bitmap_or(to_bitmap(3),to_bitmap(1000)));" + sql "insert into ${tbName} values(3,bitmap_or(to_bitmap(999),to_bitmap(1000),to_bitmap(888888)));" + qt_sql "select k,bitmap_count(id_bitmap),bitmap_to_string(id_bitmap) from ${tbName} order by k;" + + sql "insert into ${tbName} values(3,bitmap_from_string('1,0,1,2,3,1,5,99,876,2445'));" + sql "insert into ${tbName} values(1,bitmap_or(bitmap_from_string('90,5,876'),to_bitmap(1000)));" + qt_sql "select k,bitmap_count(id_bitmap),bitmap_to_string(id_bitmap) from ${tbName} order by k;" + + sql "DROP TABLE ${tbName};" + } + } } diff --git a/regression-test/suites/data_model_p0/unique/test_unique_hll.groovy b/regression-test/suites/data_model_p0/unique/test_unique_hll.groovy new file mode 100644 index 0000000000..a4551e0ccf --- /dev/null +++ b/regression-test/suites/data_model_p0/unique/test_unique_hll.groovy @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_unique_table_hll") { + + for (def enable_mow : [true, false]) { + for (def use_nereids : [true, false]) { + if (use_nereids) { + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_nereids_dml=true;" + } else { + sql "set enable_nereids_planner=false" + sql "set enable_nereids_dml=false;" + } + sql "sync;" + + def tbName = "test_unique_hll1" + sql "DROP TABLE IF EXISTS ${tbName}" + sql """ CREATE TABLE IF NOT EXISTS ${tbName} ( k int, v hll ) UNIQUE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1", "enable_unique_key_merge_on_write" = "${enable_mow}"); """ + + def result = sql "show create table ${tbName}" + logger.info("${result}") + assertTrue(result.toString().containsIgnoreCase('`v` HLL NOT NULL')) + + def tbNameAgg = "test_unique_hll_agg1" + sql "DROP TABLE IF EXISTS ${tbNameAgg}" + sql """ CREATE TABLE IF NOT EXISTS ${tbNameAgg} ( k int, v hll hll_union ) AGGREGATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1"); """ + + sql """ insert into ${tbNameAgg} values + (1,hll_empty()),(2, hll_hash(100)), + (2,hll_hash(0)),(2, hll_hash(4875)), + (2,hll_hash(9234)),(2, hll_hash(45)), + (2,hll_hash(0)),(2,hll_hash(100000)), + (3,hll_hash(0)),(3,hll_hash(1)); """ + qt_sql "select k, hll_cardinality(v) from ${tbNameAgg} order by k;" + qt_sql "select HLL_UNION_AGG(v) from ${tbNameAgg};" + + // 1. insert from aggregate table + sql "insert into ${tbName} select * from ${tbNameAgg};" + qt_from_agg "select k, hll_cardinality(v) from ${tbName} order by k, hll_cardinality(v);" + qt_from_agg "select HLL_UNION_AGG(v) from ${tbName};" + // 2. insert into values + sql """ insert into ${tbName} values (4, hll_hash(100)), (1, hll_hash(999)), (2, hll_hash(0));""" + qt_from_values "select k, hll_cardinality(v) from ${tbName} order by k, hll_cardinality(v);" + qt_from_values "select k, hll_cardinality(hll_union(v)) from ${tbName} group by k order by k, hll_cardinality(hll_union(v));" + qt_from_values "select HLL_UNION_AGG(v) from ${tbName};" + // 3. insert from UNIQUE table + sql "insert into ${tbName} select * from ${tbName};" + qt_from_uniq "select k, hll_cardinality(v) from ${tbName} order by k, hll_cardinality(v);" + qt_from_uniq "select k, hll_cardinality(hll_union(v)) from ${tbName} group by k order by k, hll_cardinality(hll_union(v));" + qt_from_uniq "select HLL_UNION_AGG(v) from ${tbName};" + + sql "DROP TABLE IF EXISTS ${tbName};" + sql "DROP TABLE IF EXISTS ${tbNameAgg};" + + + tbName = "test_unique_hll3" + sql "DROP TABLE IF EXISTS ${tbName}" + test { + sql """ CREATE TABLE IF NOT EXISTS ${tbName} (k hll, v int) UNIQUE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1"); """ + exception "Key column can not set complex type:k" + } + } + } +} diff --git a/regression-test/suites/data_model_p0/unique/test_unique_quantile_state.groovy b/regression-test/suites/data_model_p0/unique/test_unique_quantile_state.groovy new file mode 100644 index 0000000000..f7eeed8d82 --- /dev/null +++ b/regression-test/suites/data_model_p0/unique/test_unique_quantile_state.groovy @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_unique_table_quantile_state") { + + for (def enable_mow : [true, false]) { + for (def use_nereids : [true, false]) { + if (use_nereids) { + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_nereids_dml=true;" + } else { + sql "set enable_nereids_planner=false" + sql "set enable_nereids_dml=false;" + } + sql "sync;" + + def tbName = "test_unique_quantile_state1" + sql "DROP TABLE IF EXISTS ${tbName}" + sql """ CREATE TABLE IF NOT EXISTS ${tbName} ( k int, v QUANTILE_STATE ) UNIQUE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1", "enable_unique_key_merge_on_write" = "${enable_mow}"); """ + + def result = sql "show create table ${tbName}" + logger.info("${result}") + assertTrue(result.toString().containsIgnoreCase('`v` QUANTILE_STATE NOT NULL')) + + def tbNameAgg = "test_unique_quantile_state_agg1" + sql "DROP TABLE IF EXISTS ${tbNameAgg}" + sql """ CREATE TABLE IF NOT EXISTS ${tbNameAgg} ( k int, v QUANTILE_STATE QUANTILE_UNION NOT NULL ) AGGREGATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1"); """ + + sql """ insert into ${tbNameAgg} values + (1,to_quantile_state(-1, 2048)),(2,to_quantile_state(0, 2048)), + (2,to_quantile_state(1, 2048)),(3,to_quantile_state(0, 2048)), + (3,to_quantile_state(1, 2048)),(3,to_quantile_state(2, 2048));""" + qt_sql "select k, quantile_percent(v, 0), quantile_percent(v, 0.5), quantile_percent(v, 1) from ${tbNameAgg} order by k;" + + // 1. insert from aggregate table + sql "insert into ${tbName} select * from ${tbNameAgg};" + qt_from_agg "select k, quantile_percent(v, 0) c1, quantile_percent(v, 0.5) c2, quantile_percent(v, 1) c3 from ${tbName} order by k, c1, c2, c3;" + // 2. insert into values + sql """ insert into ${tbName} values (1, to_quantile_state(-2, 2048)), (1, to_quantile_state(0, 2048)), (2, to_quantile_state(-100, 2048));""" + qt_from_values "select k, quantile_percent(v, 0) c1, quantile_percent(v, 0.5) c2, quantile_percent(v, 1) c3 from ${tbName} order by k, c1, c2, c3;" + qt_from_values """ select k, quantile_percent(QUANTILE_UNION(v), 0) c1, quantile_percent(QUANTILE_UNION(v), 0.5) c2, quantile_percent(QUANTILE_UNION(v), 1) c3 + from ${tbName} group by k order by k, c1, c2, c3; """ + // 3. insert from UNIQUE table + sql "insert into ${tbName} select * from ${tbName};" + qt_from_uniq "select k, quantile_percent(v, 0) c1, quantile_percent(v, 0.5) c2, quantile_percent(v, 1) c3 from ${tbName} order by k, c1, c2, c3;" + qt_from_uniq """ select k, quantile_percent(QUANTILE_UNION(v), 0) c1, quantile_percent(QUANTILE_UNION(v), 0.5) c2, quantile_percent(QUANTILE_UNION(v), 1) c3 + from ${tbName} group by k order by k, c1, c2, c3; """ + + sql "DROP TABLE IF EXISTS ${tbName};" + sql "DROP TABLE IF EXISTS ${tbNameAgg};" + + + tbName = "test_unique_quantile_state3" + sql "DROP TABLE IF EXISTS ${tbName}" + test { + sql """ CREATE TABLE IF NOT EXISTS ${tbName} (k QUANTILE_STATE, v int) UNIQUE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1"); """ + exception "Key column can not set complex type:k" + } + } + } +}