database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improvement](dynamic partition) Fix dynamic partition no bucket (#18300)

Browse Source 
Signed-off-by: Jack Drogon <jack.xsuperman@gmail.com>
This commit is contained in:
Jack Drogon
2023-04-02 15:51:21 +08:00
committed by GitHub
parent 97aab138aa
commit 7d49d9cf99
6 changed files with 110 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
View File

@ -1755,7 +1755,7 @@ public class SchemaChangeHandler extends AlterHandler {
if (!olapTable.dynamicPartitionExists()) {
try {
DynamicPartitionUtil.checkInputDynamicPartitionProperties(properties,
olapTable.getPartitionInfo());
olapTable);
} catch (DdlException e) {
// This table is not a dynamic partition table
// and didn't supply all dynamic partition properties

12
fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.common.util;
import org.apache.doris.analysis.TimestampArithmeticExpr.TimeUnit;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.DistributionInfo;
import org.apache.doris.catalog.DynamicPartitionProperty;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
@ -399,10 +400,12 @@ public class DynamicPartitionUtil {
// Check if all requried properties has been set.
// And also check all optional properties, if not set, set them to default value.
public static boolean checkInputDynamicPartitionProperties(Map<String, String> properties,
PartitionInfo partitionInfo) throws DdlException {
OlapTable olapTable) throws DdlException {
if (properties == null || properties.isEmpty()) {
return false;
}
PartitionInfo partitionInfo = olapTable.getPartitionInfo();
if (partitionInfo.getType() != PartitionType.RANGE || partitionInfo.isMultiColumnPartition()) {
throw new DdlException("Dynamic partition only support single-column range partition");
}
@ -443,7 +446,9 @@ public class DynamicPartitionUtil {
throw new DdlException("Must assign dynamic_partition.end properties");
}
if (Strings.isNullOrEmpty(buckets)) {
throw new DdlException("Must assign dynamic_partition.buckets properties");
DistributionInfo distributionInfo = olapTable.getDefaultDistributionInfo();
buckets = String.valueOf(distributionInfo.getBucketNum());
properties.put(DynamicPartitionProperty.BUCKETS, buckets);
}
if (Strings.isNullOrEmpty(timeZone)) {
properties.put(DynamicPartitionProperty.TIME_ZONE, TimeUtils.getSystemTimeZone().getID());
@ -506,6 +511,7 @@ public class DynamicPartitionUtil {
properties.remove(DynamicPartitionProperty.BUCKETS);
analyzedProperties.put(DynamicPartitionProperty.BUCKETS, bucketsValue);
}
if (properties.containsKey(DynamicPartitionProperty.ENABLE)) {
String enableValue = properties.get(DynamicPartitionProperty.ENABLE);
checkEnable(enableValue);
@ -675,7 +681,7 @@ public class DynamicPartitionUtil {
*/
public static void checkAndSetDynamicPartitionProperty(OlapTable olapTable, Map<String, String> properties,
Database db) throws UserException {
if (DynamicPartitionUtil.checkInputDynamicPartitionProperties(properties, olapTable.getPartitionInfo())) {
if (DynamicPartitionUtil.checkInputDynamicPartitionProperties(properties, olapTable)) {
Map<String, String> dynamicPartitionProperties =
DynamicPartitionUtil.analyzeDynamicPartition(properties, olapTable, db);
TableProperty tableProperty = olapTable.getTableProperty();

6
fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
View File

@ -258,7 +258,7 @@ public class DynamicPartitionTableTest {
@Test
public void testMissBuckets() throws Exception {
String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_buckets` (\n"
String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_miss_buckets` (\n"
+ " `k1` date NULL COMMENT \"\",\n"
+ " `k2` int NULL COMMENT \"\",\n"
+ " `k3` smallint NULL COMMENT \"\",\n"
@ -282,14 +282,12 @@ public class DynamicPartitionTableTest {
+ "\"dynamic_partition.time_unit\" = \"day\",\n"
+ "\"dynamic_partition.prefix\" = \"p\"\n"
+ ");";
expectedException.expect(DdlException.class);
expectedException.expectMessage("errCode = 2, detailMessage = Must assign dynamic_partition.buckets properties");
createTable(createOlapTblStmt);
}
@Test
public void testNotAllowed() throws Exception {
String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_buckets` (\n"
String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_not_allowed` (\n"
+ " `k1` date NULL COMMENT \"\",\n"
+ " `k2` int NULL COMMENT \"\",\n"
+ " `k3` smallint NULL COMMENT \"\",\n"

3
regression-test/suites/autobucket/test_autobucket.groovy
View File

@ -25,7 +25,7 @@ suite("test_autobucket") {
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`user_id`) BUCKETS AUTO
PROPERTIES (
"replication_allocation" = "tag.location.default: 1"
"replication_allocation" = "tag.location.default: 1"
)
"""
@ -35,6 +35,7 @@ suite("test_autobucket") {
result = sql "show partitions from autobucket_test"
logger.info("${result}")
// XXX: buckets at pos(8), next maybe impl by sql meta
// 10 is the default buckets without partition size
assertEquals(Integer.valueOf(result.get(0).get(8)), 10)
sql "drop table if exists autobucket_test"

48
regression-test/suites/autobucket/test_autobucket_dynamic_partition.groovy Normal file
View File

@ -0,0 +1,48 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_autobucket_dynamic_partition") {
sql "drop table if exists test_autobucket_dynamic_partition"
result = sql """
CREATE TABLE
test_autobucket_dynamic_partition (k1 DATETIME)
PARTITION BY
RANGE (k1) () DISTRIBUTED BY HASH (k1) BUCKETS AUTO
PROPERTIES (
"dynamic_partition.enable" = "true",
"dynamic_partition.time_unit" = "WEEK",
"dynamic_partition.start" = "-2",
"dynamic_partition.end" = "2",
"dynamic_partition.prefix" = "p",
"replication_allocation" = "tag.location.default: 1"
)
"""
result = sql "show create table test_autobucket_dynamic_partition"
assertTrue(result.toString().containsIgnoreCase("BUCKETS AUTO"))
result = sql "show partitions from test_autobucket_dynamic_partition"
logger.info("${result}")
// XXX: buckets at pos(8), next maybe impl by sql meta
// 10 is the default buckets without partition size
assertEquals(result.size(), 3)
assertEquals(Integer.valueOf(result.get(0).get(8)), 10)
assertEquals(Integer.valueOf(result.get(1).get(8)), 10)
assertEquals(Integer.valueOf(result.get(2).get(8)), 10)
sql "drop table if exists test_autobucket_dynamic_partition"
}

73
regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy
View File

@ -14,44 +14,68 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_dynamic_partition") {
// todo: test dynamic partition
sql "drop table if exists dy_par"
sql """
CREATE TABLE IF NOT EXISTS dy_par ( k1 date NOT NULL, k2 varchar(20) NOT NULL, k3 int sum NOT NULL )
AGGREGATE KEY(k1,k2)
PARTITION BY RANGE(k1) ( )
DISTRIBUTED BY HASH(k1) BUCKETS 3
PROPERTIES (
"dynamic_partition.enable"="true",
"dynamic_partition.end"="3",
"dynamic_partition.buckets"="10",
"dynamic_partition.start"="-3",
"dynamic_partition.prefix"="p",
"dynamic_partition.time_unit"="DAY",
CREATE TABLE IF NOT EXISTS dy_par ( k1 date NOT NULL, k2 varchar(20) NOT NULL, k3 int sum NOT NULL )
AGGREGATE KEY(k1,k2)
PARTITION BY RANGE(k1) ( )
DISTRIBUTED BY HASH(k1) BUCKETS 3
PROPERTIES (
"dynamic_partition.enable"="true",
"dynamic_partition.end"="3",
"dynamic_partition.buckets"="10",
"dynamic_partition.start"="-3",
"dynamic_partition.prefix"="p",
"dynamic_partition.time_unit"="DAY",
"dynamic_partition.create_history_partition"="true",
"dynamic_partition.replication_allocation" = "tag.location.default: 1")
"""
List<List<Object>> result = sql "show tables like 'dy_par'"
logger.info("${result}")
assertEquals(result.size(), 1)
result = sql "show partitions from dy_par"
// XXX: buckets at pos(8), next maybe impl by sql meta
assertEquals(Integer.valueOf(result.get(0).get(8)), 10)
sql "drop table dy_par"
//
sql "drop table if exists dy_par_bucket_set_by_distribution"
sql """
CREATE TABLE IF NOT EXISTS dy_par_bucket_set_by_distribution
( k1 date NOT NULL, k2 varchar(20) NOT NULL, k3 int sum NOT NULL )
AGGREGATE KEY(k1,k2)
PARTITION BY RANGE(k1) ( )
DISTRIBUTED BY HASH(k1) BUCKETS 3
PROPERTIES (
"dynamic_partition.enable"="true",
"dynamic_partition.end"="3",
"dynamic_partition.start"="-3",
"dynamic_partition.prefix"="p",
"dynamic_partition.time_unit"="DAY",
"dynamic_partition.create_history_partition"="true",
"dynamic_partition.replication_allocation" = "tag.location.default: 1")
"""
result = sql "show tables like 'dy_par_bucket_set_by_distribution'"
logger.info("${result}")
assertEquals(result.size(), 1)
result = sql "show partitions from dy_par_bucket_set_by_distribution"
// XXX: buckets at pos(8), next maybe impl by sql meta
assertEquals(Integer.valueOf(result.get(0).get(8)), 3)
sql "drop table dy_par_bucket_set_by_distribution"
sql "drop table if exists dy_par_bad"
test {
sql """
CREATE TABLE IF NOT EXISTS dy_par_bad ( k1 date NOT NULL, k2 varchar(20) NOT NULL, k3 int sum NOT NULL )
AGGREGATE KEY(k1,k2)
PARTITION BY RANGE(k1) ( )
DISTRIBUTED BY HASH(k1) BUCKETS 3
PROPERTIES (
"dynamic_partition.enable"="true",
"dynamic_partition.end"="3",
"dynamic_partition.buckets"="10",
"dynamic_partition.start"="-3",
"dynamic_partition.prefix"="p",
"dynamic_partition.time_unit"="DAY",
CREATE TABLE IF NOT EXISTS dy_par_bad ( k1 date NOT NULL, k2 varchar(20) NOT NULL, k3 int sum NOT NULL )
AGGREGATE KEY(k1,k2)
PARTITION BY RANGE(k1) ( )
DISTRIBUTED BY HASH(k1) BUCKETS 3
PROPERTIES (
"dynamic_partition.enable"="true",
"dynamic_partition.end"="3",
"dynamic_partition.buckets"="10",
"dynamic_partition.start"="-3",
"dynamic_partition.prefix"="p",
"dynamic_partition.time_unit"="DAY",
"dynamic_partition.create_history_partition"="true",
"dynamic_partition.replication_allocation" = "tag.location.not_exist_tag: 1")
"""
@ -59,7 +83,6 @@ suite("test_dynamic_partition") {
exception "errCode = 2,"
}
sql "drop table if exists dy_par_bad"
sql """
CREATE TABLE IF NOT EXISTS dy_par ( k1 datev2 NOT NULL, k2 varchar(20) NOT NULL, k3 int sum NOT NULL )
AGGREGATE KEY(k1,k2)