database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhance](mtmv)support replace materialized view (#36749) (#37147)

Browse Source 
pick: https://github.com/apache/doris/pull/36749
This commit is contained in:
zhangdong
2024-07-02 23:00:55 +08:00
committed by GitHub
parent b3eaf0e4d2
commit 65375b48fc
12 changed files with 501 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
View File

@ -108,6 +108,7 @@ statementBase
| REFRESH MATERIALIZED VIEW mvName=multipartIdentifier (partitionSpec | COMPLETE | AUTO) #refreshMTMV
| ALTER MATERIALIZED VIEW mvName=multipartIdentifier ((RENAME newName=identifier)
| (REFRESH (refreshMethod | refreshTrigger | refreshMethod refreshTrigger))
| REPLACE WITH MATERIALIZED VIEW newName=identifier propertyClause?
| (SET LEFT_PAREN fileProperties=propertyItemList RIGHT_PAREN)) #alterMTMV
| DROP MATERIALIZED VIEW (IF EXISTS)? mvName=multipartIdentifier #dropMTMV
| PAUSE MATERIALIZED VIEW JOB ON mvName=multipartIdentifier #pauseMTMV

8
fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
View File

@ -525,6 +525,11 @@ public class Alter {
ReplaceTableClause clause = (ReplaceTableClause) alterClauses.get(0);
String newTblName = clause.getTblName();
boolean swapTable = clause.isSwapTable();
processReplaceTable(db, origTable, newTblName, swapTable);
}
public void processReplaceTable(Database db, OlapTable origTable, String newTblName, boolean swapTable)
throws UserException {
db.writeLockOrDdlException();
try {
List<TableType> tableTypes = Lists.newArrayList(TableType.OLAP, TableType.MATERIALIZED_VIEW);
@ -611,6 +616,9 @@ public class Alter {
} else {
// not swap, the origin table is not used anymore, need to drop all its tablets.
Env.getCurrentEnv().onEraseOlapTable(origTable, isReplay);
if (origTable.getType() == TableType.MATERIALIZED_VIEW) {
Env.getCurrentEnv().getMtmvService().deregisterMTMV((MTMV) origTable);
}
}
}

6
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
View File

@ -390,6 +390,7 @@ import org.apache.doris.nereids.trees.plans.commands.info.AlterMTMVInfo;
import org.apache.doris.nereids.trees.plans.commands.info.AlterMTMVPropertyInfo;
import org.apache.doris.nereids.trees.plans.commands.info.AlterMTMVRefreshInfo;
import org.apache.doris.nereids.trees.plans.commands.info.AlterMTMVRenameInfo;
import org.apache.doris.nereids.trees.plans.commands.info.AlterMTMVReplaceInfo;
import org.apache.doris.nereids.trees.plans.commands.info.AlterViewInfo;
import org.apache.doris.nereids.trees.plans.commands.info.BulkLoadDataDesc;
import org.apache.doris.nereids.trees.plans.commands.info.BulkStorageDesc;
@ -831,6 +832,11 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
} else if (ctx.SET() != null) {
alterMTMVInfo = new AlterMTMVPropertyInfo(mvName,
Maps.newHashMap(visitPropertyItemList(ctx.fileProperties)));
} else if (ctx.REPLACE() != null) {
String newName = ctx.newName.getText();
Map<String, String> properties = ctx.propertyClause() != null
? Maps.newHashMap(visitPropertyClause(ctx.propertyClause())) : Maps.newHashMap();
alterMTMVInfo = new AlterMTMVReplaceInfo(mvName, newName, properties);
}
return new AlterMTMVCommand(alterMTMVInfo);
}

10
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterMTMVInfo.java
View File

@ -18,6 +18,8 @@
package org.apache.doris.nereids.trees.plans.commands.info;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.UserException;
import org.apache.doris.mysql.privilege.PrivPredicate;
@ -51,6 +53,14 @@ public abstract class AlterMTMVInfo {
mvName.getDb() + ": " + mvName.getTbl());
throw new AnalysisException(message);
}
// check mv exist
try {
Env.getCurrentInternalCatalog().getDbOrAnalysisException(mvName.getDb())
.getTableOrDdlException(mvName.getTbl(),
TableType.MATERIALIZED_VIEW);
} catch (DdlException | org.apache.doris.common.AnalysisException e) {
throw new AnalysisException(e.getMessage(), e);
}
}
public abstract void run() throws UserException;

1
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterMTMVRenameInfo.java
View File

@ -61,5 +61,6 @@ public class AlterMTMVRenameInfo extends AlterMTMVInfo {
Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(mvName.getDb());
Table table = db.getTableOrDdlException(mvName.getTbl());
Env.getCurrentEnv().renameTable(db, table, newName);
Env.getCurrentEnv().getMtmvService().alterTable(table);
}
}

101
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterMTMVReplaceInfo.java Normal file
View File

@ -0,0 +1,101 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.plans.commands.info;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.qe.ConnectContext;
import java.util.Map;
import java.util.Objects;
/**
* replace
*/
public class AlterMTMVReplaceInfo extends AlterMTMVInfo {
protected final String newName;
private final Map<String, String> properties;
// parsed from properties.
// if false, after replace, there will be only one table exist with.
// if true, the new table and the old table will be exchanged.
// default is true.
private boolean swapTable;
/**
* constructor for alter MTMV
*/
public AlterMTMVReplaceInfo(TableNameInfo mvName, String newName, Map<String, String> properties) {
super(mvName);
this.newName = Objects.requireNonNull(newName, "require newName object");
this.properties = Objects.requireNonNull(properties, "require properties object");
}
/**
* analyze
*
* @param ctx ctx
* @throws AnalysisException AnalysisException
*/
public void analyze(ConnectContext ctx) throws AnalysisException {
super.analyze(ctx);
if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(ctx, mvName.getCtl(), mvName.getDb(),
newName, PrivPredicate.ALTER)) {
String message = ErrorCode.ERR_TABLEACCESS_DENIED_ERROR.formatErrorMsg("ALTER",
ctx.getQualifiedUser(), ctx.getRemoteIP(),
mvName.getDb() + ": " + newName);
throw new AnalysisException(message);
}
this.swapTable = PropertyAnalyzer.analyzeBooleanProp(properties, PropertyAnalyzer.PROPERTIES_SWAP_TABLE, true);
if (properties != null && !properties.isEmpty()) {
throw new AnalysisException("Unknown properties: " + properties.keySet());
}
// check new mv exist
try {
Env.getCurrentInternalCatalog().getDbOrAnalysisException(mvName.getDb())
.getTableOrDdlException(newName,
TableType.MATERIALIZED_VIEW);
} catch (DdlException | org.apache.doris.common.AnalysisException e) {
throw new AnalysisException(e.getMessage(), e);
}
}
@Override
public void run() throws UserException {
Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(mvName.getDb());
MTMV mtmv = (MTMV) db.getTableOrDdlException(mvName.getTbl(), TableType.MATERIALIZED_VIEW);
MTMV newMtmv = (MTMV) db.getTableOrDdlException(newName, TableType.MATERIALIZED_VIEW);
Env.getCurrentEnv().getAlterInstance().processReplaceTable(db, mtmv, newName, swapTable);
Env.getCurrentEnv().getMtmvService().alterTable(newMtmv);
if (swapTable) {
Env.getCurrentEnv().getMtmvService().alterTable(mtmv);
} else {
Env.getCurrentEnv().getMtmvService().dropMTMV(mtmv);
Env.getCurrentEnv().getMtmvService().dropTable(mtmv);
}
}
}

4
regression-test/data/mtmv_p0/test_multi_level_rename_mtmv.out Normal file
View File

@ -0,0 +1,4 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !status --
test_multi_level_rename_mtmv_mv2 SCHEMA_CHANGE

4
regression-test/data/mtmv_p0/test_multi_level_replace_mtmv.out Normal file
View File

@ -0,0 +1,4 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !status --
test_multi_level_replace_mtmv_mv2 SCHEMA_CHANGE

25
regression-test/data/mtmv_p0/test_replace_mtmv.out Normal file
View File

@ -0,0 +1,25 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !mv1 --
1 1
-- !mv2 --
2 2
-- !mv1_replace --
2 2
-- !mv2_replace --
1 1
-- !mv1_refresh --
2 2
4 4
-- !mv2_refresh --
1 1
3 3
-- !mv1_replace_not_swap --
1 1
3 3

88
regression-test/suites/mtmv_p0/test_multi_level_rename_mtmv.groovy Normal file
View File

@ -0,0 +1,88 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
import org.junit.Assert;
suite("test_multi_level_rename_mtmv","mtmv") {
String dbName = context.config.getDbNameByFile(context.file)
String suiteName = "test_multi_level_rename_mtmv"
String tableName1 = "${suiteName}_table1"
String tableName2 = "${suiteName}_table2"
String mvName1 = "${suiteName}_mv1"
String mvName1_rename = "${suiteName}_mv1_rename"
String mvName2 = "${suiteName}_mv2"
sql """drop table if exists `${tableName1}`"""
sql """drop table if exists `${tableName2}`"""
sql """drop materialized view if exists ${mvName1};"""
sql """drop materialized view if exists ${mvName1_rename};"""
sql """drop materialized view if exists ${mvName2};"""
sql """
CREATE TABLE ${tableName1}
(
k1 TINYINT,
k2 INT not null
)
DISTRIBUTED BY HASH(k2) BUCKETS 2
PROPERTIES (
"replication_num" = "1"
);
"""
sql """
CREATE TABLE ${tableName2}
(
k3 TINYINT,
k4 INT not null
)
DISTRIBUTED BY HASH(k4) BUCKETS 2
PROPERTIES (
"replication_num" = "1"
);
"""
sql """
CREATE MATERIALIZED VIEW ${mvName1}
BUILD DEFERRED REFRESH AUTO ON MANUAL
DISTRIBUTED BY RANDOM BUCKETS 2
PROPERTIES (
'replication_num' = '1'
)
AS
SELECT * from ${tableName1};
"""
sql """
CREATE MATERIALIZED VIEW ${mvName2}
BUILD DEFERRED REFRESH AUTO ON MANUAL
DISTRIBUTED BY RANDOM BUCKETS 2
PROPERTIES (
'replication_num' = '1'
)
AS
SELECT * from ${mvName1};
"""
sql """
alter MATERIALIZED VIEW ${mvName1} rename ${mvName1_rename};
"""
order_qt_status "select Name,State from mv_infos('database'='${dbName}') where Name='${mvName2}'"
sql """drop table if exists `${tableName1}`"""
sql """drop table if exists `${tableName2}`"""
sql """drop materialized view if exists ${mvName1};"""
sql """drop materialized view if exists ${mvName1_rename};"""
sql """drop materialized view if exists ${mvName2};"""
}

98
regression-test/suites/mtmv_p0/test_multi_level_replace_mtmv.groovy Normal file
View File

@ -0,0 +1,98 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
import org.junit.Assert;
suite("test_multi_level_replace_mtmv","mtmv") {
String dbName = context.config.getDbNameByFile(context.file)
String suiteName = "test_multi_level_replace_mtmv"
String tableName1 = "${suiteName}_table1"
String tableName2 = "${suiteName}_table2"
String mvName1 = "${suiteName}_mv1"
String mvName1_replace = "${suiteName}_mv1_replace"
String mvName2 = "${suiteName}_mv2"
sql """drop table if exists `${tableName1}`"""
sql """drop table if exists `${tableName2}`"""
sql """drop materialized view if exists ${mvName1};"""
sql """drop materialized view if exists ${mvName1_replace};"""
sql """drop materialized view if exists ${mvName2};"""
sql """
CREATE TABLE ${tableName1}
(
k1 TINYINT,
k2 INT not null
)
DISTRIBUTED BY HASH(k2) BUCKETS 2
PROPERTIES (
"replication_num" = "1"
);
"""
sql """
CREATE TABLE ${tableName2}
(
k3 TINYINT,
k4 INT not null
)
DISTRIBUTED BY HASH(k4) BUCKETS 2
PROPERTIES (
"replication_num" = "1"
);
"""
sql """
CREATE MATERIALIZED VIEW ${mvName1}
BUILD DEFERRED REFRESH AUTO ON MANUAL
DISTRIBUTED BY RANDOM BUCKETS 2
PROPERTIES (
'replication_num' = '1'
)
AS
SELECT * from ${tableName1};
"""
sql """
CREATE MATERIALIZED VIEW ${mvName1_replace}
BUILD DEFERRED REFRESH AUTO ON MANUAL
DISTRIBUTED BY RANDOM BUCKETS 2
PROPERTIES (
'replication_num' = '1'
)
AS
SELECT * from ${tableName1};
"""
sql """
CREATE MATERIALIZED VIEW ${mvName2}
BUILD DEFERRED REFRESH AUTO ON MANUAL
DISTRIBUTED BY RANDOM BUCKETS 2
PROPERTIES (
'replication_num' = '1'
)
AS
SELECT * from ${mvName1};
"""
sql """
alter MATERIALIZED VIEW ${mvName1} replace with MATERIALIZED VIEW ${mvName1_replace};
"""
order_qt_status "select Name,State from mv_infos('database'='${dbName}') where Name='${mvName2}'"
sql """drop table if exists `${tableName1}`"""
sql """drop table if exists `${tableName2}`"""
sql """drop materialized view if exists ${mvName1};"""
sql """drop materialized view if exists ${mvName1_replace};"""
sql """drop materialized view if exists ${mvName2};"""
}

155
regression-test/suites/mtmv_p0/test_replace_mtmv.groovy Normal file
View File

@ -0,0 +1,155 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
import org.junit.Assert;
suite("test_replace_mtmv","mtmv") {
String dbName = context.config.getDbNameByFile(context.file)
String suiteName = "test_replace_mtmv"
String tableName1 = "${suiteName}_table1"
String tableName2 = "${suiteName}_table2"
String mvName1 = "${suiteName}_mv1"
String mvName2 = "${suiteName}_mv2"
sql """drop table if exists `${tableName1}`"""
sql """drop table if exists `${tableName2}`"""
sql """drop materialized view if exists ${mvName1};"""
sql """drop materialized view if exists ${mvName2};"""
sql """
CREATE TABLE ${tableName1}
(
k1 TINYINT,
k2 INT not null
)
DISTRIBUTED BY HASH(k2) BUCKETS 2
PROPERTIES (
"replication_num" = "1"
);
"""
sql """
CREATE TABLE ${tableName2}
(
k3 TINYINT,
k4 INT not null
)
DISTRIBUTED BY HASH(k4) BUCKETS 2
PROPERTIES (
"replication_num" = "1"
);
"""
sql """
CREATE MATERIALIZED VIEW ${mvName1}
BUILD DEFERRED REFRESH AUTO ON MANUAL
DISTRIBUTED BY RANDOM BUCKETS 2
PROPERTIES (
'replication_num' = '1'
)
AS
SELECT * from ${tableName1};
"""
sql """
CREATE MATERIALIZED VIEW ${mvName2}
BUILD DEFERRED REFRESH AUTO ON MANUAL
DISTRIBUTED BY RANDOM BUCKETS 2
PROPERTIES (
'replication_num' = '1'
)
AS
SELECT * from ${tableName2};
"""
sql """
insert into ${tableName1} values(1,1);
"""
sql """
REFRESH MATERIALIZED VIEW ${mvName1} AUTO
"""
waitingMTMVTaskFinishedByMvName(mvName1)
order_qt_mv1 "SELECT * FROM ${mvName1}"
sql """
insert into ${tableName2} values(2,2);
"""
sql """
REFRESH MATERIALIZED VIEW ${mvName2} AUTO
"""
waitingMTMVTaskFinishedByMvName(mvName2)
order_qt_mv2 "SELECT * FROM ${mvName2}"
test {
sql """
alter MATERIALIZED VIEW ${mvName1} replace with MATERIALIZED VIEW ${tableName1};
"""
exception "MATERIALIZED_VIEW"
}
test {
sql """
alter MATERIALIZED VIEW ${tableName1} replace with MATERIALIZED VIEW ${mvName1};
"""
exception "MATERIALIZED_VIEW"
}
sql """
alter MATERIALIZED VIEW ${mvName1} replace with MATERIALIZED VIEW ${mvName2};
"""
order_qt_mv1_replace "SELECT * FROM ${mvName1}"
order_qt_mv2_replace "SELECT * FROM ${mvName2}"
sql """
insert into ${tableName1} values(3,3);
"""
sql """
insert into ${tableName2} values(4,4);
"""
sql """
REFRESH MATERIALIZED VIEW ${mvName1} AUTO
"""
waitingMTMVTaskFinishedByMvName(mvName1)
order_qt_mv1_refresh "SELECT * FROM ${mvName1}"
sql """
REFRESH MATERIALIZED VIEW ${mvName2} AUTO
"""
waitingMTMVTaskFinishedByMvName(mvName2)
order_qt_mv2_refresh "SELECT * FROM ${mvName2}"
sql """
alter MATERIALIZED VIEW ${mvName1} replace with MATERIALIZED VIEW ${mvName2} PROPERTIES('swap' = 'false');
"""
def mvResult = sql """select Name from mv_infos("database"="${dbName}");"""
logger.info("mvResult: " + mvResult.toString())
assertFalse(mvResult.toString().contains("${mvName2}"))
def jobResult = sql """select MvName from jobs("type"="mv");"""
logger.info("jobResult: " + jobResult.toString())
assertFalse(jobResult.toString().contains("${mvName2}"))
order_qt_mv1_replace_not_swap "SELECT * FROM ${mvName1}"
sql """drop table if exists `${tableName1}`"""
sql """drop table if exists `${tableName2}`"""
sql """drop materialized view if exists ${mvName1};"""
sql """drop materialized view if exists ${mvName2};"""
}