database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](hierarchical-storage) Fix bug that storage medium property change back to SSD (#9158)

Browse Source 
1. fix bug described in #9159
2. fix a `fill_tuple` bug introduced from #9173
This commit is contained in:
Mingyu Chen
2022-04-26 10:15:19 +08:00
committed by GitHub
parent 62b38d7a75
commit 7cfebd05fd
25 changed files with 96 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
View File

@ -295,7 +295,6 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
public class Catalog {
@ -4997,7 +4996,8 @@ public class Catalog {
if (dataProperty.getStorageMedium() == TStorageMedium.SSD
&& dataProperty.getCooldownTimeMs() < currentTimeMs) {
// expire. change to HDD.
partitionInfo.setDataProperty(partition.getId(), new DataProperty(TStorageMedium.HDD));
DataProperty hddProperty = new DataProperty(TStorageMedium.HDD);
partitionInfo.setDataProperty(partition.getId(), hddProperty);
storageMediumMap.put(partitionId, TStorageMedium.HDD);
LOG.debug("partition[{}-{}-{}] storage medium changed from SSD to HDD",
dbId, tableId, partitionId);
@ -5006,7 +5006,7 @@ public class Catalog {
ModifyPartitionInfo info =
new ModifyPartitionInfo(db.getId(), olapTable.getId(),
partition.getId(),
DataProperty.DEFAULT_DATA_PROPERTY,
hddProperty,
ReplicaAllocation.NOT_SET,
partitionInfo.getIsInMemory(partition.getId()));
editLog.logModifyPartition(info);

6
fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java
View File

@ -67,7 +67,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@ -129,6 +128,9 @@ public class TableQueryPlanAction extends RestBaseController {
// status code should conforms to HTTP semantic
resultMap.put("status", e.getCode().code());
resultMap.put("exception", e.getMessage());
} catch (Exception e) {
resultMap.put("status", "1");
resultMap.put("exception", e.getMessage());
}
return ResponseEntityBuilder.ok(resultMap);
}
@ -252,7 +254,7 @@ public class TableQueryPlanAction extends RestBaseController {
for (TScanRangeLocations scanRangeLocations : scanRangeLocationsList) {
// only process palo(doris) scan range
TPaloScanRange scanRange = scanRangeLocations.scan_range.palo_scan_range;
Node tabletRouting = new Node(Long.parseLong(scanRange.version), Integer.parseInt(scanRange.schema_hash));
Node tabletRouting = new Node(Long.parseLong(scanRange.version), 0 /* schema hash is not used */);
for (TNetworkAddress address : scanRange.hosts) {
tabletRouting.addRouting(address.hostname + ":" + address.port);
}

182
fe/fe-core/src/test/java/org/apache/doris/analysis/AlterViewStmtTest.java
View File

@ -1,182 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.analysis;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.SinglePartitionInfo;
import org.apache.doris.catalog.View;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.common.util.SqlParserUtils;
import org.apache.doris.mysql.privilege.PaloAuth;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.persist.AlterViewInfo;
import org.apache.doris.persist.CreateTableInfo;
import org.apache.doris.persist.EditLog;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.Lists;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.io.StringReader;
import java.util.LinkedList;
import java.util.List;
import mockit.Expectations;
import mockit.Mock;
import mockit.MockUp;
import mockit.Mocked;
public class AlterViewStmtTest {
private Analyzer analyzer;
private Catalog catalog;
@Mocked
EditLog editLog;
@Mocked
private ConnectContext connectContext;
@Mocked
private PaloAuth auth;
@Before
public void setUp() {
catalog = Deencapsulation.newInstance(Catalog.class);
analyzer = new Analyzer(catalog, connectContext);
Database db = new Database(50000L, "testCluster:testDb");
Column column1 = new Column("col1", PrimitiveType.BIGINT);
Column column2 = new Column("col2", PrimitiveType.DOUBLE);
List<Column> baseSchema = new LinkedList<Column>();
baseSchema.add(column1);
baseSchema.add(column2);
OlapTable table = new OlapTable(30000, "testTbl",
baseSchema, KeysType.AGG_KEYS, new SinglePartitionInfo(), null);
db.createTable(table);
new Expectations(auth) {
{
auth.checkGlobalPriv((ConnectContext) any, (PrivPredicate) any);
minTimes = 0;
result = true;
auth.checkDbPriv((ConnectContext) any, anyString, (PrivPredicate) any);
minTimes = 0;
result = true;
auth.checkTblPriv((ConnectContext) any, anyString, anyString, (PrivPredicate) any);
minTimes = 0;
result = true;
}
};
new Expectations(editLog) {
{
editLog.logCreateTable((CreateTableInfo) any);
minTimes = 0;
editLog.logModifyViewDef((AlterViewInfo) any);
minTimes = 0;
}
};
Deencapsulation.setField(catalog, "editLog", editLog);
new MockUp<Catalog>() {
@Mock
Catalog getCurrentCatalog() {
return catalog;
}
@Mock
PaloAuth getAuth() {
return auth;
}
@Mock
Database getDbOrDdlException(long dbId) {
return db;
}
@Mock
Database getDbOrDdlException(String dbName) {
return db;
}
@Mock
Database getDbOrAnalysisException(long dbId) {
return db;
}
@Mock
Database getDbOrAnalysisException(String dbName) {
return db;
}
};
new MockUp<Analyzer>() {
@Mock
String getClusterName() {
return "testCluster";
}
};
}
@Test
public void testNormal() throws Exception {
String originStmt = "select col1 as c1, sum(col2) as c2 from testDb.testTbl group by col1";
View view = new View(30000L, "testView", null);
view.setInlineViewDefWithSqlMode("select col1 as c1, sum(col2) as c2 from testDb.testTbl group by col1", 0L);
view.init();
Database db = analyzer.getCatalog().getDbOrAnalysisException("testDb");
db.createTable(view);
Assert.assertEquals(originStmt, view.getInlineViewDef());
String alterStmt = "with testTbl_cte (w1, w2) as (select col1, col2 from testDb.testTbl) select w1 as c1, sum(w2) as c2 from testTbl_cte where w1 > 10 group by w1 order by w1";
SqlParser parser = new SqlParser(new SqlScanner(new StringReader(alterStmt)));
QueryStmt alterQueryStmt = (QueryStmt) SqlParserUtils.getFirstStmt(parser);
ColWithComment col1 = new ColWithComment("h1", null);
ColWithComment col2 = new ColWithComment("h2", null);
AlterViewStmt alterViewStmt = new AlterViewStmt(new TableName("testDb", "testView"), Lists.newArrayList(col1, col2), alterQueryStmt);
alterViewStmt.analyze(analyzer);
Catalog catalog1 = analyzer.getCatalog();
if (catalog1 == null) {
System.out.println("cmy get null");
return;
}
catalog1.alterView(alterViewStmt);
View newView = (View) db.getTableOrAnalysisException("testView");
Assert.assertEquals("WITH testTbl_cte(w1, w2) AS (SELECT `col1` AS `col1`, `col2` AS `col2` FROM `testCluster:testDb`.`testTbl`)" +
" SELECT `w1` AS `h1`, sum(`w2`) AS `h2` FROM `testTbl_cte` WHERE `w1` > 10 GROUP BY `w1` ORDER BY `w1`",
newView.getInlineViewDef());
}
}

17
fe/fe-core/src/test/java/org/apache/doris/analysis/ExplainTest.java
View File

@ -22,9 +22,6 @@ import org.apache.doris.qe.ConnectContext;
import org.apache.doris.utframe.UtFrameUtils;
import org.junit.Assert;
import org.junit.Test;
import java.util.UUID;
public class ExplainTest {
@ -75,13 +72,6 @@ public class ExplainTest {
Assert.assertEquals(dropDbStmt.toSql(), dropSchemaStmt.toSql());
}
public void testExplainInsertInto() throws Exception {
String sql = "explain insert into test_explain.explain_t1 select * from test_explain.explain_t2";
String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql, true);
System.out.println(explainString);
Assert.assertTrue(explainString.contains("CAST"));
}
public void testExplainSelect() throws Exception {
String sql = "explain select * from test_explain.explain_t1 where dt = '1001';";
String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql, false);
@ -89,6 +79,13 @@ public class ExplainTest {
Assert.assertFalse(explainString.contains("CAST"));
}
public void testExplainInsertInto() throws Exception {
String sql = "explain verbose insert into test_explain.explain_t1 select * from test_explain.explain_t2";
String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql, true);
System.out.println(explainString);
Assert.assertTrue(explainString.contains("CAST"));
}
public void testExplainVerboseSelect() throws Exception {
String queryStr = "explain verbose select * from test_explain.explain_t1 where dt = '1001';";
String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, queryStr, true);

19
fe/fe-core/src/test/java/org/apache/doris/catalog/CreateViewTest.java
View File

@ -17,6 +17,7 @@
package org.apache.doris.catalog;
import org.apache.doris.analysis.AlterViewStmt;
import org.apache.doris.analysis.CreateDbStmt;
import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.analysis.CreateViewStmt;
@ -129,4 +130,22 @@ public class CreateViewTest {
System.out.println(explainString);
Assert.assertTrue(explainString.contains("OlapScanNode"));
}
@Test
public void testAlterView() throws Exception {
String originStmt = "select k1 as kc1, sum(k2) as kc2 from test.tbl1 group by kc1";
ExceptionChecker.expectThrowsNoException(
() -> createView("create view test.alter1 as " + originStmt));
Database db = Catalog.getCurrentCatalog().getDbOrDdlException("default_cluster:test");
View alter1 = (View) db.getTableOrDdlException("alter1");
Assert.assertEquals("SELECT `k1` AS `kc1`, sum(`k2`) AS `kc2` FROM `default_cluster:test`.`tbl1` GROUP BY `kc1`", alter1.getInlineViewDef());
String alterStmt = "alter view test.alter1 as with test1_cte (w1, w2) as (select k1, k2 from test.tbl1) select w1 as c1, sum(w2) as c2 from test1_cte where w1 > 10 group by w1 order by w1";
AlterViewStmt alterViewStmt = (AlterViewStmt) UtFrameUtils.parseAndAnalyzeStmt(alterStmt, connectContext);
Catalog.getCurrentCatalog().alterView(alterViewStmt);
alter1 = (View) db.getTableOrDdlException("alter1");
System.out.println(alter1.getInlineViewDef());
Assert.assertEquals("WITH test1_cte(w1, w2) AS (SELECT `k1` AS `k1`, `k2` AS `k2` FROM `default_cluster:test`.`tbl1`) SELECT `w1` AS `c1`, sum(`w2`) AS `c2` FROM `test1_cte` WHERE `w1` > 10 GROUP BY `w1` ORDER BY `w1` ASC", alter1.getInlineViewDef());
}
}

4
fe/fe-core/src/test/java/org/apache/doris/http/TableQueryPlanActionTest.java
View File

@ -48,7 +48,7 @@ public class TableQueryPlanActionTest extends DorisHttpTestCase {
}
@Test
public void testQueryPlanAction() throws IOException, TException {
RequestBody body = RequestBody.create(JSON, "{ \"sql\" : \" select k1,k2 from " + DB_NAME + "." + TABLE_NAME + " \" }");
RequestBody body = RequestBody.create("{ \"sql\" : \" select k1,k2 from " + DB_NAME + "." + TABLE_NAME + " \" }", JSON);
Request request = new Request.Builder()
.post(body)
.addHeader("Authorization", rootAuth)
@ -67,8 +67,6 @@ public class TableQueryPlanActionTest extends DorisHttpTestCase {
Assert.assertNotNull(tabletObject.get("routings"));
Assert.assertEquals(3, ((JSONArray) tabletObject.get("routings")).size());
Assert.assertEquals(testStartVersion, (long) tabletObject.get("version"));
Assert.assertEquals(testSchemaHash, (long) tabletObject.get("schemaHash"));
}
String queryPlan = (String) ((JSONObject) jsonObject.get("data")).get("opaqued_query_plan");
Assert.assertNotNull(queryPlan);

5
fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
View File

@ -1991,8 +1991,8 @@ public class QueryPlanTest {
public void testExplainInsertInto() throws Exception {
ExplainTest explainTest = new ExplainTest();
explainTest.before(connectContext);
explainTest.testExplainInsertInto();
explainTest.testExplainSelect();
explainTest.testExplainInsertInto();
explainTest.testExplainVerboseSelect();
explainTest.testExplainConcatSelect();
explainTest.testExplainVerboseConcatSelect();
@ -2097,7 +2097,8 @@ public class QueryPlanTest {
" (SELECT 4 AS bid)b ON (a.aid=b.bid)\n";
String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, queryStr);
Assert.assertFalse(explainString.contains("OUTPUT EXPRS:3 | 4"));
Assert.assertTrue(explainString.contains("OUTPUT EXPRS:CAST(`a`.`aid` AS INT) | 4"));
System.out.println(explainString);
Assert.assertTrue(explainString.contains("OUTPUT EXPRS:`a`.`aid` | 4"));
}
@Test