database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](show-stmt) fix show create table missing storage_medium info (#21757)

Browse Source
This commit is contained in:
zhengyu
2023-07-31 19:26:21 +08:00
committed by GitHub
parent f239eb4498
commit ec72383d33
6 changed files with 109 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java
View File

@ -55,6 +55,22 @@ public class DataProperty implements Writable, GsonPostProcessable {
// for persist
}
public String getStorageMediumString() {
int val = storageMedium.getValue();
switch (val) {
case 0:
return "hdd";
case 1:
return "ssd";
case 2:
return "s3";
case 3:
return "remote_cache";
default:
return "unknown";
}
}
public DataProperty(TStorageMedium medium) {
this.storageMedium = medium;
this.cooldownTimeMs = MAX_COOLDOWN_TIME_MS;

4
fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
View File

@ -3156,6 +3156,10 @@ public class Env {
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION).append("\" = \"");
sb.append(olapTable.enableSingleReplicaCompaction()).append("\"");
// storage medium
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM).append("\" = \"");
sb.append(olapTable.getStorageMedium()).append("\"");
// enable duplicate without keys by default
if (olapTable.isDuplicateWithoutKey()) {
sb.append(",\n\"")

22
fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
View File

@ -170,6 +170,8 @@ public class OlapTable extends Table {
private AutoIncrementGenerator autoIncrementGenerator;
private String storageMedium;
public OlapTable() {
// for persist
super(TableType.OLAP);
@ -1314,6 +1316,13 @@ public class OlapTable extends Table {
}
tempPartitions.write(out);
if (storageMedium == null || storageMedium.length() == 0) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
Text.writeString(out, storageMedium);
}
}
@Override
@ -1419,6 +1428,10 @@ public class OlapTable extends Table {
}
tempPartitions.unsetPartitionInfo();
if (in.readBoolean()) {
storageMedium = Text.readString(in);
}
// In the present, the fullSchema could be rebuilt by schema change while the properties is changed by MV.
// After that, some properties of fullSchema and nameToColumn may be not same as properties of base columns.
// So, here we need to rebuild the fullSchema to ensure the correctness of the properties.
@ -1840,6 +1853,15 @@ public class OlapTable extends Table {
return false;
}
public void setStorageMedium(String medium) {
storageMedium = medium;
}
public String getStorageMedium() {
return storageMedium;
}
public void setStoreRowColumn(boolean storeRowColumn) {
TableProperty tableProperty = getOrCreatTableProperty();
tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_STORE_ROW_COLUMN,

5
fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View File

@ -2162,6 +2162,7 @@ public class InternalCatalog implements CatalogIf<Database> {
try {
dataProperty = PropertyAnalyzer.analyzeDataProperty(stmt.getProperties(),
new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM));
olapTable.setStorageMedium(dataProperty.getStorageMediumString());
} catch (AnalysisException e) {
throw new DdlException(e.getMessage());
}
@ -2328,10 +2329,12 @@ public class InternalCatalog implements CatalogIf<Database> {
} else if (partitionInfo.getType() == PartitionType.RANGE
|| partitionInfo.getType() == PartitionType.LIST) {
try {
DataProperty dataProperty = null;
// just for remove entries in stmt.getProperties(),
// and then check if there still has unknown properties
PropertyAnalyzer.analyzeDataProperty(stmt.getProperties(),
dataProperty = PropertyAnalyzer.analyzeDataProperty(stmt.getProperties(),
new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM));
olapTable.setStorageMedium(dataProperty.getStorageMediumString());
if (partitionInfo.getType() == PartitionType.RANGE) {
DynamicPartitionUtil.checkAndSetDynamicPartitionProperty(olapTable, properties, db);
} else if (partitionInfo.getType() == PartitionType.LIST) {

60
fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
View File

@ -96,7 +96,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\"\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\"\n"
+ ");",
showCreateTableByName("select_decimal_table").getResultRows().get(0).get(1));
String selectFromDecimal1 =
@ -117,7 +118,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\"\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\"\n"
+ ");",
showCreateTableByName("select_decimal_table_1").getResultRows().get(0).get(1));
} else {
@ -134,7 +136,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\"\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\"\n"
+ ");",
showCreateTableByName("select_decimal_table_1").getResultRows().get(0).get(1));
}
@ -169,7 +172,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\"\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet.getResultRows().get(0).get(1));
}
@ -193,7 +197,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\"\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet1.getResultRows().get(0).get(1));
@ -219,7 +224,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\"\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet2.getResultRows().get(0).get(1));
}
@ -242,7 +248,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\"\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\"\n"
+ ");", showResultSet1.getResultRows().get(0).get(1));
String selectAlias2 = "create table `test`.`select_alias_2` PROPERTIES(\"replication_num\" = \"1\") "
+ "as select userId as alias_name, username from `test`.`varchar_table`";
@ -261,7 +268,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\"\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet2.getResultRows().get(0).get(1));
}
@ -287,7 +295,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\"\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet.getResultRows().get(0).get(1));
String selectFromJoin1 = "create table `test`.`select_join1` PROPERTIES(\"replication_num\" = \"1\") "
@ -310,7 +319,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\"\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet1.getResultRows().get(0).get(1));
}
@ -337,7 +347,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\"\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet.getResultRows().get(0).get(1));
}
@ -361,7 +372,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\"\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\"\n"
+ ");", showResultSet.getResultRows().get(0).get(1));
}
@ -384,7 +396,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\"\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet.getResultRows().get(0).get(1));
String selectFromCteAndUnion = "create table `test`.`select_cte_union` PROPERTIES(\"replication_num\" = \"1\")"
@ -403,7 +416,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\"\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\"\n"
+ ");", showResultSet1.getResultRows().get(0).get(1));
}
@ -429,7 +443,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\"\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet.getResultRows().get(0).get(1));
}
@ -454,7 +469,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\"\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet.getResultRows().get(0).get(1));
}
@ -478,7 +494,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\"\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet.getResultRows().get(0).get(1));
}
@ -503,7 +520,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\"\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet.getResultRows().get(0).get(1));
}
@ -552,7 +570,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\"\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\"\n"
+ ");",
createTableStmts.get(0));
} else {
@ -569,7 +588,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\"\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
+ "\"storage_medium\" = \"hdd\"\n"
+ ");",
createTableStmts.get(0));
}

23
fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
View File

@ -146,6 +146,16 @@ public class CreateTableTest {
.expectThrowsNoException(() -> createTable("create table test.tb7(key1 int, key2 varchar(10)) \n"
+ "distributed by hash(key1) buckets 1 properties('replication_num' = '1', 'storage_medium' = 'ssd');"));
ConfigBase.setMutableConfig("disable_storage_medium_check", "true");
ExceptionChecker
.expectThrowsNoException(() -> createTable("create table test.tb7_1(key1 int, key2 varchar(10))\n"
+ "PARTITION BY RANGE(`key1`) (\n"
+ " PARTITION `p1` VALUES LESS THAN (\"10\"),\n"
+ " PARTITION `p2` VALUES LESS THAN (\"20\"),\n"
+ " PARTITION `p3` VALUES LESS THAN (\"30\"))\n"
+ "distributed by hash(key1)\n"
+ "buckets 1 properties('replication_num' = '1', 'storage_medium' = 'ssd');"));
ExceptionChecker
.expectThrowsNoException(() -> createTable("create table test.compression1(key1 int, key2 varchar(10)) \n"
+ "distributed by hash(key1) buckets 1 \n"
@ -301,6 +311,19 @@ public class CreateTableTest {
() -> createTable("create table test.tb7(key1 int, key2 varchar(10)) distributed by hash(key1) \n"
+ "buckets 1 properties('replication_num' = '1', 'storage_medium' = 'ssd');"));
ConfigBase.setMutableConfig("disable_storage_medium_check", "false");
ExceptionChecker
.expectThrowsWithMsg(DdlException.class, "Failed to find enough backend, please check the replication num,replication tag and storage medium.\n"
+ "Create failed replications:\n"
+ "replication tag: {\"location\" : \"default\"}, replication num: 1, storage medium: SSD",
() -> createTable("create table test.tb7_1(key1 int, key2 varchar(10))\n"
+ "PARTITION BY RANGE(`key1`) (\n"
+ " PARTITION `p1` VALUES LESS THAN (\"10\"),\n"
+ " PARTITION `p2` VALUES LESS THAN (\"20\"),\n"
+ " PARTITION `p3` VALUES LESS THAN (\"30\"))\n"
+ "distributed by hash(key1)\n"
+ "buckets 1 properties('replication_num' = '1', 'storage_medium' = 'ssd');"));
ExceptionChecker
.expectThrowsWithMsg(DdlException.class, "sequence column only support UNIQUE_KEYS",
() -> createTable("create table test.atbl8\n" + "(k1 varchar(40), k2 int, v1 int sum)\n"