database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature-wip](duplicate_no_keys) add create duplicate table without keys (#18758)

Browse Source
This commit is contained in:
xueweizhang
2023-04-27 09:59:56 +08:00
committed by GitHub
parent 84d040bdbf
commit f9f5bbde6f
9 changed files with 79 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
View File

@ -344,7 +344,7 @@ public class MaterializedViewHandler extends AlterHandler {
// get rollup schema hash
int mvSchemaHash = Util.generateSchemaHash();
// get short key column count
short mvShortKeyColumnCount = Env.calcShortKeyColumnCount(mvColumns, properties);
short mvShortKeyColumnCount = Env.calcShortKeyColumnCount(mvColumns, properties, true/*isKeysRequired*/);
// get timeout
long timeoutMs = PropertyAnalyzer.analyzeTimeout(properties, Config.alter_table_timeout_second) * 1000;

2
fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
View File

@ -1431,7 +1431,7 @@ public class SchemaChangeHandler extends AlterHandler {
// 5. calc short key
short newShortKeyColumnCount = Env.calcShortKeyColumnCount(alterSchema,
indexIdToProperties.get(alterIndexId));
indexIdToProperties.get(alterIndexId), true/*isKeysRequired*/);
LOG.debug("alter index[{}] short key column count: {}", alterIndexId, newShortKeyColumnCount);
indexIdToShortKeyColumnCount.put(alterIndexId, newShortKeyColumnCount);

63
fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
View File

@ -353,39 +353,42 @@ public class CreateTableStmt extends DdlStmt {
}
keysDesc = new KeysDesc(KeysType.AGG_KEYS, keysColumnNames);
} else {
for (ColumnDef columnDef : columnDefs) {
keyLength += columnDef.getType().getIndexSize();
if (keysColumnNames.size() >= FeConstants.shortkey_max_column_count
|| keyLength > FeConstants.shortkey_maxsize_bytes) {
if (keysColumnNames.size() == 0
&& columnDef.getType().getPrimitiveType().isCharFamily()) {
keysColumnNames.add(columnDef.getName());
if (!Config.experimental_enable_duplicate_without_keys_by_default) {
for (ColumnDef columnDef : columnDefs) {
keyLength += columnDef.getType().getIndexSize();
if (keysColumnNames.size() >= FeConstants.shortkey_max_column_count
|| keyLength > FeConstants.shortkey_maxsize_bytes) {
if (keysColumnNames.size() == 0
&& columnDef.getType().getPrimitiveType().isCharFamily()) {
keysColumnNames.add(columnDef.getName());
}
break;
}
if (columnDef.getType().isFloatingPointType()) {
break;
}
if (columnDef.getType().getPrimitiveType() == PrimitiveType.STRING) {
break;
}
if (columnDef.getType().getPrimitiveType() == PrimitiveType.JSONB) {
break;
}
if (columnDef.getType().isComplexType()) {
break;
}
if (columnDef.getType().getPrimitiveType() == PrimitiveType.VARCHAR) {
keysColumnNames.add(columnDef.getName());
break;
}
break;
}
if (columnDef.getType().isFloatingPointType()) {
break;
}
if (columnDef.getType().getPrimitiveType() == PrimitiveType.STRING) {
break;
}
if (columnDef.getType().getPrimitiveType() == PrimitiveType.JSONB) {
break;
}
if (columnDef.getType().isComplexType()) {
break;
}
if (columnDef.getType().getPrimitiveType() == PrimitiveType.VARCHAR) {
keysColumnNames.add(columnDef.getName());
break;
}
keysColumnNames.add(columnDef.getName());
}
// The OLAP table must have at least one short key and the float and double should not be short key.
// So the float and double could not be the first column in OLAP table.
if (keysColumnNames.isEmpty()) {
throw new AnalysisException("The olap table first column could not be float, double, string"
+ " or array, struct, map, please use decimal or varchar instead.");
// The OLAP table must have at least one short key,
// and the float and double should not be short key,
// so the float and double could not be the first column in OLAP table.
if (keysColumnNames.isEmpty()) {
throw new AnalysisException("The olap table first column could not be float, double, string"
+ " or array, struct, map, please use decimal or varchar instead.");
}
}
keysDesc = new KeysDesc(KeysType.DUP_KEYS, keysColumnNames);
}

2
fe/fe-core/src/main/java/org/apache/doris/analysis/KeysDesc.java
View File

@ -60,7 +60,7 @@ public class KeysDesc implements Writable {
throw new AnalysisException("Keys type is null.");
}
if (keysColumnNames == null || keysColumnNames.size() == 0) {
if ((keysColumnNames == null || keysColumnNames.size() == 0) && type != KeysType.DUP_KEYS) {
throw new AnalysisException("The number of key columns is 0.");
}

28
fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
View File

@ -2849,20 +2849,24 @@ public class Env {
if (table.getType() == TableType.OLAP || table.getType() == TableType.MATERIALIZED_VIEW) {
OlapTable olapTable = (OlapTable) table;
// keys
String keySql = olapTable.getKeysType().toSql();
sb.append("\n").append(table.getType() == TableType.OLAP
if (olapTable.isDuplicateWithoutKey()) {
// after #18621, use can create a DUP_KEYS olap table without key columns
// and get a ddl schema without key type and key columns
} else {
sb.append("\n").append(table.getType() == TableType.OLAP
? keySql
: keySql.substring("DUPLICATE ".length()))
.append("(");
List<String> keysColumnNames = Lists.newArrayList();
for (Column column : olapTable.getBaseSchema()) {
if (column.isKey()) {
keysColumnNames.add("`" + column.getName() + "`");
List<String> keysColumnNames = Lists.newArrayList();
for (Column column : olapTable.getBaseSchema()) {
if (column.isKey()) {
keysColumnNames.add("`" + column.getName() + "`");
}
}
sb.append(Joiner.on(", ").join(keysColumnNames)).append(")");
}
sb.append(Joiner.on(", ").join(keysColumnNames)).append(")");
if (specificVersion != -1) {
// for copy tablet operation
@ -3707,8 +3711,8 @@ public class Env {
this.haProtocol = protocol;
}
public static short calcShortKeyColumnCount(List<Column> columns, Map<String, String> properties)
throws DdlException {
public static short calcShortKeyColumnCount(List<Column> columns, Map<String, String> properties,
boolean isKeysRequired) throws DdlException {
List<Column> indexColumns = new ArrayList<Column>();
for (Column column : columns) {
if (column.isKey()) {
@ -3716,7 +3720,9 @@ public class Env {
}
}
LOG.debug("index column size: {}", indexColumns.size());
Preconditions.checkArgument(indexColumns.size() > 0);
if (isKeysRequired) {
Preconditions.checkArgument(indexColumns.size() > 0);
}
// figure out shortKeyColumnCount
short shortKeyColumnCount = (short) -1;
@ -3770,7 +3776,7 @@ public class Env {
}
++shortKeyColumnCount;
}
if (shortKeyColumnCount == 0) {
if (isKeysRequired && shortKeyColumnCount == 0) {
throw new DdlException("The first column could not be float or double type, use decimal instead");
}

4
fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
View File

@ -1975,6 +1975,10 @@ public class OlapTable extends Table {
return tableProperty.getEnableUniqueKeyMergeOnWrite();
}
public boolean isDuplicateWithoutKey() {
return getKeysType() == KeysType.DUP_KEYS && getKeysNum() == 0;
}
// For non partitioned table:
// The table's distribute hash columns need to be a subset of the aggregate columns.
//

25
fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View File

@ -1854,9 +1854,16 @@ public class InternalCatalog implements CatalogIf<Database> {
String tableName = stmt.getTableName();
LOG.debug("begin create olap table: {}", tableName);
// get keys type
KeysDesc keysDesc = stmt.getKeysDesc();
Preconditions.checkNotNull(keysDesc);
KeysType keysType = keysDesc.getKeysType();
int keysColumnSize = keysDesc.keysColumnSize();
boolean isKeysRequired = !(keysType == KeysType.DUP_KEYS && keysColumnSize == 0);
// create columns
List<Column> baseSchema = stmt.getColumns();
validateColumns(baseSchema);
validateColumns(baseSchema, isKeysRequired);
// analyze replica allocation
ReplicaAllocation replicaAlloc = PropertyAnalyzer.analyzeReplicaAllocation(stmt.getProperties(), "");
@ -1888,18 +1895,13 @@ public class InternalCatalog implements CatalogIf<Database> {
partitionInfo = new SinglePartitionInfo();
}
// get keys type
KeysDesc keysDesc = stmt.getKeysDesc();
Preconditions.checkNotNull(keysDesc);
KeysType keysType = keysDesc.getKeysType();
// create distribution info
DistributionDesc distributionDesc = stmt.getDistributionDesc();
Preconditions.checkNotNull(distributionDesc);
DistributionInfo defaultDistributionInfo = distributionDesc.toDistributionInfo(baseSchema);
// calc short key column count
short shortKeyColumnCount = Env.calcShortKeyColumnCount(baseSchema, stmt.getProperties());
short shortKeyColumnCount = Env.calcShortKeyColumnCount(baseSchema, stmt.getProperties(), isKeysRequired);
LOG.debug("create table[{}] short key column count: {}", tableName, shortKeyColumnCount);
// create table
@ -2138,7 +2140,8 @@ public class InternalCatalog implements CatalogIf<Database> {
// set rollup index meta to olap table
List<Column> rollupColumns = Env.getCurrentEnv().getMaterializedViewHandler()
.checkAndPrepareMaterializedView(addRollupClause, olapTable, baseRollupIndex, false);
short rollupShortKeyColumnCount = Env.calcShortKeyColumnCount(rollupColumns, alterClause.getProperties());
short rollupShortKeyColumnCount = Env.calcShortKeyColumnCount(rollupColumns, alterClause.getProperties(),
true/*isKeysRequired*/);
int rollupSchemaHash = Util.generateSchemaHash();
long rollupIndexId = idGeneratorBuffer.getNextId();
olapTable.setIndexMeta(rollupIndexId, addRollupClause.getRollupName(), rollupColumns, schemaVersion,
@ -2387,7 +2390,7 @@ public class InternalCatalog implements CatalogIf<Database> {
if (baseSchema.isEmpty()) {
baseSchema = esTable.genColumnsFromEs();
}
validateColumns(baseSchema);
validateColumns(baseSchema, true);
esTable.setNewFullSchema(baseSchema);
// create partition info
@ -2593,7 +2596,7 @@ public class InternalCatalog implements CatalogIf<Database> {
/*
* generate and check columns' order and key's existence
*/
private void validateColumns(List<Column> columns) throws DdlException {
private void validateColumns(List<Column> columns, boolean isKeysRequired) throws DdlException {
if (columns.isEmpty()) {
ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_MUST_HAVE_COLUMNS);
}
@ -2611,7 +2614,7 @@ public class InternalCatalog implements CatalogIf<Database> {
}
}
if (!hasKey) {
if (!hasKey && isKeysRequired) {
ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_MUST_HAVE_KEYS);
}
}