From f9f5bbde6f0d59422c5dbe0b955345d8baa164af Mon Sep 17 00:00:00 2001 From: xueweizhang Date: Thu, 27 Apr 2023 09:59:56 +0800 Subject: [PATCH] [feature-wip](duplicate_no_keys) add create duplicate table without keys (#18758) --- .../java/org/apache/doris/common/Config.java | 7 +++ .../doris/alter/MaterializedViewHandler.java | 2 +- .../doris/alter/SchemaChangeHandler.java | 2 +- .../doris/analysis/CreateTableStmt.java | 63 ++++++++++--------- .../org/apache/doris/analysis/KeysDesc.java | 2 +- .../java/org/apache/doris/catalog/Env.java | 28 +++++---- .../org/apache/doris/catalog/OlapTable.java | 4 ++ .../doris/datasource/InternalCatalog.java | 25 ++++---- .../MultiTableMaterializedViewTest.java | 2 +- 9 files changed, 79 insertions(+), 56 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index b4fbb28aac..7c1ed56104 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2155,5 +2155,12 @@ public class Config extends ConfigBase { */ @ConfField public static boolean enable_stats = true; + + /** + * Whether create a duplicate table without keys by default + * when creating a table which not set key type and key columns + */ + @ConfField(mutable = true, masterOnly = true) + public static boolean experimental_enable_duplicate_without_keys_by_default = false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java index fc6e88c4f5..b5ee423151 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java @@ -344,7 +344,7 @@ public class MaterializedViewHandler extends AlterHandler { // get rollup schema hash int mvSchemaHash = Util.generateSchemaHash(); // get short key column count - short mvShortKeyColumnCount = Env.calcShortKeyColumnCount(mvColumns, properties); + short mvShortKeyColumnCount = Env.calcShortKeyColumnCount(mvColumns, properties, true/*isKeysRequired*/); // get timeout long timeoutMs = PropertyAnalyzer.analyzeTimeout(properties, Config.alter_table_timeout_second) * 1000; diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index c9ce361e3c..200a87b6da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -1431,7 +1431,7 @@ public class SchemaChangeHandler extends AlterHandler { // 5. calc short key short newShortKeyColumnCount = Env.calcShortKeyColumnCount(alterSchema, - indexIdToProperties.get(alterIndexId)); + indexIdToProperties.get(alterIndexId), true/*isKeysRequired*/); LOG.debug("alter index[{}] short key column count: {}", alterIndexId, newShortKeyColumnCount); indexIdToShortKeyColumnCount.put(alterIndexId, newShortKeyColumnCount); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java index 36b58e48ea..498347a207 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java @@ -353,39 +353,42 @@ public class CreateTableStmt extends DdlStmt { } keysDesc = new KeysDesc(KeysType.AGG_KEYS, keysColumnNames); } else { - for (ColumnDef columnDef : columnDefs) { - keyLength += columnDef.getType().getIndexSize(); - if (keysColumnNames.size() >= FeConstants.shortkey_max_column_count - || keyLength > FeConstants.shortkey_maxsize_bytes) { - if (keysColumnNames.size() == 0 - && columnDef.getType().getPrimitiveType().isCharFamily()) { - keysColumnNames.add(columnDef.getName()); + if (!Config.experimental_enable_duplicate_without_keys_by_default) { + for (ColumnDef columnDef : columnDefs) { + keyLength += columnDef.getType().getIndexSize(); + if (keysColumnNames.size() >= FeConstants.shortkey_max_column_count + || keyLength > FeConstants.shortkey_maxsize_bytes) { + if (keysColumnNames.size() == 0 + && columnDef.getType().getPrimitiveType().isCharFamily()) { + keysColumnNames.add(columnDef.getName()); + } + break; + } + if (columnDef.getType().isFloatingPointType()) { + break; + } + if (columnDef.getType().getPrimitiveType() == PrimitiveType.STRING) { + break; + } + if (columnDef.getType().getPrimitiveType() == PrimitiveType.JSONB) { + break; + } + if (columnDef.getType().isComplexType()) { + break; + } + if (columnDef.getType().getPrimitiveType() == PrimitiveType.VARCHAR) { + keysColumnNames.add(columnDef.getName()); + break; } - break; - } - if (columnDef.getType().isFloatingPointType()) { - break; - } - if (columnDef.getType().getPrimitiveType() == PrimitiveType.STRING) { - break; - } - if (columnDef.getType().getPrimitiveType() == PrimitiveType.JSONB) { - break; - } - if (columnDef.getType().isComplexType()) { - break; - } - if (columnDef.getType().getPrimitiveType() == PrimitiveType.VARCHAR) { keysColumnNames.add(columnDef.getName()); - break; } - keysColumnNames.add(columnDef.getName()); - } - // The OLAP table must have at least one short key and the float and double should not be short key. - // So the float and double could not be the first column in OLAP table. - if (keysColumnNames.isEmpty()) { - throw new AnalysisException("The olap table first column could not be float, double, string" - + " or array, struct, map, please use decimal or varchar instead."); + // The OLAP table must have at least one short key, + // and the float and double should not be short key, + // so the float and double could not be the first column in OLAP table. + if (keysColumnNames.isEmpty()) { + throw new AnalysisException("The olap table first column could not be float, double, string" + + " or array, struct, map, please use decimal or varchar instead."); + } } keysDesc = new KeysDesc(KeysType.DUP_KEYS, keysColumnNames); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/KeysDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/KeysDesc.java index 83fc766ecb..36e9154af1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/KeysDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/KeysDesc.java @@ -60,7 +60,7 @@ public class KeysDesc implements Writable { throw new AnalysisException("Keys type is null."); } - if (keysColumnNames == null || keysColumnNames.size() == 0) { + if ((keysColumnNames == null || keysColumnNames.size() == 0) && type != KeysType.DUP_KEYS) { throw new AnalysisException("The number of key columns is 0."); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 7509de1db0..dad70edd20 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -2849,20 +2849,24 @@ public class Env { if (table.getType() == TableType.OLAP || table.getType() == TableType.MATERIALIZED_VIEW) { OlapTable olapTable = (OlapTable) table; - // keys String keySql = olapTable.getKeysType().toSql(); - sb.append("\n").append(table.getType() == TableType.OLAP + if (olapTable.isDuplicateWithoutKey()) { + // after #18621, use can create a DUP_KEYS olap table without key columns + // and get a ddl schema without key type and key columns + } else { + sb.append("\n").append(table.getType() == TableType.OLAP ? keySql : keySql.substring("DUPLICATE ".length())) .append("("); - List keysColumnNames = Lists.newArrayList(); - for (Column column : olapTable.getBaseSchema()) { - if (column.isKey()) { - keysColumnNames.add("`" + column.getName() + "`"); + List keysColumnNames = Lists.newArrayList(); + for (Column column : olapTable.getBaseSchema()) { + if (column.isKey()) { + keysColumnNames.add("`" + column.getName() + "`"); + } } + sb.append(Joiner.on(", ").join(keysColumnNames)).append(")"); } - sb.append(Joiner.on(", ").join(keysColumnNames)).append(")"); if (specificVersion != -1) { // for copy tablet operation @@ -3707,8 +3711,8 @@ public class Env { this.haProtocol = protocol; } - public static short calcShortKeyColumnCount(List columns, Map properties) - throws DdlException { + public static short calcShortKeyColumnCount(List columns, Map properties, + boolean isKeysRequired) throws DdlException { List indexColumns = new ArrayList(); for (Column column : columns) { if (column.isKey()) { @@ -3716,7 +3720,9 @@ public class Env { } } LOG.debug("index column size: {}", indexColumns.size()); - Preconditions.checkArgument(indexColumns.size() > 0); + if (isKeysRequired) { + Preconditions.checkArgument(indexColumns.size() > 0); + } // figure out shortKeyColumnCount short shortKeyColumnCount = (short) -1; @@ -3770,7 +3776,7 @@ public class Env { } ++shortKeyColumnCount; } - if (shortKeyColumnCount == 0) { + if (isKeysRequired && shortKeyColumnCount == 0) { throw new DdlException("The first column could not be float or double type, use decimal instead"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 96eb25bc90..0794a253bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -1975,6 +1975,10 @@ public class OlapTable extends Table { return tableProperty.getEnableUniqueKeyMergeOnWrite(); } + public boolean isDuplicateWithoutKey() { + return getKeysType() == KeysType.DUP_KEYS && getKeysNum() == 0; + } + // For non partitioned table: // The table's distribute hash columns need to be a subset of the aggregate columns. // diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index e06920cde5..250cda5cce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -1854,9 +1854,16 @@ public class InternalCatalog implements CatalogIf { String tableName = stmt.getTableName(); LOG.debug("begin create olap table: {}", tableName); + // get keys type + KeysDesc keysDesc = stmt.getKeysDesc(); + Preconditions.checkNotNull(keysDesc); + KeysType keysType = keysDesc.getKeysType(); + int keysColumnSize = keysDesc.keysColumnSize(); + boolean isKeysRequired = !(keysType == KeysType.DUP_KEYS && keysColumnSize == 0); + // create columns List baseSchema = stmt.getColumns(); - validateColumns(baseSchema); + validateColumns(baseSchema, isKeysRequired); // analyze replica allocation ReplicaAllocation replicaAlloc = PropertyAnalyzer.analyzeReplicaAllocation(stmt.getProperties(), ""); @@ -1888,18 +1895,13 @@ public class InternalCatalog implements CatalogIf { partitionInfo = new SinglePartitionInfo(); } - // get keys type - KeysDesc keysDesc = stmt.getKeysDesc(); - Preconditions.checkNotNull(keysDesc); - KeysType keysType = keysDesc.getKeysType(); - // create distribution info DistributionDesc distributionDesc = stmt.getDistributionDesc(); Preconditions.checkNotNull(distributionDesc); DistributionInfo defaultDistributionInfo = distributionDesc.toDistributionInfo(baseSchema); // calc short key column count - short shortKeyColumnCount = Env.calcShortKeyColumnCount(baseSchema, stmt.getProperties()); + short shortKeyColumnCount = Env.calcShortKeyColumnCount(baseSchema, stmt.getProperties(), isKeysRequired); LOG.debug("create table[{}] short key column count: {}", tableName, shortKeyColumnCount); // create table @@ -2138,7 +2140,8 @@ public class InternalCatalog implements CatalogIf { // set rollup index meta to olap table List rollupColumns = Env.getCurrentEnv().getMaterializedViewHandler() .checkAndPrepareMaterializedView(addRollupClause, olapTable, baseRollupIndex, false); - short rollupShortKeyColumnCount = Env.calcShortKeyColumnCount(rollupColumns, alterClause.getProperties()); + short rollupShortKeyColumnCount = Env.calcShortKeyColumnCount(rollupColumns, alterClause.getProperties(), + true/*isKeysRequired*/); int rollupSchemaHash = Util.generateSchemaHash(); long rollupIndexId = idGeneratorBuffer.getNextId(); olapTable.setIndexMeta(rollupIndexId, addRollupClause.getRollupName(), rollupColumns, schemaVersion, @@ -2387,7 +2390,7 @@ public class InternalCatalog implements CatalogIf { if (baseSchema.isEmpty()) { baseSchema = esTable.genColumnsFromEs(); } - validateColumns(baseSchema); + validateColumns(baseSchema, true); esTable.setNewFullSchema(baseSchema); // create partition info @@ -2593,7 +2596,7 @@ public class InternalCatalog implements CatalogIf { /* * generate and check columns' order and key's existence */ - private void validateColumns(List columns) throws DdlException { + private void validateColumns(List columns, boolean isKeysRequired) throws DdlException { if (columns.isEmpty()) { ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_MUST_HAVE_COLUMNS); } @@ -2611,7 +2614,7 @@ public class InternalCatalog implements CatalogIf { } } - if (!hasKey) { + if (!hasKey && isKeysRequired) { ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_MUST_HAVE_KEYS); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/MultiTableMaterializedViewTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/MultiTableMaterializedViewTest.java index 4155d5914b..9ba68658fe 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/MultiTableMaterializedViewTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/MultiTableMaterializedViewTest.java @@ -135,7 +135,7 @@ public class MultiTableMaterializedViewTest extends TestWithFeService { stmt.getColumns(), 0, Util.generateSchemaHash(), - Env.calcShortKeyColumnCount(stmt.getColumns(), stmt.getProperties()), + Env.calcShortKeyColumnCount(stmt.getColumns(), stmt.getProperties(), true), TStorageType.COLUMN, stmt.getKeysDesc().getKeysType()); return mv;