database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](insert)support external hive truncate table DDL (#37659)

Browse Source 
pick: #36801
This commit is contained in:
slothever
2024-07-12 22:37:47 +08:00
committed by GitHub
parent 326b40cde2
commit f2556ba182
13 changed files with 210 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
fe/fe-core/src/main/java/org/apache/doris/analysis/TruncateTableStmt.java
View File

@ -23,7 +23,6 @@ import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.InternalDatabaseUtil;
import org.apache.doris.common.util.Util;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
@ -44,8 +43,6 @@ public class TruncateTableStmt extends DdlStmt {
public void analyze(Analyzer analyzer) throws AnalysisException, UserException {
super.analyze(analyzer);
tblRef.getName().analyze(analyzer);
// disallow external catalog
Util.prohibitExternalCatalog(tblRef.getName().getCtl(), this.getClass().getSimpleName());
if (tblRef.hasExplicitAlias()) {
throw new AnalysisException("Not support truncate table with alias");

6
fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
View File

@ -5427,8 +5427,10 @@ public class Env {
* otherwise, it will only truncate those specified partitions.
*
*/
public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlException {
getInternalCatalog().truncateTable(truncateTableStmt);
public void truncateTable(TruncateTableStmt stmt) throws DdlException {
CatalogIf<?> catalogIf = catalogMgr.getCatalogOrException(stmt.getTblRef().getName().getCtl(),
catalog -> new DdlException(("Unknown catalog " + catalog)));
catalogIf.truncateTable(stmt);
}
public void replayTruncateTable(TruncateTableInfo info) throws MetaNotFoundException {

3
fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
View File

@ -22,6 +22,7 @@ import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.analysis.DropDbStmt;
import org.apache.doris.analysis.DropTableStmt;
import org.apache.doris.analysis.TableName;
import org.apache.doris.analysis.TruncateTableStmt;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.TableIf;
@ -194,4 +195,6 @@ public interface CatalogIf<T extends DatabaseIf> {
boolean createTable(CreateTableStmt stmt) throws UserException;
void dropTable(DropTableStmt stmt) throws DdlException;
void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlException;
}

23
fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
View File

@ -22,6 +22,8 @@ import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.analysis.DropDbStmt;
import org.apache.doris.analysis.DropTableStmt;
import org.apache.doris.analysis.TableName;
import org.apache.doris.analysis.TableRef;
import org.apache.doris.analysis.TruncateTableStmt;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.InfoSchemaDb;
@ -821,4 +823,25 @@ public abstract class ExternalCatalog
}
return ret;
}
@Override
public void truncateTable(TruncateTableStmt stmt) throws DdlException {
makeSureInitialized();
if (metadataOps == null) {
throw new UnsupportedOperationException("Truncate table not supported in " + getName());
}
try {
TableRef tableRef = stmt.getTblRef();
TableName tableName = tableRef.getName();
// delete all table data if null
List<String> partitions = null;
if (tableRef.getPartitionNames() != null) {
partitions = tableRef.getPartitionNames().getPartitionNames();
}
metadataOps.truncateTable(tableName.getDb(), tableName.getTbl(), partitions);
} catch (Exception e) {
LOG.warn("Failed to drop a table", e);
throw e;
}
}
}

2
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCachedClient.java
View File

@ -94,6 +94,8 @@ public interface HMSCachedClient {
void dropTable(String dbName, String tableName);
void truncateTable(String dbName, String tblName, List<String> partitions);
void createTable(TableMetadata catalogTable, boolean ignoreIfExists);
void updateTableStatistics(

16
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
View File

@ -251,6 +251,22 @@ public class HiveMetadataOps implements ExternalMetadataOps {
}
}
@Override
public void truncateTable(String dbName, String tblName, List<String> partitions) throws DdlException {
ExternalDatabase<?> db = catalog.getDbNullable(dbName);
if (db == null) {
throw new DdlException("Failed to get database: '" + dbName + "' in catalog: " + catalog.getName());
}
try {
client.truncateTable(dbName, tblName, partitions);
} catch (Exception e) {
throw new DdlException(e.getMessage(), e);
}
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(catalog.getId(), dbName, tblName);
db.setLastUpdateTime(System.currentTimeMillis());
db.setUnInitialized(true);
}
@Override
public List<String> listTableNames(String dbName) {
return client.getAllTables(dbName);

4
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PostgreSQLJdbcHMSCachedClient.java
View File

@ -546,6 +546,10 @@ public class PostgreSQLJdbcHMSCachedClient extends JdbcHMSCachedClient {
throw new NotImplementedException("PostgreSQL dropDatabase not implemented");
}
public void truncateTable(String dbName, String tblName, List<String> partitions) {
throw new NotImplementedException("PostgreSQL truncateTable not implemented");
}
public void createTable(TableMetadata hiveTable, boolean ignoreIfExists) {
throw new NotImplementedException("PostgreSQL createTable not implemented");
}

19
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java
View File

@ -244,6 +244,23 @@ public class ThriftHMSCachedClient implements HMSCachedClient {
}
}
@Override
public void truncateTable(String dbName, String tblName, List<String> partitions) {
try (ThriftHMSClient client = getClient()) {
try {
ugiDoAs(() -> {
client.client.truncateTable(dbName, tblName, partitions);
return null;
});
} catch (Exception e) {
client.setThrowable(e);
throw e;
}
} catch (Exception e) {
throw new HMSClientException("failed to truncate table %s in db %s.", e, tblName, dbName);
}
}
@Override
public boolean tableExists(String dbName, String tblName) {
try (ThriftHMSClient client = getClient()) {
@ -272,7 +289,7 @@ public class ThriftHMSCachedClient implements HMSCachedClient {
throw e;
}
} catch (Exception e) {
throw new HMSClientException("failed to check if table %s in db %s exists", e, tblName, dbName);
throw new HMSClientException("failed to list partitions in table '%s.%s'.", e, dbName, tblName);
}
}

5
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
View File

@ -173,4 +173,9 @@ public class IcebergMetadataOps implements ExternalMetadataOps {
catalog.dropTable(TableIdentifier.of(dbName, tableName));
db.setUnInitialized(true);
}
@Override
public void truncateTable(String dbName, String tblName, List<String> partitions) {
throw new UnsupportedOperationException("Truncate Iceberg table is not supported.");
}
}

8
fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java
View File

@ -60,6 +60,14 @@ public interface ExternalMetadataOps {
*/
void dropTable(DropTableStmt stmt) throws DdlException;
/**
*
* @param dbName
* @param tblName
* @param partitions
*/
void truncateTable(String dbName, String tblName, List<String> partitions) throws DdlException;
/**
*
* @return