database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[refactor](mysql compatibility) An abstract class for all databases created for mysql compatibility (#23087)

Browse Source 
Better code structure for mysql compatibility databases.
This commit is contained in:
hzq
2023-08-18 09:16:23 +08:00
committed by GitHub
parent de98324ea7
commit 38c182100a
11 changed files with 140 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
View File

@ -18,7 +18,6 @@
package org.apache.doris.catalog;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
@ -758,14 +757,6 @@ public class Database extends MetaObject implements Writable, DatabaseIf<Table>
return FunctionUtil.getFunctions(name2Function);
}
public boolean isInfoSchemaDb() {
return ClusterNamespace.getNameFromFullName(fullQualifiedName).equalsIgnoreCase(InfoSchemaDb.DATABASE_NAME);
}
public boolean isMysqlDb() {
return ClusterNamespace.getNameFromFullName(fullQualifiedName).equalsIgnoreCase(MysqlDb.DATABASE_NAME);
}
public synchronized void addEncryptKey(EncryptKey encryptKey, boolean ifNotExists) throws UserException {
if (addEncryptKeyImpl(encryptKey, false, ifNotExists)) {
Env.getCurrentEnv().getEditLog().logAddEncryptKey(encryptKey);
@ -907,4 +898,11 @@ public class Database extends MetaObject implements Writable, DatabaseIf<Table>
public String toString() {
return toJson();
}
// Return ture if database is created for mysql compatibility.
// Currently, we have two dbs that are created for this purpose, InformationSchemaDb and MysqlDb,
public boolean isMysqlCompatibleDatabase() {
return false;
}
}

55
fe/fe-core/src/main/java/org/apache/doris/catalog/InfoSchemaDb.java
View File

@ -18,76 +18,29 @@
package org.apache.doris.catalog;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.Pair;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
// Information schema used for MySQL compatible.
public class InfoSchemaDb extends Database {
public class InfoSchemaDb extends MysqlCompatibleDatabase {
public static final String DATABASE_NAME = "information_schema";
public static final long DATABASE_ID = 0L;
public InfoSchemaDb() {
super(DATABASE_ID, DATABASE_NAME);
initTables();
}
public InfoSchemaDb(String cluster) {
super(DATABASE_ID, ClusterNamespace.getFullName(cluster, DATABASE_NAME));
initTables();
}
@Override
public Pair<Boolean, Boolean> createTableWithLock(Table table, boolean isReplay, boolean setIfNotExist) {
return Pair.of(false, false);
}
@Override
public boolean createTable(Table table) {
// Do nothing.
return false;
}
@Override
public void dropTable(String name) {
// Do nothing.
}
@Override
public void write(DataOutput out) throws IOException {
// Do nothing
}
public void readFields(DataInput in) throws IOException {
throw new IOException("Not support.");
}
private void initTables() {
protected void initTables() {
for (Table table : SchemaTable.TABLE_MAP.values()) {
super.createTable(table);
}
}
@Override
public Table getTableNullable(String name) {
return super.getTableNullable(name.toLowerCase());
}
public static String getFullInfoSchemaDbName(String cluster) {
return ClusterNamespace.getFullName(cluster, DATABASE_NAME);
}
public static boolean isInfoSchemaDb(String dbName) {
if (dbName == null) {
return false;
}
String[] ele = dbName.split(ClusterNamespace.CLUSTER_DELIMITER);
String newDbName = dbName;
if (ele.length == 2) {
newDbName = ele[1];
}
return DATABASE_NAME.equalsIgnoreCase(newDbName);
public boolean createTable(Table table) {
return false;
}
}

101
fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlCompatibleDatabase.java Normal file
View File

@ -0,0 +1,101 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.catalog;
import org.apache.doris.alter.Alter;
import org.apache.doris.analysis.AlterTableStmt;
import org.apache.doris.analysis.CreateViewStmt;
import org.apache.doris.common.Pair;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
/**
* Abstract class for all databases created for mysql compatibility.
*/
public abstract class MysqlCompatibleDatabase extends Database {
public static int COUNT = 0;
public MysqlCompatibleDatabase(long id, String name) {
super(id, name);
initTables();
}
/**
* Internal database is not persisted to bdb, it will be created when fe starts.
* So driven class should implement this function to create table.
*/
protected abstract void initTables();
/**
* Currently, rename a table of InfoSchemaDb will throw exception
* {@link Alter#processAlterTable(AlterTableStmt)}
* so we follow this design.
* @note: Rename a table of mysql database in MYSQL ls allowed.
*/
@Override
public boolean createTable(Table table) {
return super.createTable(table);
}
@Override
public void dropTable(String name) {
// Do nothing
}
/**
* MysqlCompatibleDatabase will not be persisted to bdb.
* It will be constructed everytime the fe starts. See
* {@link org.apache.doris.datasource.InternalCatalog#InternalCatalog()}
*/
@Override
public void write(DataOutput out) throws IOException {
throw new IOException("Not support");
}
/**
* MysqlCompatibleDatabase should not be read from bdb.
*/
@Override
public void readFields(DataInput in) throws IOException {
throw new IOException("Not support.");
}
@Override
public boolean isMysqlCompatibleDatabase() {
return true;
}
/**
* This method must be re-implemented since {@link Env#createView(CreateViewStmt)}
* will call this method. And create view should not succeed under this database.
*/
@Override
public Pair<Boolean, Boolean> createTableWithLock(Table table, boolean isReplay, boolean setIfNotExist) {
return Pair.of(false, false);
}
/**
* All tables of mysql compatible database has case-insensitive name
* */
@Override
public Table getTableNullable(String name) {
return super.getTableNullable(name.toLowerCase());
}
}

73
fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlDb.java
View File

@ -17,15 +17,7 @@
package org.apache.doris.catalog;
import org.apache.doris.alter.Alter;
import org.apache.doris.analysis.AlterTableStmt;
import org.apache.doris.analysis.CreateViewStmt;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.Pair;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
/**
* This class is used for MySQL compatibility.
@ -40,7 +32,7 @@ import java.io.IOException;
* but currently we do not create any tables under mysql database of doris.
* We will add useful system tables in the future.
*/
public class MysqlDb extends Database {
public class MysqlDb extends MysqlCompatibleDatabase {
public static final String DATABASE_NAME = "mysql";
/**
* Database created by user will have database id starting from 10000 {@link Env#NEXT_ID_INIT_VALUE}.
@ -53,82 +45,21 @@ public class MysqlDb extends Database {
*/
public MysqlDb() {
super(DATABASE_ID, DATABASE_NAME);
initTables();
}
public MysqlDb(String cluster) {
super(DATABASE_ID, ClusterNamespace.getFullName(cluster, DATABASE_NAME));
initTables();
}
/**
* Do nothing for now.
* If we need tables of mysql database in the future, create a MysqlTable class like {@link SchemaTable}
*/
private void initTables() {
}
/**
* This method must be re-implemented since {@link Env#createView(CreateViewStmt)}
* will call this method. And create view should not succeed on this database.
*/
@Override
public Pair<Boolean, Boolean> createTableWithLock(Table table, boolean isReplay, boolean setIfNotExist) {
return Pair.of(false, false);
}
public void initTables() {}
/**
* Currently, rename a table of InfoSchemaDb will throw exception
* {@link Alter#processAlterTable(AlterTableStmt)}
* so we follow this design.
* @note: Rename a table of mysql database in MYSQL ls allowed.
*/
@Override
public boolean createTable(Table table) {
return false;
}
@Override
public void dropTable(String name) {
// Do nothing.
}
/**
* MysqlDb is not persistent to bdb. It will be constructed everytime the fe starts.
* {@link org.apache.doris.datasource.InternalCatalog#InternalCatalog()}
*/
@Override
public void write(DataOutput out) throws IOException {
// Do nothing
}
/**
* Same with {@link InfoSchemaDb#readFields(DataInput)}
*/
@Override
public void readFields(DataInput in) throws IOException {
throw new IOException("Not support.");
}
/**
* Same with {@link InfoSchemaDb#getTableNullable(String)}
*/
@Override
public Table getTableNullable(String name) {
return super.getTableNullable(name.toLowerCase());
}
public static boolean isMysqlDb(String dbName) {
if (dbName == null) {
return false;
}
String[] elements = dbName.split(ClusterNamespace.CLUSTER_DELIMITER);
String newDbName = dbName;
if (elements.length == 2) {
newDbName = elements[1];
}
return DATABASE_NAME.equalsIgnoreCase(newDbName);
}
}

2
fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java
View File

@ -281,7 +281,7 @@ public class TabletChecker extends MasterDaemon {
continue;
}
if (db.isInfoSchemaDb() || db.isMysqlDb()) {
if (db.isMysqlCompatibleDatabase()) {
continue;
}

30
fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View File

@ -80,6 +80,7 @@ import org.apache.doris.catalog.MaterializedIndex.IndexState;
import org.apache.doris.catalog.MaterializedIndexMeta;
import org.apache.doris.catalog.MaterializedView;
import org.apache.doris.catalog.MetaIdGenerator.IdGeneratorBuffer;
import org.apache.doris.catalog.MysqlCompatibleDatabase;
import org.apache.doris.catalog.MysqlDb;
import org.apache.doris.catalog.MysqlTable;
import org.apache.doris.catalog.OdbcTable;
@ -187,7 +188,6 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
@ -217,11 +217,12 @@ public class InternalCatalog implements CatalogIf<Database> {
public InternalCatalog() {
// create internal databases
List<Database> internalDbs = new ArrayList<>();
internalDbs.add(new InfoSchemaDb(SystemInfoService.DEFAULT_CLUSTER));
internalDbs.add(new MysqlDb(SystemInfoService.DEFAULT_CLUSTER));
List<MysqlCompatibleDatabase> mysqlCompatibleDatabases = new ArrayList<>();
mysqlCompatibleDatabases.add(new InfoSchemaDb(SystemInfoService.DEFAULT_CLUSTER));
mysqlCompatibleDatabases.add(new MysqlDb(SystemInfoService.DEFAULT_CLUSTER));
MysqlCompatibleDatabase.COUNT = 2;
for (Database idb : internalDbs) {
for (MysqlCompatibleDatabase idb : mysqlCompatibleDatabases) {
// do not call unprotectedCreateDb, because it will cause loop recursive when initializing Env singleton
idToDb.put(idb.getId(), idb);
fullNameToDb.put(idb.getFullName(), idb);
@ -856,6 +857,10 @@ public class InternalCatalog implements CatalogIf<Database> {
// check database
Database db = (Database) getDbOrDdlException(dbName);
if (db.isMysqlCompatibleDatabase()) {
throw new DdlException("Drop table from this database is not allowed.");
}
db.writeLockOrDdlException();
try {
Table table = db.getTableNullable(tableName);
@ -1074,7 +1079,7 @@ public class InternalCatalog implements CatalogIf<Database> {
// check if db exists
Database db = getDbOrDdlException(dbName);
// InfoSchemaDb and MysqlDb can not create table manually
if (db instanceof InfoSchemaDb || db instanceof MysqlDb) {
if (db.isMysqlCompatibleDatabase()) {
ErrorReport.reportDdlException(ErrorCode.ERR_CANT_CREATE_TABLE, tableName,
ErrorCode.ERR_CANT_CREATE_TABLE.getCode(), "not supported create table in this database");
}
@ -3080,14 +3085,19 @@ public class InternalCatalog implements CatalogIf<Database> {
public long saveDb(CountingDataOutputStream dos, long checksum) throws IOException {
// 2 is for information_schema db & mysql db, which does not need to be persisted.
int dbCount = idToDb.size() - 2;
// And internal database could not be dropped, so we assert dbCount >= 0
int dbCount = idToDb.size() - MysqlCompatibleDatabase.COUNT;
if (dbCount < 0) {
throw new IOException("Invalid database count");
}
checksum ^= dbCount;
dos.writeInt(dbCount);
for (Map.Entry<Long, Database> entry : idToDb.entrySet()) {
Database db = entry.getValue();
String dbName = db.getFullName();
// Don't write information_schema & mysql db meta
if (!InfoSchemaDb.isInfoSchemaDb(dbName) && !MysqlDb.isMysqlDb(dbName)) {
// Don't write internal database meta.
if (!db.isMysqlCompatibleDatabase()) {
checksum ^= entry.getKey();
db.write(dos);
}

3
fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java
View File

@ -198,8 +198,7 @@ public class ShowAction extends RestBaseController {
} else {
for (long dbId : Env.getCurrentInternalCatalog().getDbIds()) {
DatabaseIf db = Env.getCurrentInternalCatalog().getDbNullable(dbId);
if (db == null || !(db instanceof Database) || ((Database) db).isInfoSchemaDb()
|| ((Database) db).isMysqlDb()) {
if (db == null || !(db instanceof Database) || ((Database) db).isMysqlCompatibleDatabase()) {
continue;
}
totalSize += getDataSizeOfDatabase(db);

2
fe/fe-core/src/main/java/org/apache/doris/master/PartitionInMemoryInfoCollector.java
View File

@ -56,7 +56,7 @@ public class PartitionInMemoryInfoCollector extends MasterDaemon {
LOG.warn("Database [" + dbId + "] does not exist, skip to update database used data quota");
continue;
}
if (db.isInfoSchemaDb() || db.isMysqlDb()) {
if (db.isMysqlCompatibleDatabase()) {
continue;
}
try {

2
fe/fe-core/src/main/java/org/apache/doris/transaction/DbUsedDataQuotaInfoCollector.java
View File

@ -50,7 +50,7 @@ public class DbUsedDataQuotaInfoCollector extends MasterDaemon {
LOG.warn("Database [" + dbId + "] does not exist, skip to update database used data quota");
continue;
}
if (db.isInfoSchemaDb() || db.isMysqlDb()) {
if (db.isMysqlCompatibleDatabase()) {
continue;
}
try {

2
fe/fe-core/src/test/java/org/apache/doris/catalog/InfoSchemaDbTest.java
View File

@ -32,7 +32,7 @@ public class InfoSchemaDbTest {
Assert.assertFalse(db.createTable(null));
Assert.assertFalse(db.createTableWithLock(null, false, false).first);
db.dropTable("authors");
db.write(null);
Assert.assertThrows(IOException.class, () -> db.write(null));
Assert.assertNull(db.getTableNullable("authors"));
}
}

2
fe/fe-core/src/test/java/org/apache/doris/catalog/MysqlDbTest.java
View File

@ -32,7 +32,7 @@ public class MysqlDbTest {
Assert.assertFalse(db.createTable(null));
Assert.assertFalse(db.createTableWithLock(null, false, false).first);
db.dropTable("authors");
db.write(null);
Assert.assertThrows(IOException.class, () -> db.write(null));
Assert.assertNull(db.getTableNullable("authors"));
}