database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Add a session variable to show or hide hidden columns (#4510)

Browse Source 
* add session variable to show hidden columns
This commit is contained in:
Zhengguo Yang
2020-09-10 13:07:43 +08:00
committed by GitHub
parent c4feb82229
commit fe0260e54f
20 changed files with 370 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
View File

@ -958,7 +958,7 @@ public class SelectStmt extends QueryStmt {
* refs for each column to selectListExprs.
*/
private void expandStar(TableName tblName, TupleDescriptor desc) {
for (Column col : desc.getTable().getBaseSchema(false)) {
for (Column col : desc.getTable().getBaseSchema()) {
resultExprs.add(new SlotRef(tblName, col.getName()));
colLabels.add(col.getName());
}

2
fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
View File

@ -3947,7 +3947,7 @@ public class Catalog {
sb.append("TABLE ");
sb.append("`").append(table.getName()).append("` (\n");
int idx = 0;
for (Column column : table.getBaseSchema(false)) {
for (Column column : table.getBaseSchema()) {
if (idx++ != 0) {
sb.append(",\n");
}

11
fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java
View File

@ -40,6 +40,7 @@ import java.io.IOException;
import java.io.StringReader;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
public class MaterializedIndexMeta implements Writable, GsonPostProcessable {
@SerializedName(value = "indexId")
@ -93,7 +94,15 @@ public class MaterializedIndexMeta implements Writable, GsonPostProcessable {
}
public List<Column> getSchema() {
return schema;
return getSchema(true);
}
public List<Column> getSchema(boolean full) {
if (full) {
return schema;
} else {
return schema.stream().filter(column -> column.isVisible()).collect(Collectors.toList());
}
}
public int getSchemaHash() {

4
fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
View File

@ -524,13 +524,13 @@ public class OlapTable extends Table {
public Map<Long, List<Column>> getIndexIdToSchema() {
Map<Long, List<Column>> result = Maps.newHashMap();
for (Map.Entry<Long, MaterializedIndexMeta> entry : indexIdToMeta.entrySet()) {
result.put(entry.getKey(), entry.getValue().getSchema());
result.put(entry.getKey(), entry.getValue().getSchema(Util.showHiddenColumns()));
}
return result;
}
public List<Column> getSchemaByIndexId(Long indexId) {
return indexIdToMeta.get(indexId).getSchema();
return getSchemaByIndexId(indexId, Util.showHiddenColumns());
}
public List<Column> getSchemaByIndexId(Long indexId, boolean full) {

2
fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java
View File

@ -76,7 +76,7 @@ public class SchemaTable extends Table {
"tables",
TableType.SCHEMA,
builder()
.column("TABLE_CATALOG", ScalarType.createVarchar(FN_REFLEN))
.column(" TABLE_CATALOG", ScalarType.createVarchar(FN_REFLEN))
.column("TABLE_SCHEMA", ScalarType.createVarchar(NAME_CHAR_LEN))
.column("TABLE_NAME", ScalarType.createVarchar(NAME_CHAR_LEN))
.column("TABLE_TYPE", ScalarType.createVarchar(NAME_CHAR_LEN))

5
fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
View File

@ -21,6 +21,7 @@ import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.Util;
import org.apache.doris.thrift.TTableDescriptor;
import com.google.common.base.Preconditions;
@ -141,12 +142,12 @@ public class Table extends MetaObject implements Writable {
}
public List<Column> getFullSchema() {
return fullSchema;
return getBaseSchema();
}
// should override in subclass if necessary
public List<Column> getBaseSchema() {
return fullSchema;
return getBaseSchema(Util.showHiddenColumns());
}
public List<Column> getBaseSchema(boolean full) {
if (full) {

2
fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java
View File

@ -124,7 +124,7 @@ public class IndexInfoProcDir implements ProcDirInterface {
Set<String> bfColumns = null;
if (table.getType() == TableType.OLAP) {
OlapTable olapTable = (OlapTable) table;
schema = olapTable.getSchemaByIndexId(idxId, false);
schema = olapTable.getSchemaByIndexId(idxId);
if (schema == null) {
throw new AnalysisException("Index " + idxId + " does not exist");
}

5
fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.common.util;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.qe.ConnectContext;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
@ -477,5 +478,9 @@ public class Util {
conn.setReadTimeout(readTimeoutMs);
return conn.getInputStream();
}
public static boolean showHiddenColumns() {
return ConnectContext.get() != null && ConnectContext.get().getSessionVariable().showHiddenColumns();
}
}

2
fe/fe-core/src/main/java/org/apache/doris/planner/SchemaScanNode.java
View File

@ -22,6 +22,7 @@ import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.catalog.SchemaTable;
import org.apache.doris.common.Config;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.Util;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.service.FrontendOptions;
import org.apache.doris.thrift.TPlanNode;
@ -91,6 +92,7 @@ public class SchemaScanNode extends ScanNode {
msg.schema_scan_node.setDb("SESSION");
}
}
msg.schema_scan_node.show_hidden_cloumns = Util.showHiddenColumns();
if (schemaTable != null) {
msg.schema_scan_node.setTable(schemaTable);

3
fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
View File

@ -60,6 +60,7 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.Reference;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.Util;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
@ -1364,7 +1365,7 @@ public class SingleNodePlanner {
switch (tblRef.getTable().getType()) {
case OLAP:
OlapScanNode olapNode = new OlapScanNode(ctx_.getNextNodeId(), tblRef.getDesc(), "OlapScanNode");
if (((OlapTable) tblRef.getTable()).hasDeleteSign()) {
if (Util.showHiddenColumns() && ((OlapTable) tblRef.getTable()).hasDeleteSign()) {
Expr conjunct = new BinaryPredicate(BinaryPredicate.Operator.EQ,
new SlotRef(tblRef.getAliasAsName(), Column.DELETE_SIGN), new IntLiteral(0));
conjunct.analyze(analyzer);

14
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View File

@ -85,6 +85,7 @@ public class SessionVariable implements Serializable, Writable {
public static final String FORWARD_TO_MASTER = "forward_to_master";
// user can set instance num after exchange, no need to be equal to nums of before exchange
public static final String PARALLEL_EXCHANGE_INSTANCE_NUM = "parallel_exchange_instance_num";
public static final String SHOW_HIDDEN_COLUMNS = "show_hidden_columns";
/*
* configure the mem limit of load process on BE.
* Previously users used exec_mem_limit to set memory limits.
@ -262,6 +263,8 @@ public class SessionVariable implements Serializable, Writable {
private int maxScanKeyNum = -1;
@VariableMgr.VarAttr(name = MAX_PUSHDOWN_CONDITIONS_PER_COLUMN)
private int maxPushdownConditionsPerColumn = -1;
@VariableMgr.VarAttr(name = SHOW_HIDDEN_COLUMNS, flag = VariableMgr.SESSION_ONLY)
private boolean showHiddenColumns = false;
public long getMaxExecMemByte() {
return maxExecMemByte;
@ -444,7 +447,7 @@ public class SessionVariable implements Serializable, Writable {
public void setEnablePartitionCache(boolean enablePartitionCache) {
this.enablePartitionCache = enablePartitionCache;
}
// Serialize to thrift object
public boolean getForwardToMaster() {
return forwardToMaster;
@ -509,6 +512,15 @@ public class SessionVariable implements Serializable, Writable {
this.maxPushdownConditionsPerColumn = maxPushdownConditionsPerColumn;
}
public boolean showHiddenColumns() {
return showHiddenColumns;
}
public void setShowHiddenColumns(boolean showHiddenColumns) {
this.showHiddenColumns = showHiddenColumns;
}
// Serialize to thrift object
// used for rest api
public TQueryOptions toThrift() {

2
fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
View File

@ -307,7 +307,7 @@ public class FrontendServiceImpl implements FrontendService.Iface {
try {
Table table = db.getTable(params.getTableName());
if (table != null) {
for (Column column : table.getBaseSchema()) {
for (Column column : table.getBaseSchema(params.isShowHiddenColumns())) {
final TColumnDesc desc = new TColumnDesc(column.getName(), column.getDataType().toThrift());
final Integer precision = column.getOriginType().getPrecision();
if (precision != null) {

12
fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java
View File

@ -19,8 +19,9 @@ package org.apache.doris.analysis;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.common.util.Util;
import org.apache.doris.planner.Planner;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.VariableMgr;
import org.apache.doris.utframe.DorisAssert;
import org.apache.doris.utframe.UtFrameUtils;
@ -34,6 +35,9 @@ import org.junit.rules.ExpectedException;
import java.util.UUID;
import mockit.Mock;
import mockit.MockUp;
public class SelectStmtTest {
private static String runningDir = "fe/mocked/DemoTest/" + UUID.randomUUID().toString() + "/";
private static DorisAssert dorisAssert;
@ -48,6 +52,12 @@ public class SelectStmtTest {
@BeforeClass
public static void setUp() throws Exception {
new MockUp<Util>() {
@Mock
public boolean showHiddenColumns() {
return true;
}
};
Config.enable_batch_delete_by_default = true;
UtFrameUtils.createMinDorisCluster(runningDir);
String createTblStmtStr = "create table db1.tbl1(k1 varchar(32), k2 varchar(32), k3 varchar(32), k4 int) "

10
fe/fe-core/src/test/java/org/apache/doris/qe/PartitionCacheTest.java
View File

@ -23,6 +23,7 @@ import org.apache.doris.common.util.SqlParserUtils;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.Util;
import org.apache.doris.thrift.TStorageType;
import org.apache.doris.qe.ConnectScheduler;
@ -77,6 +78,8 @@ import org.apache.doris.qe.ConnectContext;
import org.apache.doris.system.SystemInfoService;
import org.apache.doris.thrift.TUniqueId;
import mockit.Mock;
import mockit.MockUp;
import mockit.Mocked;
import mockit.Tested;
import mockit.Injectable;
@ -147,7 +150,12 @@ public class PartitionCacheTest {
public void setUp() {
MockedAuth.mockedAuth(auth);
MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1");
new MockUp<Util>() {
@Mock
public boolean showHiddenColumns() {
return true;
}
};
db = new Database(1L, fullDbName);
OlapTable tbl1 = createOrderTable();