database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Reduce AlterJobV2/TruncateTable binlog size (#30505)

Browse Source 
Signed-off-by: Jack Drogon <jack.xsuperman@gmail.com>
This commit is contained in:
Jack Drogon
2024-01-29 18:54:46 +08:00
committed by yiguolei
parent 15c625dcbc
commit a0100ce29f
4 changed files with 113 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/alter/AlterJobV2.java
View File

@ -165,6 +165,10 @@ public abstract class AlterJobV2 implements Writable {
this.finishedTimeMs = finishedTimeMs;
}
public String getRawSql() {
return rawSql;
}
// /api/debug_point/add/{name}?value=100
private void stateWait(final String name) {
long waitTimeMs = DebugPointUtil.getDebugParamOrDefault(name, 0);

54
fe/fe-core/src/main/java/org/apache/doris/binlog/AlterJobRecord.java Normal file
View File

@ -0,0 +1,54 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.binlog;
import org.apache.doris.alter.AlterJobV2;
import org.apache.doris.persist.gson.GsonUtils;
import com.google.gson.annotations.SerializedName;
public class AlterJobRecord {
@SerializedName(value = "type")
private AlterJobV2.JobType type;
@SerializedName(value = "dbId")
private long dbId;
@SerializedName(value = "tableId")
private long tableId;
@SerializedName(value = "tableName")
private String tableName;
@SerializedName(value = "jobId")
private long jobId;
@SerializedName(value = "jobState")
private AlterJobV2.JobState jobState;
@SerializedName(value = "rawSql")
private String rawSql;
public AlterJobRecord(AlterJobV2 job) {
this.type = job.getType();
this.dbId = job.getDbId();
this.tableId = job.getTableId();
this.tableName = job.getTableName();
this.jobId = job.getJobId();
this.jobState = job.getJobState();
this.rawSql = job.getRawSql();
}
public String toJson() {
return GsonUtils.GSON.toJson(this);
}
}

6
fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java
View File

@ -213,7 +213,8 @@ public class BinlogManager {
tableIds.add(alterJob.getTableId());
long timestamp = -1;
TBinlogType type = TBinlogType.ALTER_JOB;
String data = alterJob.toJson();
AlterJobRecord alterJobRecord = new AlterJobRecord(alterJob);
String data = alterJobRecord.toJson();
addBinlog(dbId, tableIds, commitSeq, timestamp, type, data, false);
}
@ -303,7 +304,8 @@ public class BinlogManager {
tableIds.add(info.getTblId());
long timestamp = -1;
TBinlogType type = TBinlogType.TRUNCATE_TABLE;
String data = info.toJson();
TruncateTableRecord record = new TruncateTableRecord(info);
String data = record.toJson();
addBinlog(dbId, tableIds, commitSeq, timestamp, type, data, false);
}

51
fe/fe-core/src/main/java/org/apache/doris/binlog/TruncateTableRecord.java Normal file
View File

@ -0,0 +1,51 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.binlog;
import org.apache.doris.persist.TruncateTableInfo;
import org.apache.doris.persist.gson.GsonUtils;
import com.google.gson.annotations.SerializedName;
public class TruncateTableRecord {
@SerializedName(value = "dbId")
private long dbId;
@SerializedName(value = "db")
private String db;
@SerializedName(value = "tblId")
private long tblId;
@SerializedName(value = "table")
private String table;
@SerializedName(value = "isEntireTable")
private boolean isEntireTable = false;
@SerializedName(value = "rawSql")
private String rawSql = "";
public TruncateTableRecord(TruncateTableInfo info) {
this.dbId = info.getDbId();
this.db = info.getDb();
this.tblId = info.getTblId();
this.table = info.getTable();
this.isEntireTable = info.isEntireTable();
this.rawSql = info.getRawSql();
}
public String toJson() {
return GsonUtils.GSON.toJson(this);
}
}