[feature](stmt) add ADMIN COPY TABLET stmt for local debug (#12176)
Add a new stmt ADMIN COPY TABLET for easy copy a tablet to local env to reproduce problem. See document for more details.
This commit is contained in:
Mingyu Chen
@ -78,7 +78,7 @@ void SnapshotAction::handle(HttpRequest* req) {
|
||||
VLOG_ROW << "get make snapshot tablet info: " << tablet_id << "-" << schema_hash;
|
||||
|
||||
std::string snapshot_path;
|
||||
int64_t ret = make_snapshot(tablet_id, schema_hash, &snapshot_path);
|
||||
int64_t ret = _make_snapshot(tablet_id, schema_hash, &snapshot_path);
|
||||
if (ret != 0L) {
|
||||
std::string error_msg = std::string("make snapshot failed");
|
||||
HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, error_msg);
|
||||
@ -93,8 +93,8 @@ void SnapshotAction::handle(HttpRequest* req) {
|
||||
LOG(INFO) << "deal with snapshot request finished! tablet id: " << tablet_id;
|
||||
}
|
||||
|
||||
int64_t SnapshotAction::make_snapshot(int64_t tablet_id, int32_t schema_hash,
|
||||
std::string* snapshot_path) {
|
||||
int64_t SnapshotAction::_make_snapshot(int64_t tablet_id, int32_t schema_hash,
|
||||
std::string* snapshot_path) {
|
||||
TSnapshotRequest request;
|
||||
request.tablet_id = tablet_id;
|
||||
request.schema_hash = schema_hash;
|
||||
|
||||
@ -37,7 +37,7 @@ public:
|
||||
void handle(HttpRequest* req) override;
|
||||
|
||||
private:
|
||||
int64_t make_snapshot(int64_t tablet_id, int schema_hash, std::string* snapshot_path);
|
||||
int64_t _make_snapshot(int64_t tablet_id, int schema_hash, std::string* snapshot_path);
|
||||
}; // end class SnapshotAction
|
||||
|
||||
} // end namespace doris
|
||||
|
||||
@ -303,10 +303,10 @@ Status SnapshotManager::_calc_snapshot_id_path(const TabletSharedPtr& tablet, in
|
||||
return res;
|
||||
}
|
||||
|
||||
std::unique_lock<std::mutex> auto_lock(
|
||||
_snapshot_mutex); // will automatically unlock when function return.
|
||||
std::unique_lock<std::mutex> auto_lock(_snapshot_mutex);
|
||||
uint64_t sid = _snapshot_base_id++;
|
||||
*out_path = fmt::format("{}/{}/{}.{}.{}", tablet->data_dir()->path(), SNAPSHOT_PREFIX, time_str,
|
||||
_snapshot_base_id++, timeout_s);
|
||||
sid, timeout_s);
|
||||
return res;
|
||||
}
|
||||
|
||||
@ -322,6 +322,11 @@ std::string SnapshotManager::_get_header_full_path(const TabletSharedPtr& ref_ta
|
||||
return fmt::format("{}/{}.hdr", schema_hash_path, ref_tablet->tablet_id());
|
||||
}
|
||||
|
||||
std::string SnapshotManager::_get_json_header_full_path(const TabletSharedPtr& ref_tablet,
|
||||
const std::string& schema_hash_path) const {
|
||||
return fmt::format("{}/{}.hdr.json", schema_hash_path, ref_tablet->tablet_id());
|
||||
}
|
||||
|
||||
Status SnapshotManager::_link_index_and_data_files(
|
||||
const std::string& schema_hash_path, const TabletSharedPtr& ref_tablet,
|
||||
const std::vector<RowsetSharedPtr>& consistent_rowsets) {
|
||||
@ -348,11 +353,11 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet
|
||||
|
||||
// snapshot_id_path:
|
||||
// /data/shard_id/tablet_id/snapshot/time_str/id.timeout/
|
||||
std::string snapshot_id_path;
|
||||
int64_t timeout_s = config::snapshot_expire_time_sec;
|
||||
if (request.__isset.timeout) {
|
||||
timeout_s = request.timeout;
|
||||
}
|
||||
std::string snapshot_id_path;
|
||||
res = _calc_snapshot_id_path(ref_tablet, timeout_s, &snapshot_id_path);
|
||||
if (!res.ok()) {
|
||||
LOG(WARNING) << "failed to calc snapshot_id_path, ref tablet="
|
||||
@ -366,6 +371,8 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet
|
||||
// header_path:
|
||||
// /schema_full_path/tablet_id.hdr
|
||||
auto header_path = _get_header_full_path(ref_tablet, schema_full_path);
|
||||
// /schema_full_path/tablet_id.hdr.json
|
||||
auto json_header_path = _get_json_header_full_path(ref_tablet, schema_full_path);
|
||||
if (FileUtils::check_exist(schema_full_path)) {
|
||||
VLOG_TRACE << "remove the old schema_full_path.";
|
||||
FileUtils::remove_all(schema_full_path);
|
||||
@ -515,6 +522,9 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet
|
||||
|
||||
if (snapshot_version == g_Types_constants.TSNAPSHOT_REQ_VERSION2) {
|
||||
res = new_tablet_meta->save(header_path);
|
||||
if (res.ok() && request.__isset.is_copy_tablet_task && request.is_copy_tablet_task) {
|
||||
res = new_tablet_meta->save_as_json(json_header_path, ref_tablet->data_dir());
|
||||
}
|
||||
} else {
|
||||
res = Status::OLAPInternalError(OLAP_ERR_INVALID_SNAPSHOT_VERSION);
|
||||
}
|
||||
|
||||
@ -73,6 +73,9 @@ private:
|
||||
std::string _get_header_full_path(const TabletSharedPtr& ref_tablet,
|
||||
const std::string& schema_hash_path) const;
|
||||
|
||||
std::string _get_json_header_full_path(const TabletSharedPtr& ref_tablet,
|
||||
const std::string& schema_hash_path) const;
|
||||
|
||||
Status _link_index_and_data_files(const std::string& header_path,
|
||||
const TabletSharedPtr& ref_tablet,
|
||||
const std::vector<RowsetSharedPtr>& consistent_rowsets);
|
||||
|
||||
@ -313,6 +313,20 @@ std::string TabletMeta::construct_header_file_path(const string& schema_hash_pat
|
||||
return header_name_stream.str();
|
||||
}
|
||||
|
||||
Status TabletMeta::save_as_json(const string& file_path, DataDir* dir) {
|
||||
std::string json_meta;
|
||||
json2pb::Pb2JsonOptions json_options;
|
||||
json_options.pretty_json = true;
|
||||
json_options.bytes_to_base64 = true;
|
||||
to_json(&json_meta, json_options);
|
||||
// save to file
|
||||
io::FileWriterPtr file_writer;
|
||||
RETURN_IF_ERROR(dir->fs()->create_file(file_path, &file_writer));
|
||||
RETURN_IF_ERROR(file_writer->append(json_meta));
|
||||
RETURN_IF_ERROR(file_writer->close());
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status TabletMeta::save(const string& file_path) {
|
||||
TabletMetaPB tablet_meta_pb;
|
||||
to_meta_pb(&tablet_meta_pb);
|
||||
|
||||
@ -100,6 +100,7 @@ public:
|
||||
// Previous tablet_meta is a physical file in tablet dir, which is not stored in rocksdb.
|
||||
Status create_from_file(const std::string& file_path);
|
||||
Status save(const std::string& file_path);
|
||||
Status save_as_json(const string& file_path, DataDir* dir);
|
||||
static Status save(const std::string& file_path, const TabletMetaPB& tablet_meta_pb);
|
||||
static Status reset_tablet_uid(const std::string& file_path);
|
||||
static std::string construct_header_file_path(const std::string& schema_hash_path,
|
||||
|
||||
@ -1,81 +1,224 @@
|
||||
---
|
||||
{
|
||||
"title": "Tablet Local Debug",
|
||||
"language": "zh-CN"
|
||||
}
|
||||
---
|
||||
|
||||
During the online operation of Doris, various bugs may occur for various reasons, e.g., inconsistent replicas, version diffs in the data, etc. At this point, it is necessary to replicate the online tablet data to the local environment and then locate the problem.
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
In this case, you need to copy the online tablet copy data to the local environment for replication and then locate the problem.
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
## 1\. Prepare the environment
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
Deploy a single-node Doris cluster locally, with the same deployment version as the online cluster.
|
||||
# Tablet Local Debug
|
||||
|
||||
If the online deployment is DORIS-1.0.1, deploy DORIS-1.0.1 in the local environment as well.
|
||||
During the online operation of Doris, various bugs may occur due to various reasons. For example: the replica is inconsistent, the data exists in the version diff, etc.
|
||||
|
||||
After deploying the cluster, create a local table that is the same as the online one, but the changes that need to be made are that the local table has only one copy of the tablet, and the version, version_hash need to be specified.
|
||||
At this time, it is necessary to copy the copy data of the tablet online to the local environment for reproduction, and then locate the problem.
|
||||
|
||||
If the problem is to locate inconsistent data, you can build three different tables corresponding to each copy online.
|
||||
## 1. Get information about the tablet
|
||||
|
||||
## 2\. Copy Data
|
||||
The tablet id can be confirmed by the BE log, and then the information can be obtained by the following command (assuming the tablet id is 10020).
|
||||
|
||||
To find the machine where the copy of online tablet is located, find the method, you can find the machine where the corresponding copy is located with the following two commands.
|
||||
|
||||
* show tablet meta data
|
||||
```show tablet 10011```
|
||||
```
|
||||
mysql> show tablet 10011;
|
||||
+-------------------------------+-----------+---------------+-----------+-------+---------+-------------+---------+--------+-------+------------------------------------------------------------+
|
||||
| DbName | TableName | PartitionName | IndexName | DbId | TableId | PartitionId | IndexId | IsSync | Order | DetailCmd |
|
||||
+-------------------------------+-----------+---------------+-----------+-------+---------+-------------+---------+--------+-------+------------------------------------------------------------+
|
||||
| default_cluster:test_query_qa | baseall | baseall | baseall | 10007 | 10009 | 10008 | 10010 | true | 0 | SHOW PROC '/dbs/10007/10009/partitions/10008/10010/10011'; |
|
||||
+-------------------------------+-----------+---------------+-----------+-------+---------+-------------+---------+--------+-------+------------------------------------------------------------+
|
||||
1 row in set (0.00 sec)
|
||||
```
|
||||
* Execute the `DetailCmd` command, locate the BE ip：
|
||||
|
||||
```SHOW PROC '/dbs/10007/10009/partitions/10008/10010/10011';```
|
||||
```
|
||||
mysql> SHOW PROC '/dbs/10007/10009/partitions/10008/10010/10011';
|
||||
+-----------+-----------+---------+-------------------+------------------+---------------+------------+----------+----------+--------+-------+--------------+----------------------+-------------------------------------------------+---------------------------------------------------------------+
|
||||
| ReplicaId | BackendId | Version | LstSuccessVersion | LstFailedVersion | LstFailedTime | SchemaHash | DataSize | RowCount | State | IsBad | VersionCount | PathHash | MetaUrl | CompactionStatus |
|
||||
+-----------+-----------+---------+-------------------+------------------+---------------+------------+----------+----------+--------+-------+--------------+----------------------+-------------------------------------------------+---------------------------------------------------------------+
|
||||
| 10012 | 10003 | 2 | 2 | -1 | NULL | 945014548 | 2195 | 5 | NORMAL | false | 2 | 844142863681807094 | http://192.168.0.2:8001/api/meta/header/10011 | http://192.168.0.2:8001/api/compaction/show?tablet_id=10011 |
|
||||
| 10013 | 10002 | 2 | 2 | -1 | NULL | 945014548 | 2195 | 5 | NORMAL | false | 2 | -6740067817150249792 | http://192.168.0.1:8001/api/meta/header/10011 | http://192.168.0.1:8001/api/compaction/show?tablet_id=10011 |
|
||||
| 10014 | 10005 | 2 | 2 | -1 | NULL | 945014548 | 2195 | 5 | NORMAL | false | 2 | 4758004238194195485 | http://192.168.0.3:8001/api/meta/header/10011 | http://192.168.0.3:8001/api/compaction/show?tablet_id=10011 |
|
||||
+-----------+-----------+---------+-------------------+------------------+---------------+------------+----------+----------+--------+-------+--------------+----------------------+-------------------------------------------------+---------------------------------------------------------------+
|
||||
3 rows in set (0.01 sec)
|
||||
```
|
||||
* Log in to the corresponding machine and find the directory where the replica is located.
|
||||
Get information such as DbId/TableId/PartitionId where the tablet is located.
|
||||
|
||||
```
|
||||
ll ./data.HDD/data/0/10011/945014548/
|
||||
total 4
|
||||
-rw-rw-r-- 1 palo-qa palo-qa 2195 Jul 14 20:16 0200000000000018bb4a69226c414ace42487209dc145dbb_0.dat
|
||||
mysql> show tablet 10020\G
|
||||
*************************** 1. row ***************************
|
||||
DbName: default_cluster:db1
|
||||
TableName: tbl1
|
||||
PartitionName: tbl1
|
||||
IndexName: tbl1
|
||||
DbId: 10004
|
||||
TableId: 10016
|
||||
PartitionId: 10015
|
||||
IndexId: 10017
|
||||
IsSync: true
|
||||
Order: 1
|
||||
DetailCmd: SHOW PROC '/dbs/10004/10016/partitions/10015/10017/10020';
|
||||
```
|
||||
|
||||
Then replica the directory to the local counterpart with the scp command.
|
||||
|
||||
## 3\. Download meta data
|
||||
|
||||
Get the metadata of the replica
|
||||
Execute `DetailCmd` in the previous step to obtain information such as BackendId/SchemHash.
|
||||
|
||||
```
|
||||
wget http://host:be_http_port/api/meta/header/$tablet_id?byte_to_base64=true -O meta_data
|
||||
mysql> SHOW PROC '/dbs/10004/10016/partitions/10015/10017/10020'\G
|
||||
*************************** 1. row ***************************
|
||||
ReplicaId: 10021
|
||||
BackendId: 10003
|
||||
Version: 3
|
||||
LstSuccessVersion: 3
|
||||
LstFailedVersion: -1
|
||||
LstFailedTime: NULL
|
||||
SchemaHash: 785778507
|
||||
LocalDataSize: 780
|
||||
RemoteDataSize: 0
|
||||
RowCount: 2
|
||||
State: NORMAL
|
||||
IsBad: false
|
||||
VersionCount: 3
|
||||
PathHash: 7390150550643804973
|
||||
MetaUrl: http://192.168.10.1:8040/api/meta/header/10020
|
||||
CompactionStatus: http://192.168.10.1:8040/api/compaction/show?tablet_id=10020
|
||||
```
|
||||
|
||||
## 4\. Modify meta data
|
||||
|
||||
Take the metadata downloaded online and modify it to identify the online data copied in step 2.
|
||||
|
||||
In the same way download the tablet metadata from the local deployment cluster, and according to the correspondence, change the `table_id, tablet_id, partition_id, schema_hash, shard_id` in the metadata to the same case as local, other fields do not need to be changed.
|
||||
|
||||
## 5\. Take effect
|
||||
|
||||
(1) Stop the local be in order to take effect the metadata.
|
||||
|
||||
(2) Delete the metadata in the local be by the command, and take effect the new metadata at the same time.
|
||||
```
|
||||
./lib/meta_tool --root_path=/home/doris/be --operation=get_meta --tablet_id=10027 --schema_hash=112641656
|
||||
./lib/meta_tool --root_path=/home/doris/be --operation=delete_meta --tablet_id=10027 --schema_hash=112641656
|
||||
./lib/meta_tool --root_path=/home/doris/be --operation=load_meta --json_meta_path=/home/doris/error/tablet/112641656_1
|
||||
Create tablet snapshot and get table creation statement
|
||||
|
||||
```
|
||||
mysql> admin copy tablet 10020 properties("backend_id" = "10003", "version" = "2")\G
|
||||
*************************** 1. row ***************************
|
||||
TabletId: 10020
|
||||
BackendId: 10003
|
||||
Ip: 192.168.10.1
|
||||
Path: /path/to/be/storage/snapshot/20220830101353.2.3600
|
||||
ExpirationMinutes: 60
|
||||
CreateTableStmt: CREATE TABLE `tbl1` (
|
||||
`k1` int(11) NULL,
|
||||
`k2` int(11) NULL
|
||||
) ENGINE=OLAP
|
||||
DUPLICATE KEY(`k1`, `k2`)
|
||||
DISTRIBUTED BY HASH(k1) BUCKETS 1
|
||||
PROPERTIES (
|
||||
"replication_num" = "1",
|
||||
"version_info" = "2"
|
||||
);
|
||||
```
|
||||
|
||||
(3) Restart be query the corresponding data.
|
||||
The `admin copy tablet` command can generate a snapshot file of the corresponding replica and version for the specified tablet. Snapshot files are stored in the `Path` directory of the BE node indicated by the `Ip` field.
|
||||
|
||||
There will be a directory named tablet id under this directory, which will be packaged as a whole for later use. (Note that the directory is kept for a maximum of 60 minutes, after which it is automatically deleted).
|
||||
|
||||
```
|
||||
cd /path/to/be/storage/snapshot/20220830101353.2.3600
|
||||
tar xzf 10020.tar.gz 10020/
|
||||
```
|
||||
|
||||
The command will also generate the table creation statement corresponding to the tablet at the same time. Note that this table creation statement is not the original table creation statement, its bucket number and replica number are both 1, and the `versionInfo` field is specified. This table building statement is used later when loading the tablet locally.
|
||||
|
||||
So far, we have obtained all the necessary information, the list is as follows:
|
||||
|
||||
1. Packaged tablet data, such as 10020.tar.gz.
|
||||
2. Create a table statement.
|
||||
|
||||
## 2. Load Tablet locally
|
||||
|
||||
1. Build a local debugging environment
|
||||
|
||||
Deploy a single-node Doris cluster (1FE, 1BE) locally, and the deployment version is the same as the online cluster. If the online deployment version is DORIS-1.1.1, the local environment also deploys the DORIS-1.1.1 version.
|
||||
|
||||
2. Create a table
|
||||
|
||||
Create a table in the local environment using the create table statement from the previous step.
|
||||
|
||||
3. Get the tablet information of the newly created table
|
||||
|
||||
Because the number of buckets and replicas of the newly created table is 1, there will only be one tablet with one replica:
|
||||
|
||||
```
|
||||
mysql> show tablets from tbl1\G
|
||||
*************************** 1. row ***************************
|
||||
TabletId: 10017
|
||||
ReplicaId: 10018
|
||||
BackendId: 10003
|
||||
SchemaHash: 44622287
|
||||
Version: 1
|
||||
LstSuccessVersion: 1
|
||||
LstFailedVersion: -1
|
||||
LstFailedTime: NULL
|
||||
LocalDataSize: 0
|
||||
RemoteDataSize: 0
|
||||
RowCount: 0
|
||||
State: NORMAL
|
||||
LstConsistencyCheckTime: NULL
|
||||
CheckVersion: -1
|
||||
VersionCount: -1
|
||||
PathHash: 7390150550643804973
|
||||
MetaUrl: http://192.168.10.1:8040/api/meta/header/10017
|
||||
CompactionStatus: http://192.168.10.1:8040/api/compaction/show?tablet_id=10017
|
||||
```
|
||||
|
||||
```
|
||||
mysql> show tablet 10017\G
|
||||
*************************** 1. row ***************************
|
||||
DbName: default_cluster:db1
|
||||
TableName: tbl1
|
||||
PartitionName: tbl1
|
||||
IndexName: tbl1
|
||||
DbId: 10004
|
||||
TableId: 10015
|
||||
PartitionId: 10014
|
||||
IndexId: 10016
|
||||
IsSync: true
|
||||
Order: 0
|
||||
DetailCmd: SHOW PROC '/dbs/10004/10015/partitions/10014/10016/10017';
|
||||
```
|
||||
|
||||
Here we will record the following information:
|
||||
|
||||
* TableId
|
||||
* PartitionId
|
||||
* TabletId
|
||||
* SchemaHash
|
||||
|
||||
At the same time, we also need to go to the data directory of the BE node in the debugging environment to confirm the shard id where the new tablet is located:
|
||||
|
||||
```
|
||||
cd /path/to/storage/data/*/10017 && pwd
|
||||
```
|
||||
|
||||
This command will enter the directory where the tablet 10017 is located and display the path. Here we will see a path similar to the following:
|
||||
|
||||
```
|
||||
/path/to/storage/data/0/10017
|
||||
```
|
||||
|
||||
where `0` is the shard id.
|
||||
|
||||
4. Modify Tablet Data
|
||||
|
||||
Unzip the tablet data package obtained in the first step. The editor opens the 10017.hdr.json file, and modifies the following fields to the information obtained in the previous step:
|
||||
|
||||
```
|
||||
"table_id":10015
|
||||
"partition_id":10014
|
||||
"tablet_id":10017
|
||||
"schema_hash":44622287
|
||||
"shard_id":0
|
||||
```
|
||||
|
||||
5. Load the tablet
|
||||
|
||||
First, stop the debug environment's BE process (./bin/stop_be.sh). Then copy all the .dat files in the same level directory of the 10017.hdr.json file to the `/path/to/storage/data/0/10017/44622287` directory. This directory is the directory where the debugging environment tablet we obtained in step 3 is located. `10017/44622287` are the tablet id and schema hash respectively.
|
||||
|
||||
Delete the original tablet meta with the `meta_tool` tool. The tool is located in the `be/lib` directory.
|
||||
|
||||
```
|
||||
./lib/meta_tool --root_path=/path/to/storage --operation=delete_meta --tablet_id=10017 --schema_hash=44622287
|
||||
```
|
||||
|
||||
Where `/path/to/storage` is the data root directory of BE. If the deletion is successful, the delete successfully log will appear.
|
||||
|
||||
Load the new tablet meta via the `meta_tool` tool.
|
||||
|
||||
```
|
||||
./lib/meta_tool --root_path=/path/to/storage --operation=load_meta --json_meta_path=/path/to/10017.hdr.json
|
||||
```
|
||||
|
||||
If the load is successful, the load successfully log will appear.
|
||||
|
||||
6. Verification
|
||||
|
||||
Restart the debug environment's BE process (./bin/start_be.sh). Query the table, if correct, you can query the data of the loaded tablet, or reproduce the online problem.
|
||||
@ -0,0 +1,101 @@
|
||||
---
|
||||
{
|
||||
"title": "ADMIN-COPY-TABLET"
|
||||
"language": "en"
|
||||
}
|
||||
---
|
||||
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
## ADMIN-COPY-TABLET
|
||||
|
||||
### Name
|
||||
|
||||
ADMIN COPY TABLET
|
||||
|
||||
### Description
|
||||
|
||||
This statement is used to make a snapshot for the specified tablet, mainly used to load the tablet locally to reproduce the problem.
|
||||
|
||||
syntax:
|
||||
|
||||
```sql
|
||||
ADMIN COPY TABLET tablet_id PROPERTIES("xxx");
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
This command requires ROOT privileges.
|
||||
|
||||
PROPERTIES supports the following properties:
|
||||
|
||||
1. backend_id: Specifies the id of the BE node where the replica is located. If not specified, a replica is randomly selected.
|
||||
|
||||
2. version: Specifies the version of the snapshot. The version must be less than or equal to the largest version of the replica. If not specified, the largest version is used.
|
||||
|
||||
3. expiration_minutes: Snapshot retention time. The default is 1 hour. It will automatically clean up after a timeout. Unit minutes.
|
||||
|
||||
The results are shown below:
|
||||
|
||||
```
|
||||
TabletId: 10020
|
||||
BackendId: 10003
|
||||
Ip: 192.168.10.1
|
||||
Path: /path/to/be/storage/snapshot/20220830101353.2.3600
|
||||
ExpirationMinutes: 60
|
||||
CreateTableStmt: CREATE TABLE `tbl1` (
|
||||
`k1` int(11) NULL,
|
||||
`k2` int(11) NULL
|
||||
) ENGINE=OLAP
|
||||
DUPLICATE KEY(`k1`, `k2`)
|
||||
DISTRIBUTED BY HASH(k1) BUCKETS 1
|
||||
PROPERTIES (
|
||||
"replication_num" = "1",
|
||||
"version_info" = "2"
|
||||
);
|
||||
```
|
||||
|
||||
* TabletId: tablet id
|
||||
* BackendId: BE node id
|
||||
* Ip: BE node ip
|
||||
* Path: The directory where the snapshot is located
|
||||
* ExpirationMinutes: snapshot expiration time
|
||||
* CreateTableStmt: The table creation statement for the table corresponding to the tablet. This statement is not the original table-building statement, but a simplified table-building statement for later loading the tablet locally.
|
||||
|
||||
### Example
|
||||
|
||||
1. Take a snapshot of the replica on the specified BE node
|
||||
|
||||
```sql
|
||||
ADMIN COPY TABLET 10010 PROPERTIES("backend_id" = "10001");
|
||||
```
|
||||
|
||||
2. Take a snapshot of the specified version of the replica on the specified BE node
|
||||
|
||||
```sql
|
||||
ADMIN COPY TABLET 10010 PROPERTIES("backend_id" = "10001", "version" = "10");
|
||||
```
|
||||
|
||||
### Keywords
|
||||
|
||||
ADMIN, COPY, TABLET
|
||||
|
||||
### Best Practice
|
||||
|
||||
@ -701,6 +701,7 @@
|
||||
"sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SET-CONFIG",
|
||||
"sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SHOW-TABLET-STORAGE-FORMAT",
|
||||
"sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SHOW-REPLICA-STATUS"
|
||||
"sql-manual/sql-reference/Database-Administration-Statements/ADMIN-COPY-TABLET"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@ -1,81 +1,224 @@
|
||||
---
|
||||
{
|
||||
"title": "Tablet 本地调试",
|
||||
"language": "zh-CN"
|
||||
}
|
||||
---
|
||||
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
# Tablet 本地调试
|
||||
|
||||
Doris线上运行过程中，因为各种原因，可能出现各种各样的bug。例如：副本不一致，数据存在版本diff等。
|
||||
|
||||
这时候需要将线上的tablet的副本数据拷贝到本地环境进行复现，然后进行问题定位。
|
||||
|
||||
## 1\. 准备环境
|
||||
## 1. 获取有问题的 Tablet 的信息
|
||||
|
||||
在本地部署一个单节点的Doris集群，部署版本和线上集群保持一致。
|
||||
可以通过 BE 日志确认 tablet id，然后通过以下命令获取信息（假设 tablet id 为 10020）。
|
||||
|
||||
如果线上部署的版本是DORIS-1.0.1, 本地环境也同样部署DORIS-1.0.1的版本。
|
||||
|
||||
部署好集群之后，在本地建一个和线上同样的表，但是需要作出的改动就是，本地建的表只有一个副本的tablet，同时需要指定version, version_hash。
|
||||
|
||||
如果是定位数据不一致的问题，可以建三个不同的表对应线上的各个副本。
|
||||
|
||||
## 2\. 拷贝数据
|
||||
|
||||
找到线上tablet的副本所在的机器，寻找方法，可以下面两个命令找到副本所在的机器。
|
||||
|
||||
* 查看tablet元数据
|
||||
```show tablet 10011```
|
||||
```
|
||||
mysql> show tablet 10011;
|
||||
+-------------------------------+-----------+---------------+-----------+-------+---------+-------------+---------+--------+-------+------------------------------------------------------------+
|
||||
| DbName | TableName | PartitionName | IndexName | DbId | TableId | PartitionId | IndexId | IsSync | Order | DetailCmd |
|
||||
+-------------------------------+-----------+---------------+-----------+-------+---------+-------------+---------+--------+-------+------------------------------------------------------------+
|
||||
| default_cluster:test_query_qa | baseall | baseall | baseall | 10007 | 10009 | 10008 | 10010 | true | 0 | SHOW PROC '/dbs/10007/10009/partitions/10008/10010/10011'; |
|
||||
+-------------------------------+-----------+---------------+-----------+-------+---------+-------------+---------+--------+-------+------------------------------------------------------------+
|
||||
1 row in set (0.00 sec)
|
||||
```
|
||||
* 执行`DetailCmd`命令, 定位BE：
|
||||
|
||||
```SHOW PROC '/dbs/10007/10009/partitions/10008/10010/10011';```
|
||||
```
|
||||
mysql> SHOW PROC '/dbs/10007/10009/partitions/10008/10010/10011';
|
||||
+-----------+-----------+---------+-------------------+------------------+---------------+------------+----------+----------+--------+-------+--------------+----------------------+-------------------------------------------------+---------------------------------------------------------------+
|
||||
| ReplicaId | BackendId | Version | LstSuccessVersion | LstFailedVersion | LstFailedTime | SchemaHash | DataSize | RowCount | State | IsBad | VersionCount | PathHash | MetaUrl | CompactionStatus |
|
||||
+-----------+-----------+---------+-------------------+------------------+---------------+------------+----------+----------+--------+-------+--------------+----------------------+-------------------------------------------------+---------------------------------------------------------------+
|
||||
| 10012 | 10003 | 2 | 2 | -1 | NULL | 945014548 | 2195 | 5 | NORMAL | false | 2 | 844142863681807094 | http://192.168.0.2:8001/api/meta/header/10011 | http://192.168.0.2:8001/api/compaction/show?tablet_id=10011 |
|
||||
| 10013 | 10002 | 2 | 2 | -1 | NULL | 945014548 | 2195 | 5 | NORMAL | false | 2 | -6740067817150249792 | http://192.168.0.1:8001/api/meta/header/10011 | http://192.168.0.1:8001/api/compaction/show?tablet_id=10011 |
|
||||
| 10014 | 10005 | 2 | 2 | -1 | NULL | 945014548 | 2195 | 5 | NORMAL | false | 2 | 4758004238194195485 | http://192.168.0.3:8001/api/meta/header/10011 | http://192.168.0.3:8001/api/compaction/show?tablet_id=10011 |
|
||||
+-----------+-----------+---------+-------------------+------------------+---------------+------------+----------+----------+--------+-------+--------------+----------------------+-------------------------------------------------+---------------------------------------------------------------+
|
||||
3 rows in set (0.01 sec)
|
||||
```
|
||||
* 登录到对应机器上，找到副本所在的目录。
|
||||
获获取 tablet 所在的 DbId/TableId/PartitionId 等信息。
|
||||
|
||||
```
|
||||
ll ./data.HDD/data/0/10011/945014548/
|
||||
total 4
|
||||
-rw-rw-r-- 1 palo-qa palo-qa 2195 Jul 14 20:16 0200000000000018bb4a69226c414ace42487209dc145dbb_0.dat
|
||||
mysql> show tablet 10020\G
|
||||
*************************** 1. row ***************************
|
||||
DbName: default_cluster:db1
|
||||
TableName: tbl1
|
||||
PartitionName: tbl1
|
||||
IndexName: tbl1
|
||||
DbId: 10004
|
||||
TableId: 10016
|
||||
PartitionId: 10015
|
||||
IndexId: 10017
|
||||
IsSync: true
|
||||
Order: 1
|
||||
DetailCmd: SHOW PROC '/dbs/10004/10016/partitions/10015/10017/10020';
|
||||
```
|
||||
|
||||
然后通过scp命令，把目录拷贝到本地对应的目录下。
|
||||
|
||||
## 3\. 下载元数据
|
||||
|
||||
获取副本的元数据
|
||||
执行上一步中的 `DetailCmd` 获取 BackendId/SchemHash 等信息。
|
||||
|
||||
```
|
||||
wget http://host:be_http_port/api/meta/header/$tablet_id?byte_to_base64=true -O meta_data
|
||||
mysql> SHOW PROC '/dbs/10004/10016/partitions/10015/10017/10020'\G
|
||||
*************************** 1. row ***************************
|
||||
ReplicaId: 10021
|
||||
BackendId: 10003
|
||||
Version: 3
|
||||
LstSuccessVersion: 3
|
||||
LstFailedVersion: -1
|
||||
LstFailedTime: NULL
|
||||
SchemaHash: 785778507
|
||||
LocalDataSize: 780
|
||||
RemoteDataSize: 0
|
||||
RowCount: 2
|
||||
State: NORMAL
|
||||
IsBad: false
|
||||
VersionCount: 3
|
||||
PathHash: 7390150550643804973
|
||||
MetaUrl: http://192.168.10.1:8040/api/meta/header/10020
|
||||
CompactionStatus: http://192.168.10.1:8040/api/compaction/show?tablet_id=10020
|
||||
```
|
||||
|
||||
## 4\. 修改元数据
|
||||
|
||||
将线上下载下来的元数据，进行修改，修改的目的是为了识别第2步拷贝的线上数据。
|
||||
|
||||
同样的方式将本地部署集群的tablet元数据下载下来，根据对应关系，将元数据中的`table_id, tablet_id, partition_id, schema_hash，shard_id`改成和本地一样的情况，其他字段不用改。
|
||||
|
||||
## 5\. 生效
|
||||
|
||||
(1) 停掉本地的be，才能生效元数据。
|
||||
|
||||
(2) 通过命令删除本地be中的元数据，同时生效新的元数据。
|
||||
```
|
||||
./lib/meta_tool --root_path=/home/doris/be --operation=get_meta --tablet_id=10027 --schema_hash=112641656
|
||||
./lib/meta_tool --root_path=/home/doris/be --operation=delete_meta --tablet_id=10027 --schema_hash=112641656
|
||||
./lib/meta_tool --root_path=/home/doris/be --operation=load_meta --json_meta_path=/home/doris/error/tablet/112641656_1
|
||||
创建 tablet 快照并获取建表语句
|
||||
|
||||
```
|
||||
mysql> admin copy tablet 10020 properties("backend_id" = "10003", "version" = "2")\G
|
||||
*************************** 1. row ***************************
|
||||
TabletId: 10020
|
||||
BackendId: 10003
|
||||
Ip: 192.168.10.1
|
||||
Path: /path/to/be/storage/snapshot/20220830101353.2.3600
|
||||
ExpirationMinutes: 60
|
||||
CreateTableStmt: CREATE TABLE `tbl1` (
|
||||
`k1` int(11) NULL,
|
||||
`k2` int(11) NULL
|
||||
) ENGINE=OLAP
|
||||
DUPLICATE KEY(`k1`, `k2`)
|
||||
DISTRIBUTED BY HASH(k1) BUCKETS 1
|
||||
PROPERTIES (
|
||||
"replication_num" = "1",
|
||||
"version_info" = "2"
|
||||
);
|
||||
```
|
||||
|
||||
(3) 重启be, 查询对应的数据。
|
||||
`admin copy tablet` 命令可以为指定的 tablet 生成对应副本和版本的快照文件。快照文件存储在 `Ip` 字段所示节点的 `Path` 目录下。
|
||||
|
||||
该目下会有一个 tablet id 命名的目录，将这个目录整体打包后备用。（注意，该目录最多保留 60 分钟，之后会自动删除）。
|
||||
|
||||
```
|
||||
cd /path/to/be/storage/snapshot/20220830101353.2.3600
|
||||
tar xzf 10020.tar.gz 10020/
|
||||
```
|
||||
|
||||
该命令还会同时生成这个 tablet 对应的建表语句。注意，这个建表语句并不是原始的建表语句，他的分桶数和副本数都是1，并且指定了 `versionInfo` 字段。该建表语句是用于之后在本地加载 tablet 时使用的。
|
||||
|
||||
至此，我们已经获取到所有必要的信息，清单如下：
|
||||
|
||||
1. 打包好的 tablet 数据，如 10020.tar.gz。
|
||||
2. 建表语句。
|
||||
|
||||
## 2. 本地加载 Tablet
|
||||
|
||||
1. 搭建本地调试环境
|
||||
|
||||
在本地部署一个单节点的Doris集群（1FE、1BE），部署版本和线上集群保持一致。如线上部署的版本是DORIS-1.1.1, 本地环境也同样部署DORIS-1.1.1的版本。
|
||||
|
||||
2. 建表
|
||||
|
||||
使用上一步中得到的建表语句，在本地环境中创建一张表。
|
||||
|
||||
3. 获取新建的表的 tablet 的信息
|
||||
|
||||
因为新建表的分桶数和副本数都为1，所以只会有一个一副本的 tablet：
|
||||
|
||||
```
|
||||
mysql> show tablets from tbl1\G
|
||||
*************************** 1. row ***************************
|
||||
TabletId: 10017
|
||||
ReplicaId: 10018
|
||||
BackendId: 10003
|
||||
SchemaHash: 44622287
|
||||
Version: 1
|
||||
LstSuccessVersion: 1
|
||||
LstFailedVersion: -1
|
||||
LstFailedTime: NULL
|
||||
LocalDataSize: 0
|
||||
RemoteDataSize: 0
|
||||
RowCount: 0
|
||||
State: NORMAL
|
||||
LstConsistencyCheckTime: NULL
|
||||
CheckVersion: -1
|
||||
VersionCount: -1
|
||||
PathHash: 7390150550643804973
|
||||
MetaUrl: http://192.168.10.1:8040/api/meta/header/10017
|
||||
CompactionStatus: http://192.168.10.1:8040/api/compaction/show?tablet_id=10017
|
||||
```
|
||||
|
||||
```
|
||||
mysql> show tablet 10017\G
|
||||
*************************** 1. row ***************************
|
||||
DbName: default_cluster:db1
|
||||
TableName: tbl1
|
||||
PartitionName: tbl1
|
||||
IndexName: tbl1
|
||||
DbId: 10004
|
||||
TableId: 10015
|
||||
PartitionId: 10014
|
||||
IndexId: 10016
|
||||
IsSync: true
|
||||
Order: 0
|
||||
DetailCmd: SHOW PROC '/dbs/10004/10015/partitions/10014/10016/10017';
|
||||
```
|
||||
|
||||
这里我们要记录如下信息：
|
||||
|
||||
* TableId
|
||||
* PartitionId
|
||||
* TabletId
|
||||
* SchemaHash
|
||||
|
||||
同时，我们还需要到调试环境BE节点的数据目录下，确认新的 tablet 所在的 shard id：
|
||||
|
||||
```
|
||||
cd /path/to/storage/data/*/10017 && pwd
|
||||
```
|
||||
|
||||
这个命令会进入 10017 这个 tablet 所在目录并展示路径。这里我们会看到类似如下的路径：
|
||||
|
||||
```
|
||||
/path/to/storage/data/0/10017
|
||||
```
|
||||
|
||||
其中 `0` 既是 shard id。
|
||||
|
||||
4. 修改 Tablet 数据
|
||||
|
||||
解压第一步中获取到的 tablet 数据包。编辑器打开其中的 10017.hdr.json 文件，并修改以下字段为上一步中获取到的信息：
|
||||
|
||||
```
|
||||
"table_id":10015
|
||||
"partition_id":10014
|
||||
"tablet_id":10017
|
||||
"schema_hash":44622287
|
||||
"shard_id":0
|
||||
```
|
||||
|
||||
5. 加载新 tablet
|
||||
|
||||
首先，停止调试环境的 BE 进程（./bin/stop_be.sh）。然后将 10017.hdr.json 文件同级目录所在的所有 .dat 文件，拷贝到 `/path/to/storage/data/0/10017/44622287` 目录下。这个目录既是在第3步中，我们获取到的调试环境tablet所在目录。`10017/44622287` 分别是 tablet id 和 schema hash。
|
||||
|
||||
通过 `meta_tool` 工具删除原来的 tablet meta。该工具位于 `be/lib` 目录下。
|
||||
|
||||
```
|
||||
./lib/meta_tool --root_path=/path/to/storage --operation=delete_meta --tablet_id=10017 --schema_hash=44622287
|
||||
```
|
||||
|
||||
其中 `/path/to/storage` 为 BE 的数据根目录。如删除成功，会出现 delete successfully 日志。
|
||||
|
||||
通过 `meta_tool` 工具加载新的 tablet meta。
|
||||
|
||||
```
|
||||
./lib/meta_tool --root_path=/path/to/storage --operation=load_meta --json_meta_path=/path/to/10017.hdr.json
|
||||
```
|
||||
|
||||
如加载成功，会出现 load successfully 日志。
|
||||
|
||||
6. 验证
|
||||
|
||||
重新启动调试环境的 BE 进程(./bin/start_be.sh)。对表进行查询，如果正确，则可以查询出加载的 tablet的数据，或复现线上问题。
|
||||
@ -0,0 +1,101 @@
|
||||
---
|
||||
{
|
||||
"title": "ADMIN-COPY-TABLET"
|
||||
"language": "zh-CN"
|
||||
}
|
||||
---
|
||||
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
## ADMIN-COPY-TABLET
|
||||
|
||||
### Name
|
||||
|
||||
ADMIN COPY TABLET
|
||||
|
||||
### Description
|
||||
|
||||
该语句用于为指定的 tablet 制作快照，主要用于本地加载 tablet 来复现问题。
|
||||
|
||||
语法：
|
||||
|
||||
```sql
|
||||
ADMIN COPY TABLET tablet_id PROPERTIES("xxx");
|
||||
```
|
||||
|
||||
说明：
|
||||
|
||||
该命令需要 ROOT 权限。
|
||||
|
||||
PROPERTIES 支持如下属性：
|
||||
|
||||
1. backend_id：指定副本所在的 BE 节点的 id。如果不指定，则随机选择一个副本。
|
||||
|
||||
2. version：指定快照的版本。该版本需小于等于副本的最大版本。如不指定，则使用最大版本。
|
||||
|
||||
3. expiration_minutes：快照保留时长。默认为1小时。超时后会自动清理。单位分钟。
|
||||
|
||||
结果展示如下：
|
||||
|
||||
```
|
||||
TabletId: 10020
|
||||
BackendId: 10003
|
||||
Ip: 192.168.10.1
|
||||
Path: /path/to/be/storage/snapshot/20220830101353.2.3600
|
||||
ExpirationMinutes: 60
|
||||
CreateTableStmt: CREATE TABLE `tbl1` (
|
||||
`k1` int(11) NULL,
|
||||
`k2` int(11) NULL
|
||||
) ENGINE=OLAP
|
||||
DUPLICATE KEY(`k1`, `k2`)
|
||||
DISTRIBUTED BY HASH(k1) BUCKETS 1
|
||||
PROPERTIES (
|
||||
"replication_num" = "1",
|
||||
"version_info" = "2"
|
||||
);
|
||||
```
|
||||
|
||||
* TabletId: tablet id
|
||||
* BackendId: BE 节点 id
|
||||
* Ip: BE 节点 ip
|
||||
* Path: 快照所在目录
|
||||
* ExpirationMinutes: 快照过期时间
|
||||
* CreateTableStmt: tablet 对应的表的建表语句。该语句不是原始的建表语句，而是用于之后本地加载 tablet 的简化后的建表语句。
|
||||
|
||||
### Example
|
||||
|
||||
1. 对指定 BE 节点上的副本做快照
|
||||
|
||||
```sql
|
||||
ADMIN COPY TABLET 10010 PROPERTIES("backend_id" = "10001");
|
||||
```
|
||||
|
||||
2. 对指定 BE 节点上的副本，做指定版本的快照
|
||||
|
||||
```sql
|
||||
ADMIN COPY TABLET 10010 PROPERTIES("backend_id" = "10001", "version" = "10");
|
||||
```
|
||||
|
||||
### Keywords
|
||||
|
||||
ADMIN, COPY, TABLET
|
||||
|
||||
### Best Practice
|
||||
|
||||
@ -243,7 +243,7 @@ terminal String KW_ADD, KW_ADMIN, KW_AFTER, KW_AGGREGATE, KW_ALIAS, KW_ALL, KW_A
|
||||
KW_BACKEND, KW_BACKUP, KW_BETWEEN, KW_BEGIN, KW_BIGINT, KW_BINLOG, KW_BITMAP, KW_BITMAP_UNION, KW_QUANTILE_STATE, KW_QUANTILE_UNION, KW_BLOB, KW_BOOLEAN, KW_BROKER, KW_BACKENDS, KW_BY, KW_BUILD, KW_BUILTIN,
|
||||
KW_CANCEL, KW_CASE, KW_CAST, KW_CHAIN, KW_CHAR, KW_CHARSET, KW_CHECK, KW_CLUSTER, KW_CLUSTERS, KW_CLEAN, KW_CURRENT_TIMESTAMP,
|
||||
KW_COLLATE, KW_COLLATION, KW_COLUMN, KW_COLUMNS, KW_COMMENT, KW_COMMIT, KW_COMMITTED, KW_COMPACT, KW_COMPLETE,
|
||||
KW_CONFIG, KW_CONNECTION, KW_CONNECTION_ID, KW_CONSISTENT, KW_CONVERT, KW_COUNT, KW_CREATE, KW_CREATION, KW_CROSS, KW_CUBE, KW_CURRENT, KW_CURRENT_USER,
|
||||
KW_CONFIG, KW_CONNECTION, KW_CONNECTION_ID, KW_CONSISTENT, KW_CONVERT, KW_COPY, KW_COUNT, KW_CREATE, KW_CREATION, KW_CROSS, KW_CUBE, KW_CURRENT, KW_CURRENT_USER,
|
||||
KW_DATA, KW_DATABASE, KW_DATABASES, KW_DATE, KW_DATETIME, KW_DATEV2, KW_DATETIMEV2, KW_DAY, KW_DECIMAL, KW_DECIMALV3, KW_DECOMMISSION, KW_DEFAULT, KW_DEFERRED, KW_DEMAND, KW_DESC, KW_DESCRIBE,
|
||||
KW_DELETE, KW_UPDATE, KW_DIAGNOSE, KW_DISK, KW_DISTINCT, KW_DISTINCTPC, KW_DISTINCTPCSA, KW_DISTRIBUTED, KW_DISTRIBUTION, KW_DYNAMIC, KW_BUCKETS, KW_DIV, KW_DOUBLE, KW_DROP, KW_DROPP, KW_DUPLICATE,
|
||||
KW_ELSE, KW_ENABLE, KW_ENCRYPTKEY, KW_ENCRYPTKEYS, KW_END, KW_ENGINE, KW_ENGINES, KW_ENTER, KW_ERRORS, KW_EVENTS, KW_EXCEPT, KW_EXCLUDE,
|
||||
@ -5624,6 +5624,10 @@ admin_stmt ::=
|
||||
{:
|
||||
RESULT = new AdminShowTabletStorageFormatStmt(true);
|
||||
:}
|
||||
| KW_ADMIN KW_COPY KW_TABLET INTEGER_LITERAL:tabletId opt_properties:properties
|
||||
{:
|
||||
RESULT = new AdminCopyTabletStmt(tabletId, properties);
|
||||
:}
|
||||
;
|
||||
|
||||
truncate_stmt ::=
|
||||
@ -5797,6 +5801,8 @@ keyword ::=
|
||||
{: RESULT = id; :}
|
||||
| KW_CONVERT:id
|
||||
{: RESULT = id; :}
|
||||
| KW_COPY:id
|
||||
{: RESULT = id; :}
|
||||
| KW_CREATION:id
|
||||
{: RESULT = id; :}
|
||||
| KW_DATA:id
|
||||
|
||||
@ -0,0 +1,120 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.analysis;
|
||||
|
||||
import org.apache.doris.catalog.Column;
|
||||
import org.apache.doris.catalog.Env;
|
||||
import org.apache.doris.catalog.ScalarType;
|
||||
import org.apache.doris.common.AnalysisException;
|
||||
import org.apache.doris.common.ErrorCode;
|
||||
import org.apache.doris.common.ErrorReport;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
import org.apache.doris.qe.ShowResultSetMetaData;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.Maps;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
|
||||
// ADMIN COPY TABLET 10110 PROPERTIES('version' = '1000', backend_id = '10001');
|
||||
public class AdminCopyTabletStmt extends ShowStmt {
|
||||
public static final String PROP_VERSION = "version";
|
||||
public static final String PROP_BACKEND_ID = "backend_id";
|
||||
public static final String PROP_EXPIRATION = "expiration_minutes";
|
||||
private static final long DEFAULT_EXPIRATION_MINUTES = 60;
|
||||
public static final ImmutableList<String> TITLE_NAMES = new ImmutableList.Builder<String>().add("TabletId")
|
||||
.add("BackendId").add("Ip").add("Path").add("ExpirationMinutes").add("CreateTableStmt").build();
|
||||
|
||||
private long tabletId;
|
||||
private Map<String, String> properties = Maps.newHashMap();
|
||||
private long version = -1;
|
||||
private long backendId = -1;
|
||||
private long expirationMinutes = DEFAULT_EXPIRATION_MINUTES; // default 60min
|
||||
|
||||
public AdminCopyTabletStmt(long tabletId, Map<String, String> properties) {
|
||||
this.tabletId = tabletId;
|
||||
this.properties = properties;
|
||||
}
|
||||
|
||||
public long getTabletId() {
|
||||
return tabletId;
|
||||
}
|
||||
|
||||
public long getVersion() {
|
||||
return version;
|
||||
}
|
||||
|
||||
public long getBackendId() {
|
||||
return backendId;
|
||||
}
|
||||
|
||||
public long getExpirationMinutes() {
|
||||
return expirationMinutes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void analyze(Analyzer analyzer) throws AnalysisException {
|
||||
if (!Env.getCurrentEnv().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.OPERATOR)) {
|
||||
ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "NODE");
|
||||
}
|
||||
|
||||
try {
|
||||
Iterator<Map.Entry<String, String>> iter = properties.entrySet().iterator();
|
||||
while (iter.hasNext()) {
|
||||
Map.Entry<String, String> entry = iter.next();
|
||||
if (entry.getKey().equalsIgnoreCase(PROP_VERSION)) {
|
||||
version = Long.valueOf(entry.getValue());
|
||||
iter.remove();
|
||||
continue;
|
||||
} else if (entry.getKey().equalsIgnoreCase(PROP_BACKEND_ID)) {
|
||||
backendId = Long.valueOf(entry.getValue());
|
||||
iter.remove();
|
||||
continue;
|
||||
} else if (entry.getKey().equalsIgnoreCase(PROP_EXPIRATION)) {
|
||||
expirationMinutes = Long.valueOf(entry.getValue());
|
||||
expirationMinutes = Math.min(DEFAULT_EXPIRATION_MINUTES, expirationMinutes);
|
||||
iter.remove();
|
||||
continue;
|
||||
}
|
||||
}
|
||||
} catch (NumberFormatException e) {
|
||||
throw new AnalysisException("Invalid property: " + e.getMessage());
|
||||
}
|
||||
|
||||
if (!properties.isEmpty()) {
|
||||
throw new AnalysisException("Unknown property: " + properties);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public ShowResultSetMetaData getMetaData() {
|
||||
ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
|
||||
for (String title : TITLE_NAMES) {
|
||||
builder.addColumn(new Column(title, ScalarType.createStringType()));
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public RedirectStatus getRedirectStatus() {
|
||||
return RedirectStatus.NO_FORWARD;
|
||||
}
|
||||
}
|
||||
|
||||
@ -74,7 +74,6 @@ import org.apache.doris.analysis.ReplacePartitionClause;
|
||||
import org.apache.doris.analysis.RestoreStmt;
|
||||
import org.apache.doris.analysis.RollupRenameClause;
|
||||
import org.apache.doris.analysis.ShowAlterStmt.AlterType;
|
||||
import org.apache.doris.analysis.TableName;
|
||||
import org.apache.doris.analysis.TableRenameClause;
|
||||
import org.apache.doris.analysis.TruncateTableStmt;
|
||||
import org.apache.doris.analysis.UninstallPluginStmt;
|
||||
@ -2683,9 +2682,9 @@ public class Env {
|
||||
}
|
||||
|
||||
public static void getDdlStmt(TableIf table, List<String> createTableStmt, List<String> addPartitionStmt,
|
||||
List<String> createRollupStmt, boolean separatePartition, boolean hidePassword) {
|
||||
List<String> createRollupStmt, boolean separatePartition, boolean hidePassword, long specificVersion) {
|
||||
getDdlStmt(null, null, table, createTableStmt, addPartitionStmt, createRollupStmt, separatePartition,
|
||||
hidePassword, false);
|
||||
hidePassword, false, specificVersion);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -2695,7 +2694,7 @@ public class Env {
|
||||
*/
|
||||
public static void getDdlStmt(DdlStmt ddlStmt, String dbName, TableIf table, List<String> createTableStmt,
|
||||
List<String> addPartitionStmt, List<String> createRollupStmt, boolean separatePartition,
|
||||
boolean hidePassword, boolean getDdlForLike) {
|
||||
boolean hidePassword, boolean getDdlForLike, long specificVersion) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
// 1. create table
|
||||
@ -2764,6 +2763,17 @@ public class Env {
|
||||
}
|
||||
sb.append(Joiner.on(", ").join(keysColumnNames)).append(")");
|
||||
|
||||
if (specificVersion != -1) {
|
||||
// for copy tablet operation
|
||||
sb.append("\nDISTRIBUTED BY HASH(").append(olapTable.getBaseSchema().get(0).getName())
|
||||
.append(") BUCKETS 1");
|
||||
sb.append("\nPROPERTIES (\n" + "\"replication_num\" = \"1\",\n" + "\"version_info\" = \""
|
||||
+ specificVersion + "\"\n" + ")");
|
||||
createTableStmt.add(sb + ";");
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
addTableComment(olapTable, sb);
|
||||
|
||||
// partition
|
||||
@ -5007,15 +5017,4 @@ public class Env {
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
public TableName getTableNameByTableId(Long tableId) {
|
||||
for (String dbName : getInternalCatalog().getDbNames()) {
|
||||
DatabaseIf db = getInternalCatalog().getDbNullable(dbName);
|
||||
Optional<Table> table = db.getTable(tableId);
|
||||
if (table.isPresent()) {
|
||||
return new TableName(InternalCatalog.INTERNAL_CATALOG_NAME, db.getFullName(), table.get().getName());
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1084,7 +1084,7 @@ public class InternalCatalog implements CatalogIf<Database> {
|
||||
throw new DdlException("Table[" + table.getName() + "] is external, not support rollup copy");
|
||||
}
|
||||
|
||||
Env.getDdlStmt(stmt, stmt.getDbName(), table, createTableStmt, null, null, false, false, true);
|
||||
Env.getDdlStmt(stmt, stmt.getDbName(), table, createTableStmt, null, null, false, false, true, -1L);
|
||||
if (createTableStmt.isEmpty()) {
|
||||
ErrorReport.reportDdlException(ErrorCode.ERROR_CREATE_TABLE_LIKE_EMPTY, "CREATE");
|
||||
}
|
||||
@ -1588,6 +1588,7 @@ public class InternalCatalog implements CatalogIf<Database> {
|
||||
// version and version hash
|
||||
if (versionInfo != null) {
|
||||
partition.updateVisibleVersion(versionInfo);
|
||||
partition.setNextVersion(versionInfo + 1);
|
||||
}
|
||||
long version = partition.getVisibleVersion();
|
||||
|
||||
|
||||
@ -76,8 +76,8 @@ public class GetDdlStmtAction extends RestBaseController {
|
||||
|
||||
table.readLock();
|
||||
try {
|
||||
Env.getDdlStmt(table, createTableStmt, addPartitionStmt,
|
||||
createRollupStmt, true, false /* show password */);
|
||||
Env.getDdlStmt(table, createTableStmt, addPartitionStmt, createRollupStmt, true, false /* show password */,
|
||||
-1L);
|
||||
} finally {
|
||||
table.readUnlock();
|
||||
}
|
||||
|
||||
@ -179,7 +179,7 @@ public class StmtExecutionAction extends RestBaseController {
|
||||
List<String> createStmts = Lists.newArrayList();
|
||||
for (TableIf tbl : tableMap.values()) {
|
||||
List<String> createTableStmts = Lists.newArrayList();
|
||||
Env.getDdlStmt(tbl, createTableStmts, null, null, false, true);
|
||||
Env.getDdlStmt(tbl, createTableStmts, null, null, false, true, -1L);
|
||||
if (!createTableStmts.isEmpty()) {
|
||||
createStmts.add(createTableStmts.get(0));
|
||||
}
|
||||
|
||||
@ -541,10 +541,14 @@ public class MasterImpl {
|
||||
|
||||
private void finishMakeSnapshot(AgentTask task, TFinishTaskRequest request) {
|
||||
SnapshotTask snapshotTask = (SnapshotTask) task;
|
||||
if (Env.getCurrentEnv().getBackupHandler().handleFinishedSnapshotTask(snapshotTask, request)) {
|
||||
AgentTaskQueue.removeTask(task.getBackendId(), TTaskType.MAKE_SNAPSHOT, task.getSignature());
|
||||
if (snapshotTask.isCopyTabletTask()) {
|
||||
snapshotTask.setResultSnapshotPath(request.getSnapshotPath());
|
||||
snapshotTask.countDown(task.getBackendId(), task.getTabletId());
|
||||
} else {
|
||||
if (Env.getCurrentEnv().getBackupHandler().handleFinishedSnapshotTask(snapshotTask, request)) {
|
||||
AgentTaskQueue.removeTask(task.getBackendId(), TTaskType.MAKE_SNAPSHOT, task.getSignature());
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private void finishUpload(AgentTask task, TFinishTaskRequest request) {
|
||||
|
||||
@ -17,6 +17,7 @@
|
||||
|
||||
package org.apache.doris.qe;
|
||||
|
||||
import org.apache.doris.analysis.AdminCopyTabletStmt;
|
||||
import org.apache.doris.analysis.AdminDiagnoseTabletStmt;
|
||||
import org.apache.doris.analysis.AdminShowConfigStmt;
|
||||
import org.apache.doris.analysis.AdminShowReplicaDistributionStmt;
|
||||
@ -133,6 +134,7 @@ import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.common.ErrorCode;
|
||||
import org.apache.doris.common.ErrorReport;
|
||||
import org.apache.doris.common.FeConstants;
|
||||
import org.apache.doris.common.MarkedCountDownLatch;
|
||||
import org.apache.doris.common.MetaNotFoundException;
|
||||
import org.apache.doris.common.PatternMatcher;
|
||||
import org.apache.doris.common.proc.BackendsProcDir;
|
||||
@ -172,8 +174,13 @@ import org.apache.doris.statistics.StatisticsJobManager;
|
||||
import org.apache.doris.system.Backend;
|
||||
import org.apache.doris.system.Diagnoser;
|
||||
import org.apache.doris.system.SystemInfoService;
|
||||
import org.apache.doris.task.AgentBatchTask;
|
||||
import org.apache.doris.task.AgentClient;
|
||||
import org.apache.doris.task.AgentTaskExecutor;
|
||||
import org.apache.doris.task.AgentTaskQueue;
|
||||
import org.apache.doris.task.SnapshotTask;
|
||||
import org.apache.doris.thrift.TCheckStorageFormatResult;
|
||||
import org.apache.doris.thrift.TTaskType;
|
||||
import org.apache.doris.thrift.TUnit;
|
||||
import org.apache.doris.transaction.GlobalTransactionMgr;
|
||||
import org.apache.doris.transaction.TransactionStatus;
|
||||
@ -199,6 +206,7 @@ import java.util.Comparator;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
// Execute one show statement.
|
||||
@ -361,6 +369,8 @@ public class ShowExecutor {
|
||||
handleShowCatalogs();
|
||||
} else if (stmt instanceof ShowAnalyzeStmt) {
|
||||
handleShowAnalyze();
|
||||
} else if (stmt instanceof AdminCopyTabletStmt) {
|
||||
handleCopyTablet();
|
||||
} else {
|
||||
handleEmtpy();
|
||||
}
|
||||
@ -853,7 +863,7 @@ public class ShowExecutor {
|
||||
return;
|
||||
}
|
||||
List<String> createTableStmt = Lists.newArrayList();
|
||||
Env.getDdlStmt(table, createTableStmt, null, null, false, true /* hide password */);
|
||||
Env.getDdlStmt(table, createTableStmt, null, null, false, true /* hide password */, -1L);
|
||||
if (createTableStmt.isEmpty()) {
|
||||
resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
|
||||
return;
|
||||
@ -964,7 +974,7 @@ public class ShowExecutor {
|
||||
view.readLock();
|
||||
try {
|
||||
List<String> createViewStmt = Lists.newArrayList();
|
||||
Env.getDdlStmt(view, createViewStmt, null, null, false, true /* hide password */);
|
||||
Env.getDdlStmt(view, createViewStmt, null, null, false, true /* hide password */, -1L);
|
||||
if (!createViewStmt.isEmpty()) {
|
||||
rows.add(Lists.newArrayList(view.getName(), createViewStmt.get(0)));
|
||||
}
|
||||
@ -2263,9 +2273,107 @@ public class ShowExecutor {
|
||||
|
||||
private void handleShowAnalyze() throws AnalysisException {
|
||||
ShowAnalyzeStmt showStmt = (ShowAnalyzeStmt) stmt;
|
||||
StatisticsJobManager jobManager = Env.getCurrentEnv()
|
||||
.getStatisticsJobManager();
|
||||
StatisticsJobManager jobManager = Env.getCurrentEnv().getStatisticsJobManager();
|
||||
List<List<String>> results = jobManager.getAnalyzeJobInfos(showStmt);
|
||||
resultSet = new ShowResultSet(showStmt.getMetaData(), results);
|
||||
}
|
||||
|
||||
private void handleCopyTablet() throws AnalysisException {
|
||||
AdminCopyTabletStmt copyStmt = (AdminCopyTabletStmt) stmt;
|
||||
long tabletId = copyStmt.getTabletId();
|
||||
long version = copyStmt.getVersion();
|
||||
long backendId = copyStmt.getBackendId();
|
||||
|
||||
TabletInvertedIndex invertedIndex = Env.getCurrentInvertedIndex();
|
||||
TabletMeta tabletMeta = invertedIndex.getTabletMeta(tabletId);
|
||||
if (tabletMeta == null) {
|
||||
throw new AnalysisException("Unknown tablet: " + tabletId);
|
||||
}
|
||||
|
||||
// 1. find replica
|
||||
Replica replica = null;
|
||||
if (backendId != -1) {
|
||||
replica = invertedIndex.getReplica(tabletId, backendId);
|
||||
} else {
|
||||
List<Replica> replicas = invertedIndex.getReplicasByTabletId(tabletId);
|
||||
if (!replicas.isEmpty()) {
|
||||
replica = replicas.get(0);
|
||||
}
|
||||
}
|
||||
if (replica == null) {
|
||||
throw new AnalysisException("Replica not found on backend: " + backendId);
|
||||
}
|
||||
backendId = replica.getBackendId();
|
||||
Backend be = Env.getCurrentSystemInfo().getBackend(backendId);
|
||||
if (be == null || !be.isAlive()) {
|
||||
throw new AnalysisException("Unavailable backend: " + backendId);
|
||||
}
|
||||
|
||||
// 2. find version
|
||||
if (version != -1 && replica.getVersion() < version) {
|
||||
throw new AnalysisException("Version is larger than replica max version: " + replica.getVersion());
|
||||
}
|
||||
version = version == -1 ? replica.getVersion() : version;
|
||||
|
||||
// 3. get create table stmt
|
||||
Database db = Env.getCurrentInternalCatalog().getDbOrAnalysisException(tabletMeta.getDbId());
|
||||
OlapTable tbl = (OlapTable) db.getTableNullable(tabletMeta.getTableId());
|
||||
if (tbl == null) {
|
||||
throw new AnalysisException("Failed to find table: " + tabletMeta.getTableId());
|
||||
}
|
||||
|
||||
List<String> createTableStmt = Lists.newArrayList();
|
||||
tbl.readLock();
|
||||
try {
|
||||
Env.getDdlStmt(tbl, createTableStmt, null, null, false, true /* hide password */, version);
|
||||
} finally {
|
||||
tbl.readUnlock();
|
||||
}
|
||||
|
||||
// 4. create snapshot task
|
||||
SnapshotTask task = new SnapshotTask(null, backendId, tabletId, -1, tabletMeta.getDbId(),
|
||||
tabletMeta.getTableId(), tabletMeta.getPartitionId(), tabletMeta.getIndexId(), tabletId, version, 0,
|
||||
copyStmt.getExpirationMinutes() * 60 * 1000, false);
|
||||
task.setIsCopyTabletTask(true);
|
||||
MarkedCountDownLatch<Long, Long> countDownLatch = new MarkedCountDownLatch<Long, Long>(1);
|
||||
countDownLatch.addMark(backendId, tabletId);
|
||||
task.setCountDownLatch(countDownLatch);
|
||||
|
||||
// 5. send task and wait
|
||||
AgentBatchTask batchTask = new AgentBatchTask();
|
||||
batchTask.addTask(task);
|
||||
try {
|
||||
AgentTaskQueue.addBatchTask(batchTask);
|
||||
AgentTaskExecutor.submit(batchTask);
|
||||
|
||||
boolean ok = false;
|
||||
try {
|
||||
ok = countDownLatch.await(10, TimeUnit.SECONDS);
|
||||
} catch (InterruptedException e) {
|
||||
LOG.warn("InterruptedException: ", e);
|
||||
ok = false;
|
||||
}
|
||||
|
||||
if (!ok) {
|
||||
throw new AnalysisException(
|
||||
"Failed to make snapshot for tablet " + tabletId + " on backend: " + backendId);
|
||||
}
|
||||
|
||||
// send result
|
||||
List<List<String>> resultRowSet = Lists.newArrayList();
|
||||
List<String> row = Lists.newArrayList();
|
||||
row.add(String.valueOf(tabletId));
|
||||
row.add(String.valueOf(backendId));
|
||||
row.add(be.getHost());
|
||||
row.add(task.getResultSnapshotPath());
|
||||
row.add(String.valueOf(copyStmt.getExpirationMinutes()));
|
||||
row.add(createTableStmt.get(0));
|
||||
resultRowSet.add(row);
|
||||
|
||||
ShowResultSetMetaData showMetaData = copyStmt.getMetaData();
|
||||
resultSet = new ShowResultSet(showMetaData, resultRowSet);
|
||||
} finally {
|
||||
AgentTaskQueue.removeBatchTask(batchTask, TTaskType.MAKE_SNAPSHOT);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -17,6 +17,7 @@
|
||||
|
||||
package org.apache.doris.task;
|
||||
|
||||
import org.apache.doris.common.MarkedCountDownLatch;
|
||||
import org.apache.doris.thrift.TResourceInfo;
|
||||
import org.apache.doris.thrift.TSnapshotRequest;
|
||||
import org.apache.doris.thrift.TTaskType;
|
||||
@ -24,18 +25,22 @@ import org.apache.doris.thrift.TypesConstants;
|
||||
|
||||
public class SnapshotTask extends AgentTask {
|
||||
private long jobId;
|
||||
|
||||
private long version;
|
||||
|
||||
private int schemaHash;
|
||||
|
||||
private long timeoutMs;
|
||||
|
||||
private boolean isRestoreTask;
|
||||
|
||||
public SnapshotTask(TResourceInfo resourceInfo, long backendId, long signature, long jobId,
|
||||
long dbId, long tableId, long partitionId, long indexId, long tabletId,
|
||||
long version, int schemaHash, long timeoutMs, boolean isRestoreTask) {
|
||||
// Set to true if this task for AdminCopyTablet.
|
||||
// Otherwise, it is for Backup/Restore operation.
|
||||
private boolean isCopyTabletTask = false;
|
||||
private MarkedCountDownLatch<Long, Long> countDownLatch;
|
||||
// Only for copy tablet task.
|
||||
// Save the snapshot path.
|
||||
private String resultSnapshotPath;
|
||||
|
||||
public SnapshotTask(TResourceInfo resourceInfo, long backendId, long signature, long jobId, long dbId, long tableId,
|
||||
long partitionId, long indexId, long tabletId, long version, int schemaHash, long timeoutMs,
|
||||
boolean isRestoreTask) {
|
||||
super(resourceInfo, backendId, TTaskType.MAKE_SNAPSHOT, dbId, tableId, partitionId, indexId, tabletId,
|
||||
signature);
|
||||
|
||||
@ -49,6 +54,22 @@ public class SnapshotTask extends AgentTask {
|
||||
this.isRestoreTask = isRestoreTask;
|
||||
}
|
||||
|
||||
public void setIsCopyTabletTask(boolean value) {
|
||||
this.isCopyTabletTask = value;
|
||||
}
|
||||
|
||||
public boolean isCopyTabletTask() {
|
||||
return isCopyTabletTask;
|
||||
}
|
||||
|
||||
public void setCountDownLatch(MarkedCountDownLatch<Long, Long> countDownLatch) {
|
||||
this.countDownLatch = countDownLatch;
|
||||
}
|
||||
|
||||
public void countDown(long backendId, long tabletId) {
|
||||
this.countDownLatch.markedCountDown(backendId, tabletId);
|
||||
}
|
||||
|
||||
public long getJobId() {
|
||||
return jobId;
|
||||
}
|
||||
@ -69,12 +90,21 @@ public class SnapshotTask extends AgentTask {
|
||||
return isRestoreTask;
|
||||
}
|
||||
|
||||
public void setResultSnapshotPath(String resultSnapshotPath) {
|
||||
this.resultSnapshotPath = resultSnapshotPath;
|
||||
}
|
||||
|
||||
public String getResultSnapshotPath() {
|
||||
return resultSnapshotPath;
|
||||
}
|
||||
|
||||
public TSnapshotRequest toThrift() {
|
||||
TSnapshotRequest request = new TSnapshotRequest(tabletId, schemaHash);
|
||||
request.setVersion(version);
|
||||
request.setListFiles(true);
|
||||
request.setPreferredSnapshotVersion(TypesConstants.TPREFER_SNAPSHOT_REQ_VERSION);
|
||||
request.setTimeout(timeoutMs / 1000);
|
||||
request.setIsCopyTabletTask(isCopyTabletTask);
|
||||
return request;
|
||||
}
|
||||
}
|
||||
|
||||
@ -149,6 +149,7 @@ import org.apache.doris.qe.SqlModeHelper;
|
||||
keywordMap.put("connection_id", new Integer(SqlParserSymbols.KW_CONNECTION_ID));
|
||||
keywordMap.put("consistent", new Integer(SqlParserSymbols.KW_CONSISTENT));
|
||||
keywordMap.put("convert", new Integer(SqlParserSymbols.KW_CONVERT));
|
||||
keywordMap.put("copy", new Integer(SqlParserSymbols.KW_COPY));
|
||||
keywordMap.put("count", new Integer(SqlParserSymbols.KW_COUNT));
|
||||
keywordMap.put("create", new Integer(SqlParserSymbols.KW_CREATE));
|
||||
keywordMap.put("creation", new Integer(SqlParserSymbols.KW_CREATION));
|
||||
|
||||
@ -333,7 +333,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
|
||||
List<String> createStmts = Lists.newArrayList();
|
||||
for (TableIf tbl : tableMap.values()) {
|
||||
List<String> createTableStmts = Lists.newArrayList();
|
||||
Env.getDdlStmt(tbl, createTableStmts, null, null, false, true);
|
||||
Env.getDdlStmt(tbl, createTableStmts, null, null, false, true, -1L);
|
||||
createStmts.add(createTableStmts.get(0));
|
||||
if (tbl.getName().equals("qs1")) {
|
||||
Assert.assertEquals("CREATE TABLE `qs1` (\n" + " `k1` int(11) NULL,\n" + " `k2` int(11) NULL\n"
|
||||
|
||||
@ -80,10 +80,11 @@ public class CreateTableLikeTest {
|
||||
private static void checkTableEqual(Table newTable, Table existedTable, int rollupSize) {
|
||||
List<String> newCreateTableStmt = Lists.newArrayList();
|
||||
List<String> newAddRollupStmt = Lists.newArrayList();
|
||||
Env.getDdlStmt(newTable, newCreateTableStmt, null, newAddRollupStmt, false, true /* hide password */);
|
||||
Env.getDdlStmt(newTable, newCreateTableStmt, null, newAddRollupStmt, false, true /* hide password */, -1L);
|
||||
List<String> existedTableStmt = Lists.newArrayList();
|
||||
List<String> existedAddRollupStmt = Lists.newArrayList();
|
||||
Env.getDdlStmt(existedTable, existedTableStmt, null, existedAddRollupStmt, false, true /* hide password */);
|
||||
Env.getDdlStmt(existedTable, existedTableStmt, null, existedAddRollupStmt, false, true /* hide password */,
|
||||
-1L);
|
||||
Assert.assertEquals(newCreateTableStmt.get(0).replace(newTable.getName(), existedTable.getName()),
|
||||
existedTableStmt.get(0));
|
||||
checkTableRollup(existedAddRollupStmt, newAddRollupStmt, newTable.getName(), existedTable.getName(),
|
||||
|
||||
@ -233,10 +233,10 @@ public class ShowExecutorTest {
|
||||
minTimes = 0;
|
||||
result = env;
|
||||
|
||||
Env.getDdlStmt((Table) any, (List) any, (List) any, (List) any, anyBoolean, anyBoolean);
|
||||
Env.getDdlStmt((Table) any, (List) any, (List) any, (List) any, anyBoolean, anyBoolean, anyLong);
|
||||
minTimes = 0;
|
||||
|
||||
Env.getDdlStmt((Table) any, (List) any, null, null, anyBoolean, anyBoolean);
|
||||
Env.getDdlStmt((Table) any, (List) any, null, null, anyBoolean, anyBoolean, anyLong);
|
||||
minTimes = 0;
|
||||
|
||||
env.getCatalogMgr();
|
||||
|
||||
@ -270,6 +270,7 @@ struct TSnapshotRequest {
|
||||
// Deprecated since version 0.13
|
||||
8: optional bool allow_incremental_clone
|
||||
9: optional i32 preferred_snapshot_version = Types.TPREFER_SNAPSHOT_REQ_VERSION
|
||||
10: optional bool is_copy_tablet_task
|
||||
}
|
||||
|
||||
struct TReleaseSnapshotRequest {
|
||||
|
||||
Reference in New Issue
Block a user