[docs] Split the FAQ And Revert auto-label action (#7770)

This commit is contained in:
jakevin
2022-01-17 10:34:56 +08:00
committed by GitHub
parent e80c34b6fe
commit ebc27a40d7
9 changed files with 444 additions and 477 deletions

View File

@ -1,88 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
name: Add label automatically based on PR title
on:
pull_request:
types: [opened, edited]
jobs:
auto-filling-pull-request:
runs-on: ubuntu-latest
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
with:
persist-credentials: false
submodules: recursive
- name: Add labels based on PR title
uses: ./.github/actions/set-label-based-on-pr-title
with:
words: >-
[fix];
[feature];
[improvement];
[docs];
[typo];
[refactor];
[performance];
[test];
[chore];
[revert];
[community];
(vectorized);
(planner);
(storage);
(load);
(stream-load);
(broker-load);
(routine-load);
(export);
(spark-connector);
(flink-connector);
(datax);
(config)
labels: >-
kind/fix;
kind/feature;
kind/improvement;
kind/docs;
kind/typo;
kind/refactor;
kind/performance;
kind/test;
kind/chore;
kind/revert;
kind/community;
area/vectorized;
area/planner;
area/storage;
area/load;
area/stream-load;
area/broker-load;
area/routine-load;
area/export;
area/spark-connector;
area/flink-connector;
area/datax;
area/config
repo-token: "${{ secrets.GITHUB_TOKEN }}"

4
.gitmodules vendored
View File

@ -7,7 +7,3 @@
[submodule ".github/actions/clang-format-lint-action"]
path = .github/actions/clang-format-lint-action
url = https://github.com/DoozyX/clang-format-lint-action.git
[submodule ".github/actions/set-label-based-on-pr-title"]
path = .github/actions/set-label-based-on-pr-title
url = https://github.com/jackwener/set-label-based-on-pr-title.git

View File

@ -89,51 +89,51 @@ module.exports = [
title: "FE",
directoryPath: "fe/",
children: [
{
title: "MANAGER",
directoryPath: "manager/",
children: [
"cluster-action",
"node-action",
"query-profile-action",
],
},
"backends-action",
"bootstrap-action",
"cancel-load-action",
"check-decommission-action",
"check-storage-type-action",
"config-action",
"connection-action",
"get-ddl-stmt-action",
"get-load-info-action",
"get-load-state",
"get-log-file-action",
"get-small-file",
"ha-action",
"hardware-info-action",
"health-action",
"log-action",
"logout-action",
"meta-action",
"meta-info-action",
"meta-replay-state-action",
"profile-action",
"query-detail-action",
"query-profile-action",
"row-count-action",
"session-action",
"set-config-action",
"show-data-action",
"show-meta-info-action",
"show-proc-action",
"show-runtime-info-action",
"statement-execution-action",
"system-action",
"table-query-plan-action",
"table-row-count-action",
"table-schema-action",
"upload-action",
{
title: "MANAGER",
directoryPath: "manager/",
children: [
"cluster-action",
"node-action",
"query-profile-action",
],
},
"backends-action",
"bootstrap-action",
"cancel-load-action",
"check-decommission-action",
"check-storage-type-action",
"config-action",
"connection-action",
"get-ddl-stmt-action",
"get-load-info-action",
"get-load-state",
"get-log-file-action",
"get-small-file",
"ha-action",
"hardware-info-action",
"health-action",
"log-action",
"logout-action",
"meta-action",
"meta-info-action",
"meta-replay-state-action",
"profile-action",
"query-detail-action",
"query-profile-action",
"row-count-action",
"session-action",
"set-config-action",
"show-data-action",
"show-meta-info-action",
"show-proc-action",
"show-runtime-info-action",
"statement-execution-action",
"system-action",
"table-query-plan-action",
"table-row-count-action",
"table-schema-action",
"upload-action",
],
},
"cancel-label",
@ -437,12 +437,12 @@ module.exports = [
title: "Encryption and Digest Functions",
directoryPath: "encrypt-digest-functions/",
children: [
"aes",
"md5",
"md5sum",
"sm4",
"sm3",
"sm3sum"
"aes",
"md5",
"md5sum",
"sm4",
"sm3",
"sm3sum"
],
},
{
@ -648,31 +648,32 @@ module.exports = [
title: "Doris User",
directoryPath: "case-user/",
children: [
"case-user",
"case-user",
],
},
{
title: "Developer Guide",
directoryPath: "developer-guide/",
children: [
"debug-tool",
"docker-dev",
"benchmark-tool",
"fe-eclipse-dev",
"fe-idea-dev",
"be-vscode-dev",
"java-format-code",
"cpp-format-code",
"How-to-Share-blogs",
"minidump",
"bitmap-hll-file-format",
"debug-tool",
"docker-dev",
"benchmark-tool",
"fe-eclipse-dev",
"fe-idea-dev",
"be-vscode-dev",
"java-format-code",
"cpp-format-code",
"How-to-Share-blogs",
"minidump",
"bitmap-hll-file-format",
],
},
{
title: "FAQ",
directoryPath: "faq/",
children: [
"faq"
"faq",
"error"
],
},
{

View File

@ -88,51 +88,51 @@ module.exports = [
title: "FE",
directoryPath: "fe/",
children: [
{
title: "MANAGER",
directoryPath: "manager/",
children: [
"cluster-action",
"node-action",
"query-profile-action",
],
},
"backends-action",
"bootstrap-action",
"cancel-load-action",
"check-decommission-action",
"check-storage-type-action",
"config-action",
"connection-action",
"get-ddl-stmt-action",
"get-load-info-action",
"get-load-state",
"get-log-file-action",
"get-small-file",
"ha-action",
"hardware-info-action",
"health-action",
"log-action",
"logout-action",
"meta-action",
"meta-info-action",
"meta-replay-state-action",
"profile-action",
"query-detail-action",
"query-profile-action",
"row-count-action",
"session-action",
"set-config-action",
"show-data-action",
"show-meta-info-action",
"show-proc-action",
"show-runtime-info-action",
"statement-execution-action",
"system-action",
"table-query-plan-action",
"table-row-count-action",
"table-schema-action",
"upload-action",
{
title: "MANAGER",
directoryPath: "manager/",
children: [
"cluster-action",
"node-action",
"query-profile-action",
],
},
"backends-action",
"bootstrap-action",
"cancel-load-action",
"check-decommission-action",
"check-storage-type-action",
"config-action",
"connection-action",
"get-ddl-stmt-action",
"get-load-info-action",
"get-load-state",
"get-log-file-action",
"get-small-file",
"ha-action",
"hardware-info-action",
"health-action",
"log-action",
"logout-action",
"meta-action",
"meta-info-action",
"meta-replay-state-action",
"profile-action",
"query-detail-action",
"query-profile-action",
"row-count-action",
"session-action",
"set-config-action",
"show-data-action",
"show-meta-info-action",
"show-proc-action",
"show-runtime-info-action",
"statement-execution-action",
"system-action",
"table-query-plan-action",
"table-row-count-action",
"table-schema-action",
"upload-action",
],
},
"cancel-label",
@ -652,31 +652,32 @@ module.exports = [
title: "Doris用户",
directoryPath: "case-user/",
children: [
"case-user",
"case-user",
],
},
{
title: "开发者手册",
directoryPath: "developer-guide/",
children: [
"debug-tool",
"benchmark-tool",
"docker-dev",
"fe-eclipse-dev",
"fe-idea-dev",
"be-vscode-dev",
"java-format-code",
"cpp-format-code",
"How-to-Share-blogs",
"minidump",
"bitmap-hll-file-format",
"debug-tool",
"benchmark-tool",
"docker-dev",
"fe-eclipse-dev",
"fe-idea-dev",
"be-vscode-dev",
"java-format-code",
"cpp-format-code",
"How-to-Share-blogs",
"minidump",
"bitmap-hll-file-format",
],
},
{
title: "FAQ 常见问题",
directoryPath: "faq/",
children: [
"faq"
"faq",
"error"
],
},
{

149
docs/en/faq/error.md Normal file
View File

@ -0,0 +1,149 @@
---
{
"title": "Common Error",
"language": "en"
}
---
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Common Error
This document is mainly used to record the errors reported during the use of Doris. If you encounter some errors, you are welcome to contribute to us for updates.
### E1. Query error: Failed to get scan range, no queryable replica found in tablet: xxxx
This situation is because the corresponding tablet does not find a copy that can be queried, usually because the BE is down, the copy is missing, and so on. You can use the `show tablet tablet_id` statement first, and then execute the following `show proc` statement to view the copy information corresponding to this tablet, and check whether the copy is complete. At the same time, you can use the `show proc "/cluster_balance"` information to query the progress of replica scheduling and repair in the cluster.
For commands related to data copy management, please refer to [Data Copy Management](../administrator-guide/operation/tablet-repair-and-balance.md).
### E2. FE failed to start, fe.log keeps scrolling "wait catalog to be ready. FE type UNKNOWN"
There are usually two reasons for this problem:
1. The local IP obtained when the FE is started this time is inconsistent with the last time, usually because the `priority_network` is not set correctly, the wrong IP address is matched when the FE is started. Need to modify `priority_network` and restart FE.
2. Most Follower FE nodes in the cluster are not started. For example, there are 3 Followers and only one is started. At this time, at least one other FE needs to be also activated, and the FE electable group can elect the Master to provide services.
If none of the above conditions can be resolved, you can follow the [Metadata Operation and Maintenance Document] (../administrator-guide/operation/metadata-operation.md) in the Doris official website to restore.
### E3. tablet writer write failed, tablet_id=27306172, txn_id=28573520, err=-235 or -215 or -238
This error usually occurs during data import operations. The error code of the new version is -235, and the error code of the old version may be -215. The meaning of this error is that the data version of the corresponding tablet exceeds the maximum limit (default 500, controlled by the BE parameter `max_tablet_version_num`), and subsequent writes will be rejected. For example, the error in the question means that the data version of the tablet 27306172 exceeds the limit.
This error is usually because the import frequency is too high, which is greater than the compaction speed of the background data, which causes the version to accumulate and eventually exceeds the limit. At this point, we can first use the show tablet 27306172 statement, and then execute the show proc statement in the result to view the status of each copy of the tablet. The versionCount in the result represents the number of versions. If you find that there are too many versions of a copy, you need to reduce the import frequency or stop importing, and observe whether the number of versions drops. If the version number still does not decrease after the import is stopped, you need to go to the corresponding BE node to check the be.INFO log, search for the tablet id and compaction keywords, and check whether the compaction is running normally. For compaction tuning related, you can refer to the ApacheDoris public account article: Doris Best Practice-Compaction Tuning (3)
The -238 error usually occurs when the amount of imported data in the same batch is too large, which leads to too many Segment files for a certain tablet (the default is 200, which is controlled by the BE parameter `max_segment_num_per_rowset`). At this time, it is recommended to reduce the amount of data imported in one batch, or to appropriately increase the value of the BE configuration parameter to solve the problem.
### E4. tablet 110309738 has few replicas: 1, alive backends: [10003]
This error may occur during query or import operation. It usually means that the copy of the tablet is abnormal.
At this point, you can first check whether the BE node is down by using the show backends command, such as the isAlive field is false, or LastStartTime is the most recent time (indicating that it has been restarted recently). If the BE is down, you need to go to the node corresponding to the BE and check the be.out log. If the BE is down due to an exception, usually the exception stack will be printed in be.out to help troubleshoot the problem. If there is no error stack in be.out. You can use the linux command dmesg -T to check whether the process is killed by the system because of OOM.
If no BE node is down, you need to use the show tablet 110309738 statement, and then execute the show proc statement in the result to check the status of each copy of the tablet for further investigation.
### E5. disk xxxxx on backend xxx exceed limit usage
It usually appears in operations such as import and Alter. This error means that the usage of the corresponding disk corresponding to the BE exceeds the threshold (95% by default). At this time, you can use the show backends command first, where MaxDiskUsedPct shows the usage of the disk with the highest usage on the corresponding BE. If If it exceeds 95%, this error will be reported.
At this time, you need to go to the corresponding BE node to check the usage in the data directory. The trash directory and snapshot directory can be manually cleaned up to free up space. If the data directory occupies a lot, you need to consider deleting some data to free up space. For details, please refer to [Disk Space Management](../administrator-guide/operation/disk-capacity.md).
### E6. invalid cluster id: xxxx
This error may appear in the results of the show backends or show frontends commands. It usually appears in the error message column of a certain FE or BE node. The meaning of this error is that after Master FE sends heartbeat information to this node, the node finds that the cluster id carried in the heartbeat information is different from the cluster id stored locally, so it refuses to respond to the heartbeat.
Doris' Master FE node will actively send a heartbeat to each FE or BE node, and will carry a cluster_id in the heartbeat information. The cluster_id is the unique cluster ID generated by the Master FE when a cluster is initialized. When the FE or BE receives the heartbeat information for the first time, it will save the cluster_id locally in the form of a file. The FE file is in the image/ directory of the metadata directory, and BE has a cluster_id file in all data directories. After that, every time a node receives a heartbeat, it will compare the content of the local cluster_id with the content in the heartbeat. If it is inconsistent, it will refuse to respond to the heartbeat.
This mechanism is a node authentication mechanism to prevent receiving wrong heartbeat information from nodes outside the cluster.
If you need to recover from this error. First, confirm whether all nodes are the correct nodes in the cluster. After that, for the FE node, you can try to modify the cluster_id value in the image/VERSION file in the metadata directory and restart the FE. For BE nodes, you can delete cluster_id files in all data directories and restart BE.
### E7. Import data by calling stream load through a Java program. When a batch of data is large, a Broken Pipe error may be reported
In addition to Broken Pipe, there may be other strange errors.
This situation usually occurs after opening httpv2. Because httpv2 is an http service implemented using spring boot, and uses tomcat as the default built-in container. But jetty's handling of 307 forwarding seems to have some problems, so the built-in container will be modified to jetty later. In addition, the version of apache http client in the java program needs to use a version later than 4.5.13. In the previous version, there were also some problems with the processing of forwarding.
So this problem can be solved in two ways:
1. Turn off httpv2
Add enable_http_server_v2=false in fe.conf and restart FE. However, the new UI interface can no longer be used in this way, and some new interfaces based on httpv2 cannot be used later. (Normal import queries are not affected).
2. Upgrade
You can upgrade to Doris 0.15 and later versions, this problem has been fixed.
### E8. `Lost connection to MySQL server at'reading initial communication packet', system error: 0`
If the following problems occur when using the MySQL client to connect to Doris, this is usually caused by the difference between the jdk version used when compiling FE and the jdk version used when running FE.
Note that when using docker image to compile, the default JDK version is openjdk 11, you can switch to openjdk 8 by command (see the compilation document for details).
### E9. -214 error
When performing operations such as load and query, you may encounter the following errors:
```
failed to initialize storage reader. tablet=63416.1050661139.aa4d304e7a7aff9c-f0fa7579928c85a0, res=-214, backend=192.168.100.10
```
A -214 error means that the data version of the corresponding tablet is missing. For example, the above error indicates that the data version of the replica of tablet 63416 on the BE of 192.168.100.10 is missing. (There may be other similar error codes, which can be checked and repaired in the following ways).
Normally, if your data has multiple replicas, the system will automatically repair these problematic replicas. You can troubleshoot through the following steps:
First, use the `show tablet 63416` statement and execute the `show proc xxx` statement in the result to view the status of each replica of the corresponding tablet. Usually we need to care about the data in the `Version` column.
Under normal circumstances, the Version of multiple replicas of a tablet should be the same. And it is the same as the VisibleVersion of the corresponding partition.
You can use `show partitions from tblx` to view the corresponding partition version (the partition corresponding to the tablet can be obtained in the `show tablet` statement.)
At the same time, you can also visit the URL in the CompactionStatus column of the `show proc` statement (just open it in the browser) to view more specific version information, to check which version is missing.
If there is no automatic repair for a long time, you need to use the `show proc "/cluster_balance"` statement to view the tablet repair and scheduling tasks currently being performed by the system. It may be because there are a large number of tablets waiting to be scheduled, which leads to a long repair time. You can follow the records in `pending_tablets` and `running_tablets`.
Furthermore, you can use the `admin repair` statement to specify the priority to repair a table or partition. For details, please refer to `help admin repair`;
If it still cannot be repaired, then in the case of multiple replicas, we use the `admin set replica status` command to force the replica to go offline. For details, please refer to the example of `help admin set replica status` to set the status of the replica to bad. (After set to bad, the replica will not be accessed again. And will be automatically repaired later. But before the operation, you should make sure that the other replicas are normal)
### E10. Not connected to 192.168.100.1:8060 yet, server_id=384
We may encounter this error when loading or querying. If you go to the corresponding BE log to check, you may also find similar errors.
This is an RPC error, and there are usually two possibilities: 1. The corresponding BE node is down. 2. rpc congestion or other errors.
If the BE node is down, you need to check the specific reason for the downtime. Only the problem of rpc congestion is discussed here.
One situation is OVERCROWDED, which means that a large amount of unsent data at the rpc client exceeds the threshold. BE has two parameters related to it:
1. `brpc_socket_max_unwritten_bytes`: The default is 1GB. If the unwritten data exceeds this value, an error will be reported. You can modify this value appropriately to avoid OVERCROWDED errors. (But this cures the symptoms rather than the root cause, essentially congestion still occurs).
2. `tablet_writer_ignore_eovercrowded`: The default is false. If set to true, Doris will ignore OVERCROWDED errors during the load process. This parameter is mainly used to avoid load failure and improve the stability of load.
The second is that the packet size of rpc exceeds `max_body_size`. This problem may occur if the query contains a very large String type or a Bitmap type. It can be circumvented by modifying the following BE parameters:
1. `brpc_max_body_size`: The default is 3GB.
### E11. `recoveryTracker should overlap or follow on disk last VLSN of 4,422,880 recoveryFirst= 4,422,882 UNEXPECTED_STATE_FATAL`
Sometimes when restarting the Fe, the above error will occur (usually only in the case of multiple followers), and the difference between the two values in the error is 2. As a result, the Fe startup fails.
This is a bug in bdbje that has not been resolved. In this case, metadata can only be recovered through fault recovery in [metadata operation and maintenance manual](../administrator-guide/operation/metadata-operation.md).

View File

@ -30,28 +30,11 @@ This document is mainly used to record common problems in the use of Doris. Will
### Q1. Use Stream Load to access the public network address of FE to import data, and it is redirected to the internal network IP?
When the connection target of stream load is the http port of FE, FE will only randomly select a BE node for http 307 redirect operation, so the user's request is actually sent to a BE designated by FE. The redirect returns the ip of BE, which is the intranet IP. So if you send the request through the public IP of FE, it is very likely that you will not be able to connect because you are redirected to the intranet address.
The usual approach is to ensure that you can access the intranet IP address, or assume a load balance for all BE upper layers, and then directly send the stream load request to the load balancer, and the load balancer transparently transmits the request to the BE node .
### Q2. Query error: Failed to get scan range, no queryable replica found in tablet: xxxx
This situation is because the corresponding tablet does not find a copy that can be queried, usually because the BE is down, the copy is missing, and so on. You can use the `show tablet tablet_id` statement first, and then execute the following `show proc` statement to view the copy information corresponding to this tablet, and check whether the copy is complete. At the same time, you can use the `show proc "/cluster_balance"` information to query the progress of replica scheduling and repair in the cluster.
For commands related to data copy management, please refer to [Data Copy Management](../administrator-guide/operation/tablet-repair-and-balance.md).
### Q3. FE failed to start, fe.log keeps scrolling "wait catalog to be ready. FE type UNKNOWN"
There are usually two reasons for this problem:
1. The local IP obtained when the FE is started this time is inconsistent with the last time, usually because the `priority_network` is not set correctly, the wrong IP address is matched when the FE is started. Need to modify `priority_network` and restart FE.
2. Most Follower FE nodes in the cluster are not started. For example, there are 3 Followers and only one is started. At this time, at least one other FE needs to be also activated, and the FE electable group can elect the Master to provide services.
If none of the above conditions can be resolved, you can follow the [Metadata Operation and Maintenance Document] (../administrator-guide/operation/metadata-operation.md) in the Doris official website to restore.
### Q4. When the BE node is offline through DECOMMISSION, why is there always some tablet remaining?
### Q2. When the BE node is offline through DECOMMISSION, why is there always some tablet remaining?
During the offline process, check the tabletNum of the offline node through show backends, and you will observe that the number of tabletNum is decreasing, indicating that the data fragments are migrating from this node. When the number is reduced to 0, the system will automatically delete this node. But in some cases, tabletNum does not change after it drops to a certain value. This can usually have the following two reasons:
@ -62,7 +45,7 @@ During the offline process, check the tabletNum of the offline node through show
For the above situation, you can first check whether the cluster still has unhealthy shards through show proc "/statistic". If it is 0, you can delete the BE directly through the drop backend statement. Otherwise, you need to check the copy status of unhealthy shards.
### Q5. How should priorty_network be set?
### Q3. How should priorty_network be set?
Priorty_network is a configuration parameter for both FE and BE. This parameter is mainly used to help the system choose the correct network card IP as its own IP. It is recommended to set this parameter explicitly under any circumstances to prevent the problem of incorrect IP selection caused by the addition of a new network card to the subsequent machine.
@ -70,7 +53,7 @@ The value of priorty_network is expressed in CIDR format. It is divided into two
The reason for using the CIDR format instead of directly specifying a specific IP is to ensure that all nodes can use uniform configuration values. For example, there are two nodes: 10.168.10.1 and 10.168.10.2, then we can use 10.168.10.0/24 as the value of priorty_network.
### Q6. What are FE's Master, Follower and Observer?
### Q4. What are FE's Master, Follower and Observer?
First of all, make it clear that FE has only two roles: Follower and Observer. The Master is just an FE selected from a group of Follower nodes. Master can be regarded as a special kind of Follower. So when we were asked how many FEs in a cluster and what roles are they, the correct answer should be the number of all FE nodes, the number of Follower roles, and the number of Observer roles.
@ -82,39 +65,7 @@ The role of Observer is the same as the meaning of this word. It only acts as an
Normally, you can deploy 1 Follower + 2 Observer or 3 Follower + N Observer. The former is simple to operate and maintain, and there will be almost no consensus agreement between Followers to cause this complicated error situation (most of Baidu's internal clusters use this method). The latter can ensure the high availability of metadata writing. If it is a high-concurrency query scenario, you can appropriately increase the Observer.
### Q7. tablet writer write failed, tablet_id=27306172, txn_id=28573520, err=-235 or -215 or -238
This error usually occurs during data import operations. The error code of the new version is -235, and the error code of the old version may be -215. The meaning of this error is that the data version of the corresponding tablet exceeds the maximum limit (default 500, controlled by the BE parameter `max_tablet_version_num`), and subsequent writes will be rejected. For example, the error in the question means that the data version of the tablet 27306172 exceeds the limit.
This error is usually because the import frequency is too high, which is greater than the compaction speed of the background data, which causes the version to accumulate and eventually exceeds the limit. At this point, we can first use the show tablet 27306172 statement, and then execute the show proc statement in the result to view the status of each copy of the tablet. The versionCount in the result represents the number of versions. If you find that there are too many versions of a copy, you need to reduce the import frequency or stop importing, and observe whether the number of versions drops. If the version number still does not decrease after the import is stopped, you need to go to the corresponding BE node to check the be.INFO log, search for the tablet id and compaction keywords, and check whether the compaction is running normally. For compaction tuning related, you can refer to the ApacheDoris public account article: Doris Best Practice-Compaction Tuning (3)
The -238 error usually occurs when the amount of imported data in the same batch is too large, which leads to too many Segment files for a certain tablet (the default is 200, which is controlled by the BE parameter `max_segment_num_per_rowset`). At this time, it is recommended to reduce the amount of data imported in one batch, or to appropriately increase the value of the BE configuration parameter to solve the problem.
### Q8. tablet 110309738 has few replicas: 1, alive backends: [10003]
This error may occur during query or import operation. It usually means that the copy of the tablet is abnormal.
At this point, you can first check whether the BE node is down by using the show backends command, such as the isAlive field is false, or LastStartTime is the most recent time (indicating that it has been restarted recently). If the BE is down, you need to go to the node corresponding to the BE and check the be.out log. If the BE is down due to an exception, usually the exception stack will be printed in be.out to help troubleshoot the problem. If there is no error stack in be.out. You can use the linux command dmesg -T to check whether the process is killed by the system because of OOM.
If no BE node is down, you need to use the show tablet 110309738 statement, and then execute the show proc statement in the result to check the status of each copy of the tablet for further investigation.
### Q9. disk xxxxx on backend xxx exceed limit usage
It usually appears in operations such as import and Alter. This error means that the usage of the corresponding disk corresponding to the BE exceeds the threshold (95% by default). At this time, you can use the show backends command first, where MaxDiskUsedPct shows the usage of the disk with the highest usage on the corresponding BE. If If it exceeds 95%, this error will be reported.
At this time, you need to go to the corresponding BE node to check the usage in the data directory. The trash directory and snapshot directory can be manually cleaned up to free up space. If the data directory occupies a lot, you need to consider deleting some data to free up space. For details, please refer to [Disk Space Management](../administrator-guide/operation/disk-capacity.md).
### Q10. invalid cluster id: xxxx
This error may appear in the results of the show backends or show frontends commands. It usually appears in the error message column of a certain FE or BE node. The meaning of this error is that after Master FE sends heartbeat information to this node, the node finds that the cluster id carried in the heartbeat information is different from the cluster id stored locally, so it refuses to respond to the heartbeat.
Doris' Master FE node will actively send a heartbeat to each FE or BE node, and will carry a cluster_id in the heartbeat information. The cluster_id is the unique cluster ID generated by the Master FE when a cluster is initialized. When the FE or BE receives the heartbeat information for the first time, it will save the cluster_id locally in the form of a file. The FE file is in the image/ directory of the metadata directory, and BE has a cluster_id file in all data directories. After that, every time a node receives a heartbeat, it will compare the content of the local cluster_id with the content in the heartbeat. If it is inconsistent, it will refuse to respond to the heartbeat.
This mechanism is a node authentication mechanism to prevent receiving wrong heartbeat information from nodes outside the cluster.
If you need to recover from this error. First, confirm whether all nodes are the correct nodes in the cluster. After that, for the FE node, you can try to modify the cluster_id value in the image/VERSION file in the metadata directory and restart the FE. For BE nodes, you can delete cluster_id files in all data directories and restart BE.
### Q11. Does Doris support modifying column names?
### Q5. Does Doris support modifying column names?
Does not support modifying column names.
@ -124,7 +75,7 @@ For some historical reasons, the column names are currently written directly int
We do not rule out the subsequent use of some compatible means to support lightweight column name modification operations.
### Q12. Does the table of the Unique Key model support the creation of materialized views?
### Q6. Does the table of the Unique Key model support the creation of materialized views?
not support.
@ -132,7 +83,7 @@ The table of the Unique Key model is a business-friendly table. Because of its u
Unfortunately, the table of the Unique Key model cannot create a materialized view. The reason is that the nature of the materialized view is to "pre-calculate" the data through pre-calculation, so that the calculated data is directly returned during the query to speed up the query. In the materialized view, the "pre-calculated" data is usually some aggregated indicators, such as summation and count. At this time, if the data changes, such as udpate or delete, because the pre-calculated data has lost the detailed information, it cannot be updated synchronously. For example, a sum of 5 may be 1+4 or 2+3. Because of the loss of detailed information, we cannot distinguish how the sum is calculated, and therefore cannot meet the update requirements.
### Q13. show backends/frontends Viewed information is incomplete
### Q7. show backends/frontends Viewed information is incomplete
After executing certain statements such as `show backends/frontends`, some columns may be incomplete in the results. For example, the disk capacity information cannot be seen in the show backends results.
@ -140,23 +91,7 @@ This problem usually occurs when there are multiple FEs in the cluster. If users
Of course, the user can also execute `set forward_to_master=true;` before executing these statements. After the session variable is set to true, some information viewing statements executed later will be automatically forwarded to the Master FE to obtain the results. In this way, no matter which FE the user is connected to, the complete result can be obtained.
### Q14. Import data by calling stream load through a Java program. When a batch of data is large, a Broken Pipe error may be reported
In addition to Broken Pipe, there may be other strange errors.
This situation usually occurs after opening httpv2. Because httpv2 is an http service implemented using spring boot, and uses tomcat as the default built-in container. But jetty's handling of 307 forwarding seems to have some problems, so the built-in container will be modified to jetty later. In addition, the version of apache http client in the java program needs to use a version later than 4.5.13. In the previous version, there were also some problems with the processing of forwarding.
So this problem can be solved in two ways:
1. Turn off httpv2
Add enable_http_server_v2=false in fe.conf and restart FE. However, the new UI interface can no longer be used in this way, and some new interfaces based on httpv2 cannot be used later. (Normal import queries are not affected).
2. Upgrade
You can upgrade to Doris 0.15 and later versions, this problem has been fixed.
### Q15. A new disk is added to the node. Why is the data not balanced on the new disk?
### Q8. A new disk is added to the node. Why is the data not balanced on the new disk?
The current balance strategy of Doris is based on nodes. In other words, the cluster load is judged according to the overall load index of the node (the number of shards and the total disk utilization). And migrate data fragments from high-load nodes to low-load nodes. If each node adds a disk, from the perspective of the node as a whole, the load has not changed, so the balancing logic cannot be triggered.
@ -182,7 +117,7 @@ Here we provide 3 ways to solve this problem:
Doris provides [HTTP API](../administrator-guide/http-actions/tablet-migration-action.md), which allows you to manually specify data fragments on one disk to migrate to another disk.
### Q16. How to read FE/BE log correctly?
### Q9. How to read FE/BE log correctly?
In many cases, we need to troubleshoot problems through logs. Here is an explanation of the format and viewing method of the FE/BE log.
@ -230,7 +165,7 @@ In many cases, we need to troubleshoot problems through logs. Here is an explana
Normally, we mainly check the be.INFO log. Under special circumstances, such as BE downtime, you need to check be.out.
### Q17. How to troubleshoot the cause of FE/BE node down?
### Q10. How to troubleshoot the cause of FE/BE node down?
1. BE
@ -260,7 +195,7 @@ In many cases, we need to troubleshoot problems through logs. Here is an explana
FE is a java process, and its robustness depends on the C/C++ program. Usually, the cause of FE hanging may be OOM (Out-of-Memory) or metadata writing failure. These errors usually have an error stack in fe.log or fe.out. You need to investigate further based on the error stack information.
### Q18. About the configuration of the data directory SSD and HDD.
### Q11. About the configuration of the data directory SSD and HDD.
Doris supports a BE node to configure multiple storage paths. Normally, it is sufficient to configure one storage path for each disk. At the same time, Doris supports storage media attributes of specified paths, such as SSD or HDD. SSD stands for high-speed storage devices, and HDD stands for low-speed storage devices.
@ -270,55 +205,7 @@ It should be noted that Doris does not automatically perceive the actual storage
In other words, ".HDD" and ".SSD" are only used to identify the "relative" "low speed" and "high speed" of the storage directory, not the actual storage medium type. Therefore, if the storage path on the BE node has no difference in media, there is no need to fill in the suffix.
### Q19. `Lost connection to MySQL server at'reading initial communication packet', system error: 0`
If the following problems occur when using the MySQL client to connect to Doris, this is usually caused by the difference between the jdk version used when compiling FE and the jdk version used when running FE.
Note that when using docker image to compile, the default JDK version is openjdk 11, you can switch to openjdk 8 by command (see the compilation document for details).
### Q20. -214 error
When performing operations such as load and query, you may encounter the following errors:
```
failed to initialize storage reader. tablet=63416.1050661139.aa4d304e7a7aff9c-f0fa7579928c85a0, res=-214, backend=192.168.100.10
```
A -214 error means that the data version of the corresponding tablet is missing. For example, the above error indicates that the data version of the replica of tablet 63416 on the BE of 192.168.100.10 is missing. (There may be other similar error codes, which can be checked and repaired in the following ways).
Normally, if your data has multiple replicas, the system will automatically repair these problematic replicas. You can troubleshoot through the following steps:
First, use the `show tablet 63416` statement and execute the `show proc xxx` statement in the result to view the status of each replica of the corresponding tablet. Usually we need to care about the data in the `Version` column.
Under normal circumstances, the Version of multiple replicas of a tablet should be the same. And it is the same as the VisibleVersion of the corresponding partition.
You can use `show partitions from tblx` to view the corresponding partition version (the partition corresponding to the tablet can be obtained in the `show tablet` statement.)
At the same time, you can also visit the URL in the CompactionStatus column of the `show proc` statement (just open it in the browser) to view more specific version information, to check which version is missing.
If there is no automatic repair for a long time, you need to use the `show proc "/cluster_balance"` statement to view the tablet repair and scheduling tasks currently being performed by the system. It may be because there are a large number of tablets waiting to be scheduled, which leads to a long repair time. You can follow the records in `pending_tablets` and `running_tablets`.
Furthermore, you can use the `admin repair` statement to specify the priority to repair a table or partition. For details, please refer to `help admin repair`;
If it still cannot be repaired, then in the case of multiple replicas, we use the `admin set replica status` command to force the replica to go offline. For details, please refer to the example of `help admin set replica status` to set the status of the replica to bad. (After set to bad, the replica will not be accessed again. And will be automatically repaired later. But before the operation, you should make sure that the other replicas are normal)
### Q21. Not connected to 192.168.100.1:8060 yet, server_id=384
We may encounter this error when loading or querying. If you go to the corresponding BE log to check, you may also find similar errors.
This is an RPC error, and there are usually two possibilities: 1. The corresponding BE node is down. 2. rpc congestion or other errors.
If the BE node is down, you need to check the specific reason for the downtime. Only the problem of rpc congestion is discussed here.
One situation is OVERCROWDED, which means that a large amount of unsent data at the rpc client exceeds the threshold. BE has two parameters related to it:
1. `brpc_socket_max_unwritten_bytes`: The default is 1GB. If the unwritten data exceeds this value, an error will be reported. You can modify this value appropriately to avoid OVERCROWDED errors. (But this cures the symptoms rather than the root cause, essentially congestion still occurs).
2. `tablet_writer_ignore_eovercrowded`: The default is false. If set to true, Doris will ignore OVERCROWDED errors during the load process. This parameter is mainly used to avoid load failure and improve the stability of load.
The second is that the packet size of rpc exceeds `max_body_size`. This problem may occur if the query contains a very large String type or a Bitmap type. It can be circumvented by modifying the following BE parameters:
1. `brpc_max_body_size`: The default is 3GB.
### Q22. The query results of unique key model are inconsistent
### Q12. The query results of unique key model are inconsistent
In some cases, when users use the same SQL to query a table of a unique key model, inconsistent query results may occur. And the query results always change between 2-3 kinds.
@ -335,13 +222,7 @@ Then the result of replica 1 may be '1, "ABC', while the result of replica 2 may
To ensure the unique data order between different replicas, refer to the [Sequence Column](../administrator-guide/load-data/sequence-column-manual.md) function.
### Q23. `recoveryTracker should overlap or follow on disk last VLSN of 4,422,880 recoveryFirst= 4,422,882 UNEXPECTED_STATE_FATAL`
Sometimes when restarting the Fe, the above error will occur (usually only in the case of multiple followers), and the difference between the two values in the error is 2. As a result, the Fe startup fails.
This is a bug in bdbje that has not been resolved. In this case, metadata can only be recovered through fault recovery in [metadata operation and maintenance manual](../administrator-guide/operation/metadata-operation.md).
### Q24. Multiple FEs cannot log in when using Nginx to implement web UI load balancing
### Q13. Multiple FEs cannot log in when using Nginx to implement web UI load balancing
Doris can deploy multiple FEs. When accessing the Web UI, if you use Nginx for load balancing, you will be prompted to log in again because of Session problems. This problem is actually a session sharing problem. Nginx provides centralized session sharing. The solution, here we use the ip_hash technology in nginx, ip_hash can direct the request of a certain ip to the same backend, so that a certain client and a certain backend under this ip can establish a stable The session, ip_hash is defined in the upstream configuration:

147
docs/zh-CN/faq/error.md Normal file
View File

@ -0,0 +1,147 @@
---
{
"title": "常见报错",
"language": "zh-CN"
}
---
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# 常见报错
本文档主要用于记录 Doris 使用过程中的报错,如果您有遇见一些报错,欢迎贡献给我们更新。
### E1. 查询报错:Failed to get scan range, no queryable replica found in tablet: xxxx
这种情况是因为对应的 tablet 没有找到可以查询的副本,通常原因可能是 BE 宕机、副本缺失等。可以先通过 `show tablet tablet_id` 语句,然后执行后面的 `show proc` 语句,查看这个 tablet 对应的副本信息,检查副本是否完整。同时还可以通过 `show proc "/cluster_balance"` 信息来查询集群内副本调度和修复的进度。
关于数据副本管理相关的命令,可以参阅 [数据副本管理](../administrator-guide/operation/tablet-repair-and-balance.md)。
### E2. FE启动失败,fe.log中一直滚动 "wait catalog to be ready. FE type UNKNOWN"
这种问题通常有两个原因:
1. 本次FE启动时获取到的本机IP和上次启动不一致,通常是因为没有正确设置 `priority_network` 而导致 FE 启动时匹配到了错误的 IP 地址。需修改 `priority_network` 后重启 FE。
2. 集群内多数 Follower FE 节点未启动。比如有 3 个 Follower,只启动了一个。此时需要将另外至少一个 FE 也启动,FE 可选举组方能选举出 Master 已提供服务。
如果以上情况都不能解决,可以按照 Doris 官网文档中的[元数据运维文档](../administrator-guide/operation/metadata-operation.md)进行恢复:
### E3. tablet writer write failed, tablet_id=27306172, txn_id=28573520, err=-235 or -215 or -238
这个错误通常发生在数据导入操作中。新版错误码为 -235,老版本错误码可能是 -215。这个错误的含义是,对应tablet的数据版本超过了最大限制(默认500,由 BE 参数 `max_tablet_version_num` 控制),后续写入将被拒绝。比如问题中这个错误,即表示 27306172 这个tablet的数据版本超过了限制。
这个错误通常是因为导入的频率过高,大于后台数据的compaction速度,导致版本堆积并最终超过了限制。此时,我们可以先通过show tablet 27306172 语句,然后执行结果中的 show proc 语句,查看tablet各个副本的情况。结果中的 versionCount即表示版本数量。如果发现某个副本的版本数量过多,则需要降低导入频率或停止导入,并观察版本数是否有下降。如果停止导入后,版本数依然没有下降,则需要去对应的BE节点查看be.INFO日志,搜索tablet id以及 compaction关键词,检查compaction是否正常运行。关于compaction调优相关,可以参阅 ApacheDoris 公众号文章:Doris 最佳实践-Compaction调优(3)
-238 错误通常出现在同一批导入数据量过大的情况,从而导致某一个 tablet 的 Segment 文件过多(默认是 200,由 BE 参数 `max_segment_num_per_rowset` 控制)。此时建议减少一批次导入的数据量,或者适当提高 BE 配置参数值来解决。
### E4. tablet 110309738 has few replicas: 1, alive backends: [10003]
这个错误可能发生在查询或者导入操作中。通常意味着对应tablet的副本出现了异常。
此时,可以先通过 show backends 命令检查BE节点是否有宕机,如 isAlive 字段为false,或者 LastStartTime 是最近的某个时间(表示最近重启过)。如果BE有宕机,则需要去BE对应的节点,查看be.out日志。如果BE是因为异常原因宕机,通常be.out中会打印异常堆栈,帮助排查问题。如果be.out中没有错误堆栈。则可以通过linux命令dmesg -T 检查是否是因为OOM导致进程被系统kill掉。
如果没有BE节点宕机,则需要通过show tablet 110309738 语句,然后执行结果中的 show proc 语句,查看tablet各个副本的情况,进一步排查。
### E5. disk xxxxx on backend xxx exceed limit usage
通常出现在导入、Alter等操作中。这个错误意味着对应BE的对应磁盘的使用量超过了阈值(默认95%)此时可以先通过 show backends 命令,其中MaxDiskUsedPct展示的是对应BE上,使用率最高的那块磁盘的使用率,如果超过95%,则会报这个错误。
此时需要前往对应BE节点,查看数据目录下的使用量情况。其中trash目录和snapshot目录可以手动清理以释放空间。如果是data目录占用较大,则需要考虑删除部分数据以释放空间了。具体可以参阅[磁盘空间管理](../administrator-guide/operation/disk-capacity.md)。
### E6. invalid cluster id: xxxx
这个错误可能会在show backends 或 show frontends 命令的结果中出现。通常出现在某个FE或BE节点的错误信息列中。这个错误的含义是,Master FE向这个节点发送心跳信息后,该节点发现心跳信息中携带的 cluster id和本地存储的 cluster id不同,所以拒绝回应心跳。
Doris的 Master FE 节点会主动发送心跳给各个FE或BE节点,并且在心跳信息中会携带一个cluster_id。cluster_id是在一个集群初始化时,由Master FE生成的唯一集群标识。当FE或BE第一次收到心跳信息后,则会将cluster_id以文件的形式保存在本地。FE的该文件在元数据目录的image/目录下,BE则在所有数据目录下都有一个cluster_id文件。之后,每次节点收到心跳后,都会用本地cluster_id的内容和心跳中的内容作比对,如果不一致,则拒绝响应心跳。
该机制是一个节点认证机制,以防止接收到集群外的节点发送来的错误的心跳信息。
如果需要恢复这个错误。首先要先确认所有节点是否都是正确的集群中的节点。之后,对于FE节点,可以尝试修改元数据目录下的 image/VERSION 文件中的 cluster_id 值后重启FE。对于BE节点,则可以删除所有数据目录下的 cluster_id 文件后重启 BE。
### E7. 通过 Java 程序调用 stream load 导入数据,在一批次数据量较大时,可能会报错 Broken Pipe
除了 Broken Pipe 外,还可能出现一些其他的奇怪的错误。
这个情况通常出现在开启httpv2后。因为httpv2是使用spring boot实现的http 服务,并且使用tomcat作为默认内置容器。但是jetty对307转发的处理似乎有些问题,所以后面将内置容器修改为了jetty。此外,在java程序中的 apache http client的版本需要使用4.5.13以后的版本。之前的版本,对转发的处理也存在一些问题。
所以这个问题可以有两种解决方式:
1. 关闭httpv2
在fe.conf中添加 enable_http_server_v2=false后重启FE。但是这样无法再使用新版UI界面,并且之后的一些基于httpv2的新接口也无法使用。(正常的导入查询不受影响)。
2. 升级
可以升级到 Doris 0.15 及之后的版本,已修复这个问题。
### E8. `Lost connection to MySQL server at 'reading initial communication packet', system error: 0`
如果使用 MySQL 客户端连接 Doris 时出现如下问题,这通常是因为编译 FE 时使用的 jdk 版本和运行 FE 时使用的 jdk 版本不同导致的。
注意使用 docker 编译镜像编译时,默认的 JDK 版本是 openjdk 11,可以通过命令切换到 openjdk 8(详见编译文档)。
### E9. -214 错误
在执行导入、查询等操作时,可能会遇到如下错误:
```
failed to initialize storage reader. tablet=63416.1050661139.aa4d304e7a7aff9c-f0fa7579928c85a0, res=-214, backend=192.168.100.10
```
-214 错误意味着对应 tablet 的数据版本缺失。比如如上错误,表示 tablet 63416 在 192.168.100.10 这个 BE 上的副本的数据版本有缺失。(可能还有其他类似错误码,都可以用如下方式进行排查和修复)。
通常情况下,如果你的数据是多副本的,那么系统会自动修复这些有问题的副本。可以通过以下步骤进行排查:
首先通过 `show tablet 63416` 语句并执行结果中的 `show proc xxx` 语句来查看对应 tablet 的各个副本情况。通常我们需要关心 `Version` 这一列的数据。
正常情况下,一个 tablet 的多个副本的 Version 应该是相同的。并且和对应分区的 VisibleVersion 版本相同。
你可以通过 `show partitions from tblx` 来查看对应的分区版本(tablet 对应的分区可以在 `show tablet` 语句中获取。)
同时,你也可以访问 `show proc` 语句中的 CompactionStatus 列中的 URL(在浏览器打开即可)来查看更具体的版本信息,来检查具体丢失的是哪些版本。
如果长时间没有自动修复,则需要通过 `show proc "/cluster_balance"` 语句,查看当前系统正在执行的 tablet 修复和调度任务。可能是因为有大量的 tablet 在等待被调度,导致修复时间较长。可以关注 `pending_tablets``running_tablets` 中的记录。
更进一步的,可以通过 `admin repair` 语句来指定优先修复某个表或分区,具体可以参阅 `help admin repair`;
如果依然无法修复,那么在多副本的情况下,我们使用 `admin set replica status` 命令强制将有问题的副本下线。具体可参阅 `help admin set replica status` 中将副本状态置为 bad 的示例。(置为 bad 后,副本将不会再被访问。并且会后续自动修复。但在操作前,应先确保其他副本是正常的)
### E10. Not connected to 192.168.100.1:8060 yet, server_id=384
在导入或者查询时,我们可能遇到这个错误。如果你去对应的 BE 日志中查看,也可能会找到类似错误。
这是一个 RPC 错误,通常由两种可能:1. 对应的 BE 节点宕机。2. rpc 拥塞或其他错误。
如果是 BE 节点宕机,则需要查看具体的宕机原因。这里只讨论 rpc 拥塞的问题。
一种情况是 OVERCROWDED,即表示 rpc 源端有大量未发送的数据超过了阈值。BE 有两个参数与之相关:
1. `brpc_socket_max_unwritten_bytes`:默认 1GB,如果未发送数据超过这个值,则会报错。可以适当修改这个值以避免 OVERCROWDED 错误。(但这个治标不治本,本质上还是有拥塞发生)。
2. `tablet_writer_ignore_eovercrowded`:默认为 false。如果设为true,则 Doris 会忽略导入过程中出现的 OVERCROWDED 错误。这个参数主要为了避免导入失败,以提高导入的稳定性。
第二种是 rpc 的包大小超过 max_body_size。如果查询中带有超大 String 类型,或者 bitmap 类型时,可能出现这个问题。可以通过修改以下 BE 参数规避:
1. `brpc_max_body_size`:默认 3GB.
### E11. `recoveryTracker should overlap or follow on disk last VLSN of 4,422,880 recoveryFirst= 4,422,882 UNEXPECTED_STATE_FATAL`
有时重启 FE,会出现如上错误(通常只会出现在多 Follower 的情况下)。并且错误中的两个数值相差2。导致 FE 启动失败。
这是 bdbje 的一个 bug,尚未解决。遇到这种情况,只能通过 [元数据运维手册](../administrator-guide/operation/metadata-operation.md) 中的 故障恢复 进行操作来恢复元数据了。

View File

@ -30,28 +30,11 @@ under the License.
### Q1. 使用 Stream Load 访问 FE 的公网地址导入数据,被重定向到内网 IP?
当 stream load 的连接目标为FE的http端口时,FE仅会随机选择一台BE节点做http 307 redirect 操作,因此用户的请求实际是发送给FE指派的某一个BE的。而redirect返回的是BE的ip,也即内网IP。所以如果你是通过FE的公网IP发送的请求,很有可能因为redirect到内网地址而无法连接。
通常的做法,一种是确保自己能够访问内网IP地址,或者是给所有BE上层假设一个负载均衡,然后直接将 stream load 请求发送到负载均衡器上,由负载均衡将请求透传到BE节点。
### Q2. 查询报错:Failed to get scan range, no queryable replica found in tablet: xxxx
这种情况是因为对应的 tablet 没有找到可以查询的副本,通常原因可能是 BE 宕机、副本缺失等。可以先通过 `show tablet tablet_id` 语句,然后执行后面的 `show proc` 语句,查看这个 tablet 对应的副本信息,检查副本是否完整。同时还可以通过 `show proc "/cluster_balance"` 信息来查询集群内副本调度和修复的进度。
关于数据副本管理相关的命令,可以参阅 [数据副本管理](../administrator-guide/operation/tablet-repair-and-balance.md)。
### Q3. FE启动失败,fe.log中一直滚动 "wait catalog to be ready. FE type UNKNOWN"
这种问题通常有两个原因:
1. 本次FE启动时获取到的本机IP和上次启动不一致,通常是因为没有正确设置 `priority_network` 而导致 FE 启动时匹配到了错误的 IP 地址。需修改 `priority_network` 后重启 FE。
2. 集群内多数 Follower FE 节点未启动。比如有 3 个 Follower,只启动了一个。此时需要将另外至少一个 FE 也启动,FE 可选举组方能选举出 Master 已提供服务。
如果以上情况都不能解决,可以按照 Doris 官网文档中的[元数据运维文档](../administrator-guide/operation/metadata-operation.md)进行恢复:
### Q4. 通过 DECOMMISSION 下线BE节点时,为什么总会有部分tablet残留?
### Q2. 通过 DECOMMISSION 下线BE节点时,为什么总会有部分tablet残留?
在下线过程中,通过 show backends 查看下线节点的 tabletNum ,会观察到 tabletNum 数量在减少,说明数据分片正在从这个节点迁移走。当数量减到0时,系统会自动删除这个节点。但某些情况下,tabletNum 下降到一定数值后就不变化。这通常可能有以下两种原因:
@ -62,7 +45,7 @@ under the License.
对于以上情况,可以先通过 show proc "/statistic" 查看集群是否还有 unhealthy 的分片,如果为0,则可以直接通过 drop backend 语句删除这个 BE 。否则,还需要具体查看不健康分片的副本情况。
### Q5. priorty_network 应该如何设置?
### Q3. priorty_network 应该如何设置?
priorty_network 是 FE、BE 都有的配置参数。这个参数主要用于帮助系统选择正确的网卡 IP 作为自己的 IP 。建议任何情况下,都显式的设置这个参数,以防止后续机器增加新网卡导致IP选择不正确的问题。
@ -70,7 +53,7 @@ priorty_network 的值是 CIDR 格式表示的。分为两部分,第一部分
之所以使用 CIDR 格式而不是直接指定一个具体 IP,是为了保证所有节点都可以使用统一的配置值。比如有两个节点:10.168.10.1 和 10.168.10.2,则我们可以使用 10.168.10.0/24 来作为 priorty_network 的值。
### Q6. FE的Master、Follower、Observer都是什么?
### Q4. FE的Master、Follower、Observer都是什么?
首先明确一点,FE 只有两种角色:Follower 和 Observer。而 Master 只是一组 Follower 节点中选择出来的一个 FE。Master 可以看成是一种特殊的 Follower。所以当我们被问及一个集群有多少 FE,都是什么角色时,正确的回答当时应该是所有 FE 节点的个数,以及 Follower 角色的个数和 Observer 角色的个数。
@ -82,39 +65,7 @@ Observer 角色和这个单词的含义一样,仅仅作为观察者来同步
通常情况下,可以部署 1 Follower + 2 Observer 或者 3 Follower + N Observer。前者运维简单,几乎不会出现 Follower 之间的一致性协议导致这种复杂错误情况(百度内部集群大多使用这种方式)。后者可以保证元数据写的高可用,如果是高并发查询场景,可以适当增加 Observer。
### Q7. tablet writer write failed, tablet_id=27306172, txn_id=28573520, err=-235 or -215 or -238
这个错误通常发生在数据导入操作中。新版错误码为 -235,老版本错误码可能是 -215。这个错误的含义是,对应tablet的数据版本超过了最大限制(默认500,由 BE 参数 `max_tablet_version_num` 控制),后续写入将被拒绝。比如问题中这个错误,即表示 27306172 这个tablet的数据版本超过了限制。
这个错误通常是因为导入的频率过高,大于后台数据的compaction速度,导致版本堆积并最终超过了限制。此时,我们可以先通过show tablet 27306172 语句,然后执行结果中的 show proc 语句,查看tablet各个副本的情况。结果中的 versionCount即表示版本数量。如果发现某个副本的版本数量过多,则需要降低导入频率或停止导入,并观察版本数是否有下降。如果停止导入后,版本数依然没有下降,则需要去对应的BE节点查看be.INFO日志,搜索tablet id以及 compaction关键词,检查compaction是否正常运行。关于compaction调优相关,可以参阅 ApacheDoris 公众号文章:Doris 最佳实践-Compaction调优(3)
-238 错误通常出现在同一批导入数据量过大的情况,从而导致某一个 tablet 的 Segment 文件过多(默认是 200,由 BE 参数 `max_segment_num_per_rowset` 控制)。此时建议减少一批次导入的数据量,或者适当提高 BE 配置参数值来解决。
### Q8. tablet 110309738 has few replicas: 1, alive backends: [10003]
这个错误可能发生在查询或者导入操作中。通常意味着对应tablet的副本出现了异常。
此时,可以先通过 show backends 命令检查BE节点是否有宕机,如 isAlive 字段为false,或者 LastStartTime 是最近的某个时间(表示最近重启过)。如果BE有宕机,则需要去BE对应的节点,查看be.out日志。如果BE是因为异常原因宕机,通常be.out中会打印异常堆栈,帮助排查问题。如果be.out中没有错误堆栈。则可以通过linux命令dmesg -T 检查是否是因为OOM导致进程被系统kill掉。
如果没有BE节点宕机,则需要通过show tablet 110309738 语句,然后执行结果中的 show proc 语句,查看tablet各个副本的情况,进一步排查。
### Q9. disk xxxxx on backend xxx exceed limit usage
通常出现在导入、Alter等操作中。这个错误意味着对应BE的对应磁盘的使用量超过了阈值(默认95%)此时可以先通过 show backends 命令,其中MaxDiskUsedPct展示的是对应BE上,使用率最高的那块磁盘的使用率,如果超过95%,则会报这个错误。
此时需要前往对应BE节点,查看数据目录下的使用量情况。其中trash目录和snapshot目录可以手动清理以释放空间。如果是data目录占用较大,则需要考虑删除部分数据以释放空间了。具体可以参阅[磁盘空间管理](../administrator-guide/operation/disk-capacity.md)。
### Q10. invalid cluster id: xxxx
这个错误可能会在show backends 或 show frontends 命令的结果中出现。通常出现在某个FE或BE节点的错误信息列中。这个错误的含义是,Master FE向这个节点发送心跳信息后,该节点发现心跳信息中携带的 cluster id和本地存储的 cluster id不同,所以拒绝回应心跳。
Doris的 Master FE 节点会主动发送心跳给各个FE或BE节点,并且在心跳信息中会携带一个cluster_id。cluster_id是在一个集群初始化时,由Master FE生成的唯一集群标识。当FE或BE第一次收到心跳信息后,则会将cluster_id以文件的形式保存在本地。FE的该文件在元数据目录的image/目录下,BE则在所有数据目录下都有一个cluster_id文件。之后,每次节点收到心跳后,都会用本地cluster_id的内容和心跳中的内容作比对,如果不一致,则拒绝响应心跳。
该机制是一个节点认证机制,以防止接收到集群外的节点发送来的错误的心跳信息。
如果需要恢复这个错误。首先要先确认所有节点是否都是正确的集群中的节点。之后,对于FE节点,可以尝试修改元数据目录下的 image/VERSION 文件中的 cluster_id 值后重启FE。对于BE节点,则可以删除所有数据目录下的 cluster_id 文件后重启 BE。
### Q11. Doris 是否支持修改列名?
### Q5. Doris 是否支持修改列名?
不支持修改列名。
@ -124,7 +75,7 @@ Doris支持修改数据库名、表名、分区名、物化视图(Rollup)名
我们不排除后续通过一些兼容手段来支持轻量化的列名修改操作。
### Q12. Unique Key模型的表是否支持创建物化视图?
### Q6. Unique Key模型的表是否支持创建物化视图?
不支持。
@ -132,7 +83,7 @@ Unique Key模型的表是一个对业务比较友好的表,因为其特有的
但遗憾的是,Unique Key模型的表是无法建立物化视图的。原因在于,物化视图的本质,是通过预计算来将数据“预先算好”,这样在查询时直接返回已经计算好的数据,来加速查询。在物化视图中,“预计算”的数据通常是一些聚合指标,比如求和、求count。这时,如果数据发生变更,如udpate或delete,因为预计算的数据已经丢失了明细信息,因此无法同步的进行更新。比如一个求和值5,可能是 1+4,也可能是2+3。因为明细信息的丢失,我们无法区分这个求和值是如何计算出来的,因此也就无法满足更新的需求。
### Q13. show backends/frontends 查看到的信息不完整
### Q7. show backends/frontends 查看到的信息不完整
在执行如` show backends/frontends` 等某些语句后,结果中可能会发现有部分列内容不全。比如show backends结果中看不到磁盘容量信息等。
@ -140,23 +91,7 @@ Unique Key模型的表是一个对业务比较友好的表,因为其特有的
当然,用户也可以在执行这些语句前,先执行 `set forward_to_master=true;` 这个会话变量设置为true后,后续执行的一些信息查看类语句会自动转发到Master FE获取结果。这样,不论用户连接的是哪个FE,都可以获取到完整结果了。
### Q14. 通过Java程序调用stream load导入数据,在一批次数据量较大时,可能会报错Broken Pipe
除了Broken Pipe外,还可能出现一些其他的奇怪的错误。
这个情况通常出现在开启httpv2后。因为httpv2是使用spring boot实现的http 服务,并且使用tomcat作为默认内置容器。但是jetty对307转发的处理似乎有些问题,所以后面将内置容器修改为了jetty。此外,在java程序中的 apache http client的版本需要使用4.5.13以后的版本。之前的版本,对转发的处理也存在一些问题。
所以这个问题可以有两种解决方式:
1. 关闭httpv2
在fe.conf中添加 enable_http_server_v2=false后重启FE。但是这样无法再使用新版UI界面,并且之后的一些基于httpv2的新接口也无法使用。(正常的导入查询不受影响)。
2. 升级
可以升级到 Doris 0.15 及之后的版本,已修复这个问题。
### Q15. 节点新增加了新的磁盘,为什么数据没有均衡到新的磁盘上?
### Q8. 节点新增加了新的磁盘,为什么数据没有均衡到新的磁盘上?
当前Doris的均衡策略是以节点为单位的。也就是说,是按照节点整体的负载指标(分片数量和总磁盘利用率)来判断集群负载。并且将数据分片从高负载节点迁移到低负载节点。如果每个节点都增加了一块磁盘,则从节点整体角度看,负载并没有改变,所以无法触发均衡逻辑。
@ -182,7 +117,7 @@ Unique Key模型的表是一个对业务比较友好的表,因为其特有的
Doris提供了[HTTP API](../administrator-guide/http-actions/tablet-migration-action.md),可以手动指定一个磁盘上的数据分片迁移到另一个磁盘上。
### Q16. 如何正确阅读 FE/BE 日志?
### Q9. 如何正确阅读 FE/BE 日志?
很多情况下我们需要通过日志来排查问题。这里说明一下FE/BE日志的格式和查看方式。
@ -230,7 +165,7 @@ Unique Key模型的表是一个对业务比较友好的表,因为其特有的
通常情况下我们主要查看be.INFO日志。特殊情况下,如BE宕机,则需要查看be.out。
### Q17. FE/BE 节点挂了应该如何排查原因?
### Q10. FE/BE 节点挂了应该如何排查原因?
1. BE
@ -260,7 +195,7 @@ Unique Key模型的表是一个对业务比较友好的表,因为其特有的
FE 是 java 进程,健壮程度要由于 C/C++ 程序。通常FE 挂掉的原因可能是 OOM(Out-of-Memory)或者是元数据写入失败。这些错误通常在 fe.log 或者 fe.out 中有错误堆栈。需要根据错误堆栈信息进一步排查。
### Q18. 关于数据目录SSD和HDD的配置。
### Q11. 关于数据目录SSD和HDD的配置。
Doris支持一个BE节点配置多个存储路径。通常情况下,每块盘配置一个存储路径即可。同时,Doris支持指定路径的存储介质属性,如SSD或HDD。SSD代表高速存储设备,HDD代表低速存储设备。
@ -270,55 +205,7 @@ Doris支持一个BE节点配置多个存储路径。通常情况下,每块盘
换句话说,".HDD" 和 ".SSD" 只是用于标识存储目录“相对”的“低速”和“高速”之分,而并不是标识实际的存储介质类型。所以如果BE节点上的存储路径没有介质区别,则无需填写后缀。
### Q19. `Lost connection to MySQL server at 'reading initial communication packet', system error: 0`
如果使用 MySQL 客户端连接 Doris 时出现如下问题,这通常是因为编译 FE 时使用的 jdk 版本和运行 FE 时使用的 jdk 版本不同导致的。
注意使用 docker 编译镜像编译时,默认的 JDK 版本是 openjdk 11,可以通过命令切换到 openjdk 8(详见编译文档)。
### Q20. -214 错误
在执行导入、查询等操作时,可能会遇到如下错误:
```
failed to initialize storage reader. tablet=63416.1050661139.aa4d304e7a7aff9c-f0fa7579928c85a0, res=-214, backend=192.168.100.10
```
-214 错误意味着对应 tablet 的数据版本缺失。比如如上错误,表示 tablet 63416 在 192.168.100.10 这个 BE 上的副本的数据版本有缺失。(可能还有其他类似错误码,都可以用如下方式进行排查和修复)。
通常情况下,如果你的数据是多副本的,那么系统会自动修复这些有问题的副本。可以通过以下步骤进行排查:
首先通过 `show tablet 63416` 语句并执行结果中的 `show proc xxx` 语句来查看对应 tablet 的各个副本情况。通常我们需要关心 `Version` 这一列的数据。
正常情况下,一个 tablet 的多个副本的 Version 应该是相同的。并且和对应分区的 VisibleVersion 版本相同。
你可以通过 `show partitions from tblx` 来查看对应的分区版本(tablet 对应的分区可以在 `show tablet` 语句中获取。)
同时,你也可以访问 `show proc` 语句中的 CompactionStatus 列中的 URL(在浏览器打开即可)来查看更具体的版本信息,来检查具体丢失的是哪些版本。
如果长时间没有自动修复,则需要通过 `show proc "/cluster_balance"` 语句,查看当前系统正在执行的 tablet 修复和调度任务。可能是因为有大量的 tablet 在等待被调度,导致修复时间较长。可以关注 `pending_tablets` 和 `running_tablets` 中的记录。
更进一步的,可以通过 `admin repair` 语句来指定优先修复某个表或分区,具体可以参阅 `help admin repair`;
如果依然无法修复,那么在多副本的情况下,我们使用 `admin set replica status` 命令强制将有问题的副本下线。具体可参阅 `help admin set replica status` 中将副本状态置为 bad 的示例。(置为 bad 后,副本将不会再被访问。并且会后续自动修复。但在操作前,应先确保其他副本是正常的)
### Q21. Not connected to 192.168.100.1:8060 yet, server_id=384
在导入或者查询时,我们可能遇到这个错误。如果你去对应的 BE 日志中查看,也可能会找到类似错误。
这是一个 RPC 错误,通常由两种可能:1. 对应的 BE 节点宕机。2. rpc 拥塞或其他错误。
如果是 BE 节点宕机,则需要查看具体的宕机原因。这里只讨论 rpc 拥塞的问题。
一种情况是 OVERCROWDED,即表示 rpc 源端有大量未发送的数据超过了阈值。BE 有两个参数与之相关:
1. `brpc_socket_max_unwritten_bytes`:默认 1GB,如果未发送数据超过这个值,则会报错。可以适当修改这个值以避免 OVERCROWDED 错误。(但这个治标不治本,本质上还是有拥塞发生)。
2. `tablet_writer_ignore_eovercrowded`:默认为 false。如果设为true,则 Doris 会忽略导入过程中出现的 OVERCROWDED 错误。这个参数主要为了避免导入失败,以提高导入的稳定性。
第二种是 rpc 的包大小超过 max_body_size。如果查询中带有超大 String 类型,或者 bitmap 类型时,可能出现这个问题。可以通过修改以下 BE 参数规避:
1. `brpc_max_body_size`:默认 3GB.
### Q22. Unique Key 模型查询结果不一致
### Q12. Unique Key 模型查询结果不一致
某些情况下,当用户使用相同的 SQL 查询一个 Unique Key 模型的表时,可能会出现多次查询结果不一致的现象。并且查询结果总在 2-3 种之间变化。
@ -335,13 +222,7 @@ failed to initialize storage reader. tablet=63416.1050661139.aa4d304e7a7aff9c-f0
为了确保不同副本之间的数据先后顺序唯一,可以参考 [Sequence Column](../administrator-guide/load-data/sequence-column-manual.md) 功能。
### Q23. `recoveryTracker should overlap or follow on disk last VLSN of 4,422,880 recoveryFirst= 4,422,882 UNEXPECTED_STATE_FATAL`
有时重启 FE,会出现如上错误(通常只会出现在多 Follower 的情况下)。并且错误中的两个数值相差2。导致 FE 启动失败。
这是 bdbje 的一个 bug,尚未解决。遇到这种情况,只能通过 [元数据运维手册](../administrator-guide/operation/metadata-operation.md) 中的 故障恢复 进行操作来恢复元数据了。
### Q24. 多个FE,在使用Nginx实现web UI负载均衡时,无法登录
### Q13. 多个FE,在使用Nginx实现web UI负载均衡时,无法登录
Doris 可以部署多个FE,在访问Web UI的时候,如果使用Nginx进行负载均衡,因为Session问题会出现不停的提示要重新登录,这个问题其实是Session共享的问题,Nginx提供了集中Session共享的解决方案,这里我们使用的是nginx中的ip_hash技术,ip_hash能够将某个ip的请求定向到同一台后端,这样一来这个ip下的某个客户端和某个后端就能建立起稳固的session,ip_hash是在upstream配置中定义的: