[enhancement](session-variable)add a use_fix_replica session variable to fix query replica (#17101)

Add use_fix_replica session variable, so that we can be better debug replica inconsistencies problem.
If use_fix_replica default is -1, which means not fix,
else we will choose the {use_fix_replica} smallest replica.

docs/en/docs/advanced/variables.md

@@ -590,3 +590,7 @@ Translated with www.DeepL.com/Translator (free version)
     <version since="dev"></version>
 
     Controls whether to show each user's implicit roles in the results of `show roles`. Default is false.
+
+* `use_fix_replica`
+
+    Use a fixed replica to query. If use_fix_replica is 1, the smallest one is used, if use_fix_replica is 2, the second smallest one is used, and so on. The default value is -1, which means it is not enabled.

docs/zh-CN/docs/advanced/variables.md

@@ -577,3 +577,7 @@ SELECT /*+ SET_VAR(query_timeout = 1, enable_partition_cache=true) */ sleep(3);
     <version since="dev"></version>
 
     控制是否在 `show roles` 的结果里显示每个用户隐式对应的角色。默认为 false。
+
+* `use_fix_replica`
+
+    使用固定的replica进行查询，该值表示固定使用第几小的replica，默认为-1表示不启用。

fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java

@@ -36,6 +36,7 @@ import java.util.Comparator;
 public class Replica implements Writable {
     private static final Logger LOG = LogManager.getLogger(Replica.class);
     public static final VersionComparator<Replica> VERSION_DESC_COMPARATOR = new VersionComparator<Replica>();
+    public static final IdComparator<Replica> ID_COMPARATOR = new IdComparator<Replica>();
 
     public enum ReplicaState {
         NORMAL,
@@ -538,6 +539,22 @@ public class Replica implements Writable {
         }
     }
 
+    private static class IdComparator<T extends Replica> implements Comparator<T> {
+        public IdComparator() {
+        }
+
+        @Override
+        public int compare(T replica1, T replica2) {
+            if (replica1.getId() < replica2.getId()) {
+                return -1;
+            } else if (replica1.getId() == replica2.getId()) {
+                return 0;
+            } else {
+                return 1;
+            }
+        }
+    }
+
     public void setWatermarkTxnId(long watermarkTxnId) {
         this.watermarkTxnId = watermarkTxnId;
     }

fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java

@@ -684,7 +684,20 @@ public class OlapScanNode extends ScanNode {
                 throw new UserException("Failed to get scan range, no queryable replica found in tablet: " + tabletId);
             }
 
-            Collections.shuffle(replicas);
+            int useFixReplica = -1;
+            if (ConnectContext.get() != null) {
+                useFixReplica = ConnectContext.get().getSessionVariable().useFixReplica;
+            }
+            if (useFixReplica == -1) {
+                Collections.shuffle(replicas);
+            } else {
+                LOG.debug("use fix replica, value: {}, replica num: {}", useFixReplica, replicas.size());
+                // sort by replica id
+                replicas.sort(Replica.ID_COMPARATOR);
+                Replica replica = replicas.get(useFixReplica >= replicas.size() ? replicas.size() - 1 : useFixReplica);
+                replicas.clear();
+                replicas.add(replica);
+            }
             boolean tabletIsNull = true;
             boolean collectedStat = false;
             List<String> errs = Lists.newArrayList();

fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java

@@ -275,6 +275,9 @@ public class SessionVariable implements Serializable, Writable {
 
     public static final String SHOW_USER_DEFAULT_ROLE = "show_user_default_role";
 
+    // fix replica to query. If num = 1, query the smallest replica, if 2 is the second smallest replica.
+    public static final String USE_FIX_REPLICA = "use_fix_replica";
+
     // session origin value
     public Map<Field, String> sessionOriginValue = new HashMap<Field, String>();
     // check stmt is or not [select /*+ SET_VAR(...)*/ ...]
@@ -730,6 +733,10 @@ public class SessionVariable implements Serializable, Writable {
     @VariableMgr.VarAttr(name = SHOW_USER_DEFAULT_ROLE, needForward = true)
     public boolean showUserDefaultRole = false;
 
+    // Default value is -1, which means not fix replica
+    @VariableMgr.VarAttr(name = USE_FIX_REPLICA)
+    public int useFixReplica = -1;
+
     // If this fe is in fuzzy mode, then will use initFuzzyModeVariables to generate some variables,
     // not the default value set in the code.
     public void initFuzzyModeVariables() {