diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go
index 9cce66aae3..d4e2003c41 100644
--- a/planner/core/find_best_task.go
+++ b/planner/core/find_best_task.go
@@ -2198,6 +2198,7 @@ func (ds *DataSource) getOriginalPhysicalTableScan(prop *property.PhysicalProper
 		HandleCols:      ds.handleCols,
 		tblCols:         ds.TblCols,
 		tblColHists:     ds.TblColHists,
+		prop:            prop,
 	}.Init(ds.ctx, ds.blockOffset)
 	ts.filterCondition = make([]expression.Expression, len(path.TableFilters))
 	copy(ts.filterCondition, path.TableFilters)
@@ -2267,6 +2268,7 @@ func (ds *DataSource) getOriginalPhysicalIndexScan(prop *property.PhysicalProper
 		physicalTableID:  ds.physicalTableID,
 		tblColHists:      ds.TblColHists,
 		pkIsHandleCol:    ds.getPKIsHandleCol(),
+		prop:             prop,
 	}.Init(ds.ctx, ds.blockOffset)
 	statsTbl := ds.statisticTable
 	if statsTbl.Indices[idx.ID] != nil {
diff --git a/planner/core/physical_plans.go b/planner/core/physical_plans.go
index e1b43e6bf0..1aa700f9de 100644
--- a/planner/core/physical_plans.go
+++ b/planner/core/physical_plans.go
@@ -468,6 +468,7 @@ type PhysicalIndexScan struct {
 	// tblColHists contains all columns before pruning, which are used to calculate row-size
 	tblColHists   *statistics.HistColl
 	pkIsHandleCol *expression.Column
+	prop          *property.PhysicalProperty
 }
 
 // Clone implements PhysicalPlan interface.
@@ -569,6 +570,7 @@ type PhysicalTableScan struct {
 	// tblCols and tblColHists contains all columns before pruning, which are used to calculate row-size
 	tblCols     []*expression.Column
 	tblColHists *statistics.HistColl
+	prop        *property.PhysicalProperty
 }
 
 // Clone implements PhysicalPlan interface.
diff --git a/planner/core/plan_cost.go b/planner/core/plan_cost.go
index 6dc4132fa2..3a5f5aff90 100644
--- a/planner/core/plan_cost.go
+++ b/planner/core/plan_cost.go
@@ -404,7 +404,7 @@ func (p *PhysicalTableScan) GetPlanCost(taskType property.TaskType, costFlag uin
 	switch p.ctx.GetSessionVars().CostModelVersion {
 	case modelVer1: // scan cost: rows * row-size * scan-factor
 		scanFactor := p.ctx.GetSessionVars().GetScanFactor(p.Table)
-		if p.Desc {
+		if p.Desc && p.prop != nil && p.prop.ExpectedCnt >= smallScanThreshold {
 			scanFactor = p.ctx.GetSessionVars().GetDescScanFactor(p.Table)
 		}
 		selfCost = getCardinality(p, costFlag) * p.getScanRowSize() * scanFactor
@@ -441,7 +441,7 @@ func (p *PhysicalIndexScan) GetPlanCost(taskType property.TaskType, costFlag uin
 	switch p.ctx.GetSessionVars().CostModelVersion {
 	case modelVer1: // scan cost: rows * row-size * scan-factor
 		scanFactor := p.ctx.GetSessionVars().GetScanFactor(p.Table)
-		if p.Desc {
+		if p.Desc && p.prop != nil && p.prop.ExpectedCnt >= smallScanThreshold {
 			scanFactor = p.ctx.GetSessionVars().GetDescScanFactor(p.Table)
 		}
 		selfCost = getCardinality(p, costFlag) * p.getScanRowSize() * scanFactor