database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

branch-2.1: [fix](nereids) Fix query err when topn push through union_all #54158 (#54258)

Browse Source 
Cherry-picked from #54158

Co-authored-by: seawinde <wusi@selectdb.com>
This commit is contained in:
github-actions[bot]
2025-08-06 11:47:37 +08:00
committed by GitHub
parent ae3a41db29
commit ebbfe1dfa2
3 changed files with 266 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughUnion.java
View File

@ -22,6 +22,7 @@ import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
import org.apache.doris.nereids.trees.plans.logical.LogicalTopN;
@ -63,11 +64,13 @@ public class PushDownTopNThroughUnion implements RewriteRuleFactory {
.then(topN -> {
LogicalUnion union = topN.child();
List<Plan> newChildren = new ArrayList<>();
for (Plan child : union.children()) {
for (int j = 0; j < union.children().size(); j++) {
Plan child = union.child(j);
List<SlotReference> regularChildOutput = union.getRegularChildOutput(j);
Map<Expression, Expression> replaceMap = new HashMap<>();
for (int i = 0; i < union.getOutputs().size(); ++i) {
for (int i = 0; i < regularChildOutput.size(); ++i) {
NamedExpression output = union.getOutputs().get(i);
replaceMap.put(output, child.getOutput().get(i));
replaceMap.put(output, regularChildOutput.get(i));
}
List<OrderKey> orderKeys = topN.getOrderKeys().stream()

22
regression-test/data/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.out
View File

@ -129,3 +129,25 @@ PhysicalResultSink
------------filter((t2.id < 5) and (t2.score < 20))
--------------PhysicalOlapScan[table1]
-- !union_all_push_down_top_n --
error_code=500&message=Database+connection+failed {"component":"database","severity":"critical"} 500 Database connection error 192.168.1.103 /api/v1/products 192.168.1.103 api.example.com {"app":"api-gateway","dependencies":["mysql","redis"],"env":"production"} {"city":"Tokyo","coordinates":{"lat":35.6762,"lon":139.6503},"country":"JP"} {"Content-Type":"application/json","X-Correlation-ID":"corr_456mno"} {"region":"ap-northeast-1","server":"api-node-05"} https://app.example.com/products Mozilla/5.0 (Linux; Android 13; SM-S901B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36 {"source":"mobile","tags":["error","database"]} {"context":{"attempts":3,"database":"products_db","duration_ms":5000,"error":"Connection timeout"},"level":"ERROR","message":"Failed to connect to database","timestamp":"2025-07-31T16:45:33Z"} 2025-07-31T16:45:33
error_code=500&message=Internal+Server+Error {"severity":"high","type":"error"} 500 Internal server error occurred 192.168.1.104 /api/v1/users 192.168.1.104 api.example.com {"app":"web","env":"production"} {"city":"Berlin","country":"Germany"} {"Content-Type":"application/json","X-Request-ID":"req123"} {"hostname":"web-server-02"} https://client.example.com Mozilla/5.0 (Windows NT 10.0; Win64; x64) {"input_type":"error","tags":["error","prod"]} {"level":"error","message":"server error","timestamp":"2025-07-31T18:15:55Z"} 2025-07-31T18:15:55
order_id=789&amount=99.99 {"method":"credit_card","type":"payment"} 201 Payment processed 192.168.1.102 /api/v1/payments 192.168.1.102 pay.example.com {"app":"payment","env":"production"} {"city":"London","country":"UK"} {"Content-Type":"application/json","X-CSRF-Token":"xyz789"} {"hostname":"payment-gateway-01"} https://checkout.example.com Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) {"input_type":"api","tags":["payment","prod"]} {"level":"info","message":"payment success","timestamp":"2025-07-31T14:45:20Z"} 2025-07-31T14:45:20
product_id=456&category=electronics {"device":"mobile","type":"product_view"} 304 Product page viewed 192.168.1.101 /products/456 192.168.1.101 shop.example.com {"app":"ecommerce","env":"staging"} {"city":"New York","country":"USA"} {"Cache-Control":"max-age=3600","Content-Type":"text/html"} {"hostname":"shop-server-02"} https://search.example.com Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) {"input_type":"http","tags":["shop","mobile"]} {"level":"info","message":"product view","timestamp":"2025-07-31T11:30:45Z"} 2025-07-31T11:30:45
product_id=789&quantity=2&price=59.99 {"event":"purchase","payment_method":"credit_card"} 201 Order processed successfully 192.168.1.101 /api/v1/orders 192.168.1.101 shop.example.com {"app":"ecommerce","env":"production","version":"3.2.1"} {"city":"New York","coordinates":{"lat":40.7128,"lon":-74.006},"country":"US"} {"Accept":"application/json","Content-Type":"application/json","X-CSRF-Token":"csrf_abc123"} {"region":"us-east-1","server":"shop-node-03"} https://shop.example.com/product/789 Mozilla/5.0 (iPhone; CPU iPhone OS 16_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Mobile/15E148 Safari/604.1 {"source":"mobile","tags":["transaction","order"]} {"context":{"amount":119.98,"currency":"USD","customer_id":"cust_123abc","order_id":"ord_789xyz"},"level":"INFO","message":"New order created","timestamp":"2025-07-31T11:30:45Z"} 2025-07-31T11:30:45
query=smartphone&page=1&sort=price_asc {"filters":["in_stock","free_shipping"],"search_type":"product"} 200 Search results returned 192.168.1.104 /api/v1/search 192.168.1.104 search.example.com {"app":"search","env":"production","features":["autocomplete","faceted_search"]} {"city":"Berlin","coordinates":{"lat":52.52,"lon":13.405},"country":"DE"} {"Content-Type":"application/json","X-Forwarded-For":"192.168.1.104, 203.0.113.5"} {"region":"eu-central-1","server":"search-node-01"} https://www.example.com/search Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/125.0 {"source":"web","tags":["search","query"]} {"context":{"filters_applied":2,"query":"smartphone","response_time_ms":125,"results_count":42},"level":"INFO","message":"Search query processed","timestamp":"2025-07-31T19:05:07Z"} 2025-07-31T19:05:07
search_term=laptop&page=2 {"results_count":25,"type":"search"} 200 Search results returned 192.168.1.103 /search 192.168.1.103 search.example.com {"app":"search","env":"production"} {"city":"Tokyo","country":"Japan"} {"Accept-Language":"en-US","Content-Type":"application/json"} {"hostname":"search-server-03"} https://www.example.com Mozilla/5.0 (Linux; Android 11; SM-G991B) {"input_type":"search","tags":["search","prod"]} {"level":"info","message":"search query","timestamp":"2025-07-31T16:30:10Z"} 2025-07-31T16:30:10
transaction_id=txn_987654&amount=199.99 {"currency":"USD","gateway":"stripe","type":"payment"} 202 Payment processing started 192.168.1.102 /api/v1/payments 192.168.1.102 pay.example.com {"app":"payment","env":"production","features":["3d_secure","refunds"]} {"city":"London","coordinates":{"lat":51.5074,"lon":-0.1278},"country":"GB"} {"Content-Type":"application/json","Idempotency-Key":"idemp_123","X-Request-ID":"req_987xyz"} {"region":"eu-west-2","server":"payment-gw-02"} https://checkout.example.com/payment Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 {"source":"web","tags":["finance","payment"]} {"context":{"amount":199.99,"currency":"USD","customer":"cust_456def","transaction_id":"txn_987654"},"level":"INFO","message":"Payment initiated","timestamp":"2025-07-31T14:20:18Z"} 2025-07-31T14:20:18
-- !union_all_push_down_top_n_shape --
------------PhysicalOlapScan[t1]
------------PhysicalOlapScan[t2]
----------PhysicalTopN[LOCAL_SORT]
----------PhysicalTopN[LOCAL_SORT]
--------PhysicalTopN[MERGE_SORT]
--------PhysicalTopN[MERGE_SORT]
------PhysicalUnion
----PhysicalTopN[LOCAL_SORT]
--PhysicalTopN[MERGE_SORT]
PhysicalResultSink

238
regression-test/suites/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.groovy
View File

@ -74,4 +74,242 @@ suite("push_down_top_n_through_union") {
qt_push_down_topn_union_complex_conditions """
explain shape plan select * from (select * from table1 t1 where t1.score > 10 and t1.name = 'Test' union all select * from table1 t2 where t2.id < 5 and t2.score < 20) sub order by id limit 10;
"""
sql "DROP TABLE IF EXISTS t1";
sql """
CREATE TABLE `t1` (
`logTimestamp` datetime NULL,
`args1` varchar(65533) NULL,
`args2` variant NULL,
`args3` int NULL,
`args4` text NULL,
`args5` varchar(65533) NULL,
`args6` varchar(65533) NULL,
`args7` varchar(200) NULL,
`args8` varchar(65533) NULL,
`args9` variant NULL,
`args10` variant NULL,
`args11` variant NULL,
`args12` variant NULL,
`args13` varchar(65533) NULL,
`args14` text NULL,
`args15` variant NULL,
`log` variant NULL
) ENGINE=OLAP
DISTRIBUTED BY RANDOM BUCKETS 10
PROPERTIES (
"replication_num" = "1"
);"""
sql """
INSERT INTO t1 (
`logTimestamp`, `args1`, `args2`, `args3`, `args4`,
`args5`, `args6`, `args7`, `args8`, `args9`,
`args10`, `args11`, `args12`, `args13`, `args14`,
`args15`, `log`
) VALUES
('2025-07-31 09:15:22', 'username=admin&password=*****', '{"action":"login","status":"success"}', 200, 'User authentication successful',
'192.168.1.100', '/api/v1/auth/login', '192.168.1.100', 'auth.example.com', '{"app":"authentication","env":"production"}',
'{"country":"CN","city":"Beijing","coordinates":{"lat":39.9042,"lon":116.4074}}', '{"Content-Type":"application/json","Authorization":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"}', '{"server":"auth-node-01","region":"east-1"}', 'https://portal.example.com/login', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
'{"source":"web","tags":["security","auth"]}', '{"timestamp":"2025-07-31T09:15:22Z","level":"INFO","message":"Successful login for user admin","context":{"ip":"192.168.1.100","user_agent":"Chrome/125.0.0.0","session_id":"sess_xyz123"}}'),
('2025-07-31 11:30:45', 'product_id=789&quantity=2&price=59.99', '{"event":"purchase","payment_method":"credit_card"}', 201, 'Order processed successfully',
'192.168.1.101', '/api/v1/orders', '192.168.1.101', 'shop.example.com', '{"app":"ecommerce","env":"production","version":"3.2.1"}',
'{"country":"US","city":"New York","coordinates":{"lat":40.7128,"lon":-74.0060}}', '{"Content-Type":"application/json","X-CSRF-Token":"csrf_abc123","Accept":"application/json"}', '{"server":"shop-node-03","region":"us-east-1"}', 'https://shop.example.com/product/789', 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Mobile/15E148 Safari/604.1',
'{"source":"mobile","tags":["transaction","order"]}', '{"timestamp":"2025-07-31T11:30:45Z","level":"INFO","message":"New order created","context":{"order_id":"ord_789xyz","amount":119.98,"currency":"USD","customer_id":"cust_123abc"}}'),
('2025-07-31 14:20:18', 'transaction_id=txn_987654&amount=199.99', '{"type":"payment","gateway":"stripe","currency":"USD"}', 202, 'Payment processing started',
'192.168.1.102', '/api/v1/payments', '192.168.1.102', 'pay.example.com', '{"app":"payment","env":"production","features":["3d_secure","refunds"]}',
'{"country":"GB","city":"London","coordinates":{"lat":51.5074,"lon":-0.1278}}', '{"Content-Type":"application/json","X-Request-ID":"req_987xyz","Idempotency-Key":"idemp_123"}', '{"server":"payment-gw-02","region":"eu-west-2"}', 'https://checkout.example.com/payment', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
'{"source":"web","tags":["finance","payment"]}', '{"timestamp":"2025-07-31T14:20:18Z","level":"INFO","message":"Payment initiated","context":{"transaction_id":"txn_987654","amount":199.99,"currency":"USD","customer":"cust_456def"}}'),
('2025-07-31 16:45:33', 'error_code=500&message=Database+connection+failed', '{"severity":"critical","component":"database"}', 500, 'Database connection error',
'192.168.1.103', '/api/v1/products', '192.168.1.103', 'api.example.com', '{"app":"api-gateway","env":"production","dependencies":["mysql","redis"]}',
'{"country":"JP","city":"Tokyo","coordinates":{"lat":35.6762,"lon":139.6503}}', '{"Content-Type":"application/json","X-Correlation-ID":"corr_456mno"}', '{"server":"api-node-05","region":"ap-northeast-1"}', 'https://app.example.com/products', 'Mozilla/5.0 (Linux; Android 13; SM-S901B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36',
'{"source":"mobile","tags":["error","database"]}', '{"timestamp":"2025-07-31T16:45:33Z","level":"ERROR","message":"Failed to connect to database","context":{"error":"Connection timeout","attempts":3,"database":"products_db","duration_ms":5000}}'),
('2025-07-31 19:05:07', 'query=smartphone&page=1&sort=price_asc', '{"search_type":"product","filters":["in_stock","free_shipping"]}', 200, 'Search results returned',
'192.168.1.104', '/api/v1/search', '192.168.1.104', 'search.example.com', '{"app":"search","env":"production","features":["autocomplete","faceted_search"]}',
'{"country":"DE","city":"Berlin","coordinates":{"lat":52.5200,"lon":13.4050}}', '{"Content-Type":"application/json","X-Forwarded-For":"192.168.1.104, 203.0.113.5"}', '{"server":"search-node-01","region":"eu-central-1"}', 'https://www.example.com/search', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/125.0',
'{"source":"web","tags":["search","query"]}', '{"timestamp":"2025-07-31T19:05:07Z","level":"INFO","message":"Search query processed","context":{"query":"smartphone","results_count":42,"response_time_ms":125,"filters_applied":2}}');
"""
sql """
DROP TABLE IF EXISTS t2;
"""
sql """
CREATE TABLE `t2` (
`logTimestamp` datetime NULL,
`args1` varchar(65533) NULL,
`args2` variant NULL,
`args3` int NULL,
`args4` text NULL,
`args5` varchar(65533) NULL,
`args6` varchar(65533) NULL,
`args7` varchar(200) NULL,
`args8` varchar(65533) NULL,
`args9` variant NULL,
`args10` variant NULL,
`args11` variant NULL,
`args12` variant NULL,
`args13` varchar(65533) NULL,
`args14` text NULL,
`args15` variant NULL,
`log` variant NULL
) ENGINE=OLAP
DUPLICATE KEY(`logTimestamp`)
DISTRIBUTED BY RANDOM BUCKETS 10
PROPERTIES (
"replication_num" = "1"
);
"""
sql """
INSERT INTO t2 (
`logTimestamp`, `args1`, `args2`, `args3`, `args4`,
`args5`, `args6`, `args7`, `args8`, `args9`,
`args10`, `args11`, `args12`, `args13`, `args14`,
`args15`, `log`
) VALUES
('2025-07-31 10:15:30', 'user_id=123&action=login', '{"type":"authentication","status":"success"}', 200, 'User login successful',
'192.168.1.100', '/api/v1/login', '192.168.1.100', 'api.example.com', '{"app":"web","env":"production"}',
'{"country":"China","city":"Beijing"}', '{"Content-Type":"application/json","Authorization":"Bearer abc123"}', '{"hostname":"web-server-01"}', 'https://referer.example.com', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)',
'{"input_type":"http","tags":["auth","prod"]}', '{"timestamp":"2025-07-31T10:15:30Z","level":"info","message":"login success"}'),
('2025-07-31 11:30:45', 'product_id=456&category=electronics', '{"type":"product_view","device":"mobile"}', 304, 'Product page viewed',
'192.168.1.101', '/products/456', '192.168.1.101', 'shop.example.com', '{"app":"ecommerce","env":"staging"}',
'{"country":"USA","city":"New York"}', '{"Content-Type":"text/html","Cache-Control":"max-age=3600"}', '{"hostname":"shop-server-02"}', 'https://search.example.com', 'Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X)',
'{"input_type":"http","tags":["shop","mobile"]}', '{"timestamp":"2025-07-31T11:30:45Z","level":"info","message":"product view"}'),
('2025-07-31 14:45:20', 'order_id=789&amount=99.99', '{"type":"payment","method":"credit_card"}', 201, 'Payment processed',
'192.168.1.102', '/api/v1/payments', '192.168.1.102', 'pay.example.com', '{"app":"payment","env":"production"}',
'{"country":"UK","city":"London"}', '{"Content-Type":"application/json","X-CSRF-Token":"xyz789"}', '{"hostname":"payment-gateway-01"}', 'https://checkout.example.com', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)',
'{"input_type":"api","tags":["payment","prod"]}', '{"timestamp":"2025-07-31T14:45:20Z","level":"info","message":"payment success"}'),
('2025-07-31 16:30:10', 'search_term=laptop&page=2', '{"type":"search","results_count":25}', 200, 'Search results returned',
'192.168.1.103', '/search', '192.168.1.103', 'search.example.com', '{"app":"search","env":"production"}',
'{"country":"Japan","city":"Tokyo"}', '{"Content-Type":"application/json","Accept-Language":"en-US"}', '{"hostname":"search-server-03"}', 'https://www.example.com', 'Mozilla/5.0 (Linux; Android 11; SM-G991B)',
'{"input_type":"search","tags":["search","prod"]}', '{"timestamp":"2025-07-31T16:30:10Z","level":"info","message":"search query"}'),
('2025-07-31 18:15:55', 'error_code=500&message=Internal+Server+Error', '{"type":"error","severity":"high"}', 500, 'Internal server error occurred',
'192.168.1.104', '/api/v1/users', '192.168.1.104', 'api.example.com', '{"app":"web","env":"production"}',
'{"country":"Germany","city":"Berlin"}', '{"Content-Type":"application/json","X-Request-ID":"req123"}', '{"hostname":"web-server-02"}', 'https://client.example.com', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)',
'{"input_type":"error","tags":["error","prod"]}', '{"timestamp":"2025-07-31T18:15:55Z","level":"error","message":"server error"}');
"""
sql "DROP VIEW IF EXISTS view_test"
sql """
CREATE VIEW `view_test`AS
SELECT t1.`logTimestamp`,
t1.`args1`,
t1.`args2`,
t1.`args3`,
t1.`args4`,
t1.`args5`,
t1.`args6`,
t1.`args7`,
t1.`args8`,
t1.`args9`,
t1.`args10`,
t1.`args11`,
t1.`args12`,
t1.`args13`,
t1.`args14`,
t1.`args15`,
t1.`log`
FROM t1
UNION all
SELECT t2.`logTimestamp`,
t2.`args1`,
t2.`args2`,
t2.`args3`,
t2.`args4`,
t2.`args5`,
t2.`args6`,
t2.`args7`,
t2.`args8`,
t2.`args9`,
t2.`args10`,
t2.`args11`,
t2.`args12`,
t2.`args13`,
t2.`args14`,
t2.`args15`,
t2.`log`
FROM t2;
"""
order_qt_union_all_push_down_top_n """
SELECT `args1`, `args2`, `args3`, `args4`, `args5`, `args6`, `args7`, `args8`, `args9`, `args10`, `args11`, `args12`, `args13`, `args14`, `args15`, `log`, logTimestamp
FROM view_test
ORDER BY logTimestamp desc
LIMIT 8;"""
order_qt_union_all_push_down_top_n_shape """
explain shape plan SELECT `args1`, `args2`, `args3`, `args4`, `args5`, `args6`, `args7`, `args8`, `args9`, `args10`, `args11`, `args12`, `args13`, `args14`, `args15`, `log`, logTimestamp
FROM view_test
ORDER BY logTimestamp desc
LIMIT 8;
"""
}