diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughUnion.java index c13c1143ef..b02d7a1e45 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughUnion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughUnion.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; @@ -63,11 +64,13 @@ public class PushDownTopNThroughUnion implements RewriteRuleFactory { .then(topN -> { LogicalUnion union = topN.child(); List newChildren = new ArrayList<>(); - for (Plan child : union.children()) { + for (int j = 0; j < union.children().size(); j++) { + Plan child = union.child(j); + List regularChildOutput = union.getRegularChildOutput(j); Map replaceMap = new HashMap<>(); - for (int i = 0; i < union.getOutputs().size(); ++i) { + for (int i = 0; i < regularChildOutput.size(); ++i) { NamedExpression output = union.getOutputs().get(i); - replaceMap.put(output, child.getOutput().get(i)); + replaceMap.put(output, regularChildOutput.get(i)); } List orderKeys = topN.getOrderKeys().stream() diff --git a/regression-test/data/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.out b/regression-test/data/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.out index be8053d834..6a32bbce85 100644 --- a/regression-test/data/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.out +++ b/regression-test/data/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.out @@ -129,3 +129,25 @@ PhysicalResultSink ------------filter((t2.id < 5) and (t2.score < 20)) --------------PhysicalOlapScan[table1] +-- !union_all_push_down_top_n -- +error_code=500&message=Database+connection+failed {"component":"database","severity":"critical"} 500 Database connection error 192.168.1.103 /api/v1/products 192.168.1.103 api.example.com {"app":"api-gateway","dependencies":["mysql","redis"],"env":"production"} {"city":"Tokyo","coordinates":{"lat":35.6762,"lon":139.6503},"country":"JP"} {"Content-Type":"application/json","X-Correlation-ID":"corr_456mno"} {"region":"ap-northeast-1","server":"api-node-05"} https://app.example.com/products Mozilla/5.0 (Linux; Android 13; SM-S901B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36 {"source":"mobile","tags":["error","database"]} {"context":{"attempts":3,"database":"products_db","duration_ms":5000,"error":"Connection timeout"},"level":"ERROR","message":"Failed to connect to database","timestamp":"2025-07-31T16:45:33Z"} 2025-07-31T16:45:33 +error_code=500&message=Internal+Server+Error {"severity":"high","type":"error"} 500 Internal server error occurred 192.168.1.104 /api/v1/users 192.168.1.104 api.example.com {"app":"web","env":"production"} {"city":"Berlin","country":"Germany"} {"Content-Type":"application/json","X-Request-ID":"req123"} {"hostname":"web-server-02"} https://client.example.com Mozilla/5.0 (Windows NT 10.0; Win64; x64) {"input_type":"error","tags":["error","prod"]} {"level":"error","message":"server error","timestamp":"2025-07-31T18:15:55Z"} 2025-07-31T18:15:55 +order_id=789&amount=99.99 {"method":"credit_card","type":"payment"} 201 Payment processed 192.168.1.102 /api/v1/payments 192.168.1.102 pay.example.com {"app":"payment","env":"production"} {"city":"London","country":"UK"} {"Content-Type":"application/json","X-CSRF-Token":"xyz789"} {"hostname":"payment-gateway-01"} https://checkout.example.com Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) {"input_type":"api","tags":["payment","prod"]} {"level":"info","message":"payment success","timestamp":"2025-07-31T14:45:20Z"} 2025-07-31T14:45:20 +product_id=456&category=electronics {"device":"mobile","type":"product_view"} 304 Product page viewed 192.168.1.101 /products/456 192.168.1.101 shop.example.com {"app":"ecommerce","env":"staging"} {"city":"New York","country":"USA"} {"Cache-Control":"max-age=3600","Content-Type":"text/html"} {"hostname":"shop-server-02"} https://search.example.com Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) {"input_type":"http","tags":["shop","mobile"]} {"level":"info","message":"product view","timestamp":"2025-07-31T11:30:45Z"} 2025-07-31T11:30:45 +product_id=789&quantity=2&price=59.99 {"event":"purchase","payment_method":"credit_card"} 201 Order processed successfully 192.168.1.101 /api/v1/orders 192.168.1.101 shop.example.com {"app":"ecommerce","env":"production","version":"3.2.1"} {"city":"New York","coordinates":{"lat":40.7128,"lon":-74.006},"country":"US"} {"Accept":"application/json","Content-Type":"application/json","X-CSRF-Token":"csrf_abc123"} {"region":"us-east-1","server":"shop-node-03"} https://shop.example.com/product/789 Mozilla/5.0 (iPhone; CPU iPhone OS 16_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Mobile/15E148 Safari/604.1 {"source":"mobile","tags":["transaction","order"]} {"context":{"amount":119.98,"currency":"USD","customer_id":"cust_123abc","order_id":"ord_789xyz"},"level":"INFO","message":"New order created","timestamp":"2025-07-31T11:30:45Z"} 2025-07-31T11:30:45 +query=smartphone&page=1&sort=price_asc {"filters":["in_stock","free_shipping"],"search_type":"product"} 200 Search results returned 192.168.1.104 /api/v1/search 192.168.1.104 search.example.com {"app":"search","env":"production","features":["autocomplete","faceted_search"]} {"city":"Berlin","coordinates":{"lat":52.52,"lon":13.405},"country":"DE"} {"Content-Type":"application/json","X-Forwarded-For":"192.168.1.104, 203.0.113.5"} {"region":"eu-central-1","server":"search-node-01"} https://www.example.com/search Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/125.0 {"source":"web","tags":["search","query"]} {"context":{"filters_applied":2,"query":"smartphone","response_time_ms":125,"results_count":42},"level":"INFO","message":"Search query processed","timestamp":"2025-07-31T19:05:07Z"} 2025-07-31T19:05:07 +search_term=laptop&page=2 {"results_count":25,"type":"search"} 200 Search results returned 192.168.1.103 /search 192.168.1.103 search.example.com {"app":"search","env":"production"} {"city":"Tokyo","country":"Japan"} {"Accept-Language":"en-US","Content-Type":"application/json"} {"hostname":"search-server-03"} https://www.example.com Mozilla/5.0 (Linux; Android 11; SM-G991B) {"input_type":"search","tags":["search","prod"]} {"level":"info","message":"search query","timestamp":"2025-07-31T16:30:10Z"} 2025-07-31T16:30:10 +transaction_id=txn_987654&amount=199.99 {"currency":"USD","gateway":"stripe","type":"payment"} 202 Payment processing started 192.168.1.102 /api/v1/payments 192.168.1.102 pay.example.com {"app":"payment","env":"production","features":["3d_secure","refunds"]} {"city":"London","coordinates":{"lat":51.5074,"lon":-0.1278},"country":"GB"} {"Content-Type":"application/json","Idempotency-Key":"idemp_123","X-Request-ID":"req_987xyz"} {"region":"eu-west-2","server":"payment-gw-02"} https://checkout.example.com/payment Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 {"source":"web","tags":["finance","payment"]} {"context":{"amount":199.99,"currency":"USD","customer":"cust_456def","transaction_id":"txn_987654"},"level":"INFO","message":"Payment initiated","timestamp":"2025-07-31T14:20:18Z"} 2025-07-31T14:20:18 + +-- !union_all_push_down_top_n_shape -- +------------PhysicalOlapScan[t1] +------------PhysicalOlapScan[t2] +----------PhysicalTopN[LOCAL_SORT] +----------PhysicalTopN[LOCAL_SORT] +--------PhysicalTopN[MERGE_SORT] +--------PhysicalTopN[MERGE_SORT] +------PhysicalUnion +----PhysicalTopN[LOCAL_SORT] +--PhysicalTopN[MERGE_SORT] +PhysicalResultSink + diff --git a/regression-test/suites/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.groovy b/regression-test/suites/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.groovy index fa0eacd5b3..b855543286 100644 --- a/regression-test/suites/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.groovy +++ b/regression-test/suites/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.groovy @@ -74,4 +74,242 @@ suite("push_down_top_n_through_union") { qt_push_down_topn_union_complex_conditions """ explain shape plan select * from (select * from table1 t1 where t1.score > 10 and t1.name = 'Test' union all select * from table1 t2 where t2.id < 5 and t2.score < 20) sub order by id limit 10; """ + + + sql "DROP TABLE IF EXISTS t1"; + + sql """ + CREATE TABLE `t1` ( + + `logTimestamp` datetime NULL, + + `args1` varchar(65533) NULL, + + `args2` variant NULL, + + `args3` int NULL, + + `args4` text NULL, + + `args5` varchar(65533) NULL, + + `args6` varchar(65533) NULL, + + `args7` varchar(200) NULL, + + `args8` varchar(65533) NULL, + + `args9` variant NULL, + + `args10` variant NULL, + + `args11` variant NULL, + + `args12` variant NULL, + + `args13` varchar(65533) NULL, + + `args14` text NULL, + + `args15` variant NULL, + + `log` variant NULL + ) ENGINE=OLAP + + DISTRIBUTED BY RANDOM BUCKETS 10 + + PROPERTIES ( + + "replication_num" = "1" + + );""" + + sql """ + INSERT INTO t1 ( + `logTimestamp`, `args1`, `args2`, `args3`, `args4`, + `args5`, `args6`, `args7`, `args8`, `args9`, + `args10`, `args11`, `args12`, `args13`, `args14`, + `args15`, `log` + ) VALUES + + ('2025-07-31 09:15:22', 'username=admin&password=*****', '{"action":"login","status":"success"}', 200, 'User authentication successful', + '192.168.1.100', '/api/v1/auth/login', '192.168.1.100', 'auth.example.com', '{"app":"authentication","env":"production"}', + '{"country":"CN","city":"Beijing","coordinates":{"lat":39.9042,"lon":116.4074}}', '{"Content-Type":"application/json","Authorization":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"}', '{"server":"auth-node-01","region":"east-1"}', 'https://portal.example.com/login', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36', + '{"source":"web","tags":["security","auth"]}', '{"timestamp":"2025-07-31T09:15:22Z","level":"INFO","message":"Successful login for user admin","context":{"ip":"192.168.1.100","user_agent":"Chrome/125.0.0.0","session_id":"sess_xyz123"}}'), + + + ('2025-07-31 11:30:45', 'product_id=789&quantity=2&price=59.99', '{"event":"purchase","payment_method":"credit_card"}', 201, 'Order processed successfully', + '192.168.1.101', '/api/v1/orders', '192.168.1.101', 'shop.example.com', '{"app":"ecommerce","env":"production","version":"3.2.1"}', + '{"country":"US","city":"New York","coordinates":{"lat":40.7128,"lon":-74.0060}}', '{"Content-Type":"application/json","X-CSRF-Token":"csrf_abc123","Accept":"application/json"}', '{"server":"shop-node-03","region":"us-east-1"}', 'https://shop.example.com/product/789', 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Mobile/15E148 Safari/604.1', + '{"source":"mobile","tags":["transaction","order"]}', '{"timestamp":"2025-07-31T11:30:45Z","level":"INFO","message":"New order created","context":{"order_id":"ord_789xyz","amount":119.98,"currency":"USD","customer_id":"cust_123abc"}}'), + + + ('2025-07-31 14:20:18', 'transaction_id=txn_987654&amount=199.99', '{"type":"payment","gateway":"stripe","currency":"USD"}', 202, 'Payment processing started', + '192.168.1.102', '/api/v1/payments', '192.168.1.102', 'pay.example.com', '{"app":"payment","env":"production","features":["3d_secure","refunds"]}', + '{"country":"GB","city":"London","coordinates":{"lat":51.5074,"lon":-0.1278}}', '{"Content-Type":"application/json","X-Request-ID":"req_987xyz","Idempotency-Key":"idemp_123"}', '{"server":"payment-gw-02","region":"eu-west-2"}', 'https://checkout.example.com/payment', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36', + '{"source":"web","tags":["finance","payment"]}', '{"timestamp":"2025-07-31T14:20:18Z","level":"INFO","message":"Payment initiated","context":{"transaction_id":"txn_987654","amount":199.99,"currency":"USD","customer":"cust_456def"}}'), + + + ('2025-07-31 16:45:33', 'error_code=500&message=Database+connection+failed', '{"severity":"critical","component":"database"}', 500, 'Database connection error', + '192.168.1.103', '/api/v1/products', '192.168.1.103', 'api.example.com', '{"app":"api-gateway","env":"production","dependencies":["mysql","redis"]}', + '{"country":"JP","city":"Tokyo","coordinates":{"lat":35.6762,"lon":139.6503}}', '{"Content-Type":"application/json","X-Correlation-ID":"corr_456mno"}', '{"server":"api-node-05","region":"ap-northeast-1"}', 'https://app.example.com/products', 'Mozilla/5.0 (Linux; Android 13; SM-S901B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36', + '{"source":"mobile","tags":["error","database"]}', '{"timestamp":"2025-07-31T16:45:33Z","level":"ERROR","message":"Failed to connect to database","context":{"error":"Connection timeout","attempts":3,"database":"products_db","duration_ms":5000}}'), + + + ('2025-07-31 19:05:07', 'query=smartphone&page=1&sort=price_asc', '{"search_type":"product","filters":["in_stock","free_shipping"]}', 200, 'Search results returned', + '192.168.1.104', '/api/v1/search', '192.168.1.104', 'search.example.com', '{"app":"search","env":"production","features":["autocomplete","faceted_search"]}', + '{"country":"DE","city":"Berlin","coordinates":{"lat":52.5200,"lon":13.4050}}', '{"Content-Type":"application/json","X-Forwarded-For":"192.168.1.104, 203.0.113.5"}', '{"server":"search-node-01","region":"eu-central-1"}', 'https://www.example.com/search', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/125.0', + '{"source":"web","tags":["search","query"]}', '{"timestamp":"2025-07-31T19:05:07Z","level":"INFO","message":"Search query processed","context":{"query":"smartphone","results_count":42,"response_time_ms":125,"filters_applied":2}}'); + """ + + sql """ + DROP TABLE IF EXISTS t2; + """ + + sql """ + CREATE TABLE `t2` ( + + `logTimestamp` datetime NULL, + + `args1` varchar(65533) NULL, + + `args2` variant NULL, + + `args3` int NULL, + + `args4` text NULL, + + `args5` varchar(65533) NULL, + + `args6` varchar(65533) NULL, + + `args7` varchar(200) NULL, + + `args8` varchar(65533) NULL, + + `args9` variant NULL, + + `args10` variant NULL, + + `args11` variant NULL, + + `args12` variant NULL, + + `args13` varchar(65533) NULL, + + `args14` text NULL, + + `args15` variant NULL, + + `log` variant NULL + + ) ENGINE=OLAP + + DUPLICATE KEY(`logTimestamp`) + + DISTRIBUTED BY RANDOM BUCKETS 10 + + PROPERTIES ( + + "replication_num" = "1" + ); + """ + + sql """ + INSERT INTO t2 ( + `logTimestamp`, `args1`, `args2`, `args3`, `args4`, + `args5`, `args6`, `args7`, `args8`, `args9`, + `args10`, `args11`, `args12`, `args13`, `args14`, + `args15`, `log` + ) VALUES + + ('2025-07-31 10:15:30', 'user_id=123&action=login', '{"type":"authentication","status":"success"}', 200, 'User login successful', + '192.168.1.100', '/api/v1/login', '192.168.1.100', 'api.example.com', '{"app":"web","env":"production"}', + '{"country":"China","city":"Beijing"}', '{"Content-Type":"application/json","Authorization":"Bearer abc123"}', '{"hostname":"web-server-01"}', 'https://referer.example.com', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)', + '{"input_type":"http","tags":["auth","prod"]}', '{"timestamp":"2025-07-31T10:15:30Z","level":"info","message":"login success"}'), + + + ('2025-07-31 11:30:45', 'product_id=456&category=electronics', '{"type":"product_view","device":"mobile"}', 304, 'Product page viewed', + '192.168.1.101', '/products/456', '192.168.1.101', 'shop.example.com', '{"app":"ecommerce","env":"staging"}', + '{"country":"USA","city":"New York"}', '{"Content-Type":"text/html","Cache-Control":"max-age=3600"}', '{"hostname":"shop-server-02"}', 'https://search.example.com', 'Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X)', + '{"input_type":"http","tags":["shop","mobile"]}', '{"timestamp":"2025-07-31T11:30:45Z","level":"info","message":"product view"}'), + + + ('2025-07-31 14:45:20', 'order_id=789&amount=99.99', '{"type":"payment","method":"credit_card"}', 201, 'Payment processed', + '192.168.1.102', '/api/v1/payments', '192.168.1.102', 'pay.example.com', '{"app":"payment","env":"production"}', + '{"country":"UK","city":"London"}', '{"Content-Type":"application/json","X-CSRF-Token":"xyz789"}', '{"hostname":"payment-gateway-01"}', 'https://checkout.example.com', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)', + '{"input_type":"api","tags":["payment","prod"]}', '{"timestamp":"2025-07-31T14:45:20Z","level":"info","message":"payment success"}'), + + + ('2025-07-31 16:30:10', 'search_term=laptop&page=2', '{"type":"search","results_count":25}', 200, 'Search results returned', + '192.168.1.103', '/search', '192.168.1.103', 'search.example.com', '{"app":"search","env":"production"}', + '{"country":"Japan","city":"Tokyo"}', '{"Content-Type":"application/json","Accept-Language":"en-US"}', '{"hostname":"search-server-03"}', 'https://www.example.com', 'Mozilla/5.0 (Linux; Android 11; SM-G991B)', + '{"input_type":"search","tags":["search","prod"]}', '{"timestamp":"2025-07-31T16:30:10Z","level":"info","message":"search query"}'), + + + ('2025-07-31 18:15:55', 'error_code=500&message=Internal+Server+Error', '{"type":"error","severity":"high"}', 500, 'Internal server error occurred', + '192.168.1.104', '/api/v1/users', '192.168.1.104', 'api.example.com', '{"app":"web","env":"production"}', + '{"country":"Germany","city":"Berlin"}', '{"Content-Type":"application/json","X-Request-ID":"req123"}', '{"hostname":"web-server-02"}', 'https://client.example.com', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)', + '{"input_type":"error","tags":["error","prod"]}', '{"timestamp":"2025-07-31T18:15:55Z","level":"error","message":"server error"}'); + """ + + + sql "DROP VIEW IF EXISTS view_test" + + + sql """ + CREATE VIEW `view_test`AS + SELECT t1.`logTimestamp`, + t1.`args1`, + t1.`args2`, + t1.`args3`, + t1.`args4`, + t1.`args5`, + t1.`args6`, + t1.`args7`, + t1.`args8`, + t1.`args9`, + t1.`args10`, + t1.`args11`, + t1.`args12`, + t1.`args13`, + t1.`args14`, + t1.`args15`, + t1.`log` + FROM t1 + UNION all + SELECT t2.`logTimestamp`, + t2.`args1`, + t2.`args2`, + t2.`args3`, + t2.`args4`, + t2.`args5`, + t2.`args6`, + t2.`args7`, + t2.`args8`, + t2.`args9`, + t2.`args10`, + t2.`args11`, + t2.`args12`, + t2.`args13`, + t2.`args14`, + t2.`args15`, + t2.`log` + FROM t2; + """ + + + order_qt_union_all_push_down_top_n """ + SELECT `args1`, `args2`, `args3`, `args4`, `args5`, `args6`, `args7`, `args8`, `args9`, `args10`, `args11`, `args12`, `args13`, `args14`, `args15`, `log`, logTimestamp + FROM view_test + ORDER BY logTimestamp desc + LIMIT 8;""" + + + order_qt_union_all_push_down_top_n_shape """ + explain shape plan SELECT `args1`, `args2`, `args3`, `args4`, `args5`, `args6`, `args7`, `args8`, `args9`, `args10`, `args11`, `args12`, `args13`, `args14`, `args15`, `log`, logTimestamp + FROM view_test + ORDER BY logTimestamp desc + LIMIT 8; + """ } \ No newline at end of file