database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[pipelineX](scan) ignore storage data distribution by default (#29192)

Browse Source
This commit is contained in:
Gabriel
2023-12-28 14:54:09 +08:00
committed by GitHub
parent fe93a8f1d0
commit 29a7c0d677
68 changed files with 6630 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
View File

@ -865,10 +865,6 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
// Set colocate info in agg node. This is a hint for local shuffling to decide which type of
// local exchanger will be used.
aggregationNode.setColocate(true);
if (aggregate.getAggMode().isFinalPhase) {
inputPlanFragment.setHasColocateFinalizeAggNode(true);
}
}
setPlanRoot(inputPlanFragment, aggregationNode, aggregate);
if (aggregate.getStats() != null) {

10
fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java
View File

@ -149,8 +149,6 @@ public class PlanFragment extends TreeNode<PlanFragment> {
// has colocate plan node
private boolean hasColocatePlanNode = false;
private boolean hasColocateFinalizeAggNode = false;
private boolean hasNullAwareLeftAntiJoin = false;
private TResultSinkType resultSinkType = TResultSinkType.MYSQL_PROTOCAL;
@ -475,14 +473,6 @@ public class PlanFragment extends TreeNode<PlanFragment> {
this.bucketNum = bucketNum;
}
public boolean isHasColocateFinalizeAggNode() {
return hasColocateFinalizeAggNode;
}
public void setHasColocateFinalizeAggNode(boolean hasColocateFinalizeAggNode) {
this.hasColocateFinalizeAggNode = hasColocateFinalizeAggNode;
}
public boolean isHasNullAwareLeftAntiJoin() {
return hasNullAwareLeftAntiJoin;
}

1
fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
View File

@ -724,7 +724,6 @@ public abstract class ScanNode extends PlanNode {
return !isKeySearch() && context != null
&& context.getSessionVariable().isIgnoreStorageDataDistribution()
&& context.getSessionVariable().getEnablePipelineXEngine()
&& !fragment.isHasColocateFinalizeAggNode()
&& !fragment.isHasNullAwareLeftAntiJoin();
}

38
fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
View File

@ -2021,16 +2021,18 @@ public class Coordinator implements CoordInterface {
return scanNode.getId().asInt() == planNodeId;
}).findFirst();
// disable shared scan optimization if one of conditions below is met:
// 1. Use non-pipeline or pipelineX engine
// 2. This fragment has a colocated scan node
// 3. This fragment has a FileScanNode
// 4. Disable shared scan optimization by session variable
/**
* Ignore storage data distribution iff:
* 1. Current fragment is not forced to use data distribution.
* 2. `parallelExecInstanceNum` is larger than scan ranges.
* 3. Use Nereids planner.
*/
boolean sharedScan = true;
int expectedInstanceNum = Math.min(parallelExecInstanceNum,
leftMostNode.getNumInstances());
if (node.isPresent() && (!node.get().shouldDisableSharedScan(context)
|| (node.get().ignoreStorageDataDistribution(context) && useNereids))) {
int expectedInstanceNum = Math.min(parallelExecInstanceNum,
leftMostNode.getNumInstances());
|| (node.get().ignoreStorageDataDistribution(context)
&& expectedInstanceNum > perNodeScanRanges.size() && useNereids))) {
expectedInstanceNum = Math.max(expectedInstanceNum, 1);
// if have limit and conjunts, only need 1 instance to save cpu and
// mem resource
@ -2040,7 +2042,7 @@ public class Coordinator implements CoordInterface {
perInstanceScanRanges = Collections.nCopies(expectedInstanceNum, perNodeScanRanges);
} else {
int expectedInstanceNum = 1;
expectedInstanceNum = 1;
if (parallelExecInstanceNum > 1) {
//the scan instance num should not larger than the tablets num
expectedInstanceNum = Math.min(perNodeScanRanges.size(), parallelExecInstanceNum);
@ -2859,10 +2861,6 @@ public class Coordinator implements CoordInterface {
BucketSeqToScanRange bucketSeqToScanRange = fragmentIdBucketSeqToScanRangeMap.get(fragmentId);
Set<Integer> scanNodeIds = fragmentIdToScanNodeIds.get(fragmentId);
boolean ignoreStorageDataDistribution = scanNodes.stream().filter(scanNode -> {
return scanNodeIds.contains(scanNode.getId().asInt());
}).allMatch(node -> node.ignoreStorageDataDistribution(context)) && useNereids;
// 1. count each node in one fragment should scan how many tablet, gather them in one list
Map<TNetworkAddress, List<Pair<Integer, Map<Integer, List<TScanRangeParams>>>>> addressToScanRanges
= Maps.newHashMap();
@ -2885,6 +2883,20 @@ public class Coordinator implements CoordInterface {
}
addressToScanRanges.get(address).add(filteredScanRanges);
}
/**
* Ignore storage data distribution iff:
* 1. Current fragment is not forced to use data distribution.
* 2. `parallelExecInstanceNum` is larger than scan ranges.
* 3. Use Nereids planner.
*/
boolean ignoreStorageDataDistribution = scanNodes.stream().filter(scanNode -> {
return scanNodeIds.contains(scanNode.getId().asInt());
}).allMatch(node -> node.ignoreStorageDataDistribution(context))
&& addressToScanRanges.entrySet().stream().allMatch(addressScanRange -> {
return addressScanRange.getValue().size() < parallelExecInstanceNum;
}) && useNereids;
FragmentScanRangeAssignment assignment = params.scanRangeAssignment;
for (Map.Entry<TNetworkAddress, List<Pair<Integer, Map<Integer, List<TScanRangeParams>>>>> addressScanRange
: addressToScanRanges.entrySet()) {

2
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View File

@ -808,7 +808,7 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = IGNORE_STORAGE_DATA_DISTRIBUTION, fuzzy = false,
varType = VariableAnnotation.EXPERIMENTAL, needForward = true)
private boolean ignoreStorageDataDistribution = false;
private boolean ignoreStorageDataDistribution = true;
@VariableMgr.VarAttr(
name = ENABLE_LOCAL_SHUFFLE, fuzzy = false, varType = VariableAnnotation.EXPERIMENTAL,

7
regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q01.out Normal file
View File

@ -0,0 +1,7 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !q01 --
A F 3774200.00 5320753880.69 5054096266.6828 5256751331.449234 25.5375 36002.1238 0.0501 147790
N F 95257.00 133737795.84 127132372.6512 132286291.229445 25.3006 35521.3269 0.0493 3765
N O 7459297.00 10512270008.90 9986238338.3847 10385578376.585467 25.5455 36000.9246 0.0500 292000
R F 3785523.00 5337950526.47 5071818532.9420 5274405503.049367 25.5259 35994.0292 0.0499 148301

47
regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q02.out Normal file
View File

@ -0,0 +1,47 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !q02 --
9828.21 Supplier#000000647 UNITED KINGDOM 13120 Manufacturer#5 x5U7MBZmwfG9 33-258-202-4782 s the slyly even ideas poach fluffily
9508.37 Supplier#000000070 FRANCE 3563 Manufacturer#1 INWNH2w,OOWgNDq0BRCcBwOMQc6PdFDc4 16-821-608-1166 ests sleep quickly express ideas. ironic ideas haggle about the final T
9508.37 Supplier#000000070 FRANCE 17268 Manufacturer#4 INWNH2w,OOWgNDq0BRCcBwOMQc6PdFDc4 16-821-608-1166 ests sleep quickly express ideas. ironic ideas haggle about the final T
9453.01 Supplier#000000802 ROMANIA 10021 Manufacturer#5 ,6HYXb4uaHITmtMBj4Ak57Pd 29-342-882-6463 gular frets. permanently special multipliers believe blithely alongs
9453.01 Supplier#000000802 ROMANIA 13275 Manufacturer#4 ,6HYXb4uaHITmtMBj4Ak57Pd 29-342-882-6463 gular frets. permanently special multipliers believe blithely alongs
9192.10 Supplier#000000115 UNITED KINGDOM 13325 Manufacturer#1 nJ 2t0f7Ve,wL1,6WzGBJLNBUCKlsV 33-597-248-1220 es across the carefully express accounts boost caref
9032.15 Supplier#000000959 GERMANY 4958 Manufacturer#4 8grA EHBnwOZhO 17-108-642-3106 nding dependencies nag furiou
8702.02 Supplier#000000333 RUSSIA 11810 Manufacturer#3 MaVf XgwPdkiX4nfJGOis8Uu2zKiIZH 32-508-202-6136 oss the deposits cajole carefully even pinto beans. regular foxes detect alo
8615.50 Supplier#000000812 FRANCE 10551 Manufacturer#2 8qh4tezyScl5bidLAysvutB,,ZI2dn6xP 16-585-724-6633 y quickly regular deposits? quickly pending packages after the caref
8615.50 Supplier#000000812 FRANCE 13811 Manufacturer#4 8qh4tezyScl5bidLAysvutB,,ZI2dn6xP 16-585-724-6633 y quickly regular deposits? quickly pending packages after the caref
8488.53 Supplier#000000367 RUSSIA 6854 Manufacturer#4 E Sv9brQVf43Mzz 32-458-198-9557 ages. carefully final excuses nag finally. carefully ironic deposits abov
8430.52 Supplier#000000646 FRANCE 11384 Manufacturer#3 IUzsmT,2oBgjhWP2TlXTL6IkJH,4h,1SJRt 16-601-220-5489 ites among the always final ideas kindle according to the theodolites. notornis in
8271.39 Supplier#000000146 RUSSIA 4637 Manufacturer#5 rBDNgCr04x0sfdzD5,gFOutCiG2 32-792-619-3155 s cajole quickly special requests. quickly enticing theodolites h
8096.98 Supplier#000000574 RUSSIA 323 Manufacturer#4 2O8 sy9g2mlBOuEjzj0pA2pevk, 32-866-246-8752 ully after the regular requests. slyly final dependencies wake slyly along the busy deposit
7392.78 Supplier#000000170 UNITED KINGDOM 7655 Manufacturer#2 RtsXQ,SunkA XHy9 33-803-340-5398 ake carefully across the quickly
7205.20 Supplier#000000477 GERMANY 10956 Manufacturer#5 VtaNKN5Mqui5yh7j2ldd5waf 17-180-144-7991 excuses wake express deposits. furiously careful asymptotes according to the carefull
6820.35 Supplier#000000007 UNITED KINGDOM 13217 Manufacturer#5 s,4TicNGB4uO6PaSqNBUq 33-990-965-2201 s unwind silently furiously regular courts. final requests are deposits. requests wake quietly blit
6721.70 Supplier#000000954 FRANCE 4191 Manufacturer#3 P3O5p UFz1QsLmZX 16-537-341-8517 ect blithely blithely final acco
6329.90 Supplier#000000996 GERMANY 10735 Manufacturer#2 Wx4dQwOAwWjfSCGupfrM 17-447-811-3282 ironic forges cajole blithely agai
6173.87 Supplier#000000408 RUSSIA 18139 Manufacturer#1 qcor1u,vJXAokjnL5,dilyYNmh 32-858-724-2950 blithely pending packages cajole furiously slyly pending notornis. slyly final
5364.99 Supplier#000000785 RUSSIA 13784 Manufacturer#4 W VkHBpQyD3qjQjWGpWicOpmILFehmEdWy67kUGY 32-297-653-2203 packages boost carefully. express ideas along
5069.27 Supplier#000000328 GERMANY 16327 Manufacturer#1 SMm24d WG62 17-231-513-5721 he unusual ideas. slyly final packages a
4941.88 Supplier#000000321 ROMANIA 7320 Manufacturer#5 pLngFl5yeMcHyov 29-573-279-1406 y final requests impress s
4672.25 Supplier#000000239 RUSSIA 12238 Manufacturer#1 XO101kgHrJagK2FL1U6QCaTE ncCsMbeuTgK6o8 32-396-654-6826 arls wake furiously deposits. even, regular depen
4586.49 Supplier#000000680 RUSSIA 5679 Manufacturer#3 UhvDfdEfJh,Qbe7VZb8uSGO2TU 0jEa6nXZXE 32-522-382-1620 the regularly regular dependencies. carefully bold excuses under th
4518.31 Supplier#000000149 FRANCE 18344 Manufacturer#5 pVyWsjOidpHKp4NfKU4yLeym 16-660-553-2456 ts detect along the foxes. final Tiresias are. idly pending deposits haggle; even, blithe pin
4315.15 Supplier#000000509 FRANCE 18972 Manufacturer#2 SF7dR8V5pK 16-298-154-3365 ronic orbits are furiously across the requests. quickly express ideas across the special, bold
3526.53 Supplier#000000553 FRANCE 8036 Manufacturer#4 a,liVofXbCJ 16-599-552-3755 lar dinos nag slyly brave
3526.53 Supplier#000000553 FRANCE 17018 Manufacturer#3 a,liVofXbCJ 16-599-552-3755 lar dinos nag slyly brave
3294.68 Supplier#000000350 GERMANY 4841 Manufacturer#4 KIFxV73eovmwhh 17-113-181-4017 e slyly special foxes. furiously unusual deposits detect carefully carefully ruthless foxes. quick
2972.26 Supplier#000000016 RUSSIA 1015 Manufacturer#4 YjP5C55zHDXL7LalK27zfQnwejdpin4AMpvh 32-822-502-4215 ously express ideas haggle quickly dugouts? fu
2963.09 Supplier#000000840 ROMANIA 3080 Manufacturer#2 iYzUIypKhC0Y 29-781-337-5584 eep blithely regular dependencies. blithely regular platelets sublate alongside o
2221.25 Supplier#000000771 ROMANIA 13981 Manufacturer#2 lwZ I15rq9kmZXUNhl 29-986-304-9006 nal foxes eat slyly about the fluffily permanent id
1381.97 Supplier#000000104 FRANCE 18103 Manufacturer#3 Dcl4yGrzqv3OPeRO49bKh78XmQEDR7PBXIs0m 16-434-972-6922 gular ideas. bravely bold deposits haggle through the carefully final deposits. slyly unusual idea
906.07 Supplier#000000138 ROMANIA 8363 Manufacturer#4 utbplAm g7RmxVfYoNdhcrQGWuzRqPe0qHSwbKw 29-533-434-6776 ickly unusual requests cajole. accounts above the furiously special excuses
765.69 Supplier#000000799 RUSSIA 11276 Manufacturer#2 jwFN7ZB3T9sMF 32-579-339-1495 nusual requests. furiously unusual epitaphs integrate. slyly
727.89 Supplier#000000470 ROMANIA 6213 Manufacturer#3 XckbzsAgBLbUkdfjgJEPjmUMTM8ebSMEvI 29-165-289-1523 gular excuses. furiously regular excuses sleep slyly caref
683.07 Supplier#000000651 RUSSIA 4888 Manufacturer#4 oWekiBV6s,1g 32-181-426-4490 ly regular requests cajole abou
167.56 Supplier#000000290 FRANCE 2037 Manufacturer#1 6Bk06GVtwZaKqg01 16-675-286-5102 the theodolites. ironic, ironic deposits above
91.39 Supplier#000000949 UNITED KINGDOM 9430 Manufacturer#2 a,UE,6nRVl2fCphkOoetR1ajIzAEJ1Aa1G1HV 33-332-697-2768 pinto beans. carefully express requests hagg
-314.06 Supplier#000000510 ROMANIA 17242 Manufacturer#4 VmXQl ,vY8JiEseo8Mv4zscvNCfsY 29-207-852-3454 bold deposits. carefully even d
-820.89 Supplier#000000409 GERMANY 2156 Manufacturer#5 LyXUYFz7aXrvy65kKAbTatGzGS,NDBcdtD 17-719-517-9836 y final, slow theodolites. furiously regular req
-845.44 Supplier#000000704 ROMANIA 9926 Manufacturer#5 hQvlBqbqqnA5Dgo1BffRBX78tkkRu 29-300-896-5991 ctions. carefully sly requ
-942.73 Supplier#000000563 GERMANY 5797 Manufacturer#1 Rc7U1cRUhYs03JD 17-108-537-2691 slyly furiously final decoys; silent, special realms poach f

13
regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q03.out Normal file
View File

@ -0,0 +1,13 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !q03 --
223140 355369.0698 1995-03-14 0
584291 354494.7318 1995-02-21 0
405063 353125.4577 1995-03-03 0
573861 351238.2770 1995-03-09 0
554757 349181.7426 1995-03-14 0
506021 321075.5810 1995-03-10 0
121604 318576.4154 1995-03-07 0
108514 314967.0754 1995-02-20 0
462502 312604.5420 1995-03-08 0
178727 309728.9306 1995-02-25 0

8
regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q04.out Normal file
View File

@ -0,0 +1,8 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !q04 --
1-URGENT 999
2-HIGH 997
3-MEDIUM 1031
4-NOT SPECIFIED 989
5-LOW 1077

8
regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q05.out Normal file
View File

@ -0,0 +1,8 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !q05 --
CHINA 7822103.0000
INDIA 6376121.5085
JAPAN 6000077.2184
INDONESIA 5580475.4027
VIETNAM 4497840.5466

4
regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q06.out Normal file
View File

@ -0,0 +1,4 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !q06 --
11803420.2534

7
regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q07.out Normal file
View File

@ -0,0 +1,7 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !q07 --
FRANCE GERMANY 1995 4637235.1501
FRANCE GERMANY 1996 5224779.5736
GERMANY FRANCE 1995 6232818.7037
GERMANY FRANCE 1996 5557312.1121

5
regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q08.out Normal file
View File

@ -0,0 +1,5 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !q08 --
1995 0.02864874
1996 0.01825027

178
regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q09.out Normal file
View File

@ -0,0 +1,178 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !q09 --
ALGERIA 1998 2321785.3682
ALGERIA 1997 3685016.8589
ALGERIA 1996 4276597.4253
ALGERIA 1995 4418370.4154
ALGERIA 1994 3864849.9521
ALGERIA 1993 3541051.3865
ALGERIA 1992 4310013.3482
ARGENTINA 1998 2685983.8005
ARGENTINA 1997 4242147.8124
ARGENTINA 1996 3907867.0103
ARGENTINA 1995 4605921.5011
ARGENTINA 1994 3542096.1564
ARGENTINA 1993 3949965.9388
ARGENTINA 1992 4521180.4695
BRAZIL 1998 2778730.3931
BRAZIL 1997 4642037.4687
BRAZIL 1996 4530304.6034
BRAZIL 1995 4502344.8657
BRAZIL 1994 4875806.5015
BRAZIL 1993 4687478.6531
BRAZIL 1992 5035200.0464
CANADA 1998 2194509.0465
CANADA 1997 3482197.9521
CANADA 1996 3712231.2814
CANADA 1995 4014814.8476
CANADA 1994 4145304.4855
CANADA 1993 3787069.6045
CANADA 1992 4168009.4201
CHINA 1998 3398578.0001
CHINA 1997 6358959.3338
CHINA 1996 6435158.3229
CHINA 1995 6174776.2113
CHINA 1994 6385751.0812
CHINA 1993 5765034.1194
CHINA 1992 6324034.2379
EGYPT 1998 2333148.3334
EGYPT 1997 3661244.2731
EGYPT 1996 3765371.2368
EGYPT 1995 4094744.2925
EGYPT 1994 3566508.0818
EGYPT 1993 3725283.7747
EGYPT 1992 3373762.3335
ETHIOPIA 1998 1953927.2682
ETHIOPIA 1997 3285786.3266
ETHIOPIA 1996 3525028.7952
ETHIOPIA 1995 3781674.8911
ETHIOPIA 1994 3037409.4360
ETHIOPIA 1993 3008978.2677
ETHIOPIA 1992 2721203.2355
FRANCE 1998 2604373.8805
FRANCE 1997 3982872.0488
FRANCE 1996 3622479.2413
FRANCE 1995 4479939.7020
FRANCE 1994 3531013.1981
FRANCE 1993 4086437.3102
FRANCE 1992 3637792.1333
GERMANY 1998 3291023.2965
GERMANY 1997 5139337.3443
GERMANY 1996 4799810.4577
GERMANY 1995 5405785.7978
GERMANY 1994 4555556.4592
GERMANY 1993 4428195.1019
GERMANY 1992 4656148.4204
INDIA 1998 2591288.1874
INDIA 1997 5159562.7033
INDIA 1996 5307258.3049
INDIA 1995 5148208.7902
INDIA 1994 5164001.9582
INDIA 1993 4321398.4388
INDIA 1992 5297703.6935
INDONESIA 1998 3094900.1597
INDONESIA 1997 5719773.0358
INDONESIA 1996 6037238.5993
INDONESIA 1995 5266783.4899
INDONESIA 1994 5470762.8729
INDONESIA 1993 6189826.6613
INDONESIA 1992 4414623.1549
IRAN 1998 3214864.1209
IRAN 1997 3688049.0691
IRAN 1996 3621649.2247
IRAN 1995 4420783.4205
IRAN 1994 4373984.6523
IRAN 1993 3731301.7814
IRAN 1992 4417133.3662
IRAQ 1998 2338859.4099
IRAQ 1997 3622681.5643
IRAQ 1996 4762291.8722
IRAQ 1995 4558092.7359
IRAQ 1994 4951604.1699
IRAQ 1993 3830077.9911
IRAQ 1992 3938636.4874
JAPAN 1998 1849535.0802
JAPAN 1997 4068688.8537
JAPAN 1996 4044774.7597
JAPAN 1995 4793005.8027
JAPAN 1994 4114717.0568
JAPAN 1993 3614468.7485
JAPAN 1992 4266694.4700
JORDAN 1998 1811488.0719
JORDAN 1997 2951297.8678
JORDAN 1996 3302528.3067
JORDAN 1995 3221813.9990
JORDAN 1994 2417892.0921
JORDAN 1993 3107641.7661
JORDAN 1992 3316379.0585
KENYA 1998 2579075.4190
KENYA 1997 2929194.2317
KENYA 1996 3569129.5619
KENYA 1995 3542889.1087
KENYA 1994 3983095.3994
KENYA 1993 3713988.9708
KENYA 1992 3304641.8340
MOROCCO 1998 1815334.8180
MOROCCO 1997 3693214.8447
MOROCCO 1996 4116175.9230
MOROCCO 1995 3515127.1402
MOROCCO 1994 4003072.1120
MOROCCO 1993 3599199.6679
MOROCCO 1992 3958335.4224
MOZAMBIQUE 1998 1620428.7346
MOZAMBIQUE 1997 2802166.6473
MOZAMBIQUE 1996 2409955.1755
MOZAMBIQUE 1995 2771602.6274
MOZAMBIQUE 1994 2548226.2158
MOZAMBIQUE 1993 2843748.9053
MOZAMBIQUE 1992 2556501.0943
PERU 1998 2036430.3602
PERU 1997 4064142.4091
PERU 1996 4068678.5671
PERU 1995 4657694.8412
PERU 1994 4731959.4655
PERU 1993 4144006.6610
PERU 1992 3754635.0078
ROMANIA 1998 1992773.6811
ROMANIA 1997 2854639.8680
ROMANIA 1996 3139337.3029
ROMANIA 1995 3222153.3776
ROMANIA 1994 3222844.3190
ROMANIA 1993 3488994.0288
ROMANIA 1992 3029274.4420
RUSSIA 1998 2339865.6635
RUSSIA 1997 4153619.5424
RUSSIA 1996 3772067.4041
RUSSIA 1995 4704988.8607
RUSSIA 1994 4479082.8694
RUSSIA 1993 4767719.9791
RUSSIA 1992 4533465.5590
SAUDI ARABIA 1998 3386948.9564
SAUDI ARABIA 1997 5425980.3373
SAUDI ARABIA 1996 5227607.1677
SAUDI ARABIA 1995 4506731.6411
SAUDI ARABIA 1994 4698658.7425
SAUDI ARABIA 1993 5493626.5285
SAUDI ARABIA 1992 4573560.0150
UNITED KINGDOM 1998 2252021.5137
UNITED KINGDOM 1997 4343926.8026
UNITED KINGDOM 1996 4189476.3065
UNITED KINGDOM 1995 4469569.8829
UNITED KINGDOM 1994 4410094.6264
UNITED KINGDOM 1993 4054677.1050
UNITED KINGDOM 1992 3978688.8831
UNITED STATES 1998 2238771.5581
UNITED STATES 1997 4135581.5734
UNITED STATES 1996 3624013.2660
UNITED STATES 1995 3892244.5172
UNITED STATES 1994 3289224.1138
UNITED STATES 1993 3626170.2028
UNITED STATES 1992 3993973.4997
VIETNAM 1998 1924313.4862
VIETNAM 1997 3436195.3709
VIETNAM 1996 4017288.8927
VIETNAM 1995 3644054.1372
VIETNAM 1994 4141277.6665
VIETNAM 1993 2556114.1693
VIETNAM 1992 4090524.4905

23
regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q10.out Normal file
View File

@ -0,0 +1,23 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !q10 --
8242 Customer#000008242 622786.7297 6322.09 ETHIOPIA P2n4nJhy,UqSo2s43YfSvYJDZ6lk 15-792-676-1184 slyly regular packages haggle carefully ironic ideas. courts are furiously. furiously unusual theodolites cajole. i
7714 Customer#000007714 557400.3053 9799.98 IRAN SnnIGB,SkmnWpX3 20-922-418-6024 arhorses according to the blithely express re
11032 Customer#000011032 512500.9641 8496.93 UNITED KINGDOM WIKHC7K3Cn7156iNOyfVG3cZ7YqkgsR,Ly 33-102-772-3533 posits-- furiously ironic accounts are again
2455 Customer#000002455 481592.4053 2070.99 GERMANY RVn1ZSRtLqPlJLIZxvpmsbgC02 17-946-225-9977 al asymptotes. finally ironic accounts cajole furiously. permanently unusual theodolites aro
12106 Customer#000012106 479414.2133 5342.11 UNITED STATES wth3twOmu6vy 34-905-346-4472 ly after the blithely regular foxes. accounts haggle carefully alongside of the blithely even ideas.
8530 Customer#000008530 457855.9467 9734.95 MOROCCO GMQyte94oDM7eD7exnkj 4hH9yq3 25-736-932-5850 slyly asymptotes. quickly final deposits in
13984 Customer#000013984 446316.5104 3482.28 IRAN qZXwuapCHvxbX 20-981-264-2952 y unusual courts could wake furiously
1966 Customer#000001966 444059.0382 1937.72 ALGERIA jPv1 UHra5JLALR5Isci5u0636RoAu7t vH 10-973-269-8886 the blithely even accounts. final deposits cajole around the blithely final packages.
11026 Customer#000011026 417913.4142 7738.76 ALGERIA XorIktoJOAEJkpNNMx 10-184-163-4632 ly even dolphins eat along the blithely even instructions. express attainments cajole slyly. busy dolphins in
8501 Customer#000008501 412797.5100 6906.70 ARGENTINA 776af4rOa mZ66hczs 11-317-552-5840 y final deposits after the fluffily even accounts are slyly final, regular
1565 Customer#000001565 412506.0062 1820.03 BRAZIL EWQO5Ck,nMuHVQimqL8dLrixRP6QKveXcz9QgorW 12-402-178-2007 ously regular accounts wake slyly ironic idea
14398 Customer#000014398 408575.3600 -602.24 UNITED STATES GWRCgIPHajtU21vICVvbJJerFu2cUk 34-814-111-5424 s. blithely even accounts cajole blithely. even foxes doubt--
1465 Customer#000001465 405055.3457 9365.93 INDIA tDRaTC7UgFbBX7VF6cVXYQA0 18-807-487-1074 s lose blithely ironic, regular packages. regular, final foxes haggle c
12595 Customer#000012595 401402.2391 -6.92 INDIA LmeaX5cR,w9NqKugl yRm98 18-186-132-3352 o the busy accounts. blithely special gifts maintain a
961 Customer#000000961 401198.1737 6963.68 JAPAN 5,81YDLFuRR47KKzv8GXdmi3zyP37PlPn 22-989-463-6089 e final requests: busily final accounts believe a
14299 Customer#000014299 400968.3751 6595.97 RUSSIA 7lFczTya0iM1bhEWT 32-156-618-1224 carefully regular requests. quickly ironic accounts against the ru
623 Customer#000000623 399883.4257 7887.60 INDONESIA HXiFb9oWlgqZXrJPUCEJ6zZIPxAM4m6 19-113-202-7085 requests. dolphins above the busily regular dependencies cajole after
9151 Customer#000009151 396562.0295 5691.95 IRAQ 7gIdRdaxB91EVdyx8DyPjShpMD 21-834-147-4906 ajole fluffily. furiously regular accounts are special, silent account
14819 Customer#000014819 396271.1036 7308.39 FRANCE w8StIbymUXmLCcUag6sx6LUIp8E3pA,Ux 16-769-398-7926 ss, final asymptotes use furiously slyly ironic dependencies. special, express dugouts according to the dep
13478 Customer#000013478 395513.1358 -778.11 KENYA 9VIsvIeZrJpC6OOdYheMC2vdtq8Ai0Rt 24-983-202-8240 r theodolites. slyly unusual pinto beans sleep fluffily against the asymptotes. quickly r

2544
regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q11.out Normal file
View File

File diff suppressed because it is too large Load Diff

5
regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q12.out Normal file
View File

@ -0,0 +1,5 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !q12 --
MAIL 647 945
SHIP 620 943

40
regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q13.out Normal file
View File

@ -0,0 +1,40 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !q13 --
0 5000
10 665
9 657
11 621
12 567
8 564
13 492
18 482
7 480
20 456
14 456
16 449
19 447
15 432
17 423
21 412
22 371
6 337
23 323
24 256
25 204
5 204
26 155
27 141
28 97
4 94
29 64
3 48
30 27
31 26
32 14
33 11
2 11
34 6
35 5
1 2
36 1

4
regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q14.out Normal file
View File

@ -0,0 +1,4 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !q14 --
16.2838556890

4
regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q15.out Normal file
View File

@ -0,0 +1,4 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !q15 --
677 Supplier#000000677 8mhrffG7D2WJBSQbOGstQ 23-290-639-3315 1614410.2928

2765
regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q16.out Normal file
View File

File diff suppressed because it is too large Load Diff

4
regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q17.out Normal file
View File

@ -0,0 +1,4 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !q17 --
23512.752857

8
regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q18.out Normal file
View File

@ -0,0 +1,8 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !q18 --
Customer#000001639 1639 502886 1994-04-12 456423.88 312.00
Customer#000006655 6655 29158 1995-10-21 452805.02 305.00
Customer#000014110 14110 565574 1995-09-24 425099.85 301.00
Customer#000001775 1775 6882 1997-04-09 408368.10 303.00
Customer#000011459 11459 551136 1993-05-19 386812.74 308.00

4
regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q19.out Normal file
View File

@ -0,0 +1,4 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !q19 --
168597.2860

12
regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q20.out Normal file
View File

@ -0,0 +1,12 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !q20 --
Supplier#000000157 ,mEGorBfVIm
Supplier#000000197 YC2Acon6kjY3zj3Fbxs2k4Vdf7X0cd2F
Supplier#000000287 7a9SP7qW5Yku5PvSg
Supplier#000000378 FfbhyCxWvcPrO8ltp9
Supplier#000000530 0qwCMwobKY OcmLyfRXlagA8ukENJv,
Supplier#000000555 TfB,a5bfl3Ah 3Z 74GqnNs6zKVGM
Supplier#000000557 jj0wUYh9K3fG5Jhdhrkuy ,4
Supplier#000000729 pqck2ppy758TQpZCUAjPvlU55K3QjfL7Bi
Supplier#000000935 ij98czM 2KzWe7dDTOxB8sq0UfCdvrX

50
regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q21.out Normal file
View File

@ -0,0 +1,50 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !q21 --
Supplier#000000445 16
Supplier#000000825 16
Supplier#000000709 15
Supplier#000000762 15
Supplier#000000357 14
Supplier#000000399 14
Supplier#000000496 14
Supplier#000000977 13
Supplier#000000144 12
Supplier#000000188 12
Supplier#000000415 12
Supplier#000000472 12
Supplier#000000633 12
Supplier#000000708 12
Supplier#000000889 12
Supplier#000000380 11
Supplier#000000602 11
Supplier#000000659 11
Supplier#000000821 11
Supplier#000000929 11
Supplier#000000262 10
Supplier#000000460 10
Supplier#000000486 10
Supplier#000000669 10
Supplier#000000718 10
Supplier#000000778 10
Supplier#000000167 9
Supplier#000000578 9
Supplier#000000673 9
Supplier#000000687 9
Supplier#000000074 8
Supplier#000000565 8
Supplier#000000648 8
Supplier#000000918 8
Supplier#000000427 7
Supplier#000000503 7
Supplier#000000610 7
Supplier#000000670 7
Supplier#000000811 7
Supplier#000000114 6
Supplier#000000379 6
Supplier#000000436 6
Supplier#000000500 6
Supplier#000000660 6
Supplier#000000788 6
Supplier#000000846 6
Supplier#000000920 4

10
regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q22.out Normal file
View File

@ -0,0 +1,10 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !q22 --
13 94 714035.05
17 96 722560.15
18 99 738012.52
23 93 708285.25
29 85 632693.46
30 87 646748.02
31 87 647372.50

17
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/customer.sql Normal file
View File

@ -0,0 +1,17 @@
CREATE TABLE IF NOT EXISTS customer (
C_CUSTKEY INTEGER NOT NULL,
C_NAME VARCHAR(25) NOT NULL,
C_ADDRESS VARCHAR(40) NOT NULL,
C_NATIONKEY INTEGER NOT NULL,
C_PHONE CHAR(15) NOT NULL,
C_ACCTBAL DECIMAL(15,2) NOT NULL,
C_MKTSEGMENT CHAR(10) NOT NULL,
C_COMMENT VARCHAR(117) NOT NULL
)
UNIQUE KEY(C_CUSTKEY, C_NAME)
DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 1
PROPERTIES (
"compression"="zstd",
"replication_num" = "1"
)

1
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/customer_delete.sql Normal file
View File

@ -0,0 +1 @@
DELETE FROM customer where C_CUSTKEY >= 0;

25
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/lineitem.sql Normal file
View File

@ -0,0 +1,25 @@
CREATE TABLE IF NOT EXISTS lineitem (
L_ORDERKEY INTEGER NOT NULL,
L_PARTKEY INTEGER NOT NULL,
L_SUPPKEY INTEGER NOT NULL,
L_LINENUMBER INTEGER NOT NULL,
L_QUANTITY DECIMAL(15,2) NOT NULL,
L_EXTENDEDPRICE DECIMAL(15,2) NOT NULL,
L_DISCOUNT DECIMAL(15,2) NOT NULL,
L_TAX DECIMAL(15,2) NOT NULL,
L_RETURNFLAG CHAR(1) NOT NULL,
L_LINESTATUS CHAR(1) NOT NULL,
L_SHIPDATE DATE NOT NULL,
L_COMMITDATE DATE NOT NULL,
L_RECEIPTDATE DATE NOT NULL,
L_SHIPINSTRUCT CHAR(25) NOT NULL,
L_SHIPMODE CHAR(10) NOT NULL,
L_COMMENT VARCHAR(44) NOT NULL
)
UNIQUE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER)
DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 1
PROPERTIES (
"compression"="zstd",
"replication_num" = "1"
)

1
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/lineitem_delete.sql Normal file
View File

@ -0,0 +1 @@
DELETE from lineitem where L_ORDERKEY >= 0;

13
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/nation.sql Normal file
View File

@ -0,0 +1,13 @@
CREATE TABLE IF NOT EXISTS nation (
N_NATIONKEY INTEGER NOT NULL,
N_NAME CHAR(25) NOT NULL,
N_REGIONKEY INTEGER NOT NULL,
N_COMMENT VARCHAR(152)
)
UNIQUE KEY(N_NATIONKEY, N_NAME)
DISTRIBUTED BY HASH(N_NATIONKEY) BUCKETS 1
PROPERTIES (
"compression"="zstd",
"replication_num" = "1"
)

1
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/nation_delete.sql Normal file
View File

@ -0,0 +1 @@
DELETE from nation where N_NATIONKEY >= 0;

18
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/orders.sql Normal file
View File

@ -0,0 +1,18 @@
CREATE TABLE IF NOT EXISTS orders (
O_ORDERKEY INTEGER NOT NULL,
O_CUSTKEY INTEGER NOT NULL,
O_ORDERSTATUS CHAR(1) NOT NULL,
O_TOTALPRICE DECIMAL(15,2) NOT NULL,
O_ORDERDATE DATE NOT NULL,
O_ORDERPRIORITY CHAR(15) NOT NULL,
O_CLERK CHAR(15) NOT NULL,
O_SHIPPRIORITY INTEGER NOT NULL,
O_COMMENT VARCHAR(79) NOT NULL
)
UNIQUE KEY(O_ORDERKEY, O_CUSTKEY)
DISTRIBUTED BY HASH(O_ORDERKEY) BUCKETS 1
PROPERTIES (
"compression"="zstd",
"replication_num" = "1"
)

1
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/orders_delete.sql Normal file
View File

@ -0,0 +1 @@
delete from orders where O_ORDERKEY >= 0;

18
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/part.sql Normal file
View File

@ -0,0 +1,18 @@
CREATE TABLE IF NOT EXISTS part (
P_PARTKEY INTEGER NOT NULL,
P_NAME VARCHAR(55) NOT NULL,
P_MFGR CHAR(25) NOT NULL,
P_BRAND CHAR(10) NOT NULL,
P_TYPE VARCHAR(25) NOT NULL,
P_SIZE INTEGER NOT NULL,
P_CONTAINER CHAR(10) NOT NULL,
P_RETAILPRICE DECIMAL(15,2) NOT NULL,
P_COMMENT VARCHAR(23) NOT NULL
)
UNIQUE KEY(P_PARTKEY, P_NAME)
DISTRIBUTED BY HASH(P_PARTKEY) BUCKETS 1
PROPERTIES (
"compression"="zstd",
"replication_num" = "1"
)

1
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/part_delete.sql Normal file
View File

@ -0,0 +1 @@
delete from part where P_PARTKEY>=0;

14
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/partsupp.sql Normal file
View File

@ -0,0 +1,14 @@
CREATE TABLE IF NOT EXISTS partsupp (
PS_PARTKEY INTEGER NOT NULL,
PS_SUPPKEY INTEGER NOT NULL,
PS_AVAILQTY INTEGER NOT NULL,
PS_SUPPLYCOST DECIMAL(15,2) NOT NULL,
PS_COMMENT VARCHAR(199) NOT NULL
)
UNIQUE KEY(PS_PARTKEY, PS_SUPPKEY)
DISTRIBUTED BY HASH(PS_PARTKEY) BUCKETS 1
PROPERTIES (
"compression"="zstd",
"replication_num" = "1"
)

1
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/partsupp_delete.sql Normal file
View File

@ -0,0 +1 @@
delete from partsupp where PS_PARTKEY>=0;

12
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/region.sql Normal file
View File

@ -0,0 +1,12 @@
CREATE TABLE IF NOT EXISTS region (
R_REGIONKEY INTEGER NOT NULL,
R_NAME CHAR(25) NOT NULL,
R_COMMENT VARCHAR(152)
)
UNIQUE KEY(R_REGIONKEY, R_NAME)
DISTRIBUTED BY HASH(R_REGIONKEY) BUCKETS 1
PROPERTIES (
"compression"="zstd",
"replication_num" = "1"
)

1
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/region_delete.sql Normal file
View File

@ -0,0 +1 @@
delete from region where R_REGIONKEY>=0;

11
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/revenue1.sql Normal file
View File

@ -0,0 +1,11 @@
CREATE VIEW IF NOT EXISTS revenue1 AS
SELECT
l_suppkey AS supplier_no,
sum(l_extendedprice * (1 - l_discount)) AS total_revenue
FROM
lineitem
WHERE
l_shipdate >= DATE '1996-01-01'
AND l_shipdate < DATE '1996-01-01' + INTERVAL '3' MONTH
GROUP BY
l_suppkey;

1
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/revenue1_delete.sql Normal file
View File

@ -0,0 +1 @@
drop view IF EXISTS revenue1;

15
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/supplier.sql Normal file
View File

@ -0,0 +1,15 @@
CREATE TABLE IF NOT EXISTS supplier (
S_SUPPKEY INTEGER NOT NULL,
S_NAME CHAR(25) NOT NULL,
S_ADDRESS VARCHAR(40) NOT NULL,
S_NATIONKEY INTEGER NOT NULL,
S_PHONE CHAR(15) NOT NULL,
S_ACCTBAL DECIMAL(15,2) NOT NULL,
S_COMMENT VARCHAR(101) NOT NULL
)
UNIQUE KEY(S_SUPPKEY, S_NAME)
DISTRIBUTED BY HASH(S_SUPPKEY) BUCKETS 1
PROPERTIES (
"compression"="zstd",
"replication_num" = "1"
)

1
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/supplier_delete.sql Normal file
View File

@ -0,0 +1 @@
delete from supplier where S_SUPPKEY>=0;

84
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/load.groovy Normal file
View File

@ -0,0 +1,84 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
// Most of the cases are copied from https://github.com/trinodb/trino/tree/master
// /testing/trino-product-tests/src/main/resources/sql-tests/testcases
// and modified by Doris.
// syntax error:
// q06 q13 q15
// Test 23 suites, failed 3 suites
// Note: To filter out tables from sql files, use the following one-liner comamnd
// sed -nr 's/.*tables: (.*)$/\1/gp' /path/to/*.sql | sed -nr 's/,/\n/gp' | sort | uniq
suite("load") {
def tables = [customer: ["c_custkey, c_name, c_address, c_nationkey, c_phone, c_acctbal, c_mktsegment, c_comment,temp"],
lineitem: ["l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag,l_linestatus, l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment,temp"],
nation : ["n_nationkey, n_name, n_regionkey, n_comment, temp"],
orders : ["o_orderkey, o_custkey, o_orderstatus, o_totalprice, o_orderdate, o_orderpriority, o_clerk, o_shippriority, o_comment, temp"],
part : ["p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment, temp"],
partsupp: ["ps_partkey,ps_suppkey,ps_availqty,ps_supplycost,ps_comment,temp"],
region : ["r_regionkey, r_name, r_comment,temp"],
supplier: ["s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment,temp"]]
tables.forEach { tableName, columns ->
sql new File("""${context.file.parent}/ddl/${tableName}.sql""").text
sql new File("""${context.file.parent}/ddl/${tableName}_delete.sql""").text
streamLoad {
// a default db 'regression_test' is specified in
// ${DORIS_HOME}/conf/regression-conf.groovy
table "${tableName}"
// default label is UUID:
// set 'label' UUID.randomUUID().toString()
// default column_separator is specify in doris fe config, usually is '\t'.
// this line change to ','
set 'column_separator', '|'
set 'compress_type', 'GZ'
set 'columns', "${columns[0]}"
// relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv.
// also, you can stream load a http stream, e.g. http://xxx/some.csv
file """${getS3Url()}/regression/tpch/sf0.1/${tableName}.tbl.gz"""
time 10000 // limit inflight 10s
// stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows
// if declared a check callback, the default check condition will ignore.
// So you must check all condition
check { result, exception, startTime, endTime ->
if (exception != null) {
throw exception
}
log.info("Stream load result: ${result}".toString())
def json = parseJson(result)
assertEquals("success", json.Status.toLowerCase())
assertEquals(json.NumberTotalRows, json.NumberLoadedRows)
assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
}
}
sql """ ANALYZE TABLE $tableName WITH SYNC """
}
def table = "revenue1"
sql new File("""${context.file.parent}/ddl/${table}_delete.sql""").text
sql new File("""${context.file.parent}/ddl/${table}.sql""").text
sql """ sync """
}

22
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q01.sql Normal file
View File

@ -0,0 +1,22 @@
-- tables: lineitem
SELECT
l_returnflag,
l_linestatus,
sum(l_quantity) AS sum_qty,
sum(l_extendedprice) AS sum_base_price,
sum(l_extendedprice * (1 - l_discount)) AS sum_disc_price,
sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) AS sum_charge,
avg(l_quantity) AS avg_qty,
avg(l_extendedprice) AS avg_price,
avg(l_discount) AS avg_disc,
count(*) AS count_order
FROM
lineitem
WHERE
l_shipdate <= DATE '1998-12-01' - INTERVAL '90' DAY
GROUP BY
l_returnflag,
l_linestatus
ORDER BY
l_returnflag,
l_linestatus

42
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q02.sql Normal file
View File

@ -0,0 +1,42 @@
-- tables: part,supplier,partsupp,nation,region
SELECT
s_acctbal,
s_name,
n_name,
p_partkey,
p_mfgr,
s_address,
s_phone,
s_comment
FROM
part,
supplier,
partsupp,
nation,
region
WHERE
p_partkey = ps_partkey
AND s_suppkey = ps_suppkey
AND p_size = 15
AND p_type LIKE '%BRASS'
AND s_nationkey = n_nationkey
AND n_regionkey = r_regionkey
AND r_name = 'EUROPE'
AND ps_supplycost = (
SELECT min(ps_supplycost)
FROM
partsupp, supplier,
nation, region
WHERE
p_partkey = ps_partkey
AND s_suppkey = ps_suppkey
AND s_nationkey = n_nationkey
AND n_regionkey = r_regionkey
AND r_name = 'EUROPE'
)
ORDER BY
s_acctbal DESC,
n_name,
s_name,
p_partkey
LIMIT 100

24
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q03.sql Normal file
View File

@ -0,0 +1,24 @@
-- tables: customer,orders,lineitem
SELECT
l_orderkey,
sum(l_extendedprice * (1 - l_discount)) AS revenue,
o_orderdate,
o_shippriority
FROM
customer,
orders,
lineitem
WHERE
c_mktsegment = 'BUILDING'
AND c_custkey = o_custkey
AND l_orderkey = o_orderkey
AND o_orderdate < DATE '1995-03-15'
AND l_shipdate > DATE '1995-03-15'
GROUP BY
l_orderkey,
o_orderdate,
o_shippriority
ORDER BY
revenue DESC,
o_orderdate
LIMIT 10

19
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q04.sql Normal file
View File

@ -0,0 +1,19 @@
-- tables: orders,lineitem
SELECT
o_orderpriority,
count(*) AS order_count
FROM orders
WHERE
o_orderdate >= DATE '1993-07-01'
AND o_orderdate < DATE '1993-07-01' + INTERVAL '3' MONTH
AND EXISTS (
SELECT *
FROM lineitem
WHERE
l_orderkey = o_orderkey
AND l_commitdate < l_receiptdate
)
GROUP BY
o_orderpriority
ORDER BY
o_orderpriority

25
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q05.sql Normal file
View File

@ -0,0 +1,25 @@
-- tables: customer,orders,lineitem,supplier,nation,region
SELECT
n_name,
sum(l_extendedprice * (1 - l_discount)) AS revenue
FROM
customer,
orders,
lineitem,
supplier,
nation,
region
WHERE
c_custkey = o_custkey
AND l_orderkey = o_orderkey
AND l_suppkey = s_suppkey
AND c_nationkey = s_nationkey
AND s_nationkey = n_nationkey
AND n_regionkey = r_regionkey
AND r_name = 'ASIA'
AND o_orderdate >= DATE '1994-01-01'
AND o_orderdate < DATE '1994-01-01' + INTERVAL '1' YEAR
GROUP BY
n_name
ORDER BY
revenue DESC

11
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q06.sql Normal file
View File

@ -0,0 +1,11 @@
-- tables: lineitem
SELECT sum(l_extendedprice * l_discount) AS revenue
FROM
lineitem
WHERE
l_shipdate >= DATE '1994-01-01'
AND l_shipdate < DATE '1994-01-01' + INTERVAL '1' YEAR
AND l_discount BETWEEN 0.06 - 0.01 AND .06 + 0.01
AND l_quantity < 24

39
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q07.sql Normal file
View File

@ -0,0 +1,39 @@
-- tables: supplier,lineitem,orders,customer,nation
SELECT
supp_nation,
cust_nation,
l_year,
sum(volume) AS revenue
FROM (
SELECT
n1.n_name AS supp_nation,
n2.n_name AS cust_nation,
extract(YEAR FROM l_shipdate) AS l_year,
l_extendedprice * (1 - l_discount) AS volume
FROM
supplier,
lineitem,
orders,
customer,
nation n1,
nation n2
WHERE
s_suppkey = l_suppkey
AND o_orderkey = l_orderkey
AND c_custkey = o_custkey
AND s_nationkey = n1.n_nationkey
AND c_nationkey = n2.n_nationkey
AND (
(n1.n_name = 'FRANCE' AND n2.n_name = 'GERMANY')
OR (n1.n_name = 'GERMANY' AND n2.n_name = 'FRANCE')
)
AND l_shipdate BETWEEN DATE '1995-01-01' AND DATE '1996-12-31'
) AS shipping
GROUP BY
supp_nation,
cust_nation,
l_year
ORDER BY
supp_nation,
cust_nation,
l_year

38
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q08.sql Normal file
View File

@ -0,0 +1,38 @@
-- tables: part,supplier,lineitem,orders,customer,nation,region
SELECT
o_year,
sum(CASE
WHEN nation = 'BRAZIL'
THEN volume
ELSE 0
END) / sum(volume) AS mkt_share
FROM (
SELECT
extract(YEAR FROM o_orderdate) AS o_year,
l_extendedprice * (1 - l_discount) AS volume,
n2.n_name AS nation
FROM
part,
supplier,
lineitem,
orders,
customer,
nation n1,
nation n2,
region
WHERE
p_partkey = l_partkey
AND s_suppkey = l_suppkey
AND l_orderkey = o_orderkey
AND o_custkey = c_custkey
AND c_nationkey = n1.n_nationkey
AND n1.n_regionkey = r_regionkey
AND r_name = 'AMERICA'
AND s_nationkey = n2.n_nationkey
AND o_orderdate BETWEEN DATE '1995-01-01' AND DATE '1996-12-31'
AND p_type = 'ECONOMY ANODIZED STEEL'
) AS all_nations
GROUP BY
o_year
ORDER BY
o_year

32
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q09.sql Normal file
View File

@ -0,0 +1,32 @@
-- tables: part,supplier,lineitem,partsupp,orders,nation
SELECT
nation,
o_year,
sum(amount) AS sum_profit
FROM (
SELECT
n_name AS nation,
extract(YEAR FROM o_orderdate) AS o_year,
l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity AS amount
FROM
part,
supplier,
lineitem,
partsupp,
orders,
nation
WHERE
s_suppkey = l_suppkey
AND ps_suppkey = l_suppkey
AND ps_partkey = l_partkey
AND p_partkey = l_partkey
AND o_orderkey = l_orderkey
AND s_nationkey = n_nationkey
AND p_name LIKE '%green%'
) AS profit
GROUP BY
nation,
o_year
ORDER BY
nation,
o_year DESC

33
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q10.sql Normal file
View File

@ -0,0 +1,33 @@
-- tables: customer,orders,lineitem,nation
SELECT
c_custkey,
c_name,
sum(l_extendedprice * (1 - l_discount)) AS revenue,
c_acctbal,
n_name,
c_address,
c_phone,
c_comment
FROM
customer,
orders,
lineitem,
nation
WHERE
c_custkey = o_custkey
AND l_orderkey = o_orderkey
AND o_orderdate >= DATE '1993-10-01'
AND o_orderdate < DATE '1993-10-01' + INTERVAL '3' MONTH
AND l_returnflag = 'R'
AND c_nationkey = n_nationkey
GROUP BY
c_custkey,
c_name,
c_acctbal,
c_phone,
n_name,
c_address,
c_comment
ORDER BY
revenue DESC
LIMIT 20

28
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q11.sql Normal file
View File

@ -0,0 +1,28 @@
-- tables: partsupp,supplier,nation
SELECT
ps_partkey,
sum(ps_supplycost * ps_availqty) AS value
FROM
partsupp,
supplier,
nation
WHERE
ps_suppkey = s_suppkey
AND s_nationkey = n_nationkey
AND n_name = 'GERMANY'
GROUP BY
ps_partkey
HAVING
sum(ps_supplycost * ps_availqty) > (
SELECT sum(ps_supplycost * ps_availqty) * 0.0001
FROM
partsupp,
supplier,
nation
WHERE
ps_suppkey = s_suppkey
AND s_nationkey = n_nationkey
AND n_name = 'GERMANY'
)
ORDER BY
value DESC

29
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q12.sql Normal file
View File

@ -0,0 +1,29 @@
-- tables: orders,lineitem
SELECT
l_shipmode,
sum(CASE
WHEN o_orderpriority = '1-URGENT'
OR o_orderpriority = '2-HIGH'
THEN 1
ELSE 0
END) AS high_line_count,
sum(CASE
WHEN o_orderpriority <> '1-URGENT'
AND o_orderpriority <> '2-HIGH'
THEN 1
ELSE 0
END) AS low_line_count
FROM
orders,
lineitem
WHERE
o_orderkey = l_orderkey
AND l_shipmode IN ('MAIL', 'SHIP')
AND l_commitdate < l_receiptdate
AND l_shipdate < l_commitdate
AND l_receiptdate >= DATE '1994-01-01'
AND l_receiptdate < DATE '1994-01-01' + INTERVAL '1' YEAR
GROUP BY
l_shipmode
ORDER BY
l_shipmode

21
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q13.sql Normal file
View File

@ -0,0 +1,21 @@
-- tables: customer
SELECT
c_count,
count(*) AS custdist
FROM (
SELECT
c_custkey,
count(o_orderkey) AS c_count
FROM
customer
LEFT OUTER JOIN orders ON
c_custkey = o_custkey
AND o_comment NOT LIKE '%special%requests%'
GROUP BY
c_custkey
) AS c_orders
GROUP BY
c_count
ORDER BY
custdist DESC,
c_count DESC

13
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q14.sql Normal file
View File

@ -0,0 +1,13 @@
-- tables: lineitem,part
SELECT 100.00 * sum(CASE
WHEN p_type LIKE 'PROMO%'
THEN l_extendedprice * (1 - l_discount)
ELSE 0
END) / sum(l_extendedprice * (1 - l_discount)) AS promo_revenue
FROM
lineitem,
part
WHERE
l_partkey = p_partkey
AND l_shipdate >= DATE '1995-09-01'
AND l_shipdate < DATE '1995-09-01' + INTERVAL '1' MONTH

20
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q15.sql Normal file
View File

@ -0,0 +1,20 @@
SELECT
s_suppkey,
s_name,
s_address,
s_phone,
total_revenue
FROM
supplier,
revenue1
WHERE
s_suppkey = supplier_no
AND total_revenue = (
SELECT max(total_revenue)
FROM
revenue1
)
ORDER BY
s_suppkey;

30
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q16.sql Normal file
View File

@ -0,0 +1,30 @@
-- tables: partsupp,part,supplier
SELECT
p_brand,
p_type,
p_size,
count(DISTINCT ps_suppkey) AS supplier_cnt
FROM
partsupp,
part
WHERE
p_partkey = ps_partkey
AND p_brand <> 'Brand#45'
AND p_type NOT LIKE 'MEDIUM POLISHED%'
AND p_size IN (49, 14, 23, 45, 19, 3, 36, 9)
AND ps_suppkey NOT IN (
SELECT s_suppkey
FROM
supplier
WHERE
s_comment LIKE '%Customer%Complaints%'
)
GROUP BY
p_brand,
p_type,
p_size
ORDER BY
supplier_cnt DESC,
p_brand,
p_type,
p_size

16
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q17.sql Normal file
View File

@ -0,0 +1,16 @@
-- tables: lineitem,part
SELECT sum(l_extendedprice) / 7.0 AS avg_yearly
FROM
lineitem,
part
WHERE
p_partkey = l_partkey
AND p_brand = 'Brand#23'
AND p_container = 'MED BOX'
AND l_quantity < (
SELECT 0.2 * avg(l_quantity)
FROM
lineitem
WHERE
l_partkey = p_partkey
)

34
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q18.sql Normal file
View File

@ -0,0 +1,34 @@
-- tables: customer,orders,lineitem
SELECT
c_name,
c_custkey,
o_orderkey,
o_orderdate,
o_totalprice,
sum(l_quantity)
FROM
customer,
orders,
lineitem
WHERE
o_orderkey IN (
SELECT l_orderkey
FROM
lineitem
GROUP BY
l_orderkey
HAVING
sum(l_quantity) > 300
)
AND c_custkey = o_custkey
AND o_orderkey = l_orderkey
GROUP BY
c_name,
c_custkey,
o_orderkey,
o_orderdate,
o_totalprice
ORDER BY
o_totalprice DESC,
o_orderdate
LIMIT 100

35
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q19.sql Normal file
View File

@ -0,0 +1,35 @@
-- tables: lineitem,part
SELECT sum(l_extendedprice * (1 - l_discount)) AS revenue
FROM
lineitem,
part
WHERE
(
p_partkey = l_partkey
AND p_brand = 'Brand#12'
AND p_container IN ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG')
AND l_quantity >= 1 AND l_quantity <= 1 + 10
AND p_size BETWEEN 1 AND 5
AND l_shipmode IN ('AIR', 'AIR REG')
AND l_shipinstruct = 'DELIVER IN PERSON'
)
OR
(
p_partkey = l_partkey
AND p_brand = 'Brand#23'
AND p_container IN ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK')
AND l_quantity >= 10 AND l_quantity <= 10 + 10
AND p_size BETWEEN 1 AND 10
AND l_shipmode IN ('AIR', 'AIR REG')
AND l_shipinstruct = 'DELIVER IN PERSON'
)
OR
(
p_partkey = l_partkey
AND p_brand = 'Brand#34'
AND p_container IN ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG')
AND l_quantity >= 20 AND l_quantity <= 20 + 10
AND p_size BETWEEN 1 AND 15
AND l_shipmode IN ('AIR', 'AIR REG')
AND l_shipinstruct = 'DELIVER IN PERSON'
)

33
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q20.sql Normal file
View File

@ -0,0 +1,33 @@
-- tables: supplier,nation,partsupp,lineitem,part
SELECT
s_name,
s_address
FROM
supplier, nation
WHERE
s_suppkey IN (
SELECT ps_suppkey
FROM
partsupp
WHERE
ps_partkey IN (
SELECT p_partkey
FROM
part
WHERE
p_name LIKE 'forest%'
)
AND ps_availqty > (
SELECT 0.5 * sum(l_quantity)
FROM
lineitem
WHERE
l_partkey = ps_partkey
AND l_suppkey = ps_suppkey
AND l_shipdate >= date('1994-01-01')
AND l_shipdate < date('1994-01-01') + interval '1' YEAR
)
)
AND s_nationkey = n_nationkey
AND n_name = 'CANADA'
ORDER BY s_name

39
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q21.sql Normal file
View File

@ -0,0 +1,39 @@
-- tables: supplier,lineitem,orders,nation
SELECT
s_name,
count(*) AS numwait
FROM
supplier,
lineitem l1,
orders,
nation
WHERE
s_suppkey = l1.l_suppkey
AND o_orderkey = l1.l_orderkey
AND o_orderstatus = 'F'
AND l1.l_receiptdate > l1.l_commitdate
AND exists(
SELECT *
FROM
lineitem l2
WHERE
l2.l_orderkey = l1.l_orderkey
AND l2.l_suppkey <> l1.l_suppkey
)
AND NOT exists(
SELECT *
FROM
lineitem l3
WHERE
l3.l_orderkey = l1.l_orderkey
AND l3.l_suppkey <> l1.l_suppkey
AND l3.l_receiptdate > l3.l_commitdate
)
AND s_nationkey = n_nationkey
AND n_name = 'SAUDI ARABIA'
GROUP BY
s_name
ORDER BY
numwait DESC,
s_name
LIMIT 100

35
regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q22.sql Normal file
View File

@ -0,0 +1,35 @@
-- tables: orders,customer
SELECT
cntrycode,
count(*) AS numcust,
sum(c_acctbal) AS totacctbal
FROM (
SELECT
substr(c_phone, 1, 2) AS cntrycode,
c_acctbal
FROM
customer
WHERE
substr(c_phone, 1, 2) IN
('13', '31', '23', '29', '30', '18', '17')
AND c_acctbal > (
SELECT avg(c_acctbal)
FROM
customer
WHERE
c_acctbal > 0.00
AND substr(c_phone, 1, 2) IN
('13', '31', '23', '29', '30', '18', '17')
)
AND NOT exists(
SELECT *
FROM
orders
WHERE
o_custkey = c_custkey
)
) AS custsale
GROUP BY
cntrycode
ORDER BY
cntrycode