[test](regression) add routine load sequence and error test (#27519)
This commit is contained in:
HHoflittlefish777
committed by
GitHub
GitHub
parent
4232327a43
commit
ff1a06abcf
@ -809,15 +809,15 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback impl
|
||||
.add("current_error_rows", this.jobStatistic.currentErrorRows)
|
||||
.add("max_error_num", maxErrorNum)
|
||||
.add("max_filter_ratio", maxFilterRatio)
|
||||
.add("msg", "current error rows is more than max error rows "
|
||||
+ "or the filter ratio is more than the max, begin to pause job")
|
||||
.add("msg", "current error rows is more than max_error_number "
|
||||
+ "or the max_filter_ratio is more than value set, begin to pause job")
|
||||
.build());
|
||||
// if this is a replay thread, the update state should already be replayed by OP_CHANGE_ROUTINE_LOAD_JOB
|
||||
if (!isReplay) {
|
||||
// remove all of task in jobs and change job state to paused
|
||||
updateState(JobState.PAUSED, new ErrorReason(InternalErrorCode.TOO_MANY_FAILURE_ROWS_ERR,
|
||||
"current error rows is more than max error num "
|
||||
+ "or the filter ratio is more than the max"), isReplay);
|
||||
"current error rows is more than max_error_number "
|
||||
+ "or the max_filter_ratio is more than the value set"), isReplay);
|
||||
}
|
||||
}
|
||||
|
||||
@ -843,14 +843,14 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback impl
|
||||
.add("current_error_rows", this.jobStatistic.currentErrorRows)
|
||||
.add("max_error_num", maxErrorNum)
|
||||
.add("max_filter_ratio", maxFilterRatio)
|
||||
.add("msg", "current error rows is more than max error rows "
|
||||
+ "or the filter ratio is more than the max, begin to pause job")
|
||||
.add("msg", "current error rows is more than max_error_number "
|
||||
+ "or the max_filter_ratio is more than the max, begin to pause job")
|
||||
.build());
|
||||
if (!isReplay) {
|
||||
// remove all of task in jobs and change job state to paused
|
||||
updateState(JobState.PAUSED, new ErrorReason(InternalErrorCode.TOO_MANY_FAILURE_ROWS_ERR,
|
||||
"current error rows is more than max error num "
|
||||
+ "or the filter ratio is more than the max"), isReplay);
|
||||
"current error rows is more than max_error_number "
|
||||
+ "or the max_filter_ratio is more than the value set"), isReplay);
|
||||
}
|
||||
// reset currentTotalNum and currentErrorNum
|
||||
this.jobStatistic.currentErrorRows = 0;
|
||||
|
||||
@ -2100,3 +2100,25 @@
|
||||
85 2023-08-11 true -7 24304 -2043877415 -2024144417867729183 \N 5363.0244 -5.78615669042831E8 -99999999.9 -99999999.9 2023-07-15T01:07:41 2023-08-13 2023-01-20T11:57:48 i WQ9dh9ajPu0y {"country":"France","capital":"Paris","population":67081000} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N
|
||||
90 2023-08-27 true 22 16456 -1476824962 -3279894870153540825 8990195191470116763 26651.906 2.06860148942546E8 -99999999.9 -99999999.9 2022-10-07T03:11:03 2023-03-18 2023-04-15T00:38:33 T L QW0GQ3GoMtHgxPQOWGfVaveynahNpsNs09siMFA1OtO6QEDBQTdivmGyq7bFzejAqwbbVQQpREAmeLjcFSXLnQuou2KbwYD {"company":"Apple","products":[{"name":"iPhone","price":1000},{"name":"MacBook","price":1500}]} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N
|
||||
91 2023-08-27 true 90 2465 702240964 6373830997821598984 305860046137409400 15991.356 1.599972327386147E9 -99999999.9 \N 2023-04-26T19:31:10 2023-07-21 \N 2 B7YKYBYT8w0YC926bZ8Yz1VzyiWw2NWDAiTlEoPVyz9AXGti2Npg1FxWqWk4hEaALw0ZBSuiAIPj41lq36g5QRpPmAjNPK {"fruit":"apple","color":"red","qty":5,"price":2.5} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N
|
||||
|
||||
-- !sql_squence --
|
||||
8 2023-08-14 true 109 -31573 -1362465190 3990845741226497177 2732763251146840270 -25698.553 1.312831962567818E9 99999999.9 99999999.9 2023-03-07T14:13:19 2022-10-18 2023-07-16T05:03:13 D PBn1wa6X8WneZYLMac11zzyhGl7tPXB5XgjmOV8L6uav9ja5oY433ktb2yhyQQIqBveZPkme {"animal":"lion","weight":200,"habitat":["savannah","grassland"]} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N
|
||||
20 2023-08-17 false -5 18158 784479801 1485484354598941738 -6632681928222776815 9708.431 -3.30432620706069E8 -99999999.9 99999999.9 2022-09-15T21:40:55 2023-02-23 2023-08-13T21:31:54 O X 2pYmX2vAhfEEHZZYPsgAmda1G7otnwx5TmUC879FPhDeIjvWI79ksBZpfFG2gp7jhCSbpZiecKGklB5SvG8tm31i5SUqe1xrWgLt4HSq7lMJWp75tx2kxD7pRIOpn {"name":"Sarah","age":30,"city":"London","isMarried":false} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N
|
||||
21 2023-08-18 false 63 -27847 -35409596 8638201997392767650 4919963231735304178 -23382.541 -1.803403621426313E9 -22009767.0 99999999.9 2023-03-31T10:56:14 2023-01-20 2023-02-18T13:37:52 N T PSiFwUEx3eVFNtjlnQ70YkgZNvKrGmQ2DN5K9yYHiSdFWeEDB1UpL3Frt8z1kEAIWRDWqXZuyi {"city":"Sydney","population":5312000,"area":2058.7} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N
|
||||
31 2023-08-27 false 17 -18849 1728109133 3266501886640700374 527195452623418935 -24062.328 -1.514348021262435E9 -99999999.9 -99999999.9 2022-10-07T03:24:23 2022-09-25 \N 0 8 yKMiAntORoRa8svnMfcxlOPwwND1m5s2fdS26Xu6cfs6HK5SAibqIp9h8sZcpjHy4 {"team":"Manchester United","players":["Ronaldo","Rooney","Giggs"],"coach":"Ole Gunnar Solskjaer"} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N
|
||||
41 2023-08-27 true -104 22750 \N 8527773271030840740 5554497317268279215 -5296.828 -1.71564688801304E9 -99999999.9 99999999.9 2022-12-02T17:56:44 2022-10-12 2023-02-19T07:02:54 V \N E9GzQdTwX1ITUQz27IVznAs6Ca4WwprKk6Odjs6SH75D2F1089QiY3HQ52LXRD1V6xAWjhLE2hWgW3EdHuAOnUDVrb5V {"food":"Sushi","price":10,"restaurant":"Sushi King"} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N
|
||||
49 2023-08-08 false \N 16275 -2144851675 -2303421957908954634 -46526938720058765 -13141.143 -6.866322332302E8 99999999.9 -99999999.9 2022-09-01T00:16:01 2023-03-25 2022-09-07T14:59:03 s yvuILR2iNxfe8RRml {"student":true,"name":"Alice","grade":9,"subjects":["math","science","history"]} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N
|
||||
50 2023-08-06 true 109 -6330 1479023892 -8630800697573159428 -1645095773540208759 17880.96 -1.453844792013949E9 -99999999.9 -99999999.9 2022-09-22T02:03:21 2023-05-14 2023-03-25T02:18:34 m JKnIgXvGVidGiWl9YRSi3mFI7wHKt1sBpWSadKF8VX3LAuElm4sdc9gtxREaUr57oikSYlU8We8h1MWqQlYNiJObl {"city":"Tokyo","temperature":20.5,"humidity":75} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N
|
||||
50 2023-08-24 true 15 14403 \N -6418906115745394180 9205303779366462513 -4331.549 -6.15112179557648E8 99999999.9 -99999999.9 2022-12-29T02:27:20 2023-06-01 2023-08-12T04:50:04 a eCl38sztIvBQvGvGKyYZmyMXy9vIJx197iu3JwP9doJGcrYUl9Uova0rz4iCCgrjlAiZU18Fs9YtCq830nhM {"band":"The Beatles","members":["John Lennon","Paul McCartney","George Harrison","Ringo Starr"]} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N
|
||||
57 2023-08-19 true 2 -25462 -74112029 6458082754318544493 -7910671781690629051 -15205.859 -3.06870797484914E8 99999999.9 -99999999.9 2023-07-10T18:39:10 2023-02-12 2023-01-27T07:26:06 y Xi9nDVrLv8m6AwEpUxmtzFAuK48sQ {"name":"John","age":25,"city":"New York"} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N
|
||||
58 2023-08-22 \N 0 -18231 1832867360 6997858407575297145 2480714305422728023 -5450.489 1.475901032138386E9 -99999999.9 -99999999.9 2023-02-02T05:13:24 2022-09-18 2023-04-23T10:51:15 k LdFXF7Kmfzgmnn2R6zLsXdmi3A2cLBLq4G4WDVNDhxvH7dYH8Kga2WA47uSIxp6NSrwPSdw0ssB1TS8RFJTDJAB0Uba3e05NL2Aiw0ja {"restaurant":"Pizza Hut","menu":["pizza","pasta","salad"]} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N
|
||||
60 2023-08-27 false -52 -2338 -757056972 1047567408607120856 6541476642780646552 6614.0894 -1.204448798517855E9 99999999.9 99999999.9 2022-12-29T14:47:30 2022-09-24 2023-08-01T12:41:59 O F RM4F1Ke7lkcnuxF2nK0j9VBW3MDcgyHR4pseBjtFnqS6GUkVFuzF6u3Cp9Nv7ab0O6UYrpP4DhU {"game":"Chess","players":2,"time":"1 hour"} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N
|
||||
62 2023-08-21 false 81 20302 -200761532 6365479976421007608 \N -29916.533 1.709141750828478E9 99999999.9 -99999999.9 2023-05-04T01:14:51 2022-09-17 2022-12-04T19:30:09 d v BKWy9dTNg1aZW7ancEJAmEDOPK5TwFsNSHbI78emu9gymeIlx5NoLmyii0QAqdzRvSQPZKiqKkwInGCTIBnK1yYkK7zD {"username":"user123","password":"pass123","email":"user123@example.com"} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N
|
||||
65 2023-08-09 false 94 31514 814994517 -297697460695940343 734910652450318597 -13061.892 6.2750847041706E7 -9808654.0 \N 2023-08-14T22:01:27 2023-05-19 2022-11-13T13:44:28 V aGeMsI24O12chGlP5ak0AHghAz7bu5MargJBStHnt0yMnChH0JnfYhsfH1u59XIHkJKMsHYktBqORkGlovu8V47E74KeFpaqxn5yLyXfDbhhzUKf {"language":"Python","version":3.9,"frameworks":["Django","Flask"]} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N
|
||||
66 2023-08-15 true -91 28378 609923317 4872185586197131212 1207709464099378591 \N -1.863683325985123E9 -99999999.9 -99999999.9 2022-09-24T10:39:23 2022-09-24 2022-10-16T18:36:43 Y z AI1BSPQdKiHJiQH1kguyLSWsDXkC7zwy7PwgWnyGSaa9tBKRex8vHBdxg2QSKZKL2mV2lHz7iI1PnsTd4MXDcIKhqiHyPuQPt2tEtgt0UgF6 {"book":{"title":"The Great Gatsby","author":"F. Scott Fitzgerald"},"year":1925} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N
|
||||
68 2023-08-23 true -73 20117 1737338128 795638676048937749 -5551546237562433901 -30627.04 6.8589475684545E7 99999999.9 99999999.9 2022-12-28T20:26:51 2022-10-04 2023-07-30T00:20:06 y keZ3JlWWpdnPBejf0cuiCQCVBBTd5gjvO08NVdcAFewqL7nRT4N9lnvSU6pWmletA5VbPQCeQapJdcnQCHfZUDCf4ulCnczyqr7SGrbGRT0XYcd7iktKM {"country":"Brazil","continent":"South America","population":211049527} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N
|
||||
80 2023-08-18 false -18 -8971 679027874 6535956962935330265 3960889045799757165 -13219.76 1.187161924505394E9 -99999999.9 -99999999.9 2023-03-11T07:40 2022-11-29 2023-01-14T07:24:07 \N D 3Nhx6xX1qdwaq7lxwLRSKMtJFbC03swWv12mpySSVysH3igGZTiGPuKMsYW7HAkf6CWc7c0nzqDsjuH3FYVMNCWRmfxMrmY8rykQCC4Ve {"car":"BMW","model":"X5","year":2020,"color":"black"} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N
|
||||
81 2023-08-23 false 106 11492 -667795397 4480250461471356146 -5346660566234294101 9082.75 3.85167225902608E8 -99999999.9 99999999.9 2023-03-20T03:33:16 2022-11-24 2023-02-16T18:29:41 G 9 Lk3eNVQNjucbekD1rZmUlGPiXS5JvcWr2LQzRU8GSGIbSag {"flower":"rose","color":"red","fragrance":true} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N
|
||||
85 2023-08-11 true -7 24304 -2043877415 -2024144417867729183 \N 5363.0244 -5.78615669042831E8 -99999999.9 -99999999.9 2023-07-15T01:07:41 2023-08-13 2023-01-20T11:57:48 i WQ9dh9ajPu0y {"country":"France","capital":"Paris","population":67081000} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N
|
||||
90 2023-08-27 true 22 16456 -1476824962 -3279894870153540825 8990195191470116763 26651.906 2.06860148942546E8 -99999999.9 -99999999.9 2022-10-07T03:11:03 2023-03-18 2023-04-15T00:38:33 T L QW0GQ3GoMtHgxPQOWGfVaveynahNpsNs09siMFA1OtO6QEDBQTdivmGyq7bFzejAqwbbVQQpREAmeLjcFSXLnQuou2KbwYD {"company":"Apple","products":[{"name":"iPhone","price":1000},{"name":"MacBook","price":1500}]} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N
|
||||
91 2023-08-27 true 90 2465 702240964 6373830997821598984 305860046137409400 15991.356 1.599972327386147E9 -99999999.9 \N 2023-04-26T19:31:10 2023-07-21 \N 2 B7YKYBYT8w0YC926bZ8Yz1VzyiWw2NWDAiTlEoPVyz9AXGti2Npg1FxWqWk4hEaALw0ZBSuiAIPj41lq36g5QRpPmAjNPK {"fruit":"apple","color":"red","qty":5,"price":2.5} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N
|
||||
|
||||
@ -0,0 +1,52 @@
|
||||
CREATE TABLE routine_load_uniq_tbl_basic_sequence
|
||||
(
|
||||
k00 INT NOT NULL,
|
||||
k01 DATE NOT NULL,
|
||||
k02 BOOLEAN NULL,
|
||||
k03 TINYINT NULL,
|
||||
k04 SMALLINT NULL,
|
||||
k05 INT NULL,
|
||||
k06 BIGINT NULL,
|
||||
k07 LARGEINT NULL,
|
||||
k08 FLOAT NULL,
|
||||
k09 DOUBLE NULL,
|
||||
k10 DECIMAL(9,1) NULL,
|
||||
k11 DECIMALV3(9,1) NULL,
|
||||
k12 DATETIME NULL,
|
||||
k13 DATEV2 NULL,
|
||||
k14 DATETIMEV2 NULL,
|
||||
k15 CHAR NULL,
|
||||
k16 VARCHAR NULL,
|
||||
k17 STRING NULL,
|
||||
k18 JSON NULL,
|
||||
kd01 BOOLEAN NOT NULL DEFAULT "TRUE",
|
||||
kd02 TINYINT NOT NULL DEFAULT "1",
|
||||
kd03 SMALLINT NOT NULL DEFAULT "2",
|
||||
kd04 INT NOT NULL DEFAULT "3",
|
||||
kd05 BIGINT NOT NULL DEFAULT "4",
|
||||
kd06 LARGEINT NOT NULL DEFAULT "5",
|
||||
kd07 FLOAT NOT NULL DEFAULT "6.0",
|
||||
kd08 DOUBLE NOT NULL DEFAULT "7.0",
|
||||
kd09 DECIMAL NOT NULL DEFAULT "888888888",
|
||||
kd10 DECIMALV3 NOT NULL DEFAULT "999999999",
|
||||
kd11 DATE NOT NULL DEFAULT "2023-08-24",
|
||||
kd12 DATETIME NOT NULL DEFAULT "2023-08-24 12:00:00",
|
||||
kd13 DATEV2 NOT NULL DEFAULT "2023-08-24",
|
||||
kd14 DATETIMEV2 NOT NULL DEFAULT "2023-08-24 12:00:00",
|
||||
kd15 CHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
|
||||
kd16 VARCHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
|
||||
kd17 STRING NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
|
||||
kd18 JSON NULL,
|
||||
|
||||
INDEX idx_bitmap_k104 (`k02`) USING BITMAP,
|
||||
INDEX idx_bitmap_k110 (`kd01`) USING BITMAP,
|
||||
INDEX idx_bitmap_k113 (`k13`) USING BITMAP,
|
||||
INDEX idx_bitmap_k114 (`k14`) USING BITMAP,
|
||||
INDEX idx_bitmap_k117 (`k17`) USING BITMAP
|
||||
)
|
||||
UNIQUE KEY(k00,k01)
|
||||
DISTRIBUTED BY HASH(k00) BUCKETS 32
|
||||
PROPERTIES (
|
||||
"function_column.sequence_col" = 'k00',
|
||||
"replication_num" = "1"
|
||||
);
|
||||
@ -0,0 +1 @@
|
||||
DROP TABLE IF EXISTS routine_load_uniq_tbl_basic_sequence;
|
||||
@ -2200,4 +2200,122 @@ suite("test_routine_load","p0") {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sequence
|
||||
if (enabled != null && enabled.equalsIgnoreCase("true")) {
|
||||
try {
|
||||
sql new File("""${context.file.parent}/ddl/uniq_tbl_basic_drop_sequence.sql""").text
|
||||
sql new File("""${context.file.parent}/ddl/uniq_tbl_basic_create_sequence.sql""").text
|
||||
|
||||
def name = "routine_load_uniq_tbl_basic_sequence"
|
||||
def job = "sequence_job"
|
||||
sql """
|
||||
CREATE ROUTINE LOAD ${job} ON ${name}
|
||||
COLUMNS(${columns[0]}),
|
||||
COLUMNS TERMINATED BY "|"
|
||||
PROPERTIES
|
||||
(
|
||||
"max_batch_interval" = "5",
|
||||
"max_batch_rows" = "300000",
|
||||
"max_batch_size" = "209715200"
|
||||
)
|
||||
FROM KAFKA
|
||||
(
|
||||
"kafka_broker_list" = "${externalEnvIp}:${kafka_port}",
|
||||
"kafka_topic" = "${topics[0]}",
|
||||
"property.kafka_default_offsets" = "OFFSET_BEGINNING"
|
||||
);
|
||||
"""
|
||||
sql "sync"
|
||||
|
||||
while (true) {
|
||||
sleep(1000)
|
||||
def res = sql "show routine load for ${job}"
|
||||
def state = res[0][8].toString()
|
||||
if (state == "NEED_SCHEDULE") {
|
||||
continue;
|
||||
}
|
||||
log.info("reason of state changed: ${res[0][17].toString()}".toString())
|
||||
assertEquals(res[0][8].toString(), "RUNNING")
|
||||
break;
|
||||
}
|
||||
|
||||
def count = 0
|
||||
while (true) {
|
||||
def res = sql "select count(*) from ${name}"
|
||||
def state = sql "show routine load for ${job}"
|
||||
log.info("routine load state: ${state[0][8].toString()}".toString())
|
||||
log.info("routine load statistic: ${state[0][14].toString()}".toString())
|
||||
log.info("reason of state changed: ${state[0][17].toString()}".toString())
|
||||
if (res[0][0] > 0) {
|
||||
break
|
||||
}
|
||||
if (count >= 120) {
|
||||
log.error("routine load can not visible for long time")
|
||||
assertEquals(20, res[0][0])
|
||||
break
|
||||
}
|
||||
sleep(5000)
|
||||
count++
|
||||
}
|
||||
qt_sql_squence "select * from routine_load_uniq_tbl_basic_sequence order by k00,k01"
|
||||
sql "stop routine load for ${job}"
|
||||
} finally {
|
||||
sql new File("""${context.file.parent}/ddl/uniq_tbl_basic_drop_sequence.sql""").text
|
||||
}
|
||||
}
|
||||
|
||||
// error command
|
||||
i = 0
|
||||
if (enabled != null && enabled.equalsIgnoreCase("true")) {
|
||||
try {
|
||||
for (String tableName in tables) {
|
||||
sql new File("""${context.file.parent}/ddl/${tableName}_drop.sql""").text
|
||||
sql new File("""${context.file.parent}/ddl/${tableName}_create.sql""").text
|
||||
|
||||
def name = "routine_load_" + tableName
|
||||
sql """
|
||||
CREATE ROUTINE LOAD ${jobs[i]} ON ${name}
|
||||
COLUMNS(${columns[i]},k30),
|
||||
COLUMNS TERMINATED BY "|"
|
||||
PROPERTIES
|
||||
(
|
||||
"max_batch_interval" = "5",
|
||||
"max_batch_rows" = "300000",
|
||||
"max_batch_size" = "209715200"
|
||||
)
|
||||
FROM KAFKA
|
||||
(
|
||||
"kafka_broker_list" = "${externalEnvIp}:${kafka_port}",
|
||||
"kafka_topic" = "${topics[i]}",
|
||||
"property.kafka_default_offsets" = "OFFSET_BEGINNING"
|
||||
);
|
||||
"""
|
||||
sql "sync"
|
||||
i++
|
||||
}
|
||||
|
||||
i = 0
|
||||
for (String tableName in tables) {
|
||||
while (true) {
|
||||
sleep(1000)
|
||||
def res = sql "show routine load for ${jobs[i]}"
|
||||
def state = res[0][8].toString()
|
||||
if (state != "PAUSED") {
|
||||
continue;
|
||||
}
|
||||
log.info("reason of state changed: ${res[0][17].toString()}".toString())
|
||||
assertEquals(res[0][17].toString(), "ErrorReason{code=errCode = 102, msg='current error rows is more than max_error_number or the max_filter_ratio is more than the value set'}")
|
||||
break;
|
||||
}
|
||||
|
||||
sql "stop routine load for ${jobs[i]}"
|
||||
i++
|
||||
}
|
||||
} finally {
|
||||
for (String tableName in tables) {
|
||||
sql new File("""${context.file.parent}/ddl/${tableName}_drop.sql""").text
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user