From 742dc796b28bd3c649298a9dbdbdad445cddc8ae Mon Sep 17 00:00:00 2001 From: lichaoyong Date: Thu, 10 Jan 2019 18:51:12 +0800 Subject: [PATCH] Fix inconsistency of three replicas belongs to one tablet (#523) There are A, B, C replicas of one tablet. A has 0 - 10 version. B has 0 - 5, 6, 7, 9, 10 version. 1. B has missed versions, so it clones 0 - 10 from A, and remove overlapped versions in its header. 2. Coincidentally, 6 is a version for delete predicate (delete where day = 20181221). When removing overlapped versions, version 6 is removed but delete predicate is not be removed. 3. Unfortunately, 0-10 cloned from A has data indicated at 20181221. 4. B performs compaction, and data generated by 20181221 is be removed falsely. --- be/src/olap/olap_header.cpp | 42 ++++++++++++++++++ be/src/olap/olap_header.h | 2 + be/src/olap/olap_table.cpp | 3 ++ be/src/olap/olap_table.h | 14 +----- .../download/1/1.1234.so.tmp | Bin 0 -> 6343 bytes 5 files changed, 48 insertions(+), 13 deletions(-) create mode 100644 be/test/runtime/test_data/user_function_cache/download/1/1.1234.so.tmp diff --git a/be/src/olap/olap_header.cpp b/be/src/olap/olap_header.cpp index da79c039f4..f8d6d43939 100644 --- a/be/src/olap/olap_header.cpp +++ b/be/src/olap/olap_header.cpp @@ -460,6 +460,48 @@ void OLAPHeader::add_delete_condition(const DeleteConditionMessage& delete_condi LOG(INFO) << "add delete condition. version=" << version; } +void OLAPHeader::delete_cond_by_version(const Version& version) { + DCHECK(version.first == version.second); + google::protobuf::RepeatedPtrField* delete_conditions + = mutable_delete_data_conditions(); + int index = 0; + for (; index < delete_conditions->size(); ++index) { + const DeleteConditionMessage& temp = delete_conditions->Get(index); + if (temp.version() == version.first) { + // log delete condtion + string del_cond_str; + const RepeatedPtrField& sub_conditions = temp.sub_conditions(); + + for (int i = 0; i != sub_conditions.size(); ++i) { + del_cond_str += sub_conditions.Get(i) + ";"; + } + + LOG(INFO) << "delete one condition. version=" << temp.version() + << ", condition=" << del_cond_str; + + // remove delete condition from PB + delete_conditions->SwapElements(index, delete_conditions->size() - 1); + delete_conditions->RemoveLast(); + } + } +} + +bool OLAPHeader::is_delete_data_version(Version version) { + if (version.first != version.second) { + return false; + } + + google::protobuf::RepeatedPtrField::const_iterator it; + it = delete_data_conditions().begin(); + for (; it != delete_data_conditions().end(); ++it) { + if (it->version() == version.first) { + return true; + } + } + + return false; +} + const PPendingDelta* OLAPHeader::get_pending_delta(int64_t transaction_id) const { for (int i = 0; i < pending_delta_size(); i++) { if (pending_delta(i).transaction_id() == transaction_id) { diff --git a/be/src/olap/olap_header.h b/be/src/olap/olap_header.h index f29bffc1bb..23f97efab5 100644 --- a/be/src/olap/olap_header.h +++ b/be/src/olap/olap_header.h @@ -79,6 +79,8 @@ public: bool empty, const std::vector* column_statistics); void add_delete_condition(const DeleteConditionMessage& delete_condition, int64_t version); + void delete_cond_by_version(const Version& version); + bool is_delete_data_version(Version version); const PPendingDelta* get_pending_delta(int64_t transaction_id) const; const PPendingSegmentGroup* get_pending_segment_group(int64_t transaction_id, int32_t pending_segment_group_id) const; diff --git a/be/src/olap/olap_table.cpp b/be/src/olap/olap_table.cpp index 0ff9487557..ecdd944873 100644 --- a/be/src/olap/olap_table.cpp +++ b/be/src/olap/olap_table.cpp @@ -1307,6 +1307,9 @@ OLAPStatus OLAPTable::clone_data(const OLAPHeader& clone_header, << " version=" << version.first << "-" << version.second << "]"; break; } + if (new_local_header.is_delete_data_version(version)) { + new_local_header.delete_cond_by_version(version); + } LOG(INFO) << "delete version from new local header when clone. [table='" << full_name() << "', version=" << version.first << "-" << version.second << "]"; } diff --git a/be/src/olap/olap_table.h b/be/src/olap/olap_table.h index ec2f4f5296..b1621957a0 100644 --- a/be/src/olap/olap_table.h +++ b/be/src/olap/olap_table.h @@ -528,19 +528,7 @@ public: } bool is_delete_data_version(Version version) { - if (version.first != version.second) { - return false; - } - - google::protobuf::RepeatedPtrField::const_iterator it; - it = _header->delete_data_conditions().begin(); - for (; it != _header->delete_data_conditions().end(); ++it) { - if (it->version() == version.first) { - return true; - } - } - - return false; + return _header->is_delete_data_version(version); } bool is_load_delete_version(Version version); diff --git a/be/test/runtime/test_data/user_function_cache/download/1/1.1234.so.tmp b/be/test/runtime/test_data/user_function_cache/download/1/1.1234.so.tmp new file mode 100644 index 0000000000000000000000000000000000000000..64cb35c2ad6e04e81f686e5fb59437d2d17a9b86 GIT binary patch literal 6343 zcmb<-^>JfjWMqH=W(GS35N`u3M8p9?F-UMj84L^z4h$9ytPBneQVil?wNRx91xTzX z><|u&W?)bN3xMokV_;x-0hNEj4iSgZ4oqNW3;|FYrVqpg=@a09$O~{l^ug!@AcGkg z7+^G18AHI+k`xvO1_leLedx3b#5e{R4N?md3Vd3U0qNIk)&jBs4oSp=VG`4ax?=21C(O0z@a{ffq_8@$y^xoFaraF z0D~xl1{bPp&qKv6(8L>{_TGb92&EYIfYpm&m?HpIj~PA;@$u=o`FZihC5c5P@$n4t znR%Hd4Do51d6^9Ho*};RDXB%N>6yhPsYM~a&N=ycsUe9;IjImi>@xAm6^S4N5_2-E zQW@gC%Ho5dCOGFL78j=$GsH)kAZ{+4 zm7bg&Uku_Ig2ad0^HLa8{DVB*J^iepw0=%z5;EJ& z1kTgbLy?4V%}lKH^9xG!lk;;6GILUk^wX1*bxrgv^o+oY@yMB(Fu418IyuK1=^5#n z!lD*jHh|J069Xd(+yss*h_X_tOil&{CWZizLQvX)#y2P*fK(_z`4>1L=>rtsu)G73 zzkwtU$}=D_5PpCpj+`}KAc-R<^$$qmFuOtK2te%zsYSL|0!bWMy+`vK4v%iuQWgaU zkJbYvtS_`dg`6291H+3(Mg|6t<|7=SBo8tgjQ^Wfu_!S7SFK@DVBnW`VEC^J;%9*5 zUOxE$|Nnp0A{GUP3{dF2yzu}3e~(_)3Xs{owplC+451#HKRi00dNlvwD`$#5?9t82 z1Cne#Q1U7E{{xVD9=)uNAfaAcDbB7#KV{pE`yFm+*LW+umnZV0aM)c9raAMg<0+ zUezUx3Jf0ItXG*87<@DzdmQ}D?9u$1(Zljau>!xm3n%~%FnV;dZe>L8`92TG(uJ$h}u zKsI$g{eL0$rTG8<|M}%XMFYc2ZZMl6)T8rh>|qF*rpGVe0t!)vvAOA}~g5oPW_Hfz+klGjT|NZ|D5;@Gkz$ndZ z4XP_ZY0l%{|No$Ra>4)q|3NXg?EnA&9bo$y7(}2nV^t6XV}$^tG!Hw+1V#pMJp(FR z8UFqMZvaxmC*Z~>;lB5rK}iXOL2M8Pu|XKbM#i9o2I6;v2nGgl9tJUIfe0l25;Q(2 zPlJT`K?DN>!xbnEtG{6N6R17}Nx|wFSp63PwHTIHe?T)E*v||M3@f1Wpt=;KKL*MN z)sY~+1c+c@U;xDji1`DWpP|MufWw#p5g(xZ3{oc!B0%L1)MA+U1gOJ6VGL3-0Yo6x z<1qXF{r{g2k}80z{}1IuoyYJ2$`66c|Ag{k@*uqffnn_>bi1cP!(kbehLykW&dyc} z8t#6fnhKyQ!I%L}z=(lCuedU|Br%CWuehWLLTA8O8HvRi40mzoiuR+N~V8lRC; z!~o|p=p~or7Zo$;r9h~xnZe;5s`FG1-Yqz0xRmM%4+ zc>yE`vI>@;U^FcMfaE|JJv|ySFff49GE6@#{laKaxdYM%lSilR7#KiBTRu!2mOo%L zq&A1-SCHK>_k+v?VP6IY22fbQ^uzKaoCYak0JR(-448hH|HIMrD}W*cDWAj21CaTk zwmG_fP&*mqE?D^kE5BegsO*Q>3?V^og|HxG4799->2HAA52IoBLsWy>zz`Owy$WKb zqv?nB$6#~?$Z;TrPz=)t4%lKFdEc1fvE-2=Ia1nNFG;z2_y%?==vMc z^d~?IRu~Pbk3l*i7#98z9s>iY{sDOrl)gb?uzn$|zliQ%bp4Z{_QUl5fc7tcK;1z| z|4cOf9MFCy2h<`c1#%h~qx0uM`$RDNVdX8X-wNxW!sKE4VeVRvWGVfBClv;v0N19Jg7e>VdI14sGcYhj zK=ZRC0|SFD)UP0USos4AW|-eW;xPOH)X`>OV7Lyl049jVek3UjCNl#g1FRlFmSkpN zg13`U#hDpk4 z7-0Dx#D`&K20;c`eujyGXl4c>23S4?@nM*mL6`xS|6pPunwdd_0hX^od>CfNG9Chw zV`hLR9uNnLLCJ`P;p2ZO7fylN-HcfLD+Nw#nEnNoCyZG93+l}?F<{1v7r5@_Vt4?} z2QZ6cq2da#iVrFQsZS8&JRo<2q?*9)5oIud_NPH*21p##-Uivb0L>hbK5YJ-12%_` z;RCds2blrFE5YKt3=g2?K1d9Nw}Zto;}_I92if}p&EA{f_5@})gUrI_p2uMGG2{Iw zSR8Ylij@)6*TPH>LSS)z1_o&T1F{c<<-y{Z=}?0a`}m$Y4smZr?BjlcIMnBY)nktP zf!ZUmHXX=)Fg%G7`?%jaMo^mvkse^`cR|I`$Nlz0?L{B=I|Wvc8J-ux>amnR$wei_ zC8cR;ddUp&@y;RsLGeDG!6Bd#z<3uFo|iLR3NaQKUy>W23>pnAW`GR>A_>LEr{u?{ z=j10P=EQ^h9`T8#6%3&9w4BtE)D%54V`C!*kVJfDN(G2#!T{;Mq?G37R>Iul>gNJ> zj$4qUuWLMv2XYRKjj#pW(E-~68CcXyPG*RA_wjdf^ojR(a|?D2i4Spf@^Ou4h>s8P zb%vXkQk);3k(ieP8qtLK8)~*oq@Sa&r!&UzD7Mi|a32isU@6jgCumF+7G$U+swm^8 Y4i3m8i5NqikP%eyuqkX56x=HW0FGqfr2qf` literal 0 HcmV?d00001