diff --git a/be/src/olap/push_handler.cpp b/be/src/olap/push_handler.cpp index b99e1f4952..9b2c7149ed 100644 --- a/be/src/olap/push_handler.cpp +++ b/be/src/olap/push_handler.cpp @@ -240,7 +240,7 @@ Status PushHandler::_convert_v2(TabletSharedPtr cur_tablet, RowsetSharedPtr* cur uint8_t* tuple_buf = reader->mem_pool()->allocate(schema->schema_size()); ContiguousRow row(schema.get(), tuple_buf); - // 4. Read data from broker and write into SegmentGroup of cur_tablet + // 4. Read data from broker and write into cur_tablet // Convert from raw to delta VLOG_NOTICE << "start to convert etl file to delta."; while (!reader->eof()) { @@ -368,7 +368,7 @@ Status PushHandler::_convert(TabletSharedPtr cur_tablet, RowsetSharedPtr* cur_ro break; } - // 5. Read data from raw file and write into SegmentGroup of cur_tablet + // 5. Read data from raw file and write into cur_tablet if (_request.__isset.http_file_path) { // Convert from raw to delta VLOG_NOTICE << "start to convert row file to delta."; diff --git a/be/src/olap/push_handler.h b/be/src/olap/push_handler.h index 02384e9f2d..1d8536f85e 100644 --- a/be/src/olap/push_handler.h +++ b/be/src/olap/push_handler.h @@ -58,7 +58,7 @@ private: Status _convert_v2(TabletSharedPtr cur_tablet, RowsetSharedPtr* cur_rowset, TabletSchemaSPtr tablet_schema); // Convert local data file to internal formatted delta, - // return new delta's SegmentGroup + // return new delta's rowset Status _convert(TabletSharedPtr cur_tablet, RowsetSharedPtr* cur_rowset, TabletSchemaSPtr tablet_schema); diff --git a/be/src/olap/rowset/segment_v2/segment.h b/be/src/olap/rowset/segment_v2/segment.h index b0dea5aeb8..b5e54dfa4f 100644 --- a/be/src/olap/rowset/segment_v2/segment.h +++ b/be/src/olap/rowset/segment_v2/segment.h @@ -37,7 +37,6 @@ namespace doris { -class SegmentGroup; class TabletSchema; class ShortKeyIndexDecoder; class Schema; diff --git a/be/test/olap/delete_handler_test.cpp b/be/test/olap/delete_handler_test.cpp index d47efff4e3..eab780df86 100644 --- a/be/test/olap/delete_handler_test.cpp +++ b/be/test/olap/delete_handler_test.cpp @@ -832,35 +832,7 @@ protected: "hi": -5350970832824939812, "lo": -6717994719194512122 }, - "creation_time": 1553765670, - "alpha_rowset_extra_meta_pb": { - "segment_groups": [ - { - "segment_group_id": 0, - "num_segments": 2, - "index_size": 132, - "data_size": 576, - "num_rows": 5, - "zone_maps": [ - { - "min": "MQ==", - "max": "NQ==", - "null_flag": false - }, - { - "min": "MQ==", - "max": "Mw==", - "null_flag": false - }, - { - "min": "J2J1c2gn", - "max": "J3RvbSc=", - "null_flag": false - } - ], - "empty": false - }] - } + "creation_time": 1553765670 })"; } diff --git a/be/test/olap/ordered_data_compaction_test.cpp b/be/test/olap/ordered_data_compaction_test.cpp index ad6813c657..1cd6c17212 100644 --- a/be/test/olap/ordered_data_compaction_test.cpp +++ b/be/test/olap/ordered_data_compaction_test.cpp @@ -258,35 +258,7 @@ protected: "hi": -5350970832824939812, "lo": -6717994719194512122 }, - "creation_time": 1553765670, - "alpha_rowset_extra_meta_pb": { - "segment_groups": [ - { - "segment_group_id": 0, - "num_segments": 2, - "index_size": 132, - "data_size": 576, - "num_rows": 5, - "zone_maps": [ - { - "min": "MQ==", - "max": "NQ==", - "null_flag": false - }, - { - "min": "MQ==", - "max": "Mw==", - "null_flag": false - }, - { - "min": "J2J1c2gn", - "max": "J3RvbSc=", - "null_flag": false - } - ], - "empty": false - }] - } + "creation_time": 1553765670 })"; pb1->init_from_json(json_rowset_meta); pb1->set_start_version(start); diff --git a/be/test/olap/rowid_conversion_test.cpp b/be/test/olap/rowid_conversion_test.cpp index 66d61044aa..3d0852fd22 100644 --- a/be/test/olap/rowid_conversion_test.cpp +++ b/be/test/olap/rowid_conversion_test.cpp @@ -218,35 +218,7 @@ protected: "hi": -5350970832824939812, "lo": -6717994719194512122 }, - "creation_time": 1553765670, - "alpha_rowset_extra_meta_pb": { - "segment_groups": [ - { - "segment_group_id": 0, - "num_segments": 2, - "index_size": 132, - "data_size": 576, - "num_rows": 5, - "zone_maps": [ - { - "min": "MQ==", - "max": "NQ==", - "null_flag": false - }, - { - "min": "MQ==", - "max": "Mw==", - "null_flag": false - }, - { - "min": "J2J1c2gn", - "max": "J3RvbSc=", - "null_flag": false - } - ], - "empty": false - }] - } + "creation_time": 1553765670 })"; pb1->init_from_json(json_rowset_meta); pb1->set_start_version(start); diff --git a/be/test/olap/tablet_test.cpp b/be/test/olap/tablet_test.cpp index 152aa00c3e..b1f949489f 100644 --- a/be/test/olap/tablet_test.cpp +++ b/be/test/olap/tablet_test.cpp @@ -63,35 +63,7 @@ public: "hi": -5350970832824939812, "lo": -6717994719194512122 }, - "creation_time": 1553765670, - "alpha_rowset_extra_meta_pb": { - "segment_groups": [ - { - "segment_group_id": 0, - "num_segments": 2, - "index_size": 132, - "data_size": 576, - "num_rows": 5, - "zone_maps": [ - { - "min": "MQ==", - "max": "NQ==", - "null_flag": false - }, - { - "min": "MQ==", - "max": "Mw==", - "null_flag": false - }, - { - "min": "J2J1c2gn", - "max": "J3RvbSc=", - "null_flag": false - } - ], - "empty": false - }] - } + "creation_time": 1553765670 })"; doris::EngineOptions options; diff --git a/be/test/olap/test_data/header_without_inc_rs.txt b/be/test/olap/test_data/header_without_inc_rs.txt index 44c191d3fc..a2beb4bb50 100644 --- a/be/test/olap/test_data/header_without_inc_rs.txt +++ b/be/test/olap/test_data/header_without_inc_rs.txt @@ -73,19 +73,7 @@ "index_disk_size": 0, "empty": true, "creation_time": 1553765664, - "num_segments": 0, - "alpha_rowset_extra_meta_pb": { - "segment_groups": [ - { - "segment_group_id": 0, - "num_segments": 0, - "index_size": 0, - "data_size": 0, - "num_rows": 0, - "empty": true - } - ] - } + "num_segments": 0 }, { "rowset_id": 540081, @@ -106,36 +94,7 @@ "lo": -6717994719194512122 }, "creation_time": 1553765670, - "num_segments": 1, - "alpha_rowset_extra_meta_pb": { - "segment_groups": [ - { - "segment_group_id": 0, - "num_segments": 1, - "index_size": 132, - "data_size": 576, - "num_rows": 5, - "zone_maps": [ - { - "min": "MQ==", - "max": "NQ==", - "null_flag": false - }, - { - "min": "MQ==", - "max": "Mw==", - "null_flag": false - }, - { - "min": "J2J1c2gn", - "max": "J3RvbSc=", - "null_flag": false - } - ], - "empty": false - } - ] - } + "num_segments": 1 } ], "in_restore_mode": false, diff --git a/be/test/olap/timestamped_version_tracker_test.cpp b/be/test/olap/timestamped_version_tracker_test.cpp index 4ca1013c40..578349e794 100644 --- a/be/test/olap/timestamped_version_tracker_test.cpp +++ b/be/test/olap/timestamped_version_tracker_test.cpp @@ -51,61 +51,7 @@ public: "hi": -5350970832824939812, "lo": -6717994719194512122 }, - "creation_time": 1553765670, - "alpha_rowset_extra_meta_pb": { - "segment_groups": [ - { - "segment_group_id": 0, - "num_segments": 1, - "index_size": 132, - "data_size": 576, - "num_rows": 5, - "zone_maps": [ - { - "min": "MQ==", - "max": "NQ==", - "null_flag": false - }, - { - "min": "MQ==", - "max": "Mw==", - "null_flag": false - }, - { - "min": "J2J1c2gn", - "max": "J3RvbSc=", - "null_flag": false - } - ], - "empty": false - }, - { - "segment_group_id": 1, - "num_segments": 1, - "index_size": 132, - "data_size": 576, - "num_rows": 5, - "zone_maps": [ - { - "min": "MQ==", - "max": "NQ==", - "null_flag": false - }, - { - "min": "MQ==", - "max": "Mw==", - "null_flag": false - }, - { - "min": "J2J1c2gn", - "max": "J3RvbSc=", - "null_flag": false - } - ], - "empty": false - } - ] - } + "creation_time": 1553765670 })"; } void TearDown() override {} diff --git a/be/test/vec/olap/vertical_compaction_test.cpp b/be/test/vec/olap/vertical_compaction_test.cpp index 5c952aa786..09d7d4656d 100644 --- a/be/test/vec/olap/vertical_compaction_test.cpp +++ b/be/test/vec/olap/vertical_compaction_test.cpp @@ -252,35 +252,7 @@ protected: "hi": -5350970832824939812, "lo": -6717994719194512122 }, - "creation_time": 1553765670, - "alpha_rowset_extra_meta_pb": { - "segment_groups": [ - { - "segment_group_id": 0, - "num_segments": 2, - "index_size": 132, - "data_size": 576, - "num_rows": 5, - "zone_maps": [ - { - "min": "MQ==", - "max": "NQ==", - "null_flag": false - }, - { - "min": "MQ==", - "max": "Mw==", - "null_flag": false - }, - { - "min": "J2J1c2gn", - "max": "J3RvbSc=", - "null_flag": false - } - ], - "empty": false - }] - } + "creation_time": 1553765670 })"; pb1->init_from_json(json_rowset_meta); pb1->set_start_version(start); diff --git a/docs/images/cpu-flame-demo.svg b/docs/images/cpu-flame-demo.svg index f08a8a7605..fd2d435a08 100644 --- a/docs/images/cpu-flame-demo.svg +++ b/docs/images/cpu-flame-demo.svg @@ -1441,10 +1441,6 @@ -doris::SegmentGroup::find_short_key (86 samples, 0.06%) - - - doris::ReadOnlyFileStream::_assure_data (46 samples, 0.03%) @@ -1489,18 +1485,10 @@ -doris::SegmentGroup::add_short_key (362 samples, 0.23%) - - - doris::ColumnData::_seek_to_row (41,249 samples, 26.61%) doris::ColumnData::_seek_to_row -doris::SegmentGroup::add_short_key (100 samples, 0.06%) - - - doris::ReadOnlyFileStream::_assure_data (178 samples, 0.11%) diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index 30b127fc78..3e20ff8345 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -109,8 +109,8 @@ message RowsetMetaPB { repeated KeyBoundsPB segments_key_bounds = 27; // tablet meta pb, for compaction optional TabletSchemaPB tablet_schema = 28; - // spare field id for future use - optional AlphaRowsetExtraMetaPB alpha_rowset_extra_meta_pb = 50; + // alpha_rowset_extra_meta_pb is deleted + reserved 50; // to indicate whether the data between the segments overlap optional SegmentsOverlapPB segments_overlap_pb = 51 [default = OVERLAP_UNKNOWN]; } @@ -122,21 +122,6 @@ message RemoteRowsetGcPB { required int64 num_segments = 3; } -message AlphaRowsetExtraMetaPB { - repeated SegmentGroupPB segment_groups = 1; -} - -message SegmentGroupPB { - required int32 segment_group_id = 1; - required int32 num_segments = 2; - required int64 index_size = 3; - required int64 data_size = 4; - required int64 num_rows = 5; - repeated ZoneMap zone_maps = 6; - optional bool empty = 7; - optional PUniqueId load_id = 8; -} - enum DataFileType { OLAP_DATA_FILE = 0; //Deprecated. Only columnar-wise format is supported. COLUMN_ORIENTED_FILE = 1;