database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[chore](BE) remove all useless segment group related code #15193

Browse Source 
The segment group is useless in current codebase, remove all the related code inside Doris. As for the related protobuf code, use reserved flag to prevent any future user from using that field.
This commit is contained in:
AlexYue
2022-12-20 17:11:47 +08:00
committed by GitHub
parent c172e2396a
commit 821c12a456
12 changed files with 13 additions and 276 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
be/src/olap/push_handler.cpp
View File

@ -240,7 +240,7 @@ Status PushHandler::_convert_v2(TabletSharedPtr cur_tablet, RowsetSharedPtr* cur
uint8_t* tuple_buf = reader->mem_pool()->allocate(schema->schema_size());
ContiguousRow row(schema.get(), tuple_buf);
// 4. Read data from broker and write into SegmentGroup of cur_tablet
// 4. Read data from broker and write into cur_tablet
// Convert from raw to delta
VLOG_NOTICE << "start to convert etl file to delta.";
while (!reader->eof()) {
@ -368,7 +368,7 @@ Status PushHandler::_convert(TabletSharedPtr cur_tablet, RowsetSharedPtr* cur_ro
break;
}
// 5. Read data from raw file and write into SegmentGroup of cur_tablet
// 5. Read data from raw file and write into cur_tablet
if (_request.__isset.http_file_path) {
// Convert from raw to delta
VLOG_NOTICE << "start to convert row file to delta.";

2
be/src/olap/push_handler.h
View File

@ -58,7 +58,7 @@ private:
Status _convert_v2(TabletSharedPtr cur_tablet, RowsetSharedPtr* cur_rowset,
TabletSchemaSPtr tablet_schema);
// Convert local data file to internal formatted delta,
// return new delta's SegmentGroup
// return new delta's rowset
Status _convert(TabletSharedPtr cur_tablet, RowsetSharedPtr* cur_rowset,
TabletSchemaSPtr tablet_schema);

1
be/src/olap/rowset/segment_v2/segment.h
View File

@ -37,7 +37,6 @@
namespace doris {
class SegmentGroup;
class TabletSchema;
class ShortKeyIndexDecoder;
class Schema;

30
be/test/olap/delete_handler_test.cpp
View File

@ -832,35 +832,7 @@ protected:
"hi": -5350970832824939812,
"lo": -6717994719194512122
},
"creation_time": 1553765670,
"alpha_rowset_extra_meta_pb": {
"segment_groups": [
{
"segment_group_id": 0,
"num_segments": 2,
"index_size": 132,
"data_size": 576,
"num_rows": 5,
"zone_maps": [
{
"min": "MQ==",
"max": "NQ==",
"null_flag": false
},
{
"min": "MQ==",
"max": "Mw==",
"null_flag": false
},
{
"min": "J2J1c2gn",
"max": "J3RvbSc=",
"null_flag": false
}
],
"empty": false
}]
}
"creation_time": 1553765670
})";
}

30
be/test/olap/ordered_data_compaction_test.cpp
View File

@ -258,35 +258,7 @@ protected:
"hi": -5350970832824939812,
"lo": -6717994719194512122
},
"creation_time": 1553765670,
"alpha_rowset_extra_meta_pb": {
"segment_groups": [
{
"segment_group_id": 0,
"num_segments": 2,
"index_size": 132,
"data_size": 576,
"num_rows": 5,
"zone_maps": [
{
"min": "MQ==",
"max": "NQ==",
"null_flag": false
},
{
"min": "MQ==",
"max": "Mw==",
"null_flag": false
},
{
"min": "J2J1c2gn",
"max": "J3RvbSc=",
"null_flag": false
}
],
"empty": false
}]
}
"creation_time": 1553765670
})";
pb1->init_from_json(json_rowset_meta);
pb1->set_start_version(start);

30
be/test/olap/rowid_conversion_test.cpp
View File

@ -218,35 +218,7 @@ protected:
"hi": -5350970832824939812,
"lo": -6717994719194512122
},
"creation_time": 1553765670,
"alpha_rowset_extra_meta_pb": {
"segment_groups": [
{
"segment_group_id": 0,
"num_segments": 2,
"index_size": 132,
"data_size": 576,
"num_rows": 5,
"zone_maps": [
{
"min": "MQ==",
"max": "NQ==",
"null_flag": false
},
{
"min": "MQ==",
"max": "Mw==",
"null_flag": false
},
{
"min": "J2J1c2gn",
"max": "J3RvbSc=",
"null_flag": false
}
],
"empty": false
}]
}
"creation_time": 1553765670
})";
pb1->init_from_json(json_rowset_meta);
pb1->set_start_version(start);

30
be/test/olap/tablet_test.cpp
View File

@ -63,35 +63,7 @@ public:
"hi": -5350970832824939812,
"lo": -6717994719194512122
},
"creation_time": 1553765670,
"alpha_rowset_extra_meta_pb": {
"segment_groups": [
{
"segment_group_id": 0,
"num_segments": 2,
"index_size": 132,
"data_size": 576,
"num_rows": 5,
"zone_maps": [
{
"min": "MQ==",
"max": "NQ==",
"null_flag": false
},
{
"min": "MQ==",
"max": "Mw==",
"null_flag": false
},
{
"min": "J2J1c2gn",
"max": "J3RvbSc=",
"null_flag": false
}
],
"empty": false
}]
}
"creation_time": 1553765670
})";
doris::EngineOptions options;

45
be/test/olap/test_data/header_without_inc_rs.txt
View File

@ -73,19 +73,7 @@
"index_disk_size": 0,
"empty": true,
"creation_time": 1553765664,
"num_segments": 0,
"alpha_rowset_extra_meta_pb": {
"segment_groups": [
{
"segment_group_id": 0,
"num_segments": 0,
"index_size": 0,
"data_size": 0,
"num_rows": 0,
"empty": true
}
]
}
"num_segments": 0
},
{
"rowset_id": 540081,
@ -106,36 +94,7 @@
"lo": -6717994719194512122
},
"creation_time": 1553765670,
"num_segments": 1,
"alpha_rowset_extra_meta_pb": {
"segment_groups": [
{
"segment_group_id": 0,
"num_segments": 1,
"index_size": 132,
"data_size": 576,
"num_rows": 5,
"zone_maps": [
{
"min": "MQ==",
"max": "NQ==",
"null_flag": false
},
{
"min": "MQ==",
"max": "Mw==",
"null_flag": false
},
{
"min": "J2J1c2gn",
"max": "J3RvbSc=",
"null_flag": false
}
],
"empty": false
}
]
}
"num_segments": 1
}
],
"in_restore_mode": false,

56
be/test/olap/timestamped_version_tracker_test.cpp
View File

@ -51,61 +51,7 @@ public:
"hi": -5350970832824939812,
"lo": -6717994719194512122
},
"creation_time": 1553765670,
"alpha_rowset_extra_meta_pb": {
"segment_groups": [
{
"segment_group_id": 0,
"num_segments": 1,
"index_size": 132,
"data_size": 576,
"num_rows": 5,
"zone_maps": [
{
"min": "MQ==",
"max": "NQ==",
"null_flag": false
},
{
"min": "MQ==",
"max": "Mw==",
"null_flag": false
},
{
"min": "J2J1c2gn",
"max": "J3RvbSc=",
"null_flag": false
}
],
"empty": false
},
{
"segment_group_id": 1,
"num_segments": 1,
"index_size": 132,
"data_size": 576,
"num_rows": 5,
"zone_maps": [
{
"min": "MQ==",
"max": "NQ==",
"null_flag": false
},
{
"min": "MQ==",
"max": "Mw==",
"null_flag": false
},
{
"min": "J2J1c2gn",
"max": "J3RvbSc=",
"null_flag": false
}
],
"empty": false
}
]
}
"creation_time": 1553765670
})";
}
void TearDown() override {}

30
be/test/vec/olap/vertical_compaction_test.cpp
View File

@ -252,35 +252,7 @@ protected:
"hi": -5350970832824939812,
"lo": -6717994719194512122
},
"creation_time": 1553765670,
"alpha_rowset_extra_meta_pb": {
"segment_groups": [
{
"segment_group_id": 0,
"num_segments": 2,
"index_size": 132,
"data_size": 576,
"num_rows": 5,
"zone_maps": [
{
"min": "MQ==",
"max": "NQ==",
"null_flag": false
},
{
"min": "MQ==",
"max": "Mw==",
"null_flag": false
},
{
"min": "J2J1c2gn",
"max": "J3RvbSc=",
"null_flag": false
}
],
"empty": false
}]
}
"creation_time": 1553765670
})";
pb1->init_from_json(json_rowset_meta);
pb1->set_start_version(start);

12
docs/images/cpu-flame-demo.svg
View File

@ -1441,10 +1441,6 @@
<text text-anchor="" x="1096.63" y="171.5" font-size="12" font-family="Verdana" fill="rgb(0,0,0)" ></text>
</g>
<g class="func_g" onmouseover="s(this)" onmouseout="c()" onclick="zoom(this)">
<title>doris::SegmentGroup::find_short_key (86 samples, 0.06%)</title><rect x="1180.9" y="193" width="0.6" height="15.0" fill="rgb(219,48,22)" rx="2" ry="2" />
<text text-anchor="" x="1183.87" y="203.5" font-size="12" font-family="Verdana" fill="rgb(0,0,0)" ></text>
</g>
<g class="func_g" onmouseover="s(this)" onmouseout="c()" onclick="zoom(this)">
<title>doris::ReadOnlyFileStream::_assure_data (46 samples, 0.03%)</title><rect x="1150.2" y="97" width="0.4" height="15.0" fill="rgb(247,11,43)" rx="2" ry="2" />
<text text-anchor="" x="1153.21" y="107.5" font-size="12" font-family="Verdana" fill="rgb(0,0,0)" ></text>
</g>
@ -1489,18 +1485,10 @@
<text text-anchor="" x="145.38" y="347.5" font-size="12" font-family="Verdana" fill="rgb(0,0,0)" ></text>
</g>
<g class="func_g" onmouseover="s(this)" onmouseout="c()" onclick="zoom(this)">
<title>doris::SegmentGroup::add_short_key (362 samples, 0.23%)</title><rect x="330.8" y="209" width="2.7" height="15.0" fill="rgb(229,195,16)" rx="2" ry="2" />
<text text-anchor="" x="333.79" y="219.5" font-size="12" font-family="Verdana" fill="rgb(0,0,0)" ></text>
</g>
<g class="func_g" onmouseover="s(this)" onmouseout="c()" onclick="zoom(this)">
<title>doris::ColumnData::_seek_to_row (41,249 samples, 26.61%)</title><rect x="874.1" y="225" width="314.0" height="15.0" fill="rgb(254,128,35)" rx="2" ry="2" />
<text text-anchor="" x="877.12" y="235.5" font-size="12" font-family="Verdana" fill="rgb(0,0,0)" >doris::ColumnData::_seek_to_row</text>
</g>
<g class="func_g" onmouseover="s(this)" onmouseout="c()" onclick="zoom(this)">
<title>doris::SegmentGroup::add_short_key (100 samples, 0.06%)</title><rect x="149.8" y="209" width="0.8" height="15.0" fill="rgb(239,109,35)" rx="2" ry="2" />
<text text-anchor="" x="152.85" y="219.5" font-size="12" font-family="Verdana" fill="rgb(0,0,0)" ></text>
</g>
<g class="func_g" onmouseover="s(this)" onmouseout="c()" onclick="zoom(this)">
<title>doris::ReadOnlyFileStream::_assure_data (178 samples, 0.11%)</title><rect x="1094.6" y="145" width="1.3" height="15.0" fill="rgb(226,175,25)" rx="2" ry="2" />
<text text-anchor="" x="1097.57" y="155.5" font-size="12" font-family="Verdana" fill="rgb(0,0,0)" ></text>
</g>

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 292 KiB

After

Width:  |  Height:  |  Size: 291 KiB

19
gensrc/proto/olap_file.proto
View File

@ -109,8 +109,8 @@ message RowsetMetaPB {
repeated KeyBoundsPB segments_key_bounds = 27;
// tablet meta pb, for compaction
optional TabletSchemaPB tablet_schema = 28;
// spare field id for future use
optional AlphaRowsetExtraMetaPB alpha_rowset_extra_meta_pb = 50;
// alpha_rowset_extra_meta_pb is deleted
reserved 50;
// to indicate whether the data between the segments overlap
optional SegmentsOverlapPB segments_overlap_pb = 51 [default = OVERLAP_UNKNOWN];
}
@ -122,21 +122,6 @@ message RemoteRowsetGcPB {
required int64 num_segments = 3;
}
message AlphaRowsetExtraMetaPB {
repeated SegmentGroupPB segment_groups = 1;
}
message SegmentGroupPB {
required int32 segment_group_id = 1;
required int32 num_segments = 2;
required int64 index_size = 3;
required int64 data_size = 4;
required int64 num_rows = 5;
repeated ZoneMap zone_maps = 6;
optional bool empty = 7;
optional PUniqueId load_id = 8;
}
enum DataFileType {
OLAP_DATA_FILE = 0; //Deprecated. Only columnar-wise format is supported.
COLUMN_ORIENTED_FILE = 1;