database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Pick 2.1](inverted index) add inverted index reader memory size into searcher cache (#36160)

Browse Source 
Pick from #35149
This commit is contained in:
airborne12
2024-06-12 14:40:20 +08:00
committed by GitHub
parent 73eda9bdb7
commit 14ece32b87
4 changed files with 8 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
be/src/clucene

Submodule be/src/clucene updated: d3de160871...25324632ba

2
be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
View File

@ -239,6 +239,8 @@ Status InvertedIndexReader::create_index_searcher(lucene::store::Directory* dir,
if (std::string(dir->getObjectName()) == "DorisCompoundReader") {
static_cast<DorisCompoundReader*>(dir)->getDorisIndexInput()->setIdxFileCache(false);
}
// NOTE: before mem_tracker hook becomes active, we caculate reader memory size by hand.
mem_tracker->consume(index_searcher_builder->get_reader_size());
return Status::OK();
};

1
be/src/olap/rowset/segment_v2/inverted_index_searcher.cpp
View File

@ -44,6 +44,7 @@ Status FulltextIndexSearcherBuilder::build(lucene::store::Directory* directory,
return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
"FulltextIndexSearcherBuilder build index_searcher error.");
}
reader_size = reader->getTermInfosRAMUsed();
// NOTE: need to cl_refcount-- here, so that directory will be deleted when
// index_searcher is destroyed
_CLDECDELETE(directory)

4
be/src/olap/rowset/segment_v2/inverted_index_searcher.h
View File

@ -61,6 +61,10 @@ public:
virtual Result<IndexSearcherPtr> get_index_searcher(lucene::store::Directory* directory);
static Result<std::unique_ptr<IndexSearcherBuilder>> create_index_searcher_builder(
InvertedIndexReaderType reader_type);
int64_t get_reader_size() const { return reader_size; }
protected:
int64_t reader_size = 0;
};
class FulltextIndexSearcherBuilder : public IndexSearcherBuilder {