e99b33c274
File meta cache on BE is used to cache the meta for external table's file such as parquet footer.
This cache is counted by number, not memory consumption.
So if the cache object is big(eg, a large parquet footer), the total memory consumption of this cache
will be large and causing OOM.
This PR mainly changes:
1. Add a new method `exceed_prune_limit()` for `CachePolicy`
For `ObjLRUCache`, it always return true so that the minor of full gc on BE will prune the cache each time.
2. Reduce the default capability of file meta cache, from 20000 to 1000
Also change the default capability of hdfs file handle cache, from 20000 to 1000
4. Change judgement of whether enable file meta cache when querying
If the number of file need to be read is larger than the 1/3 of the file meta cache's capability, file meta cache
will be disabled for this query. Because cache is useless if there are too many files.
113 lines
3.5 KiB
C++
113 lines
3.5 KiB
C++
// Licensed to the Apache Software Foundation (ASF) under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing,
|
|
// software distributed under the License is distributed on an
|
|
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
// KIND, either express or implied. See the License for the
|
|
// specific language governing permissions and limitations
|
|
// under the License.
|
|
|
|
#pragma once
|
|
|
|
#include "olap/lru_cache.h"
|
|
#include "runtime/memory/lru_cache_policy.h"
|
|
|
|
namespace doris {
|
|
|
|
// A common object cache depends on an Sharded LRU Cache.
|
|
// It has a certain capacity, which determin how many objects it can cache.
|
|
// Caller must hold a CacheHandle instance when visiting the cached object.
|
|
class ObjLRUCache : public LRUCachePolicy {
|
|
public:
|
|
struct ObjKey {
|
|
ObjKey(const std::string& key_) : key(key_) {}
|
|
|
|
std::string key;
|
|
};
|
|
|
|
template <typename T>
|
|
class ObjValue : public LRUCacheValueBase {
|
|
public:
|
|
ObjValue(const T* value)
|
|
: LRUCacheValueBase(CachePolicy::CacheType::COMMON_OBJ_LRU_CACHE), value(value) {}
|
|
~ObjValue() override {
|
|
T* v = (T*)value;
|
|
delete v;
|
|
}
|
|
|
|
const T* value;
|
|
};
|
|
|
|
class CacheHandle {
|
|
public:
|
|
CacheHandle() = default;
|
|
CacheHandle(LRUCachePolicy* cache, Cache::Handle* handle)
|
|
: _cache(cache), _handle(handle) {}
|
|
~CacheHandle() {
|
|
if (_handle != nullptr) {
|
|
_cache->release(_handle);
|
|
}
|
|
}
|
|
|
|
CacheHandle(CacheHandle&& other) noexcept {
|
|
std::swap(_cache, other._cache);
|
|
std::swap(_handle, other._handle);
|
|
}
|
|
|
|
CacheHandle& operator=(CacheHandle&& other) noexcept {
|
|
std::swap(_cache, other._cache);
|
|
std::swap(_handle, other._handle);
|
|
return *this;
|
|
}
|
|
|
|
bool valid() { return _cache != nullptr && _handle != nullptr; }
|
|
|
|
LRUCachePolicy* cache() const { return _cache; }
|
|
template <typename T>
|
|
void* data() const {
|
|
return (void*)((ObjValue<T>*)_cache->value(_handle))->value;
|
|
}
|
|
|
|
private:
|
|
LRUCachePolicy* _cache = nullptr;
|
|
Cache::Handle* _handle = nullptr;
|
|
|
|
// Don't allow copy and assign
|
|
DISALLOW_COPY_AND_ASSIGN(CacheHandle);
|
|
};
|
|
|
|
ObjLRUCache(int64_t capacity, uint32_t num_shards = DEFAULT_LRU_CACHE_NUM_SHARDS);
|
|
|
|
bool lookup(const ObjKey& key, CacheHandle* handle);
|
|
|
|
template <typename T>
|
|
void insert(const ObjKey& key, const T* value, CacheHandle* cache_handle) {
|
|
if (_enabled) {
|
|
const std::string& encoded_key = key.key;
|
|
auto* obj_value = new ObjValue<T>(value);
|
|
auto* handle = LRUCachePolicy::insert(encoded_key, obj_value, 1, sizeof(T),
|
|
CachePriority::NORMAL);
|
|
*cache_handle = CacheHandle {this, handle};
|
|
} else {
|
|
cache_handle = nullptr;
|
|
}
|
|
}
|
|
|
|
void erase(const ObjKey& key);
|
|
|
|
bool exceed_prune_limit() override;
|
|
|
|
private:
|
|
bool _enabled;
|
|
};
|
|
|
|
} // namespace doris
|