e99b33c274
File meta cache on BE is used to cache the meta for external table's file such as parquet footer.
This cache is counted by number, not memory consumption.
So if the cache object is big(eg, a large parquet footer), the total memory consumption of this cache
will be large and causing OOM.
This PR mainly changes:
1. Add a new method `exceed_prune_limit()` for `CachePolicy`
For `ObjLRUCache`, it always return true so that the minor of full gc on BE will prune the cache each time.
2. Reduce the default capability of file meta cache, from 20000 to 1000
Also change the default capability of hdfs file handle cache, from 20000 to 1000
4. Change judgement of whether enable file meta cache when querying
If the number of file need to be read is larger than the 1/3 of the file meta cache's capability, file meta cache
will be disabled for this query. Because cache is useless if there are too many files.
56 lines
1.8 KiB
C++
56 lines
1.8 KiB
C++
// Licensed to the Apache Software Foundation (ASF) under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing,
|
|
// software distributed under the License is distributed on an
|
|
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
// KIND, either express or implied. See the License for the
|
|
// specific language governing permissions and limitations
|
|
// under the License.
|
|
|
|
#include "util/obj_lru_cache.h"
|
|
|
|
namespace doris {
|
|
|
|
ObjLRUCache::ObjLRUCache(int64_t capacity, uint32_t num_shards)
|
|
: LRUCachePolicy(CachePolicy::CacheType::COMMON_OBJ_LRU_CACHE, capacity,
|
|
LRUCacheType::NUMBER, config::common_obj_lru_cache_stale_sweep_time_sec,
|
|
num_shards) {
|
|
_enabled = (capacity > 0);
|
|
}
|
|
|
|
bool ObjLRUCache::lookup(const ObjKey& key, CacheHandle* handle) {
|
|
if (!_enabled) {
|
|
return false;
|
|
}
|
|
auto* lru_handle = LRUCachePolicy::lookup(key.key);
|
|
if (!lru_handle) {
|
|
// cache miss
|
|
return false;
|
|
}
|
|
*handle = CacheHandle(this, lru_handle);
|
|
return true;
|
|
}
|
|
|
|
void ObjLRUCache::erase(const ObjKey& key) {
|
|
if (_enabled) {
|
|
LRUCachePolicy::erase(key.key);
|
|
}
|
|
}
|
|
|
|
bool ObjLRUCache::exceed_prune_limit() {
|
|
// just return true to prune all cached obj.
|
|
// Because ObjLRUCache is counted with number, not memory.
|
|
// Simple prune all
|
|
return true;
|
|
}
|
|
|
|
} // namespace doris
|