内容是新的，但是没有整理，偷懒了啊
WriteBufferManager
class WriteBufferManager final {
public:
// Parameters:
// _buffer_size: _buffer_size = 0 indicates no limit. Memory won't be capped.
// memory_usage() won't be valid and ShouldFlush() will always return true.
//
// cache_: if `cache` is provided, we'll put dummy entries in the cache and
// cost the memory allocated to the cache. It can be used even if _buffer_size
// = 0.
//
// allow_stall: if set true, it will enable stalling of writes when
// memory_usage() exceeds buffer_size. It will wait for flush to complete and
// memory usage to drop down.
explicit WriteBufferManager(size_t _buffer_size,
std::shared_ptr cache = {},
bool allow_stall = false);
// No copying allowed
WriteBufferManager(const WriteBufferManager&) = delete;
WriteBufferManager& operator=(const WriteBufferManager&) = delete;
~WriteBufferManager();
// Returns true if buffer_limit is passed to limit the total memory usage and
// is greater than 0.
bool enabled() const { return buffer_size() > 0; }
// Returns true if pointer to cache is passed.
bool cost_to_cache() const { return cache_res_mgr_ != nullptr; }
// Returns the total memory used by memtables.
// Only valid if enabled()
size_t memory_usage() const {
return memory_used_.load(std::memory_order_relaxed);
}
// Returns the total memory used by active memtables.
size_t mutable_memtable_memory_usage() const {
return memory_active_.load(std::memory_order_relaxed);
}
size_t dummy_entries_in_cache_usage() const;
// Returns the buffer_size.
size_t buffer_size() const {
return buffer_size_.load(std::memory_order_relaxed);
}
void SetBufferSize(size_t new_size) {
buffer_size_.store(new_size, std::memory_order_relaxed);
mutable_limit_.store(new_size * 7 / 8, std::memory_order_relaxed);
// Check if stall is active and can be ended.
MaybeEndWriteStall();
}
// Below functions should be called by RocksDB internally.
// Should only be called from write thread
bool ShouldFlush() const {
if (enabled()) {
if (mutable_memtable_memory_usage() >
mutable_limit_.load(std::memory_order_relaxed)) {
return true;
}
size_t local_size = buffer_size();
if (memory_usage() >= local_size &&
mutable_memtable_memory_usage() >= local_size / 2) {
// If the memory exceeds the buffer size, we trigger more aggressive
// flush. But if already more than half memory is being flushed,
// triggering more flush may not help. We will hold it instead.
return true;
}
}
return false;
}
// Returns true if total memory usage exceeded buffer_size.
// We stall the writes untill memory_usage drops below buffer_size. When the
// function returns true, all writer threads (including one checking this
// condition) across all DBs will be stalled. Stall is allowed only if user
// pass allow_stall = true during WriteBufferManager instance creation.
//
// Should only be called by RocksDB internally .
bool ShouldStall() const {
if (!allow_stall_ || !enabled()) {
return false;
}
return IsStallActive() || IsStallThresholdExceeded();
}
// Returns true if stall is active.
bool IsStallActive() const {
return stall_active_.load(std::memory_order_relaxed);
}
// Returns true if stalling condition is met.
bool IsStallThresholdExceeded() const {
return memory_usage() >= buffer_size_;
}
void ReserveMem(size_t mem);
// We are in the process of freeing `mem` bytes, so it is not considered
// when checking the soft limit.
void ScheduleFreeMem(size_t mem);
void FreeMem(size_t mem);
// Add the DB instance to the queue and block the DB.
// Should only be called by RocksDB internally.
void BeginWriteStall(StallInterface* wbm_stall);
// If stall conditions have resolved, remove DB instances from queue and
// signal them to continue.
void MaybeEndWriteStall();
void RemoveDBFromQueue(StallInterface* wbm_stall);
const std::shared_ptr& GetCache() const { return cache_; }
private:
std::atomic buffer_size_;
std::atomic mutable_limit_;
std::atomic memory_used_;
// Memory that hasn't been scheduled to free.
std::atomic memory_active_;
std::shared_ptr cache_;
std::shared_ptr cache_res_mgr_;
// Protects cache_res_mgr_
std::mutex cache_res_mgr_mu_;
std::list queue_;
// Protects the queue_ and stall_active_.
std::mutex mu_;
bool allow_stall_;
// Value should only be changed by BeginWriteStall() and MaybeEndWriteStall()
// while holding mu_, but it can be read without a lock.
std::atomic stall_active_;
void ReserveMemWithCache(size_t mem);
void FreeMemWithCache(size_t mem);
};
问题
1、如何使用WriteBufferManager
SanitizeOptions:
result.write_buffer_manager.reset(
new WriteBufferManager(result.db_write_buffer_size));
MemTable::MemTable(const InternalKeyComparator& cmp,
const ImmutableOptions& ioptions,
const MutableCFOptions& mutable_cf_options,
WriteBufferManager* write_buffer_manager,
SequenceNumber latest_seq, uint32_t column_family_id)
class AllocTracker {
public:
explicit AllocTracker(WriteBufferManager* write_buffer_manager);
void AllocTracker::Allocate(size_t bytes) {
assert(write_buffer_manager_ != nullptr);
if (write_buffer_manager_->enabled() ||
write_buffer_manager_->cost_to_cache()) {
bytes_allocated_.fetch_add(bytes, std::memory_order_relaxed);
write_buffer_manager_->ReserveMem(bytes);
}
}
2、WriteBufferManager 和 max_write_buffer_number 的关系
没什么关系，是两个不相关的内容
if (num_unflushed_memtables >= mutable_cf_options.max_write_buffer_number) {
3、WriteBufferManager 和 write_buffer_size 的关系
没什么关系，是两个不相关的内容
DBImpl::SwitchMemtable:
const auto preallocate_block_size =
GetWalPreallocateBlockSize(mutable_cf_options.write_buffer_size)
io_s = CreateWAL(new_log_number, recycle_log_number, preallocate_block_size,
&new_log);
MemTable::MemTable(const InternalKeyComparator& cmp,
const ImmutableOptions& ioptions,
const MutableCFOptions& mutable_cf_options,
WriteBufferManager* write_buffer_manager,
SequenceNumber latest_seq, uint32_t column_family_id)
: comparator_(cmp),
moptions_(ioptions, mutable_cf_options),
refs_(0),
kArenaBlockSize(OptimizeBlockSize(moptions_.arena_block_size)),
mem_tracker_(write_buffer_manager),
arena_(moptions_.arena_block_size,
(write_buffer_manager != nullptr &&
(write_buffer_manager->enabled() ||
write_buffer_manager->cost_to_cache()))
? &mem_tracker_
: nullptr,
mutable_cf_options.memtable_huge_page_size),
table_(ioptions.memtable_factory->CreateMemTableRep(
comparator_, &arena_, mutable_cf_options.prefix_extractor.get(),
ioptions.logger, column_family_id)),
range_del_table_(SkipListFactory().CreateMemTableRep(
comparator_, &arena_, nullptr /* transform */, ioptions.logger,
column_family_id)),
is_range_del_table_empty_(true),
data_size_(0),
num_entries_(0),
num_deletes_(0),
write_buffer_size_(mutable_cf_options.write_buffer_size),
AllocTracker::Allocate->WriteBufferManager::ReserveMem->WriteBufferManager::ReserveMemWithCache->cache_res_mgr_->UpdateCacheReservation
4、WriteBufferManager 如何控制内存
对内存没有硬性限制，通过ShouldFlush，判断是否需要操作释放内存
DBImpl::PreprocessWrite->if (UNLIKELY(status.ok() && write_buffer_manager_->ShouldFlush()))
5、总结 memtable 如何使用 WriteBufferManager 相关的接口
不是直接使用的，是通过MemTable构造函数中 "table_(ioptions.memtable_factory->CreateMemTableRep("使用arena_, arena_ 使用mem_tracker_， mem_tracker_ 使用write_buffer_manager，实现的
skiplist 如何使用的 WriteBufferManager
using MemtableSkipList = SkipList;
如何设置默认memtable
MemTableRepFactory
MemTable::MemTable->table_(ioptions.memtable_factory->CreateMemTableRep
SkipListFactory().CreateMemTableRep->SkipListRep
class SkipListRep : public MemTableRep
MemTable::Add->(table->InsertKeyValue)->MemTableRep::InsertKeyValue->SkipListRep::InsertKey->SkipListRep::Insert->InlineSkipList
MemTableRep::InsertKeyValue->MemTableRep::EncodeKeyValue->Allocate->(allocator_->Allocate)->
待处理问题
1、总结自己实现一个类似skiplist需要适配哪些接口
skiplist适配过程
InlineSkipList
SkipListRep
SkipListFactory::CreateMemTableRep
class SkipListFactory : public MemTableRepFactory