23#include <boost/intrusive/unordered_set.hpp>
24#include <boost/intrusive/list.hpp>
32#include <seastar/core/align.hh>
33#include <seastar/core/memory.hh>
34#include <seastar/util/assert.hh>
38static constexpr uint16_t SLAB_MAGIC_NUMBER = 0x51AB;
54 boost::intrusive::list_member_hook<> _lru_link;
55 boost::intrusive::list_member_hook<> _free_pages_link;
57 std::vector<uintptr_t> _free_objects;
61 uint8_t _slab_class_id;
63 slab_page_desc(
void *slab_page,
size_t objects,
size_t object_size, uint8_t slab_class_id, uint32_t index)
64 : _slab_page(slab_page)
67 , _magic(SLAB_MAGIC_NUMBER)
68 , _slab_class_id(slab_class_id)
70 auto object =
reinterpret_cast<uintptr_t
>(slab_page);
71 _free_objects.reserve(objects - 1);
72 for (
auto i = 1u; i < objects; i++) {
73 object += object_size;
74 _free_objects.push_back(
object);
79 return _free_objects.empty();
83 return _free_objects.size();
90 uint32_t index()
const {
94 uint16_t magic()
const {
98 uint8_t slab_class_id()
const {
99 return _slab_class_id;
102 void* slab_page()
const {
106 std::vector<uintptr_t>& free_objects() {
107 return _free_objects;
110 void* allocate_object() {
111 SEASTAR_ASSERT(!_free_objects.empty());
112 auto object =
reinterpret_cast<void*
>(_free_objects.back());
113 _free_objects.pop_back();
117 void free_object(
void *
object) {
118 _free_objects.push_back(
reinterpret_cast<uintptr_t
>(
object));
121 template<
typename Item>
123 template<
typename Item>
128 boost::intrusive::list_member_hook<> _lru_link;
130 template<
typename Item>
134template<
typename Item>
138 boost::intrusive::member_hook<slab_page_desc, boost::intrusive::list_member_hook<>,
139 &slab_page_desc::_free_pages_link>> _free_slab_pages;
141 boost::intrusive::member_hook<slab_item_base, boost::intrusive::list_member_hook<>,
142 &slab_item_base::_lru_link>> _lru;
144 uint8_t _slab_class_id;
146 template<
typename... Args>
148 Item* create_item(
void *
object, uint32_t slab_page_index, Args&&... args) {
149 Item *new_item =
new(object) Item(slab_page_index, std::forward<Args>(args)...);
155 std::pair<void *, uint32_t> evict_lru_item(std::function<
void (Item& item_ref)>& erase_func) {
157 return {
nullptr, 0U };
160 Item& victim =
reinterpret_cast<Item&
>(_lru.back());
161 uint32_t index = victim.get_slab_page_index();
162 SEASTAR_ASSERT(victim.is_unlocked());
163 _lru.erase(_lru.iterator_to(
reinterpret_cast<slab_item_base&
>(victim)));
167 return {
reinterpret_cast<void*
>(&victim), index };
170 slab_class(
size_t size, uint8_t slab_class_id)
172 , _slab_class_id(slab_class_id)
177 _free_slab_pages.clear();
181 size_t size()
const {
186 return _free_slab_pages.empty();
189 bool has_no_slab_pages()
const {
193 template<
typename... Args>
194 Item *create(Args&&... args) {
195 SEASTAR_ASSERT(!_free_slab_pages.empty());
196 auto& desc = _free_slab_pages.back();
197 auto object = desc.allocate_object();
200 _free_slab_pages.erase(_free_slab_pages.iterator_to(desc));
203 return create_item(
object, desc.index(), std::forward<Args>(args)...);
206 template<
typename... Args>
207 Item *create_from_new_page(uint64_t max_object_size, uint32_t slab_page_index,
211 constexpr size_t alignment = std::alignment_of_v<Item>;
212 void *slab_page = aligned_alloc(alignment, max_object_size);
214 throw std::bad_alloc{};
218 SEASTAR_ASSERT(_size % alignment == 0);
220 auto objects = max_object_size / _size;
221 desc =
new slab_page_desc(slab_page, objects, _size, _slab_class_id, slab_page_index);
222 }
catch (
const std::bad_alloc& e) {
224 throw std::bad_alloc{};
227 _free_slab_pages.push_front(*desc);
228 insert_slab_page_desc(*desc);
231 return create_item(slab_page, slab_page_index, std::forward<Args>(args)...);
234 template<
typename... Args>
235 Item *create_from_lru(std::function<
void (Item& item_ref)>& erase_func, Args&&... args) {
236 auto ret = evict_lru_item(erase_func);
238 throw std::bad_alloc{};
240 return create_item(ret.first, ret.second, std::forward<Args>(args)...);
245 _lru.erase(_lru.iterator_to(
reinterpret_cast<slab_item_base&
>(*item)));
246 desc.free_object(
object);
247 if (desc.size() == 1) {
249 _free_slab_pages.push_back(desc);
253 void touch_item(Item *item) {
255 _lru.erase(_lru.iterator_to(item_ref));
256 _lru.push_front(item_ref);
259 void remove_item_from_lru(Item *item) {
261 _lru.erase(_lru.iterator_to(item_ref));
264 void insert_item_into_lru(Item *item) {
266 _lru.push_front(item_ref);
270 SEASTAR_ASSERT(desc.slab_class_id() == _slab_class_id);
271 _free_slab_pages.erase(_free_slab_pages.iterator_to(desc));
275template<
typename Item>
278 std::vector<size_t> _slab_class_sizes;
279 std::vector<slab_class<Item>> _slab_classes;
282 std::function<void (Item& item_ref)> _erase_func;
283 std::vector<slab_page_desc*> _slab_pages_vector;
285 boost::intrusive::member_hook<slab_page_desc, boost::intrusive::list_member_hook<>,
286 &slab_page_desc::_lru_link>> _slab_page_desc_lru;
287 uint64_t _max_object_size;
288 uint64_t _available_slab_pages;
289 struct collectd_stats {
293 memory::reclaimer *_reclaimer =
nullptr;
294 bool _reclaimed =
false;
296 memory::reclaiming_result evict_lru_slab_page() {
297 if (_slab_page_desc_lru.empty()) {
301 return memory::reclaiming_result::reclaimed_nothing;
304 auto& desc = _slab_page_desc_lru.back();
305 SEASTAR_ASSERT(desc.refcnt() == 0);
306 uint8_t slab_class_id = desc.slab_class_id();
307 auto slab_class = get_slab_class(slab_class_id);
308 void *slab_page = desc.slab_page();
310 auto& free_objects = desc.free_objects();
315 std::sort(free_objects.begin(), free_objects.end());
318 _slab_page_desc_lru.erase(_slab_page_desc_lru.iterator_to(desc));
320 _slab_pages_vector[desc.index()] =
nullptr;
324 uintptr_t
object =
reinterpret_cast<uintptr_t
>(slab_page);
326 auto objects = _max_object_size / object_size;
327 for (
auto i = 0u; i < objects; i++,
object += object_size) {
331 if (std::binary_search(free_objects.begin(), free_objects.end(),
object)) {
335 Item* item =
reinterpret_cast<Item*
>(object);
336 SEASTAR_ASSERT(item->is_unlocked());
342 printf(
"lru slab page eviction succeeded! desc_empty?=%d\n", desc.empty());
346 return memory::reclaiming_result::reclaimed_something;
352 memory::reclaiming_result reclaim() {
357 return evict_lru_slab_page();
360 void initialize_slab_allocator(
double growth_factor, uint64_t limit) {
361 constexpr size_t alignment = std::alignment_of_v<Item>;
362 constexpr size_t initial_size = 96;
363 size_t size = initial_size;
364 uint8_t slab_class_id = 0U;
366 while (_max_object_size / size > 1) {
367 size = align_up(size, alignment);
368 _slab_class_sizes.push_back(size);
369 _slab_classes.emplace_back(size, slab_class_id);
370 size *= growth_factor;
371 SEASTAR_ASSERT(slab_class_id < std::numeric_limits<uint8_t>::max());
374 _slab_class_sizes.push_back(_max_object_size);
375 _slab_classes.emplace_back(_max_object_size, slab_class_id);
379 _reclaimer =
new memory::reclaimer([
this] {
return reclaim(); });
381 _slab_pages_vector.reserve(_available_slab_pages);
387 auto i = std::lower_bound(_slab_class_sizes.begin(), _slab_class_sizes.end(), size);
388 if (i == _slab_class_sizes.end()) {
391 auto dist = std::distance(_slab_class_sizes.begin(), i);
392 return &_slab_classes[dist];
396 SEASTAR_ASSERT(slab_class_id >= 0 && slab_class_id < _slab_classes.size());
397 return &_slab_classes[slab_class_id];
400 void register_metrics() {
403 sm::make_counter(
"malloc_total_operations", sm::description(
"Total number of slab malloc operations"), _stats.allocs),
404 sm::make_counter(
"free_total_operations", sm::description(
"Total number of slab free operations"), _stats.frees),
405 sm::make_gauge(
"malloc_objects", sm::description(
"Number of slab created objects currently in memory"), [
this] {
406 return _stats.allocs - _stats.frees;
413 auto desc = _slab_pages_vector[item->get_slab_page_index()];
414 SEASTAR_ASSERT(desc !=
nullptr);
415 SEASTAR_ASSERT(desc->magic() == SLAB_MAGIC_NUMBER);
420 return (_reclaimer && !_reclaimed) ||
421 (_available_slab_pages > 0 || sc.has_no_slab_pages());
424 slab_allocator(
double growth_factor, uint64_t limit, uint64_t max_object_size)
425 : _max_object_size(max_object_size)
426 , _available_slab_pages(limit / max_object_size)
428 initialize_slab_allocator(growth_factor, limit);
432 slab_allocator(
double growth_factor, uint64_t limit, uint64_t max_object_size,
433 std::function<
void (Item& item_ref)> erase_func)
434 : _erase_func(std::move(erase_func))
435 , _max_object_size(max_object_size)
436 , _available_slab_pages(limit / max_object_size)
438 initialize_slab_allocator(growth_factor, limit);
444 _slab_classes.clear();
445 _slab_page_desc_lru.clear();
446 for (
auto desc : _slab_pages_vector) {
450 ::free(desc->slab_page());
459 template<
typename... Args>
460 Item*
create(
const size_t size, Args&&... args) {
463 throw std::bad_alloc{};
466 Item *item =
nullptr;
468 item =
slab_class->create(std::forward<Args>(args)...);
472 auto index_to_insert = _slab_pages_vector.size();
473 item =
slab_class->create_from_new_page(_max_object_size, index_to_insert,
477 _slab_page_desc_lru.push_front(desc);
480 _slab_pages_vector.push_back(&desc);
482 std::forward<Args>(args)...);
483 if (_available_slab_pages > 0) {
484 _available_slab_pages--;
487 }
else if (_erase_func) {
488 item =
slab_class->create_from_lru(_erase_func, std::forward<Args>(args)...);
494 void lock_item(Item *item) {
495 auto& desc = get_slab_page_desc(item);
497 auto& refcnt = desc.refcnt();
501 _slab_page_desc_lru.erase(_slab_page_desc_lru.iterator_to(desc));
505 auto slab_class = get_slab_class(desc.slab_class_id());
506 slab_class->remove_item_from_lru(item);
509 void unlock_item(Item *item) {
510 auto& desc = get_slab_page_desc(item);
512 auto& refcnt = desc.refcnt();
516 _slab_page_desc_lru.push_front(desc);
520 auto slab_class = get_slab_class(desc.slab_class_id());
521 slab_class->insert_item_into_lru(item);
529 auto& desc = get_slab_page_desc(item);
530 auto slab_class = get_slab_class(desc.slab_class_id());
541 auto& desc = get_slab_page_desc(item);
542 auto slab_class = get_slab_class(desc.slab_class_id());
554 printf(
"slab[%3d]\tsize: %10lu\tper-slab-page: %5lu\n", class_id, size, _max_object_size / size);
holds the metric definition.
Definition: metrics_registration.hh:94
metric_groups & add_group(const group_name_type &name, const std::initializer_list< metric_definition > &l)
Add metrics belonging to the same group.
void touch(Item *item)
Definition: slab.hh:539
void free(Item *item)
Definition: slab.hh:527
size_t class_size(const size_t size)
Definition: slab.hh:562
void print_slab_classes()
Definition: slab.hh:550
Item * create(const size_t size, Args &&... args)
Definition: slab.hh:460
impl::metric_definition_impl make_gauge(metric_name_type name, T &&val, description d=description(), std::vector< label_instance > labels={})
Gauge are a general purpose metric.
Definition: metrics.hh:442
impl::metric_definition_impl make_counter(metric_name_type name, T &&val, description d=description(), std::vector< label_instance > labels={})
create a counter metric
Definition: metrics.hh:528
header for metrics creation.
metrics creation and registration
Definition: estimated_histogram.hh:34
Seastar API namespace.
Definition: abort_on_ebadf.hh:26