23#include <boost/intrusive/unordered_set.hpp>
24#include <boost/intrusive/list.hpp>
32#include <seastar/core/align.hh>
33#include <seastar/core/memory.hh>
37static constexpr uint16_t SLAB_MAGIC_NUMBER = 0x51AB;
53 boost::intrusive::list_member_hook<> _lru_link;
54 boost::intrusive::list_member_hook<> _free_pages_link;
56 std::vector<uintptr_t> _free_objects;
60 uint8_t _slab_class_id;
62 slab_page_desc(
void *slab_page,
size_t objects,
size_t object_size, uint8_t slab_class_id, uint32_t index)
63 : _slab_page(slab_page)
66 , _magic(SLAB_MAGIC_NUMBER)
67 , _slab_class_id(slab_class_id)
69 auto object =
reinterpret_cast<uintptr_t
>(slab_page);
70 _free_objects.reserve(objects - 1);
71 for (
auto i = 1u; i < objects; i++) {
72 object += object_size;
73 _free_objects.push_back(
object);
78 return _free_objects.empty();
82 return _free_objects.size();
89 uint32_t index()
const {
93 uint16_t magic()
const {
97 uint8_t slab_class_id()
const {
98 return _slab_class_id;
101 void* slab_page()
const {
105 std::vector<uintptr_t>& free_objects() {
106 return _free_objects;
109 void* allocate_object() {
110 assert(!_free_objects.empty());
111 auto object =
reinterpret_cast<void*
>(_free_objects.back());
112 _free_objects.pop_back();
116 void free_object(
void *
object) {
117 _free_objects.push_back(
reinterpret_cast<uintptr_t
>(
object));
120 template<
typename Item>
122 template<
typename Item>
127 boost::intrusive::list_member_hook<> _lru_link;
129 template<
typename Item>
133template<
typename Item>
137 boost::intrusive::member_hook<slab_page_desc, boost::intrusive::list_member_hook<>,
138 &slab_page_desc::_free_pages_link>> _free_slab_pages;
140 boost::intrusive::member_hook<slab_item_base, boost::intrusive::list_member_hook<>,
141 &slab_item_base::_lru_link>> _lru;
143 uint8_t _slab_class_id;
145 template<
typename... Args>
147 Item* create_item(
void *
object, uint32_t slab_page_index, Args&&... args) {
148 Item *new_item =
new(object) Item(slab_page_index, std::forward<Args>(args)...);
154 std::pair<void *, uint32_t> evict_lru_item(std::function<
void (Item& item_ref)>& erase_func) {
156 return {
nullptr, 0U };
159 Item& victim =
reinterpret_cast<Item&
>(_lru.back());
160 uint32_t index = victim.get_slab_page_index();
161 assert(victim.is_unlocked());
162 _lru.erase(_lru.iterator_to(
reinterpret_cast<slab_item_base&
>(victim)));
166 return {
reinterpret_cast<void*
>(&victim), index };
169 slab_class(
size_t size, uint8_t slab_class_id)
171 , _slab_class_id(slab_class_id)
176 _free_slab_pages.clear();
180 size_t size()
const {
185 return _free_slab_pages.empty();
188 bool has_no_slab_pages()
const {
192 template<
typename... Args>
193 Item *create(Args&&... args) {
194 assert(!_free_slab_pages.empty());
195 auto& desc = _free_slab_pages.back();
196 auto object = desc.allocate_object();
199 _free_slab_pages.erase(_free_slab_pages.iterator_to(desc));
202 return create_item(
object, desc.index(), std::forward<Args>(args)...);
205 template<
typename... Args>
206 Item *create_from_new_page(uint64_t max_object_size, uint32_t slab_page_index,
210 constexpr size_t alignment = std::alignment_of_v<Item>;
211 void *slab_page = aligned_alloc(alignment, max_object_size);
213 throw std::bad_alloc{};
217 assert(_size % alignment == 0);
219 auto objects = max_object_size / _size;
220 desc =
new slab_page_desc(slab_page, objects, _size, _slab_class_id, slab_page_index);
221 }
catch (
const std::bad_alloc& e) {
223 throw std::bad_alloc{};
226 _free_slab_pages.push_front(*desc);
227 insert_slab_page_desc(*desc);
230 return create_item(slab_page, slab_page_index, std::forward<Args>(args)...);
233 template<
typename... Args>
234 Item *create_from_lru(std::function<
void (Item& item_ref)>& erase_func, Args&&... args) {
235 auto ret = evict_lru_item(erase_func);
237 throw std::bad_alloc{};
239 return create_item(ret.first, ret.second, std::forward<Args>(args)...);
244 _lru.erase(_lru.iterator_to(
reinterpret_cast<slab_item_base&
>(*item)));
245 desc.free_object(
object);
246 if (desc.size() == 1) {
248 _free_slab_pages.push_back(desc);
252 void touch_item(Item *item) {
254 _lru.erase(_lru.iterator_to(item_ref));
255 _lru.push_front(item_ref);
258 void remove_item_from_lru(Item *item) {
260 _lru.erase(_lru.iterator_to(item_ref));
263 void insert_item_into_lru(Item *item) {
265 _lru.push_front(item_ref);
269 assert(desc.slab_class_id() == _slab_class_id);
270 _free_slab_pages.erase(_free_slab_pages.iterator_to(desc));
274template<
typename Item>
277 std::vector<size_t> _slab_class_sizes;
278 std::vector<slab_class<Item>> _slab_classes;
281 std::function<void (Item& item_ref)> _erase_func;
282 std::vector<slab_page_desc*> _slab_pages_vector;
284 boost::intrusive::member_hook<slab_page_desc, boost::intrusive::list_member_hook<>,
285 &slab_page_desc::_lru_link>> _slab_page_desc_lru;
286 uint64_t _max_object_size;
287 uint64_t _available_slab_pages;
288 struct collectd_stats {
292 memory::reclaimer *_reclaimer =
nullptr;
293 bool _reclaimed =
false;
295 memory::reclaiming_result evict_lru_slab_page() {
296 if (_slab_page_desc_lru.empty()) {
300 return memory::reclaiming_result::reclaimed_nothing;
303 auto& desc = _slab_page_desc_lru.back();
304 assert(desc.refcnt() == 0);
305 uint8_t slab_class_id = desc.slab_class_id();
306 auto slab_class = get_slab_class(slab_class_id);
307 void *slab_page = desc.slab_page();
309 auto& free_objects = desc.free_objects();
314 std::sort(free_objects.begin(), free_objects.end());
317 _slab_page_desc_lru.erase(_slab_page_desc_lru.iterator_to(desc));
319 _slab_pages_vector[desc.index()] =
nullptr;
323 uintptr_t
object =
reinterpret_cast<uintptr_t
>(slab_page);
325 auto objects = _max_object_size / object_size;
326 for (
auto i = 0u; i < objects; i++,
object += object_size) {
330 if (std::binary_search(free_objects.begin(), free_objects.end(),
object)) {
334 Item* item =
reinterpret_cast<Item*
>(object);
335 assert(item->is_unlocked());
341 printf(
"lru slab page eviction succeeded! desc_empty?=%d\n", desc.empty());
345 return memory::reclaiming_result::reclaimed_something;
351 memory::reclaiming_result reclaim() {
356 return evict_lru_slab_page();
359 void initialize_slab_allocator(
double growth_factor, uint64_t limit) {
360 constexpr size_t alignment = std::alignment_of_v<Item>;
361 constexpr size_t initial_size = 96;
362 size_t size = initial_size;
363 uint8_t slab_class_id = 0U;
365 while (_max_object_size / size > 1) {
366 size = align_up(size, alignment);
367 _slab_class_sizes.push_back(size);
368 _slab_classes.emplace_back(size, slab_class_id);
369 size *= growth_factor;
370 assert(slab_class_id < std::numeric_limits<uint8_t>::max());
373 _slab_class_sizes.push_back(_max_object_size);
374 _slab_classes.emplace_back(_max_object_size, slab_class_id);
378 _reclaimer =
new memory::reclaimer([
this] {
return reclaim(); });
380 _slab_pages_vector.reserve(_available_slab_pages);
386 auto i = std::lower_bound(_slab_class_sizes.begin(), _slab_class_sizes.end(), size);
387 if (i == _slab_class_sizes.end()) {
390 auto dist = std::distance(_slab_class_sizes.begin(), i);
391 return &_slab_classes[dist];
395 assert(slab_class_id >= 0 && slab_class_id < _slab_classes.size());
396 return &_slab_classes[slab_class_id];
399 void register_metrics() {
402 sm::make_counter(
"malloc_total_operations", sm::description(
"Total number of slab malloc operations"), _stats.allocs),
403 sm::make_counter(
"free_total_operations", sm::description(
"Total number of slab free operations"), _stats.frees),
404 sm::make_gauge(
"malloc_objects", sm::description(
"Number of slab created objects currently in memory"), [
this] {
405 return _stats.allocs - _stats.frees;
412 auto desc = _slab_pages_vector[item->get_slab_page_index()];
413 assert(desc !=
nullptr);
414 assert(desc->magic() == SLAB_MAGIC_NUMBER);
419 return (_reclaimer && !_reclaimed) ||
420 (_available_slab_pages > 0 || sc.has_no_slab_pages());
423 slab_allocator(
double growth_factor, uint64_t limit, uint64_t max_object_size)
424 : _max_object_size(max_object_size)
425 , _available_slab_pages(limit / max_object_size)
427 initialize_slab_allocator(growth_factor, limit);
431 slab_allocator(
double growth_factor, uint64_t limit, uint64_t max_object_size,
432 std::function<
void (Item& item_ref)> erase_func)
433 : _erase_func(std::move(erase_func))
434 , _max_object_size(max_object_size)
435 , _available_slab_pages(limit / max_object_size)
437 initialize_slab_allocator(growth_factor, limit);
443 _slab_classes.clear();
444 _slab_page_desc_lru.clear();
445 for (
auto desc : _slab_pages_vector) {
449 ::free(desc->slab_page());
458 template<
typename... Args>
459 Item*
create(
const size_t size, Args&&... args) {
462 throw std::bad_alloc{};
465 Item *item =
nullptr;
467 item =
slab_class->create(std::forward<Args>(args)...);
471 auto index_to_insert = _slab_pages_vector.size();
472 item =
slab_class->create_from_new_page(_max_object_size, index_to_insert,
476 _slab_page_desc_lru.push_front(desc);
479 _slab_pages_vector.push_back(&desc);
481 std::forward<Args>(args)...);
482 if (_available_slab_pages > 0) {
483 _available_slab_pages--;
486 }
else if (_erase_func) {
487 item =
slab_class->create_from_lru(_erase_func, std::forward<Args>(args)...);
493 void lock_item(Item *item) {
494 auto& desc = get_slab_page_desc(item);
496 auto& refcnt = desc.refcnt();
500 _slab_page_desc_lru.erase(_slab_page_desc_lru.iterator_to(desc));
504 auto slab_class = get_slab_class(desc.slab_class_id());
505 slab_class->remove_item_from_lru(item);
508 void unlock_item(Item *item) {
509 auto& desc = get_slab_page_desc(item);
511 auto& refcnt = desc.refcnt();
515 _slab_page_desc_lru.push_front(desc);
519 auto slab_class = get_slab_class(desc.slab_class_id());
520 slab_class->insert_item_into_lru(item);
528 auto& desc = get_slab_page_desc(item);
529 auto slab_class = get_slab_class(desc.slab_class_id());
540 auto& desc = get_slab_page_desc(item);
541 auto slab_class = get_slab_class(desc.slab_class_id());
553 printf(
"slab[%3d]\tsize: %10lu\tper-slab-page: %5lu\n", class_id, size, _max_object_size / size);
holds the metric definition.
Definition: metrics_registration.hh:94
metric_groups & add_group(const group_name_type &name, const std::initializer_list< metric_definition > &l)
Add metrics belonging to the same group.
void touch(Item *item)
Definition: slab.hh:538
void free(Item *item)
Definition: slab.hh:526
size_t class_size(const size_t size)
Definition: slab.hh:561
void print_slab_classes()
Definition: slab.hh:549
Item * create(const size_t size, Args &&... args)
Definition: slab.hh:459
impl::metric_definition_impl make_gauge(metric_name_type name, T &&val, description d=description(), std::vector< label_instance > labels={})
Gauge are a general purpose metric.
Definition: metrics.hh:442
impl::metric_definition_impl make_counter(metric_name_type name, T &&val, description d=description(), std::vector< label_instance > labels={})
create a counter metric
Definition: metrics.hh:528
header for metrics creation.
metrics creation and registration
Definition: estimated_histogram.hh:34
Seastar API namespace.
Definition: abort_on_ebadf.hh:26