Seastar
High performance C++ framework for concurrent servers
memory.hh
1/*
2 * This file is open source software, licensed to you under the terms
3 * of the Apache License, Version 2.0 (the "License"). See the NOTICE file
4 * distributed with this work for additional information regarding copyright
5 * ownership. You may not use this file except in compliance with the License.
6 *
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing,
12 * software distributed under the License is distributed on an
13 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14 * KIND, either express or implied. See the License for the
15 * specific language governing permissions and limitations
16 * under the License.
17 */
18/*
19 * Copyright (C) 2014 Cloudius Systems, Ltd.
20 */
21
22#pragma once
23
24#include <seastar/core/resource.hh>
25#include <seastar/core/bitops.hh>
26#include <seastar/util/backtrace.hh>
27#include <seastar/util/modules.hh>
28#include <seastar/util/sampler.hh>
29#ifndef SEASTAR_MODULE
30#include <new>
31#include <cstdint>
32#include <functional>
33#include <optional>
34#include <string>
35#include <vector>
36#endif
37
38namespace seastar {
39
50
124namespace memory {
125
127
128#ifdef SEASTAR_OVERRIDE_ALLOCATOR_PAGE_SIZE
129#define SEASTAR_INTERNAL_ALLOCATOR_PAGE_SIZE (SEASTAR_OVERRIDE_ALLOCATOR_PAGE_SIZE)
130#else
131#define SEASTAR_INTERNAL_ALLOCATOR_PAGE_SIZE 4096
132#endif
133
134static constexpr size_t page_size = SEASTAR_INTERNAL_ALLOCATOR_PAGE_SIZE;
135static constexpr size_t page_bits = log2ceil(page_size);
136static constexpr size_t huge_page_size =
137#if defined(__x86_64__) || defined(__i386__) || defined(__s390x__) || defined(__zarch__)
138 1 << 21; // 2M
139#elif defined(__aarch64__)
140 1 << 21; // 2M
141#elif defined(__PPC__)
142 1 << 24; // 16M
143#else
144#error "Huge page size is not defined for this architecture"
145#endif
146
147
148namespace internal {
149
150struct memory_range {
151 char* start;
152 char* end;
153 unsigned numa_node_id;
154};
155
156struct numa_layout {
157 std::vector<memory_range> ranges;
158};
159
160numa_layout merge(numa_layout one, numa_layout two);
161
162}
163
164internal::numa_layout configure(std::vector<resource::memory> m, bool mbind,
165 bool transparent_hugepages,
166 std::optional<std::string> hugetlbfs_path = {});
167
168void configure_minimal();
169
170// A deprecated alias for set_abort_on_allocation_failure(true).
171[[deprecated("use set_abort_on_allocation_failure(true) instead")]]
172void enable_abort_on_allocation_failure();
173
174class disable_abort_on_alloc_failure_temporarily {
175public:
176 disable_abort_on_alloc_failure_temporarily();
177 ~disable_abort_on_alloc_failure_temporarily() noexcept;
178};
179
180// Disables heap profiling as long as this object is alive.
181// Can be nested, in which case the profiling is re-enabled when all
182// the objects go out of scope.
183class disable_backtrace_temporarily {
185public:
186 disable_backtrace_temporarily();
187};
188
189enum class reclaiming_result {
190 reclaimed_nothing,
191 reclaimed_something
192};
193
194// Determines when reclaimer can be invoked
195enum class reclaimer_scope {
196 //
197 // Reclaimer is only invoked in its own fiber. That fiber will be
198 // given higher priority than regular application fibers.
199 //
200 async,
201
202 //
203 // Reclaimer may be invoked synchronously with allocation.
204 // It may also be invoked in async scope.
205 //
206 // Reclaimer may invoke allocation, though it is discouraged because
207 // the system may be low on memory and such allocations may fail.
208 // Reclaimers which allocate should be prepared for re-entry.
209 //
210 sync
211};
212
213class reclaimer {
214public:
215 struct request {
216 // The number of bytes which is needed to be released.
217 // The reclaimer can release a different amount.
218 // If less is released then the reclaimer may be invoked again.
219 size_t bytes_to_reclaim;
220 };
221 using reclaim_fn = std::function<reclaiming_result ()>;
222private:
223 std::function<reclaiming_result (request)> _reclaim;
224 reclaimer_scope _scope;
225public:
226 // Installs new reclaimer which will be invoked when system is falling
227 // low on memory. 'scope' determines when reclaimer can be executed.
228 reclaimer(std::function<reclaiming_result ()> reclaim, reclaimer_scope scope = reclaimer_scope::async);
229 reclaimer(std::function<reclaiming_result (request)> reclaim, reclaimer_scope scope = reclaimer_scope::async);
230 ~reclaimer();
231 reclaiming_result do_reclaim(size_t bytes_to_reclaim) { return _reclaim(request{bytes_to_reclaim}); }
232 reclaimer_scope scope() const { return _scope; }
233};
234
235extern std::pmr::polymorphic_allocator<char>* malloc_allocator;
236
237// Call periodically to recycle objects that were freed
238// on cpu other than the one they were allocated on.
239//
240// Returns @true if any work was actually performed.
241bool drain_cross_cpu_freelist();
242
243
244// We don't want the memory code calling back into the rest of
245// the system, so allow the rest of the system to tell the memory
246// code how to initiate reclaim.
247//
248// When memory is low, calling \c hook(fn) will result in fn being called
249// in a safe place wrt. allocations.
250void set_reclaim_hook(
251 std::function<void (std::function<void ()>)> hook);
252
254
255SEASTAR_MODULE_EXPORT_BEGIN
256
273
280
281class statistics;
282
285
288 uint64_t _mallocs;
289 uint64_t _frees;
290 uint64_t _cross_cpu_frees;
291 size_t _total_memory;
292 size_t _free_memory;
293 uint64_t _reclaims;
294 uint64_t _large_allocs;
295 uint64_t _failed_allocs;
296
297 uint64_t _foreign_mallocs;
298 uint64_t _foreign_frees;
299 uint64_t _foreign_cross_frees;
300private:
301 statistics(uint64_t mallocs, uint64_t frees, uint64_t cross_cpu_frees,
302 uint64_t total_memory, uint64_t free_memory, uint64_t reclaims,
303 uint64_t large_allocs, uint64_t failed_allocs,
304 uint64_t foreign_mallocs, uint64_t foreign_frees, uint64_t foreign_cross_frees)
305 : _mallocs(mallocs), _frees(frees), _cross_cpu_frees(cross_cpu_frees)
306 , _total_memory(total_memory), _free_memory(free_memory), _reclaims(reclaims)
307 , _large_allocs(large_allocs), _failed_allocs(failed_allocs)
308 , _foreign_mallocs(foreign_mallocs), _foreign_frees(foreign_frees)
309 , _foreign_cross_frees(foreign_cross_frees) {}
310public:
312 uint64_t mallocs() const { return _mallocs; }
314 uint64_t frees() const { return _frees; }
317 uint64_t cross_cpu_frees() const { return _cross_cpu_frees; }
319 size_t live_objects() const { return mallocs() - frees(); }
321 size_t free_memory() const { return _free_memory; }
323 size_t allocated_memory() const { return _total_memory - _free_memory; }
325 size_t total_memory() const { return _total_memory; }
327 uint64_t reclaims() const { return _reclaims; }
329 uint64_t large_allocations() const { return _large_allocs; }
332 uint64_t failed_allocations() const { return _failed_allocs; }
334 uint64_t foreign_mallocs() const { return _foreign_mallocs; }
336 uint64_t foreign_frees() const { return _foreign_frees; }
338 uint64_t foreign_cross_frees() const { return _foreign_cross_frees; }
340};
341
343 uintptr_t start;
344 uintptr_t end;
345};
346
347// Discover virtual address range used by the allocator on current shard.
348// Supported only when seastar allocator is enabled.
349memory::memory_layout get_memory_layout();
350
352size_t free_memory();
353
357
359void set_min_free_pages(size_t pages);
360
367
370
373
376 size_t _old_threshold;
377public:
378 explicit scoped_large_allocation_warning_threshold(size_t threshold)
379 : _old_threshold(get_large_allocation_warning_threshold()) {
381 }
385 if (_old_threshold) {
387 }
388 }
389 void operator=(const scoped_large_allocation_warning_threshold&) const = delete;
390 void operator=(scoped_large_allocation_warning_threshold&&) = delete;
391};
392
395 size_t _old_threshold;
396public:
398 : _old_threshold(get_large_allocation_warning_threshold()) {
400 }
404 if (_old_threshold) {
406 }
407 }
408 void operator=(const scoped_large_allocation_warning_disable&) const = delete;
409 void operator=(scoped_large_allocation_warning_disable&&) = delete;
410};
411
421 mutable size_t count = 0;
422 mutable size_t size = 0;
424
425 // All allocation sites are linked to each other. This can be used for easy
426 // iteration across them in gdb scripts where it's difficult to work with
427 // the C++ data structures.
428 mutable const allocation_site* next = nullptr; // next allocation site in the chain
429 mutable const allocation_site* prev = nullptr; // previous allocation site in the chain
430
431 bool operator==(const allocation_site& o) const {
432 return backtrace == o.backtrace;
433 }
434
435 bool operator!=(const allocation_site& o) const {
436 return !(*this == o);
437 }
438};
439
446std::vector<allocation_site> sampled_memory_profile();
447
461size_t sampled_memory_profile(allocation_site* output, size_t size);
462
483void set_heap_profiling_sampling_rate(size_t sample_rate);
484
488
496public:
497 scoped_heap_profiling(size_t) noexcept;
499};
500
501SEASTAR_MODULE_EXPORT_END
502
503}
504}
505
506namespace std {
507
508template<>
509struct hash<seastar::memory::allocation_site> {
510 size_t operator()(const seastar::memory::allocation_site& bi) const {
512 }
513};
514
515}
Enable heap profiling for the duration of the scope.
Definition: memory.hh:495
Disable large allocation warnings for a scope.
Definition: memory.hh:394
Set a different large allocation warning threshold for a scope.
Definition: memory.hh:375
Memory allocation statistics.
Definition: memory.hh:287
uint64_t failed_allocations() const
Definition: memory.hh:332
uint64_t foreign_frees() const
Number of foreign frees.
Definition: memory.hh:336
uint64_t cross_cpu_frees() const
Definition: memory.hh:317
size_t total_memory() const
Total memory (in bytes)
Definition: memory.hh:325
uint64_t large_allocations() const
Number of allocations which violated the large allocation threshold.
Definition: memory.hh:329
uint64_t mallocs() const
Total number of memory allocations calls since the system was started.
Definition: memory.hh:312
uint64_t reclaims() const
Number of reclaims performed due to low memory.
Definition: memory.hh:327
size_t live_objects() const
Total number of objects which were allocated but not freed.
Definition: memory.hh:319
uint64_t frees() const
Total number of memory deallocations calls since the system was started.
Definition: memory.hh:314
friend statistics stats()
Capture a snapshot of memory allocation statistics for this lcore.
uint64_t foreign_mallocs() const
Number of foreign allocations.
Definition: memory.hh:334
size_t free_memory() const
Total free memory (in bytes)
Definition: memory.hh:321
size_t allocated_memory() const
Total allocated memory (in bytes)
Definition: memory.hh:323
uint64_t foreign_cross_frees() const
Number of foreign frees on reactor threads.
Definition: memory.hh:338
Definition: backtrace.hh:81
futurize_t< std::invoke_result_t< Func, Args... > > async(thread_attributes attr, Func &&func, Args &&... args) noexcept
Definition: thread.hh:245
size_t get_heap_profiling_sample_rate()
Returns the current heap profiling sampling rate (0 means off)
size_t free_memory()
Returns the size of free memory in bytes.
void set_large_allocation_warning_threshold(size_t threshold)
void set_heap_profiling_sampling_rate(size_t sample_rate)
Enable sampled heap profiling by setting a sample rate.
size_t get_large_allocation_warning_threshold()
Gets the current large allocation warning threshold.
statistics stats()
Capture a snapshot of memory allocation statistics for this lcore.
void set_min_free_pages(size_t pages)
Sets the value of free memory low water mark in memory::page_size units.
size_t min_free_memory()
void set_abort_on_allocation_failure(bool enabled)
Set the global state of the abort on allocation failure behavior.
std::vector< allocation_site > sampled_memory_profile()
If memory sampling is on returns the current sampled memory live set.
bool is_abort_on_allocation_failure()
Determine the abort on allocation failure mode.
void disable_large_allocation_warning()
Disable large allocation warnings.
Definition: memory.hh:342
future configure(const options &opts)
set the metrics configuration
Seastar API namespace.
Definition: abort_on_ebadf.hh:26
STL namespace.
RAII class to temporarily pause sampling.
Definition: sampler.hh:91
Describes an allocation location in the code.
Definition: memory.hh:420
const allocation_site * next
call site for this allocation
Definition: memory.hh:428
simple_backtrace backtrace
amount of bytes in live objects allocated at backtrace.
Definition: memory.hh:423
size_t size
number of live objects allocated at backtrace.
Definition: memory.hh:422
Definition: backtrace.hh:168