memory.hh

/*
 * This file is open source software, licensed to you under the terms
 * of the Apache License, Version 2.0 (the "License").  See the NOTICE file
 * distributed with this work for additional information regarding copyright
 * ownership.  You may not use this file except in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
/*
 * Copyright (C) 2014 Cloudius Systems, Ltd.
 */
 
#pragma once
 
#include <seastar/core/resource.hh>
#include <seastar/core/bitops.hh>
#include <seastar/util/modules.hh>
#ifndef SEASTAR_MODULE
#include <new>
#include <cstdint>
#include <functional>
#include <optional>
#include <string>
#include <vector>
#endif
 
namespace seastar {
 
 
namespace memory {
 
 
#ifdef SEASTAR_OVERRIDE_ALLOCATOR_PAGE_SIZE
#define SEASTAR_INTERNAL_ALLOCATOR_PAGE_SIZE (SEASTAR_OVERRIDE_ALLOCATOR_PAGE_SIZE)
#else
#define SEASTAR_INTERNAL_ALLOCATOR_PAGE_SIZE 4096
#endif
 
static constexpr size_t page_size = SEASTAR_INTERNAL_ALLOCATOR_PAGE_SIZE;
static constexpr size_t page_bits = log2ceil(page_size);
static constexpr size_t huge_page_size =
#if defined(__x86_64__) || defined(__i386__) || defined(__s390x__) || defined(__zarch__)
    1 << 21; // 2M
#elif defined(__aarch64__)
    1 << 21; // 2M
#elif defined(__PPC__)
    1 << 24; // 16M
#else
#error "Huge page size is not defined for this architecture"
#endif
 
 
namespace internal {
 
struct memory_range {
    char* start;
    char* end;
    unsigned numa_node_id;
};
 
struct numa_layout {
    std::vector<memory_range> ranges;
};
 
numa_layout merge(numa_layout one, numa_layout two);
 
}
 
internal::numa_layout configure(std::vector<resource::memory> m, bool mbind,
        bool transparent_hugepages,
        std::optional<std::string> hugetlbfs_path = {});
 
// A deprecated alias for set_abort_on_allocation_failure(true).
[[deprecated("use set_abort_on_allocation_failure(true) instead")]]
void enable_abort_on_allocation_failure();
 
class disable_abort_on_alloc_failure_temporarily {
public:
    disable_abort_on_alloc_failure_temporarily();
    ~disable_abort_on_alloc_failure_temporarily() noexcept;
};
 
// Disables heap profiling as long as this object is alive.
// Can be nested, in which case the profiling is re-enabled when all
// the objects go out of scope.
class disable_backtrace_temporarily {
    bool _old;
public:
    disable_backtrace_temporarily();
    ~disable_backtrace_temporarily();
};
 
enum class reclaiming_result {
    reclaimed_nothing,
    reclaimed_something
};
 
// Determines when reclaimer can be invoked
enum class reclaimer_scope {
    //
    // Reclaimer is only invoked in its own fiber. That fiber will be
    // given higher priority than regular application fibers.
    //
    async,
 
    //
    // Reclaimer may be invoked synchronously with allocation.
    // It may also be invoked in async scope.
    //
    // Reclaimer may invoke allocation, though it is discouraged because
    // the system may be low on memory and such allocations may fail.
    // Reclaimers which allocate should be prepared for re-entry.
    //
    sync
};
 
class reclaimer {
public:
    struct request {
        // The number of bytes which is needed to be released.
        // The reclaimer can release a different amount.
        // If less is released then the reclaimer may be invoked again.
        size_t bytes_to_reclaim;
    };
    using reclaim_fn = std::function<reclaiming_result ()>;
private:
    std::function<reclaiming_result (request)> _reclaim;
    reclaimer_scope _scope;
public:
    // Installs new reclaimer which will be invoked when system is falling
    // low on memory. 'scope' determines when reclaimer can be executed.
    reclaimer(std::function<reclaiming_result ()> reclaim, reclaimer_scope scope = reclaimer_scope::async);
    reclaimer(std::function<reclaiming_result (request)> reclaim, reclaimer_scope scope = reclaimer_scope::async);
    ~reclaimer();
    reclaiming_result do_reclaim(size_t bytes_to_reclaim) { return _reclaim(request{bytes_to_reclaim}); }
    reclaimer_scope scope() const { return _scope; }
};
 
extern std::pmr::polymorphic_allocator<char>* malloc_allocator;
 
// Call periodically to recycle objects that were freed
// on cpu other than the one they were allocated on.
//
// Returns @true if any work was actually performed.
bool drain_cross_cpu_freelist();
 
 
// We don't want the memory code calling back into the rest of
// the system, so allow the rest of the system to tell the memory
// code how to initiate reclaim.
//
// When memory is low, calling \c hook(fn) will result in fn being called
// in a safe place wrt. allocations.
void set_reclaim_hook(
        std::function<void (std::function<void ()>)> hook);
 
 
SEASTAR_MODULE_EXPORT_BEGIN
 
void set_abort_on_allocation_failure(bool enabled);
 
bool is_abort_on_allocation_failure();
 
class statistics;
 
statistics stats();
 
class statistics {
    uint64_t _mallocs;
    uint64_t _frees;
    uint64_t _cross_cpu_frees;
    size_t _total_memory;
    size_t _free_memory;
    uint64_t _reclaims;
    uint64_t _large_allocs;
    uint64_t _failed_allocs;
 
    uint64_t _foreign_mallocs;
    uint64_t _foreign_frees;
    uint64_t _foreign_cross_frees;
private:
    statistics(uint64_t mallocs, uint64_t frees, uint64_t cross_cpu_frees,
            uint64_t total_memory, uint64_t free_memory, uint64_t reclaims,
            uint64_t large_allocs, uint64_t failed_allocs,
            uint64_t foreign_mallocs, uint64_t foreign_frees, uint64_t foreign_cross_frees)
        : _mallocs(mallocs), _frees(frees), _cross_cpu_frees(cross_cpu_frees)
        , _total_memory(total_memory), _free_memory(free_memory), _reclaims(reclaims)
        , _large_allocs(large_allocs), _failed_allocs(failed_allocs)
        , _foreign_mallocs(foreign_mallocs), _foreign_frees(foreign_frees)
        , _foreign_cross_frees(foreign_cross_frees) {}
public:
    uint64_t mallocs() const { return _mallocs; }
    uint64_t frees() const { return _frees; }
    uint64_t cross_cpu_frees() const { return _cross_cpu_frees; }
    size_t live_objects() const { return mallocs() - frees(); }
    size_t free_memory() const { return _free_memory; }
    size_t allocated_memory() const { return _total_memory - _free_memory; }
    size_t total_memory() const { return _total_memory; }
    uint64_t reclaims() const { return _reclaims; }
    uint64_t large_allocations() const { return _large_allocs; }
    uint64_t failed_allocations() const { return _failed_allocs; }
    uint64_t foreign_mallocs() const { return _foreign_mallocs; }
    uint64_t foreign_frees() const { return _foreign_frees; }
    uint64_t foreign_cross_frees() const { return _foreign_cross_frees; }
    friend statistics stats();
};
 
struct memory_layout {
    uintptr_t start;
    uintptr_t end;
};
 
// Discover virtual address range used by the allocator on current shard.
// Supported only when seastar allocator is enabled.
memory::memory_layout get_memory_layout();
 
size_t free_memory();
 
size_t min_free_memory();
 
void set_min_free_pages(size_t pages);
 
void set_large_allocation_warning_threshold(size_t threshold);
 
size_t get_large_allocation_warning_threshold();
 
void disable_large_allocation_warning();
 
class scoped_large_allocation_warning_threshold {
    size_t _old_threshold;
public:
    explicit scoped_large_allocation_warning_threshold(size_t threshold)
            : _old_threshold(get_large_allocation_warning_threshold()) {
        set_large_allocation_warning_threshold(threshold);
    }
    scoped_large_allocation_warning_threshold(const scoped_large_allocation_warning_threshold&) = delete;
    scoped_large_allocation_warning_threshold(scoped_large_allocation_warning_threshold&& x) = delete;
    ~scoped_large_allocation_warning_threshold() {
        if (_old_threshold) {
            set_large_allocation_warning_threshold(_old_threshold);
        }
    }
    void operator=(const scoped_large_allocation_warning_threshold&) const = delete;
    void operator=(scoped_large_allocation_warning_threshold&&) = delete;
};
 
class scoped_large_allocation_warning_disable {
    size_t _old_threshold;
public:
    scoped_large_allocation_warning_disable()
            : _old_threshold(get_large_allocation_warning_threshold()) {
        disable_large_allocation_warning();
    }
    scoped_large_allocation_warning_disable(const scoped_large_allocation_warning_disable&) = delete;
    scoped_large_allocation_warning_disable(scoped_large_allocation_warning_disable&& x) = delete;
    ~scoped_large_allocation_warning_disable() {
        if (_old_threshold) {
            set_large_allocation_warning_threshold(_old_threshold);
        }
    }
    void operator=(const scoped_large_allocation_warning_disable&) const = delete;
    void operator=(scoped_large_allocation_warning_disable&&) = delete;
};
 
void set_heap_profiling_enabled(bool);
 
class scoped_heap_profiling {
public:
    scoped_heap_profiling() noexcept;
    ~scoped_heap_profiling();
};
 
SEASTAR_MODULE_EXPORT_END
 
}
}