Seastar
High performance C++ framework for concurrent servers
prefetch.hh
1/*
2 * This file is open source software, licensed to you under the terms
3 * of the Apache License, Version 2.0 (the "License"). See the NOTICE file
4 * distributed with this work for additional information regarding copyright
5 * ownership. You may not use this file except in compliance with the License.
6 *
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing,
12 * software distributed under the License is distributed on an
13 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14 * KIND, either express or implied. See the License for the
15 * specific language governing permissions and limitations
16 * under the License.
17 */
18/*
19 * Copyright (C) 2014 Cloudius Systems, Ltd.
20 */
21
22#pragma once
23
24#ifndef SEASTAR_MODULE
25#include <algorithm>
26#include <atomic>
27#include <utility>
28#include <functional>
29#include <seastar/core/align.hh>
30#include <seastar/core/cacheline.hh>
31#include <seastar/util/modules.hh>
32#endif
33
34namespace seastar {
35
36SEASTAR_MODULE_EXPORT_BEGIN
37
38template <size_t N, int RW, int LOC>
39struct prefetcher;
40
41template<int RW, int LOC>
42struct prefetcher<0, RW, LOC> {
43 prefetcher(uintptr_t ptr) {}
44};
45
46template <size_t N, int RW, int LOC>
47struct prefetcher {
48 prefetcher(uintptr_t ptr) {
49 __builtin_prefetch(reinterpret_cast<void*>(ptr), RW, LOC);
50 std::atomic_signal_fence(std::memory_order_seq_cst);
51 prefetcher<N-cache_line_size, RW, LOC>(ptr + cache_line_size);
52 }
53};
54
55// LOC is a locality from __buitin_prefetch() gcc documentation:
56// "The value locality must be a compile-time constant integer between zero and three. A value of
57// zero means that the data has no temporal locality, so it need not be left in the cache after
58// the access. A value of three means that the data has a high degree of temporal locality and
59// should be left in all levels of cache possible. Values of one and two mean, respectively, a
60// low or moderate degree of temporal locality. The default is three."
61template<typename T, int LOC = 3>
62void prefetch(T* ptr) {
63 prefetcher<align_up(sizeof(T), cache_line_size), 0, LOC>(reinterpret_cast<uintptr_t>(ptr));
64}
65
66template<typename Iterator, int LOC = 3>
67void prefetch(Iterator begin, Iterator end) {
68 std::for_each(begin, end, [] (auto v) { prefetch<decltype(*v), LOC>(v); });
69}
70
71template<size_t C, typename T, int LOC = 3>
72void prefetch_n(T** pptr) {
73 std::invoke([&] <size_t... x> (std::index_sequence<x...>) {
74 (..., prefetch<T, LOC>(*(pptr + x)));
75 }, std::make_index_sequence<C>{});
76}
77
78template<size_t L, int LOC = 3>
79void prefetch(void* ptr) {
80 prefetcher<L*cache_line_size, 0, LOC>(reinterpret_cast<uintptr_t>(ptr));
81}
82
83template<size_t L, typename Iterator, int LOC = 3>
84void prefetch_n(Iterator begin, Iterator end) {
85 std::for_each(begin, end, [] (auto v) { prefetch<L, LOC>(v); });
86}
87
88template<size_t L, size_t C, typename T, int LOC = 3>
89void prefetch_n(T** pptr) {
90 std::invoke([&] <size_t... x> (std::index_sequence<x...>) {
91 (..., prefetch<L, LOC>(*(pptr + x)));
92 }, std::make_index_sequence<C>{});
93}
94
95template<typename T, int LOC = 3>
96void prefetchw(T* ptr) {
97 prefetcher<align_up(sizeof(T), cache_line_size), 1, LOC>(reinterpret_cast<uintptr_t>(ptr));
98}
99
100template<typename Iterator, int LOC = 3>
101void prefetchw_n(Iterator begin, Iterator end) {
102 std::for_each(begin, end, [] (auto v) { prefetchw<decltype(*v), LOC>(v); });
103}
104
105template<size_t C, typename T, int LOC = 3>
106void prefetchw_n(T** pptr) {
107 std::invoke([&] <size_t... x> (std::index_sequence<x...>) {
108 (..., prefetchw<T, LOC>(*(pptr + x)));
109 }, std::make_index_sequence<C>{});
110}
111
112template<size_t L, int LOC = 3>
113void prefetchw(void* ptr) {
114 prefetcher<L*cache_line_size, 1, LOC>(reinterpret_cast<uintptr_t>(ptr));
115}
116
117template<size_t L, typename Iterator, int LOC = 3>
118void prefetchw_n(Iterator begin, Iterator end) {
119 std::for_each(begin, end, [] (auto v) { prefetchw<L, LOC>(v); });
120}
121
122template<size_t L, size_t C, typename T, int LOC = 3>
123void prefetchw_n(T** pptr) {
124 std::invoke([&] <size_t... x> (std::index_sequence<x...>) {
125 (..., prefetchw<L, LOC>(*(pptr + x)));
126 }, std::make_index_sequence<C>{});
127}
128SEASTAR_MODULE_EXPORT_END
129
130}
Seastar API namespace.
Definition: abort_on_ebadf.hh:26
Definition: prefetch.hh:47