Seastar
High performance C++ framework for concurrent servers
prefetch.hh
1 /*
2  * This file is open source software, licensed to you under the terms
3  * of the Apache License, Version 2.0 (the "License"). See the NOTICE file
4  * distributed with this work for additional information regarding copyright
5  * ownership. You may not use this file except in compliance with the License.
6  *
7  * You may obtain a copy of the License at
8  *
9  * http://www.apache.org/licenses/LICENSE-2.0
10  *
11  * Unless required by applicable law or agreed to in writing,
12  * software distributed under the License is distributed on an
13  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14  * KIND, either express or implied. See the License for the
15  * specific language governing permissions and limitations
16  * under the License.
17  */
18 /*
19  * Copyright (C) 2014 Cloudius Systems, Ltd.
20  */
21 
22 #pragma once
23 
24 #ifndef SEASTAR_MODULE
25 #include <algorithm>
26 #include <atomic>
27 #include <boost/mpl/range_c.hpp>
28 #include <boost/mpl/for_each.hpp>
29 #include <seastar/core/align.hh>
30 #include <seastar/core/cacheline.hh>
31 #include <seastar/util/modules.hh>
32 #endif
33 
34 namespace seastar {
35 
36 SEASTAR_MODULE_EXPORT_BEGIN
37 
38 template <size_t N, int RW, int LOC>
39 struct prefetcher;
40 
41 template<int RW, int LOC>
42 struct prefetcher<0, RW, LOC> {
43  prefetcher(uintptr_t ptr) {}
44 };
45 
46 template <size_t N, int RW, int LOC>
47 struct prefetcher {
48  prefetcher(uintptr_t ptr) {
49  __builtin_prefetch(reinterpret_cast<void*>(ptr), RW, LOC);
50  std::atomic_signal_fence(std::memory_order_seq_cst);
51  prefetcher<N-cache_line_size, RW, LOC>(ptr + cache_line_size);
52  }
53 };
54 
55 // LOC is a locality from __buitin_prefetch() gcc documentation:
56 // "The value locality must be a compile-time constant integer between zero and three. A value of
57 // zero means that the data has no temporal locality, so it need not be left in the cache after
58 // the access. A value of three means that the data has a high degree of temporal locality and
59 // should be left in all levels of cache possible. Values of one and two mean, respectively, a
60 // low or moderate degree of temporal locality. The default is three."
61 template<typename T, int LOC = 3>
62 void prefetch(T* ptr) {
63  prefetcher<align_up(sizeof(T), cache_line_size), 0, LOC>(reinterpret_cast<uintptr_t>(ptr));
64 }
65 
66 template<typename Iterator, int LOC = 3>
67 void prefetch(Iterator begin, Iterator end) {
68  std::for_each(begin, end, [] (auto v) { prefetch<decltype(*v), LOC>(v); });
69 }
70 
71 template<size_t C, typename T, int LOC = 3>
72 void prefetch_n(T** pptr) {
73  boost::mpl::for_each< boost::mpl::range_c<size_t,0,C> >( [pptr] (size_t x) { prefetch<T, LOC>(*(pptr + x)); } );
74 }
75 
76 template<size_t L, int LOC = 3>
77 void prefetch(void* ptr) {
78  prefetcher<L*cache_line_size, 0, LOC>(reinterpret_cast<uintptr_t>(ptr));
79 }
80 
81 template<size_t L, typename Iterator, int LOC = 3>
82 void prefetch_n(Iterator begin, Iterator end) {
83  std::for_each(begin, end, [] (auto v) { prefetch<L, LOC>(v); });
84 }
85 
86 template<size_t L, size_t C, typename T, int LOC = 3>
87 void prefetch_n(T** pptr) {
88  boost::mpl::for_each< boost::mpl::range_c<size_t,0,C> >( [pptr] (size_t x) { prefetch<L, LOC>(*(pptr + x)); } );
89 }
90 
91 template<typename T, int LOC = 3>
92 void prefetchw(T* ptr) {
93  prefetcher<align_up(sizeof(T), cache_line_size), 1, LOC>(reinterpret_cast<uintptr_t>(ptr));
94 }
95 
96 template<typename Iterator, int LOC = 3>
97 void prefetchw_n(Iterator begin, Iterator end) {
98  std::for_each(begin, end, [] (auto v) { prefetchw<decltype(*v), LOC>(v); });
99 }
100 
101 template<size_t C, typename T, int LOC = 3>
102 void prefetchw_n(T** pptr) {
103  boost::mpl::for_each< boost::mpl::range_c<size_t,0,C> >( [pptr] (size_t x) { prefetchw<T, LOC>(*(pptr + x)); } );
104 }
105 
106 template<size_t L, int LOC = 3>
107 void prefetchw(void* ptr) {
108  prefetcher<L*cache_line_size, 1, LOC>(reinterpret_cast<uintptr_t>(ptr));
109 }
110 
111 template<size_t L, typename Iterator, int LOC = 3>
112 void prefetchw_n(Iterator begin, Iterator end) {
113  std::for_each(begin, end, [] (auto v) { prefetchw<L, LOC>(v); });
114 }
115 
116 template<size_t L, size_t C, typename T, int LOC = 3>
117 void prefetchw_n(T** pptr) {
118  boost::mpl::for_each< boost::mpl::range_c<size_t,0,C> >( [pptr] (size_t x) { prefetchw<L, LOC>(*(pptr + x)); } );
119 }
120 SEASTAR_MODULE_EXPORT_END
121 
122 }
Seastar API namespace.
Definition: abort_on_ebadf.hh:26
Definition: prefetch.hh:47