File: | deps/jemalloc/src/tcache.c |
Warning: | line 217, column 10 Value stored to 'ptr' during its initialization is never read |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | #define JEMALLOC_TCACHE_C_ |
2 | #include "jemalloc/internal/jemalloc_preamble.h" |
3 | #include "jemalloc/internal/jemalloc_internal_includes.h" |
4 | |
5 | #include "jemalloc/internal/assert.h" |
6 | #include "jemalloc/internal/mutex.h" |
7 | #include "jemalloc/internal/size_classes.h" |
8 | |
9 | /******************************************************************************/ |
10 | /* Data. */ |
11 | |
12 | bool_Bool opt_tcache = true1; |
13 | ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT15; |
14 | |
15 | cache_bin_info_t *tcache_bin_info; |
16 | static unsigned stack_nelms; /* Total stack elms per tcache. */ |
17 | |
18 | unsigned nhbins; |
19 | size_t tcache_maxclass; |
20 | |
21 | tcaches_t *tcaches; |
22 | |
23 | /* Index of first element within tcaches that has never been used. */ |
24 | static unsigned tcaches_past; |
25 | |
26 | /* Head of singly linked list tracking available tcaches elements. */ |
27 | static tcaches_t *tcaches_avail; |
28 | |
29 | /* Protects tcaches{,_past,_avail}. */ |
30 | static malloc_mutex_t tcaches_mtx; |
31 | |
32 | /******************************************************************************/ |
33 | |
34 | size_t |
35 | tcache_salloc(tsdn_t *tsdn, const void *ptr) { |
36 | return arena_salloc(tsdn, ptr); |
37 | } |
38 | |
39 | void |
40 | tcache_event_hard(tsd_t *tsd, tcache_t *tcache) { |
41 | szind_t binind = tcache->next_gc_bin; |
42 | |
43 | cache_bin_t *tbin; |
44 | if (binind < NBINS39) { |
45 | tbin = tcache_small_bin_get(tcache, binind); |
46 | } else { |
47 | tbin = tcache_large_bin_get(tcache, binind); |
48 | } |
49 | if (tbin->low_water > 0) { |
50 | /* |
51 | * Flush (ceiling) 3/4 of the objects below the low water mark. |
52 | */ |
53 | if (binind < NBINS39) { |
54 | tcache_bin_flush_small(tsd, tcache, tbin, binind, |
55 | tbin->ncached - tbin->low_water + (tbin->low_water |
56 | >> 2)); |
57 | /* |
58 | * Reduce fill count by 2X. Limit lg_fill_div such that |
59 | * the fill count is always at least 1. |
60 | */ |
61 | cache_bin_info_t *tbin_info = &tcache_bin_info[binind]; |
62 | if ((tbin_info->ncached_max >> |
63 | (tcache->lg_fill_div[binind] + 1)) >= 1) { |
64 | tcache->lg_fill_div[binind]++; |
65 | } |
66 | } else { |
67 | tcache_bin_flush_large(tsd, tbin, binind, tbin->ncached |
68 | - tbin->low_water + (tbin->low_water >> 2), tcache); |
69 | } |
70 | } else if (tbin->low_water < 0) { |
71 | /* |
72 | * Increase fill count by 2X for small bins. Make sure |
73 | * lg_fill_div stays greater than 0. |
74 | */ |
75 | if (binind < NBINS39 && tcache->lg_fill_div[binind] > 1) { |
76 | tcache->lg_fill_div[binind]--; |
77 | } |
78 | } |
79 | tbin->low_water = tbin->ncached; |
80 | |
81 | tcache->next_gc_bin++; |
82 | if (tcache->next_gc_bin == nhbins) { |
83 | tcache->next_gc_bin = 0; |
84 | } |
85 | } |
86 | |
87 | void * |
88 | tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, |
89 | cache_bin_t *tbin, szind_t binind, bool_Bool *tcache_success) { |
90 | void *ret; |
91 | |
92 | assert(tcache->arena != NULL)do { if (__builtin_expect(!!(config_debug && !(tcache ->arena != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/tcache.c", 92, "tcache->arena != NULL"); abort(); } } while (0); |
93 | arena_tcache_fill_small(tsdn, arena, tcache, tbin, binind, |
94 | config_prof ? tcache->prof_accumbytes : 0); |
95 | if (config_prof) { |
96 | tcache->prof_accumbytes = 0; |
97 | } |
98 | ret = cache_bin_alloc_easy(tbin, tcache_success); |
99 | |
100 | return ret; |
101 | } |
102 | |
103 | void |
104 | tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin, |
105 | szind_t binind, unsigned rem) { |
106 | bool_Bool merged_stats = false0; |
107 | |
108 | assert(binind < NBINS)do { if (__builtin_expect(!!(config_debug && !(binind < 39)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/tcache.c", 108, "binind < NBINS"); abort(); } } while (0); |
109 | assert((cache_bin_sz_t)rem <= tbin->ncached)do { if (__builtin_expect(!!(config_debug && !((cache_bin_sz_t )rem <= tbin->ncached)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/tcache.c", 109, "(cache_bin_sz_t)rem <= tbin->ncached" ); abort(); } } while (0); |
110 | |
111 | arena_t *arena = tcache->arena; |
112 | assert(arena != NULL)do { if (__builtin_expect(!!(config_debug && !(arena != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/tcache.c", 112, "arena != NULL"); abort(); } } while ( 0); |
113 | unsigned nflush = tbin->ncached - rem; |
114 | VARIABLE_ARRAY(extent_t *, item_extent, nflush)extent_t * item_extent[(nflush)]; |
115 | /* Look up extent once per item. */ |
116 | for (unsigned i = 0 ; i < nflush; i++) { |
117 | item_extent[i] = iealloc(tsd_tsdn(tsd), *(tbin->avail - 1 - i)); |
118 | } |
119 | |
120 | while (nflush > 0) { |
121 | /* Lock the arena bin associated with the first object. */ |
122 | extent_t *extent = item_extent[0]; |
123 | arena_t *bin_arena = extent_arena_get(extent); |
124 | bin_t *bin = &bin_arena->bins[binind]; |
125 | |
126 | if (config_prof && bin_arena == arena) { |
127 | if (arena_prof_accum(tsd_tsdn(tsd), arena, |
128 | tcache->prof_accumbytes)) { |
129 | prof_idump(tsd_tsdn(tsd)); |
130 | } |
131 | tcache->prof_accumbytes = 0; |
132 | } |
133 | |
134 | malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); |
135 | if (config_stats && bin_arena == arena) { |
136 | assert(!merged_stats)do { if (__builtin_expect(!!(config_debug && !(!merged_stats )), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/tcache.c", 136, "!merged_stats"); abort(); } } while ( 0); |
137 | merged_stats = true1; |
138 | bin->stats.nflushes++; |
139 | bin->stats.nrequests += tbin->tstats.nrequests; |
140 | tbin->tstats.nrequests = 0; |
141 | } |
142 | unsigned ndeferred = 0; |
143 | for (unsigned i = 0; i < nflush; i++) { |
144 | void *ptr = *(tbin->avail - 1 - i); |
145 | extent = item_extent[i]; |
146 | assert(ptr != NULL && extent != NULL)do { if (__builtin_expect(!!(config_debug && !(ptr != ((void*)0) && extent != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/tcache.c" , 146, "ptr != NULL && extent != NULL"); abort(); } } while (0); |
147 | |
148 | if (extent_arena_get(extent) == bin_arena) { |
149 | arena_dalloc_bin_junked_locked(tsd_tsdn(tsd), |
150 | bin_arena, extent, ptr); |
151 | } else { |
152 | /* |
153 | * This object was allocated via a different |
154 | * arena bin than the one that is currently |
155 | * locked. Stash the object, so that it can be |
156 | * handled in a future pass. |
157 | */ |
158 | *(tbin->avail - 1 - ndeferred) = ptr; |
159 | item_extent[ndeferred] = extent; |
160 | ndeferred++; |
161 | } |
162 | } |
163 | malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); |
164 | arena_decay_ticks(tsd_tsdn(tsd), bin_arena, nflush - ndeferred); |
165 | nflush = ndeferred; |
166 | } |
167 | if (config_stats && !merged_stats) { |
168 | /* |
169 | * The flush loop didn't happen to flush to this thread's |
170 | * arena, so the stats didn't get merged. Manually do so now. |
171 | */ |
172 | bin_t *bin = &arena->bins[binind]; |
173 | malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); |
174 | bin->stats.nflushes++; |
175 | bin->stats.nrequests += tbin->tstats.nrequests; |
176 | tbin->tstats.nrequests = 0; |
177 | malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); |
178 | } |
179 | |
180 | memmove(tbin->avail - rem, tbin->avail - tbin->ncached, rem * |
181 | sizeof(void *)); |
182 | tbin->ncached = rem; |
183 | if (tbin->ncached < tbin->low_water) { |
184 | tbin->low_water = tbin->ncached; |
185 | } |
186 | } |
187 | |
188 | void |
189 | tcache_bin_flush_large(tsd_t *tsd, cache_bin_t *tbin, szind_t binind, |
190 | unsigned rem, tcache_t *tcache) { |
191 | bool_Bool merged_stats = false0; |
192 | |
193 | assert(binind < nhbins)do { if (__builtin_expect(!!(config_debug && !(binind < nhbins)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/tcache.c", 193, "binind < nhbins"); abort(); } } while (0); |
194 | assert((cache_bin_sz_t)rem <= tbin->ncached)do { if (__builtin_expect(!!(config_debug && !((cache_bin_sz_t )rem <= tbin->ncached)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/tcache.c", 194, "(cache_bin_sz_t)rem <= tbin->ncached" ); abort(); } } while (0); |
195 | |
196 | arena_t *arena = tcache->arena; |
197 | assert(arena != NULL)do { if (__builtin_expect(!!(config_debug && !(arena != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/tcache.c", 197, "arena != NULL"); abort(); } } while ( 0); |
198 | unsigned nflush = tbin->ncached - rem; |
199 | VARIABLE_ARRAY(extent_t *, item_extent, nflush)extent_t * item_extent[(nflush)]; |
200 | /* Look up extent once per item. */ |
201 | for (unsigned i = 0 ; i < nflush; i++) { |
202 | item_extent[i] = iealloc(tsd_tsdn(tsd), *(tbin->avail - 1 - i)); |
203 | } |
204 | |
205 | while (nflush > 0) { |
206 | /* Lock the arena associated with the first object. */ |
207 | extent_t *extent = item_extent[0]; |
208 | arena_t *locked_arena = extent_arena_get(extent); |
209 | UNUSED__attribute__((unused)) bool_Bool idump; |
210 | |
211 | if (config_prof) { |
212 | idump = false0; |
213 | } |
214 | |
215 | malloc_mutex_lock(tsd_tsdn(tsd), &locked_arena->large_mtx); |
216 | for (unsigned i = 0; i < nflush; i++) { |
217 | void *ptr = *(tbin->avail - 1 - i); |
Value stored to 'ptr' during its initialization is never read | |
218 | assert(ptr != NULL)do { if (__builtin_expect(!!(config_debug && !(ptr != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/tcache.c", 218, "ptr != NULL"); abort(); } } while (0); |
219 | extent = item_extent[i]; |
220 | if (extent_arena_get(extent) == locked_arena) { |
221 | large_dalloc_prep_junked_locked(tsd_tsdn(tsd), |
222 | extent); |
223 | } |
224 | } |
225 | if ((config_prof || config_stats) && locked_arena == arena) { |
226 | if (config_prof) { |
227 | idump = arena_prof_accum(tsd_tsdn(tsd), arena, |
228 | tcache->prof_accumbytes); |
229 | tcache->prof_accumbytes = 0; |
230 | } |
231 | if (config_stats) { |
232 | merged_stats = true1; |
233 | arena_stats_large_nrequests_add(tsd_tsdn(tsd), |
234 | &arena->stats, binind, |
235 | tbin->tstats.nrequests); |
236 | tbin->tstats.nrequests = 0; |
237 | } |
238 | } |
239 | malloc_mutex_unlock(tsd_tsdn(tsd), &locked_arena->large_mtx); |
240 | |
241 | unsigned ndeferred = 0; |
242 | for (unsigned i = 0; i < nflush; i++) { |
243 | void *ptr = *(tbin->avail - 1 - i); |
244 | extent = item_extent[i]; |
245 | assert(ptr != NULL && extent != NULL)do { if (__builtin_expect(!!(config_debug && !(ptr != ((void*)0) && extent != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/tcache.c" , 245, "ptr != NULL && extent != NULL"); abort(); } } while (0); |
246 | |
247 | if (extent_arena_get(extent) == locked_arena) { |
248 | large_dalloc_finish(tsd_tsdn(tsd), extent); |
249 | } else { |
250 | /* |
251 | * This object was allocated via a different |
252 | * arena than the one that is currently locked. |
253 | * Stash the object, so that it can be handled |
254 | * in a future pass. |
255 | */ |
256 | *(tbin->avail - 1 - ndeferred) = ptr; |
257 | item_extent[ndeferred] = extent; |
258 | ndeferred++; |
259 | } |
260 | } |
261 | if (config_prof && idump) { |
262 | prof_idump(tsd_tsdn(tsd)); |
263 | } |
264 | arena_decay_ticks(tsd_tsdn(tsd), locked_arena, nflush - |
265 | ndeferred); |
266 | nflush = ndeferred; |
267 | } |
268 | if (config_stats && !merged_stats) { |
269 | /* |
270 | * The flush loop didn't happen to flush to this thread's |
271 | * arena, so the stats didn't get merged. Manually do so now. |
272 | */ |
273 | arena_stats_large_nrequests_add(tsd_tsdn(tsd), &arena->stats, |
274 | binind, tbin->tstats.nrequests); |
275 | tbin->tstats.nrequests = 0; |
276 | } |
277 | |
278 | memmove(tbin->avail - rem, tbin->avail - tbin->ncached, rem * |
279 | sizeof(void *)); |
280 | tbin->ncached = rem; |
281 | if (tbin->ncached < tbin->low_water) { |
282 | tbin->low_water = tbin->ncached; |
283 | } |
284 | } |
285 | |
286 | void |
287 | tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { |
288 | assert(tcache->arena == NULL)do { if (__builtin_expect(!!(config_debug && !(tcache ->arena == ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/tcache.c", 288, "tcache->arena == NULL"); abort(); } } while (0); |
289 | tcache->arena = arena; |
290 | |
291 | if (config_stats) { |
292 | /* Link into list of extant tcaches. */ |
293 | malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); |
294 | |
295 | ql_elm_new(tcache, link)do { ((tcache))->link.qre_next = ((tcache)); ((tcache))-> link.qre_prev = ((tcache)); } while (0); |
296 | ql_tail_insert(&arena->tcache_ql, tcache, link)do { if (((&arena->tcache_ql)->qlh_first) != ((void *)0)) { do { ((tcache))->link.qre_prev = (((&arena-> tcache_ql)->qlh_first))->link.qre_prev; ((tcache))-> link.qre_next = (((&arena->tcache_ql)->qlh_first)); ((tcache))->link.qre_prev->link.qre_next = ((tcache)); (((&arena->tcache_ql)->qlh_first))->link.qre_prev = ((tcache)); } while (0); } ((&arena->tcache_ql)-> qlh_first) = (((tcache))->link.qre_next); } while (0); |
297 | cache_bin_array_descriptor_init( |
298 | &tcache->cache_bin_array_descriptor, tcache->bins_small, |
299 | tcache->bins_large); |
300 | ql_tail_insert(&arena->cache_bin_array_descriptor_ql,do { if (((&arena->cache_bin_array_descriptor_ql)-> qlh_first) != ((void*)0)) { do { ((&tcache->cache_bin_array_descriptor ))->link.qre_prev = (((&arena->cache_bin_array_descriptor_ql )->qlh_first))->link.qre_prev; ((&tcache->cache_bin_array_descriptor ))->link.qre_next = (((&arena->cache_bin_array_descriptor_ql )->qlh_first)); ((&tcache->cache_bin_array_descriptor ))->link.qre_prev->link.qre_next = ((&tcache->cache_bin_array_descriptor )); (((&arena->cache_bin_array_descriptor_ql)->qlh_first ))->link.qre_prev = ((&tcache->cache_bin_array_descriptor )); } while (0); } ((&arena->cache_bin_array_descriptor_ql )->qlh_first) = (((&tcache->cache_bin_array_descriptor ))->link.qre_next); } while (0) |
301 | &tcache->cache_bin_array_descriptor, link)do { if (((&arena->cache_bin_array_descriptor_ql)-> qlh_first) != ((void*)0)) { do { ((&tcache->cache_bin_array_descriptor ))->link.qre_prev = (((&arena->cache_bin_array_descriptor_ql )->qlh_first))->link.qre_prev; ((&tcache->cache_bin_array_descriptor ))->link.qre_next = (((&arena->cache_bin_array_descriptor_ql )->qlh_first)); ((&tcache->cache_bin_array_descriptor ))->link.qre_prev->link.qre_next = ((&tcache->cache_bin_array_descriptor )); (((&arena->cache_bin_array_descriptor_ql)->qlh_first ))->link.qre_prev = ((&tcache->cache_bin_array_descriptor )); } while (0); } ((&arena->cache_bin_array_descriptor_ql )->qlh_first) = (((&tcache->cache_bin_array_descriptor ))->link.qre_next); } while (0); |
302 | |
303 | malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx); |
304 | } |
305 | } |
306 | |
307 | static void |
308 | tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache) { |
309 | arena_t *arena = tcache->arena; |
310 | assert(arena != NULL)do { if (__builtin_expect(!!(config_debug && !(arena != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/tcache.c", 310, "arena != NULL"); abort(); } } while ( 0); |
311 | if (config_stats) { |
312 | /* Unlink from list of extant tcaches. */ |
313 | malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); |
314 | if (config_debug) { |
315 | bool_Bool in_ql = false0; |
316 | tcache_t *iter; |
317 | ql_foreach(iter, &arena->tcache_ql, link)for (((iter)) = (((&arena->tcache_ql)->qlh_first)); ((iter)) != ((void*)0); ((iter)) = ((((iter))->link.qre_next != (((&arena->tcache_ql)->qlh_first))) ? ((iter))-> link.qre_next : ((void*)0))) { |
318 | if (iter == tcache) { |
319 | in_ql = true1; |
320 | break; |
321 | } |
322 | } |
323 | assert(in_ql)do { if (__builtin_expect(!!(config_debug && !(in_ql) ), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/tcache.c", 323, "in_ql"); abort(); } } while (0); |
324 | } |
325 | ql_remove(&arena->tcache_ql, tcache, link)do { if (((&arena->tcache_ql)->qlh_first) == (tcache )) { ((&arena->tcache_ql)->qlh_first) = ((((&arena ->tcache_ql)->qlh_first))->link.qre_next); } if (((& arena->tcache_ql)->qlh_first) != (tcache)) { do { ((tcache ))->link.qre_prev->link.qre_next = ((tcache))->link. qre_next; ((tcache))->link.qre_next->link.qre_prev = (( tcache))->link.qre_prev; ((tcache))->link.qre_next = (( tcache)); ((tcache))->link.qre_prev = ((tcache)); } while ( 0); } else { ((&arena->tcache_ql)->qlh_first) = ((void *)0); } } while (0); |
326 | ql_remove(&arena->cache_bin_array_descriptor_ql,do { if (((&arena->cache_bin_array_descriptor_ql)-> qlh_first) == (&tcache->cache_bin_array_descriptor)) { ((&arena->cache_bin_array_descriptor_ql)->qlh_first ) = ((((&arena->cache_bin_array_descriptor_ql)->qlh_first ))->link.qre_next); } if (((&arena->cache_bin_array_descriptor_ql )->qlh_first) != (&tcache->cache_bin_array_descriptor )) { do { ((&tcache->cache_bin_array_descriptor))-> link.qre_prev->link.qre_next = ((&tcache->cache_bin_array_descriptor ))->link.qre_next; ((&tcache->cache_bin_array_descriptor ))->link.qre_next->link.qre_prev = ((&tcache->cache_bin_array_descriptor ))->link.qre_prev; ((&tcache->cache_bin_array_descriptor ))->link.qre_next = ((&tcache->cache_bin_array_descriptor )); ((&tcache->cache_bin_array_descriptor))->link.qre_prev = ((&tcache->cache_bin_array_descriptor)); } while (0 ); } else { ((&arena->cache_bin_array_descriptor_ql)-> qlh_first) = ((void*)0); } } while (0) |
327 | &tcache->cache_bin_array_descriptor, link)do { if (((&arena->cache_bin_array_descriptor_ql)-> qlh_first) == (&tcache->cache_bin_array_descriptor)) { ((&arena->cache_bin_array_descriptor_ql)->qlh_first ) = ((((&arena->cache_bin_array_descriptor_ql)->qlh_first ))->link.qre_next); } if (((&arena->cache_bin_array_descriptor_ql )->qlh_first) != (&tcache->cache_bin_array_descriptor )) { do { ((&tcache->cache_bin_array_descriptor))-> link.qre_prev->link.qre_next = ((&tcache->cache_bin_array_descriptor ))->link.qre_next; ((&tcache->cache_bin_array_descriptor ))->link.qre_next->link.qre_prev = ((&tcache->cache_bin_array_descriptor ))->link.qre_prev; ((&tcache->cache_bin_array_descriptor ))->link.qre_next = ((&tcache->cache_bin_array_descriptor )); ((&tcache->cache_bin_array_descriptor))->link.qre_prev = ((&tcache->cache_bin_array_descriptor)); } while (0 ); } else { ((&arena->cache_bin_array_descriptor_ql)-> qlh_first) = ((void*)0); } } while (0); |
328 | tcache_stats_merge(tsdn, tcache, arena); |
329 | malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx); |
330 | } |
331 | tcache->arena = NULL((void*)0); |
332 | } |
333 | |
334 | void |
335 | tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { |
336 | tcache_arena_dissociate(tsdn, tcache); |
337 | tcache_arena_associate(tsdn, tcache, arena); |
338 | } |
339 | |
340 | bool_Bool |
341 | tsd_tcache_enabled_data_init(tsd_t *tsd) { |
342 | /* Called upon tsd initialization. */ |
343 | tsd_tcache_enabled_set(tsd, opt_tcache); |
344 | tsd_slow_update(tsd); |
345 | |
346 | if (opt_tcache) { |
347 | /* Trigger tcache init. */ |
348 | tsd_tcache_data_init(tsd); |
349 | } |
350 | |
351 | return false0; |
352 | } |
353 | |
354 | /* Initialize auto tcache (embedded in TSD). */ |
355 | static void |
356 | tcache_init(tsd_t *tsd, tcache_t *tcache, void *avail_stack) { |
357 | memset(&tcache->link, 0, sizeof(ql_elm(tcache_t)struct { tcache_t *qre_next; tcache_t *qre_prev; })); |
358 | tcache->prof_accumbytes = 0; |
359 | tcache->next_gc_bin = 0; |
360 | tcache->arena = NULL((void*)0); |
361 | |
362 | ticker_init(&tcache->gc_ticker, TCACHE_GC_INCR((8192 / 39) + ((8192 / 39 == 0) ? 0 : 1))); |
363 | |
364 | size_t stack_offset = 0; |
365 | assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0)do { if (__builtin_expect(!!(config_debug && !((200 & 1U) == 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/tcache.c", 365, "(TCACHE_NSLOTS_SMALL_MAX & 1U) == 0" ); abort(); } } while (0); |
366 | memset(tcache->bins_small, 0, sizeof(cache_bin_t) * NBINS39); |
367 | memset(tcache->bins_large, 0, sizeof(cache_bin_t) * (nhbins - NBINS39)); |
368 | unsigned i = 0; |
369 | for (; i < NBINS39; i++) { |
370 | tcache->lg_fill_div[i] = 1; |
371 | stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *); |
372 | /* |
373 | * avail points past the available space. Allocations will |
374 | * access the slots toward higher addresses (for the benefit of |
375 | * prefetch). |
376 | */ |
377 | tcache_small_bin_get(tcache, i)->avail = |
378 | (void **)((uintptr_t)avail_stack + (uintptr_t)stack_offset); |
379 | } |
380 | for (; i < nhbins; i++) { |
381 | stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *); |
382 | tcache_large_bin_get(tcache, i)->avail = |
383 | (void **)((uintptr_t)avail_stack + (uintptr_t)stack_offset); |
384 | } |
385 | assert(stack_offset == stack_nelms * sizeof(void *))do { if (__builtin_expect(!!(config_debug && !(stack_offset == stack_nelms * sizeof(void *))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/tcache.c", 385, "stack_offset == stack_nelms * sizeof(void *)" ); abort(); } } while (0); |
386 | } |
387 | |
388 | /* Initialize auto tcache (embedded in TSD). */ |
389 | bool_Bool |
390 | tsd_tcache_data_init(tsd_t *tsd) { |
391 | tcache_t *tcache = tsd_tcachep_get_unsafe(tsd); |
392 | assert(tcache_small_bin_get(tcache, 0)->avail == NULL)do { if (__builtin_expect(!!(config_debug && !(tcache_small_bin_get (tcache, 0)->avail == ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/tcache.c", 392, "tcache_small_bin_get(tcache, 0)->avail == NULL" ); abort(); } } while (0); |
393 | size_t size = stack_nelms * sizeof(void *); |
394 | /* Avoid false cacheline sharing. */ |
395 | size = sz_sa2u(size, CACHELINE64); |
396 | |
397 | void *avail_array = ipallocztm(tsd_tsdn(tsd), size, CACHELINE64, true1, |
398 | NULL((void*)0), true1, arena_get(TSDN_NULL((tsdn_t *)0), 0, true1)); |
399 | if (avail_array == NULL((void*)0)) { |
400 | return true1; |
401 | } |
402 | |
403 | tcache_init(tsd, tcache, avail_array); |
404 | /* |
405 | * Initialization is a bit tricky here. After malloc init is done, all |
406 | * threads can rely on arena_choose and associate tcache accordingly. |
407 | * However, the thread that does actual malloc bootstrapping relies on |
408 | * functional tsd, and it can only rely on a0. In that case, we |
409 | * associate its tcache to a0 temporarily, and later on |
410 | * arena_choose_hard() will re-associate properly. |
411 | */ |
412 | tcache->arena = NULL((void*)0); |
413 | arena_t *arena; |
414 | if (!malloc_initialized()) { |
415 | /* If in initialization, assign to a0. */ |
416 | arena = arena_get(tsd_tsdn(tsd), 0, false0); |
417 | tcache_arena_associate(tsd_tsdn(tsd), tcache, arena); |
418 | } else { |
419 | arena = arena_choose(tsd, NULL((void*)0)); |
420 | /* This may happen if thread.tcache.enabled is used. */ |
421 | if (tcache->arena == NULL((void*)0)) { |
422 | tcache_arena_associate(tsd_tsdn(tsd), tcache, arena); |
423 | } |
424 | } |
425 | assert(arena == tcache->arena)do { if (__builtin_expect(!!(config_debug && !(arena == tcache->arena)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/tcache.c", 425, "arena == tcache->arena"); abort(); } } while (0); |
426 | |
427 | return false0; |
428 | } |
429 | |
430 | /* Created manual tcache for tcache.create mallctl. */ |
431 | tcache_t * |
432 | tcache_create_explicit(tsd_t *tsd) { |
433 | tcache_t *tcache; |
434 | size_t size, stack_offset; |
435 | |
436 | size = sizeof(tcache_t); |
437 | /* Naturally align the pointer stacks. */ |
438 | size = PTR_CEILING(size)(((size) + ((1U << 3) - 1)) & ~((1U << 3) - 1 )); |
439 | stack_offset = size; |
440 | size += stack_nelms * sizeof(void *); |
441 | /* Avoid false cacheline sharing. */ |
442 | size = sz_sa2u(size, CACHELINE64); |
443 | |
444 | tcache = ipallocztm(tsd_tsdn(tsd), size, CACHELINE64, true1, NULL((void*)0), true1, |
445 | arena_get(TSDN_NULL((tsdn_t *)0), 0, true1)); |
446 | if (tcache == NULL((void*)0)) { |
447 | return NULL((void*)0); |
448 | } |
449 | |
450 | tcache_init(tsd, tcache, |
451 | (void *)((uintptr_t)tcache + (uintptr_t)stack_offset)); |
452 | tcache_arena_associate(tsd_tsdn(tsd), tcache, arena_ichoose(tsd, NULL((void*)0))); |
453 | |
454 | return tcache; |
455 | } |
456 | |
457 | static void |
458 | tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) { |
459 | assert(tcache->arena != NULL)do { if (__builtin_expect(!!(config_debug && !(tcache ->arena != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/tcache.c", 459, "tcache->arena != NULL"); abort(); } } while (0); |
460 | |
461 | for (unsigned i = 0; i < NBINS39; i++) { |
462 | cache_bin_t *tbin = tcache_small_bin_get(tcache, i); |
463 | tcache_bin_flush_small(tsd, tcache, tbin, i, 0); |
464 | |
465 | if (config_stats) { |
466 | assert(tbin->tstats.nrequests == 0)do { if (__builtin_expect(!!(config_debug && !(tbin-> tstats.nrequests == 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/tcache.c", 466, "tbin->tstats.nrequests == 0"); abort (); } } while (0); |
467 | } |
468 | } |
469 | for (unsigned i = NBINS39; i < nhbins; i++) { |
470 | cache_bin_t *tbin = tcache_large_bin_get(tcache, i); |
471 | tcache_bin_flush_large(tsd, tbin, i, 0, tcache); |
472 | |
473 | if (config_stats) { |
474 | assert(tbin->tstats.nrequests == 0)do { if (__builtin_expect(!!(config_debug && !(tbin-> tstats.nrequests == 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/tcache.c", 474, "tbin->tstats.nrequests == 0"); abort (); } } while (0); |
475 | } |
476 | } |
477 | |
478 | if (config_prof && tcache->prof_accumbytes > 0 && |
479 | arena_prof_accum(tsd_tsdn(tsd), tcache->arena, |
480 | tcache->prof_accumbytes)) { |
481 | prof_idump(tsd_tsdn(tsd)); |
482 | } |
483 | } |
484 | |
485 | void |
486 | tcache_flush(tsd_t *tsd) { |
487 | assert(tcache_available(tsd))do { if (__builtin_expect(!!(config_debug && !(tcache_available (tsd))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/tcache.c", 487, "tcache_available(tsd)"); abort(); } } while (0); |
488 | tcache_flush_cache(tsd, tsd_tcachep_get(tsd)); |
489 | } |
490 | |
491 | static void |
492 | tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool_Bool tsd_tcache) { |
493 | tcache_flush_cache(tsd, tcache); |
494 | tcache_arena_dissociate(tsd_tsdn(tsd), tcache); |
495 | |
496 | if (tsd_tcache) { |
497 | /* Release the avail array for the TSD embedded auto tcache. */ |
498 | void *avail_array = |
499 | (void *)((uintptr_t)tcache_small_bin_get(tcache, 0)->avail - |
500 | (uintptr_t)tcache_bin_info[0].ncached_max * sizeof(void *)); |
501 | idalloctm(tsd_tsdn(tsd), avail_array, NULL((void*)0), NULL((void*)0), true1, true1); |
502 | } else { |
503 | /* Release both the tcache struct and avail array. */ |
504 | idalloctm(tsd_tsdn(tsd), tcache, NULL((void*)0), NULL((void*)0), true1, true1); |
505 | } |
506 | } |
507 | |
508 | /* For auto tcache (embedded in TSD) only. */ |
509 | void |
510 | tcache_cleanup(tsd_t *tsd) { |
511 | tcache_t *tcache = tsd_tcachep_get(tsd); |
512 | if (!tcache_available(tsd)) { |
513 | assert(tsd_tcache_enabled_get(tsd) == false)do { if (__builtin_expect(!!(config_debug && !(tsd_tcache_enabled_get (tsd) == 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/tcache.c", 513, "tsd_tcache_enabled_get(tsd) == false" ); abort(); } } while (0); |
514 | if (config_debug) { |
515 | assert(tcache_small_bin_get(tcache, 0)->avail == NULL)do { if (__builtin_expect(!!(config_debug && !(tcache_small_bin_get (tcache, 0)->avail == ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/tcache.c", 515, "tcache_small_bin_get(tcache, 0)->avail == NULL" ); abort(); } } while (0); |
516 | } |
517 | return; |
518 | } |
519 | assert(tsd_tcache_enabled_get(tsd))do { if (__builtin_expect(!!(config_debug && !(tsd_tcache_enabled_get (tsd))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/tcache.c", 519, "tsd_tcache_enabled_get(tsd)"); abort( ); } } while (0); |
520 | assert(tcache_small_bin_get(tcache, 0)->avail != NULL)do { if (__builtin_expect(!!(config_debug && !(tcache_small_bin_get (tcache, 0)->avail != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/tcache.c", 520, "tcache_small_bin_get(tcache, 0)->avail != NULL" ); abort(); } } while (0); |
521 | |
522 | tcache_destroy(tsd, tcache, true1); |
523 | if (config_debug) { |
524 | tcache_small_bin_get(tcache, 0)->avail = NULL((void*)0); |
525 | } |
526 | } |
527 | |
528 | void |
529 | tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { |
530 | unsigned i; |
531 | |
532 | cassert(config_stats)do { if (__builtin_expect(!!(!(config_stats)), 0)) { do { if ( config_debug) { malloc_printf( "<jemalloc>: %s:%d: Unreachable code reached\n" , "src/tcache.c", 532); abort(); } __builtin_unreachable(); } while (0); } } while (0); |
533 | |
534 | /* Merge and reset tcache stats. */ |
535 | for (i = 0; i < NBINS39; i++) { |
536 | bin_t *bin = &arena->bins[i]; |
537 | cache_bin_t *tbin = tcache_small_bin_get(tcache, i); |
538 | malloc_mutex_lock(tsdn, &bin->lock); |
539 | bin->stats.nrequests += tbin->tstats.nrequests; |
540 | malloc_mutex_unlock(tsdn, &bin->lock); |
541 | tbin->tstats.nrequests = 0; |
542 | } |
543 | |
544 | for (; i < nhbins; i++) { |
545 | cache_bin_t *tbin = tcache_large_bin_get(tcache, i); |
546 | arena_stats_large_nrequests_add(tsdn, &arena->stats, i, |
547 | tbin->tstats.nrequests); |
548 | tbin->tstats.nrequests = 0; |
549 | } |
550 | } |
551 | |
552 | static bool_Bool |
553 | tcaches_create_prep(tsd_t *tsd) { |
554 | bool_Bool err; |
555 | |
556 | malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); |
557 | |
558 | if (tcaches == NULL((void*)0)) { |
559 | tcaches = base_alloc(tsd_tsdn(tsd), b0get(), sizeof(tcache_t *) |
560 | * (MALLOCX_TCACHE_MAX((1 << 12) - 3)+1), CACHELINE64); |
561 | if (tcaches == NULL((void*)0)) { |
562 | err = true1; |
563 | goto label_return; |
564 | } |
565 | } |
566 | |
567 | if (tcaches_avail == NULL((void*)0) && tcaches_past > MALLOCX_TCACHE_MAX((1 << 12) - 3)) { |
568 | err = true1; |
569 | goto label_return; |
570 | } |
571 | |
572 | err = false0; |
573 | label_return: |
574 | malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); |
575 | return err; |
576 | } |
577 | |
578 | bool_Bool |
579 | tcaches_create(tsd_t *tsd, unsigned *r_ind) { |
580 | witness_assert_depth(tsdn_witness_tsdp_get(tsd_tsdn(tsd)), 0); |
581 | |
582 | bool_Bool err; |
583 | |
584 | if (tcaches_create_prep(tsd)) { |
585 | err = true1; |
586 | goto label_return; |
587 | } |
588 | |
589 | tcache_t *tcache = tcache_create_explicit(tsd); |
590 | if (tcache == NULL((void*)0)) { |
591 | err = true1; |
592 | goto label_return; |
593 | } |
594 | |
595 | tcaches_t *elm; |
596 | malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); |
597 | if (tcaches_avail != NULL((void*)0)) { |
598 | elm = tcaches_avail; |
599 | tcaches_avail = tcaches_avail->next; |
600 | elm->tcache = tcache; |
601 | *r_ind = (unsigned)(elm - tcaches); |
602 | } else { |
603 | elm = &tcaches[tcaches_past]; |
604 | elm->tcache = tcache; |
605 | *r_ind = tcaches_past; |
606 | tcaches_past++; |
607 | } |
608 | malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); |
609 | |
610 | err = false0; |
611 | label_return: |
612 | witness_assert_depth(tsdn_witness_tsdp_get(tsd_tsdn(tsd)), 0); |
613 | return err; |
614 | } |
615 | |
616 | static tcache_t * |
617 | tcaches_elm_remove(tsd_t *tsd, tcaches_t *elm) { |
618 | malloc_mutex_assert_owner(tsd_tsdn(tsd), &tcaches_mtx); |
619 | |
620 | if (elm->tcache == NULL((void*)0)) { |
621 | return NULL((void*)0); |
622 | } |
623 | tcache_t *tcache = elm->tcache; |
624 | elm->tcache = NULL((void*)0); |
625 | return tcache; |
626 | } |
627 | |
628 | void |
629 | tcaches_flush(tsd_t *tsd, unsigned ind) { |
630 | malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); |
631 | tcache_t *tcache = tcaches_elm_remove(tsd, &tcaches[ind]); |
632 | malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); |
633 | if (tcache != NULL((void*)0)) { |
634 | tcache_destroy(tsd, tcache, false0); |
635 | } |
636 | } |
637 | |
638 | void |
639 | tcaches_destroy(tsd_t *tsd, unsigned ind) { |
640 | malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); |
641 | tcaches_t *elm = &tcaches[ind]; |
642 | tcache_t *tcache = tcaches_elm_remove(tsd, elm); |
643 | elm->next = tcaches_avail; |
644 | tcaches_avail = elm; |
645 | malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); |
646 | if (tcache != NULL((void*)0)) { |
647 | tcache_destroy(tsd, tcache, false0); |
648 | } |
649 | } |
650 | |
651 | bool_Bool |
652 | tcache_boot(tsdn_t *tsdn) { |
653 | /* If necessary, clamp opt_lg_tcache_max. */ |
654 | if (opt_lg_tcache_max < 0 || (ZU(1)((size_t)1) << opt_lg_tcache_max) < |
655 | SMALL_MAXCLASS((((size_t)1) << 13) + (((size_t)3) << 11))) { |
656 | tcache_maxclass = SMALL_MAXCLASS((((size_t)1) << 13) + (((size_t)3) << 11)); |
657 | } else { |
658 | tcache_maxclass = (ZU(1)((size_t)1) << opt_lg_tcache_max); |
659 | } |
660 | |
661 | if (malloc_mutex_init(&tcaches_mtx, "tcaches", WITNESS_RANK_TCACHES2U, |
662 | malloc_mutex_rank_exclusive)) { |
663 | return true1; |
664 | } |
665 | |
666 | nhbins = sz_size2index(tcache_maxclass) + 1; |
667 | |
668 | /* Initialize tcache_bin_info. */ |
669 | tcache_bin_info = (cache_bin_info_t *)base_alloc(tsdn, b0get(), nhbins |
670 | * sizeof(cache_bin_info_t), CACHELINE64); |
671 | if (tcache_bin_info == NULL((void*)0)) { |
672 | return true1; |
673 | } |
674 | stack_nelms = 0; |
675 | unsigned i; |
676 | for (i = 0; i < NBINS39; i++) { |
677 | if ((bin_infos[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MIN20) { |
678 | tcache_bin_info[i].ncached_max = |
679 | TCACHE_NSLOTS_SMALL_MIN20; |
680 | } else if ((bin_infos[i].nregs << 1) <= |
681 | TCACHE_NSLOTS_SMALL_MAX200) { |
682 | tcache_bin_info[i].ncached_max = |
683 | (bin_infos[i].nregs << 1); |
684 | } else { |
685 | tcache_bin_info[i].ncached_max = |
686 | TCACHE_NSLOTS_SMALL_MAX200; |
687 | } |
688 | stack_nelms += tcache_bin_info[i].ncached_max; |
689 | } |
690 | for (; i < nhbins; i++) { |
691 | tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_LARGE20; |
692 | stack_nelms += tcache_bin_info[i].ncached_max; |
693 | } |
694 | |
695 | return false0; |
696 | } |
697 | |
698 | void |
699 | tcache_prefork(tsdn_t *tsdn) { |
700 | if (!config_prof && opt_tcache) { |
701 | malloc_mutex_prefork(tsdn, &tcaches_mtx); |
702 | } |
703 | } |
704 | |
705 | void |
706 | tcache_postfork_parent(tsdn_t *tsdn) { |
707 | if (!config_prof && opt_tcache) { |
708 | malloc_mutex_postfork_parent(tsdn, &tcaches_mtx); |
709 | } |
710 | } |
711 | |
712 | void |
713 | tcache_postfork_child(tsdn_t *tsdn) { |
714 | if (!config_prof && opt_tcache) { |
715 | malloc_mutex_postfork_child(tsdn, &tcaches_mtx); |
716 | } |
717 | } |