File: | deps/jemalloc/include/jemalloc/internal/extent_inlines.h |
Warning: | line 183, column 35 The left operand of '&' is a garbage value |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | #define JEMALLOC_EXTENT_DSS_C_ | ||||
2 | #include "jemalloc/internal/jemalloc_preamble.h" | ||||
3 | #include "jemalloc/internal/jemalloc_internal_includes.h" | ||||
4 | |||||
5 | #include "jemalloc/internal/assert.h" | ||||
6 | #include "jemalloc/internal/extent_dss.h" | ||||
7 | #include "jemalloc/internal/spin.h" | ||||
8 | |||||
9 | /******************************************************************************/ | ||||
10 | /* Data. */ | ||||
11 | |||||
12 | const char *opt_dss = DSS_DEFAULT"secondary"; | ||||
13 | |||||
14 | const char *dss_prec_names[] = { | ||||
15 | "disabled", | ||||
16 | "primary", | ||||
17 | "secondary", | ||||
18 | "N/A" | ||||
19 | }; | ||||
20 | |||||
21 | /* | ||||
22 | * Current dss precedence default, used when creating new arenas. NB: This is | ||||
23 | * stored as unsigned rather than dss_prec_t because in principle there's no | ||||
24 | * guarantee that sizeof(dss_prec_t) is the same as sizeof(unsigned), and we use | ||||
25 | * atomic operations to synchronize the setting. | ||||
26 | */ | ||||
27 | static atomic_u_t dss_prec_default = ATOMIC_INIT({(unsigned)dss_prec_secondary} | ||||
28 | (unsigned)DSS_PREC_DEFAULT){(unsigned)dss_prec_secondary}; | ||||
29 | |||||
30 | /* Base address of the DSS. */ | ||||
31 | static void *dss_base; | ||||
32 | /* Atomic boolean indicating whether a thread is currently extending DSS. */ | ||||
33 | static atomic_b_t dss_extending; | ||||
34 | /* Atomic boolean indicating whether the DSS is exhausted. */ | ||||
35 | static atomic_b_t dss_exhausted; | ||||
36 | /* Atomic current upper limit on DSS addresses. */ | ||||
37 | static atomic_p_t dss_max; | ||||
38 | |||||
39 | /******************************************************************************/ | ||||
40 | |||||
41 | static void * | ||||
42 | extent_dss_sbrk(intptr_t increment) { | ||||
43 | #ifdef JEMALLOC_DSS | ||||
44 | return sbrk(increment); | ||||
45 | #else | ||||
46 | not_implemented()do { if (config_debug) { malloc_printf("<jemalloc>: %s:%d: Not implemented\n" , "src/extent_dss.c", 46); abort(); } } while (0); | ||||
47 | return NULL((void*)0); | ||||
48 | #endif | ||||
49 | } | ||||
50 | |||||
51 | dss_prec_t | ||||
52 | extent_dss_prec_get(void) { | ||||
53 | dss_prec_t ret; | ||||
54 | |||||
55 | if (!have_dss) { | ||||
56 | return dss_prec_disabled; | ||||
57 | } | ||||
58 | ret = (dss_prec_t)atomic_load_u(&dss_prec_default, ATOMIC_ACQUIREatomic_memory_order_acquire); | ||||
59 | return ret; | ||||
60 | } | ||||
61 | |||||
62 | bool_Bool | ||||
63 | extent_dss_prec_set(dss_prec_t dss_prec) { | ||||
64 | if (!have_dss) { | ||||
65 | return (dss_prec != dss_prec_disabled); | ||||
66 | } | ||||
67 | atomic_store_u(&dss_prec_default, (unsigned)dss_prec, ATOMIC_RELEASEatomic_memory_order_release); | ||||
68 | return false0; | ||||
69 | } | ||||
70 | |||||
71 | static void | ||||
72 | extent_dss_extending_start(void) { | ||||
73 | spin_t spinner = SPIN_INITIALIZER{0U}; | ||||
74 | while (true1) { | ||||
75 | bool_Bool expected = false0; | ||||
76 | if (atomic_compare_exchange_weak_b(&dss_extending, &expected, | ||||
77 | true1, ATOMIC_ACQ_RELatomic_memory_order_acq_rel, ATOMIC_RELAXEDatomic_memory_order_relaxed)) { | ||||
78 | break; | ||||
79 | } | ||||
80 | spin_adaptive(&spinner); | ||||
81 | } | ||||
82 | } | ||||
83 | |||||
84 | static void | ||||
85 | extent_dss_extending_finish(void) { | ||||
86 | assert(atomic_load_b(&dss_extending, ATOMIC_RELAXED))do { if (__builtin_expect(!!(config_debug && !(atomic_load_b (&dss_extending, atomic_memory_order_relaxed))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent_dss.c" , 86, "atomic_load_b(&dss_extending, ATOMIC_RELAXED)"); abort (); } } while (0); | ||||
87 | |||||
88 | atomic_store_b(&dss_extending, false0, ATOMIC_RELEASEatomic_memory_order_release); | ||||
89 | } | ||||
90 | |||||
91 | static void * | ||||
92 | extent_dss_max_update(void *new_addr) { | ||||
93 | /* | ||||
94 | * Get the current end of the DSS as max_cur and assure that dss_max is | ||||
95 | * up to date. | ||||
96 | */ | ||||
97 | void *max_cur = extent_dss_sbrk(0); | ||||
98 | if (max_cur == (void *)-1) { | ||||
99 | return NULL((void*)0); | ||||
100 | } | ||||
101 | atomic_store_p(&dss_max, max_cur, ATOMIC_RELEASEatomic_memory_order_release); | ||||
102 | /* Fixed new_addr can only be supported if it is at the edge of DSS. */ | ||||
103 | if (new_addr != NULL((void*)0) && max_cur != new_addr) { | ||||
104 | return NULL((void*)0); | ||||
105 | } | ||||
106 | return max_cur; | ||||
107 | } | ||||
108 | |||||
109 | void * | ||||
110 | extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, | ||||
111 | size_t alignment, bool_Bool *zero, bool_Bool *commit) { | ||||
112 | extent_t *gap; | ||||
113 | |||||
114 | cassert(have_dss)do { if (__builtin_expect(!!(!(have_dss)), 0)) { do { if (config_debug ) { malloc_printf( "<jemalloc>: %s:%d: Unreachable code reached\n" , "src/extent_dss.c", 114); abort(); } __builtin_unreachable( ); } while (0); } } while (0); | ||||
| |||||
115 | assert(size > 0)do { if (__builtin_expect(!!(config_debug && !(size > 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent_dss.c", 115, "size > 0"); abort(); } } while (0); | ||||
116 | assert(alignment > 0)do { if (__builtin_expect(!!(config_debug && !(alignment > 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent_dss.c", 116, "alignment > 0"); abort(); } } while (0); | ||||
117 | |||||
118 | /* | ||||
119 | * sbrk() uses a signed increment argument, so take care not to | ||||
120 | * interpret a large allocation request as a negative increment. | ||||
121 | */ | ||||
122 | if ((intptr_t)size
| ||||
123 | return NULL((void*)0); | ||||
124 | } | ||||
125 | |||||
126 | gap = extent_alloc(tsdn, arena); | ||||
127 | if (gap == NULL((void*)0)) { | ||||
128 | return NULL((void*)0); | ||||
129 | } | ||||
130 | |||||
131 | extent_dss_extending_start(); | ||||
132 | if (!atomic_load_b(&dss_exhausted, ATOMIC_ACQUIREatomic_memory_order_acquire)) { | ||||
133 | /* | ||||
134 | * The loop is necessary to recover from races with other | ||||
135 | * threads that are using the DSS for something other than | ||||
136 | * malloc. | ||||
137 | */ | ||||
138 | while (true1) { | ||||
139 | void *max_cur = extent_dss_max_update(new_addr); | ||||
140 | if (max_cur == NULL((void*)0)) { | ||||
141 | goto label_oom; | ||||
142 | } | ||||
143 | |||||
144 | /* | ||||
145 | * Compute how much page-aligned gap space (if any) is | ||||
146 | * necessary to satisfy alignment. This space can be | ||||
147 | * recycled for later use. | ||||
148 | */ | ||||
149 | void *gap_addr_page = (void *)(PAGE_CEILING(((((uintptr_t)max_cur) + ((size_t)(((size_t)(1U << 12)) - 1))) & ~((size_t)(((size_t)(1U << 12)) - 1))) | ||||
150 | (uintptr_t)max_cur)((((uintptr_t)max_cur) + ((size_t)(((size_t)(1U << 12)) - 1))) & ~((size_t)(((size_t)(1U << 12)) - 1)))); | ||||
151 | void *ret = (void *)ALIGNMENT_CEILING(((((uintptr_t)gap_addr_page) + (alignment - 1)) & ((~(alignment )) + 1)) | ||||
152 | (uintptr_t)gap_addr_page, alignment)((((uintptr_t)gap_addr_page) + (alignment - 1)) & ((~(alignment )) + 1)); | ||||
153 | size_t gap_size_page = (uintptr_t)ret - | ||||
154 | (uintptr_t)gap_addr_page; | ||||
155 | if (gap_size_page != 0) { | ||||
156 | extent_init(gap, arena, gap_addr_page, | ||||
157 | gap_size_page, false0, NSIZES235, | ||||
158 | arena_extent_sn_next(arena), | ||||
159 | extent_state_active, false0, true1, true1); | ||||
160 | } | ||||
161 | /* | ||||
162 | * Compute the address just past the end of the desired | ||||
163 | * allocation space. | ||||
164 | */ | ||||
165 | void *dss_next = (void *)((uintptr_t)ret + size); | ||||
166 | if ((uintptr_t)ret < (uintptr_t)max_cur || | ||||
167 | (uintptr_t)dss_next < (uintptr_t)max_cur) { | ||||
168 | goto label_oom; /* Wrap-around. */ | ||||
169 | } | ||||
170 | /* Compute the increment, including subpage bytes. */ | ||||
171 | void *gap_addr_subpage = max_cur; | ||||
172 | size_t gap_size_subpage = (uintptr_t)ret - | ||||
173 | (uintptr_t)gap_addr_subpage; | ||||
174 | intptr_t incr = gap_size_subpage + size; | ||||
175 | |||||
176 | assert((uintptr_t)max_cur + incr == (uintptr_t)ret +do { if (__builtin_expect(!!(config_debug && !((uintptr_t )max_cur + incr == (uintptr_t)ret + size)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent_dss.c" , 177, "(uintptr_t)max_cur + incr == (uintptr_t)ret + size"); abort(); } } while (0) | ||||
177 | size)do { if (__builtin_expect(!!(config_debug && !((uintptr_t )max_cur + incr == (uintptr_t)ret + size)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent_dss.c" , 177, "(uintptr_t)max_cur + incr == (uintptr_t)ret + size"); abort(); } } while (0); | ||||
178 | |||||
179 | /* Try to allocate. */ | ||||
180 | void *dss_prev = extent_dss_sbrk(incr); | ||||
181 | if (dss_prev == max_cur) { | ||||
182 | /* Success. */ | ||||
183 | atomic_store_p(&dss_max, dss_next, | ||||
184 | ATOMIC_RELEASEatomic_memory_order_release); | ||||
185 | extent_dss_extending_finish(); | ||||
186 | |||||
187 | if (gap_size_page
| ||||
188 | extent_dalloc_gap(tsdn, arena, gap); | ||||
189 | } else { | ||||
190 | extent_dalloc(tsdn, arena, gap); | ||||
191 | } | ||||
192 | if (!*commit) { | ||||
193 | *commit = pages_decommit(ret, size); | ||||
194 | } | ||||
195 | if (*zero && *commit) { | ||||
196 | extent_hooks_t *extent_hooks = | ||||
197 | EXTENT_HOOKS_INITIALIZER((void*)0); | ||||
198 | extent_t extent; | ||||
199 | |||||
200 | extent_init(&extent, arena, ret, size, | ||||
201 | size, false0, NSIZES235, | ||||
202 | extent_state_active, false0, true1, | ||||
203 | true1); | ||||
204 | if (extent_purge_forced_wrapper(tsdn, | ||||
205 | arena, &extent_hooks, &extent, 0, | ||||
206 | size)) { | ||||
207 | memset(ret, 0, size); | ||||
208 | } | ||||
209 | } | ||||
210 | return ret; | ||||
211 | } | ||||
212 | /* | ||||
213 | * Failure, whether due to OOM or a race with a raw | ||||
214 | * sbrk() call from outside the allocator. | ||||
215 | */ | ||||
216 | if (dss_prev == (void *)-1) { | ||||
217 | /* OOM. */ | ||||
218 | atomic_store_b(&dss_exhausted, true1, | ||||
219 | ATOMIC_RELEASEatomic_memory_order_release); | ||||
220 | goto label_oom; | ||||
221 | } | ||||
222 | } | ||||
223 | } | ||||
224 | label_oom: | ||||
225 | extent_dss_extending_finish(); | ||||
226 | extent_dalloc(tsdn, arena, gap); | ||||
227 | return NULL((void*)0); | ||||
228 | } | ||||
229 | |||||
230 | static bool_Bool | ||||
231 | extent_in_dss_helper(void *addr, void *max) { | ||||
232 | return ((uintptr_t)addr >= (uintptr_t)dss_base && (uintptr_t)addr < | ||||
233 | (uintptr_t)max); | ||||
234 | } | ||||
235 | |||||
236 | bool_Bool | ||||
237 | extent_in_dss(void *addr) { | ||||
238 | cassert(have_dss)do { if (__builtin_expect(!!(!(have_dss)), 0)) { do { if (config_debug ) { malloc_printf( "<jemalloc>: %s:%d: Unreachable code reached\n" , "src/extent_dss.c", 238); abort(); } __builtin_unreachable( ); } while (0); } } while (0); | ||||
239 | |||||
240 | return extent_in_dss_helper(addr, atomic_load_p(&dss_max, | ||||
241 | ATOMIC_ACQUIREatomic_memory_order_acquire)); | ||||
242 | } | ||||
243 | |||||
244 | bool_Bool | ||||
245 | extent_dss_mergeable(void *addr_a, void *addr_b) { | ||||
246 | void *max; | ||||
247 | |||||
248 | cassert(have_dss)do { if (__builtin_expect(!!(!(have_dss)), 0)) { do { if (config_debug ) { malloc_printf( "<jemalloc>: %s:%d: Unreachable code reached\n" , "src/extent_dss.c", 248); abort(); } __builtin_unreachable( ); } while (0); } } while (0); | ||||
249 | |||||
250 | if ((uintptr_t)addr_a < (uintptr_t)dss_base && (uintptr_t)addr_b < | ||||
251 | (uintptr_t)dss_base) { | ||||
252 | return true1; | ||||
253 | } | ||||
254 | |||||
255 | max = atomic_load_p(&dss_max, ATOMIC_ACQUIREatomic_memory_order_acquire); | ||||
256 | return (extent_in_dss_helper(addr_a, max) == | ||||
257 | extent_in_dss_helper(addr_b, max)); | ||||
258 | } | ||||
259 | |||||
260 | void | ||||
261 | extent_dss_boot(void) { | ||||
262 | cassert(have_dss)do { if (__builtin_expect(!!(!(have_dss)), 0)) { do { if (config_debug ) { malloc_printf( "<jemalloc>: %s:%d: Unreachable code reached\n" , "src/extent_dss.c", 262); abort(); } __builtin_unreachable( ); } while (0); } } while (0); | ||||
263 | |||||
264 | dss_base = extent_dss_sbrk(0); | ||||
265 | atomic_store_b(&dss_extending, false0, ATOMIC_RELAXEDatomic_memory_order_relaxed); | ||||
266 | atomic_store_b(&dss_exhausted, dss_base == (void *)-1, ATOMIC_RELAXEDatomic_memory_order_relaxed); | ||||
267 | atomic_store_p(&dss_max, dss_base, ATOMIC_RELAXEDatomic_memory_order_relaxed); | ||||
268 | } | ||||
269 | |||||
270 | /******************************************************************************/ |
1 | #ifndef JEMALLOC_INTERNAL_EXTENT_INLINES_H | |||
2 | #define JEMALLOC_INTERNAL_EXTENT_INLINES_H | |||
3 | ||||
4 | #include "jemalloc/internal/mutex.h" | |||
5 | #include "jemalloc/internal/mutex_pool.h" | |||
6 | #include "jemalloc/internal/pages.h" | |||
7 | #include "jemalloc/internal/prng.h" | |||
8 | #include "jemalloc/internal/ql.h" | |||
9 | #include "jemalloc/internal/sz.h" | |||
10 | ||||
11 | static inline void | |||
12 | extent_lock(tsdn_t *tsdn, extent_t *extent) { | |||
13 | assert(extent != NULL)do { if (__builtin_expect(!!(config_debug && !(extent != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 13, "extent != NULL" ); abort(); } } while (0); | |||
14 | mutex_pool_lock(tsdn, &extent_mutex_pool, (uintptr_t)extent); | |||
15 | } | |||
16 | ||||
17 | static inline void | |||
18 | extent_unlock(tsdn_t *tsdn, extent_t *extent) { | |||
19 | assert(extent != NULL)do { if (__builtin_expect(!!(config_debug && !(extent != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 19, "extent != NULL" ); abort(); } } while (0); | |||
20 | mutex_pool_unlock(tsdn, &extent_mutex_pool, (uintptr_t)extent); | |||
21 | } | |||
22 | ||||
23 | static inline void | |||
24 | extent_lock2(tsdn_t *tsdn, extent_t *extent1, extent_t *extent2) { | |||
25 | assert(extent1 != NULL && extent2 != NULL)do { if (__builtin_expect(!!(config_debug && !(extent1 != ((void*)0) && extent2 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "include/jemalloc/internal/extent_inlines.h" , 25, "extent1 != NULL && extent2 != NULL"); abort(); } } while (0); | |||
26 | mutex_pool_lock2(tsdn, &extent_mutex_pool, (uintptr_t)extent1, | |||
27 | (uintptr_t)extent2); | |||
28 | } | |||
29 | ||||
30 | static inline void | |||
31 | extent_unlock2(tsdn_t *tsdn, extent_t *extent1, extent_t *extent2) { | |||
32 | assert(extent1 != NULL && extent2 != NULL)do { if (__builtin_expect(!!(config_debug && !(extent1 != ((void*)0) && extent2 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "include/jemalloc/internal/extent_inlines.h" , 32, "extent1 != NULL && extent2 != NULL"); abort(); } } while (0); | |||
33 | mutex_pool_unlock2(tsdn, &extent_mutex_pool, (uintptr_t)extent1, | |||
34 | (uintptr_t)extent2); | |||
35 | } | |||
36 | ||||
37 | static inline arena_t * | |||
38 | extent_arena_get(const extent_t *extent) { | |||
39 | unsigned arena_ind = (unsigned)((extent->e_bits & | |||
40 | EXTENT_BITS_ARENA_MASK((((((uint64_t)0x1U) << (12)) - 1)) << (0))) >> EXTENT_BITS_ARENA_SHIFT0); | |||
41 | /* | |||
42 | * The following check is omitted because we should never actually read | |||
43 | * a NULL arena pointer. | |||
44 | */ | |||
45 | if (false0 && arena_ind >= MALLOCX_ARENA_LIMIT((1 << 12) - 1)) { | |||
46 | return NULL((void*)0); | |||
47 | } | |||
48 | assert(arena_ind < MALLOCX_ARENA_LIMIT)do { if (__builtin_expect(!!(config_debug && !(arena_ind < ((1 << 12) - 1))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 48, "arena_ind < MALLOCX_ARENA_LIMIT" ); abort(); } } while (0); | |||
49 | return (arena_t *)atomic_load_p(&arenas[arena_ind], ATOMIC_ACQUIREatomic_memory_order_acquire); | |||
50 | } | |||
51 | ||||
52 | static inline szind_t | |||
53 | extent_szind_get_maybe_invalid(const extent_t *extent) { | |||
54 | szind_t szind = (szind_t)((extent->e_bits & EXTENT_BITS_SZIND_MASK((((((uint64_t)0x1U) << (8)) - 1)) << ((2 + (1 + ( 1 + (1 + (1 + (12 + 0))))))))) >> | |||
55 | EXTENT_BITS_SZIND_SHIFT(2 + (1 + (1 + (1 + (1 + (12 + 0))))))); | |||
56 | assert(szind <= NSIZES)do { if (__builtin_expect(!!(config_debug && !(szind <= 235)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 56, "szind <= NSIZES" ); abort(); } } while (0); | |||
57 | return szind; | |||
58 | } | |||
59 | ||||
60 | static inline szind_t | |||
61 | extent_szind_get(const extent_t *extent) { | |||
62 | szind_t szind = extent_szind_get_maybe_invalid(extent); | |||
63 | assert(szind < NSIZES)do { if (__builtin_expect(!!(config_debug && !(szind < 235)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 63, "szind < NSIZES" ); abort(); } } while (0); /* Never call when "invalid". */ | |||
64 | return szind; | |||
65 | } | |||
66 | ||||
67 | static inline size_t | |||
68 | extent_usize_get(const extent_t *extent) { | |||
69 | return sz_index2size(extent_szind_get(extent)); | |||
70 | } | |||
71 | ||||
72 | static inline size_t | |||
73 | extent_sn_get(const extent_t *extent) { | |||
74 | return (size_t)((extent->e_bits & EXTENT_BITS_SN_MASK((18446744073709551615UL) << (((12 - 3) + 1) + (8 + (2 + (1 + (1 + (1 + (1 + (12 + 0)))))))))) >> | |||
75 | EXTENT_BITS_SN_SHIFT(((12 - 3) + 1) + (8 + (2 + (1 + (1 + (1 + (1 + (12 + 0)))))) ))); | |||
76 | } | |||
77 | ||||
78 | static inline extent_state_t | |||
79 | extent_state_get(const extent_t *extent) { | |||
80 | return (extent_state_t)((extent->e_bits & EXTENT_BITS_STATE_MASK((((((uint64_t)0x1U) << (2)) - 1)) << ((1 + (1 + ( 1 + (1 + (12 + 0)))))))) >> | |||
81 | EXTENT_BITS_STATE_SHIFT(1 + (1 + (1 + (1 + (12 + 0)))))); | |||
82 | } | |||
83 | ||||
84 | static inline bool_Bool | |||
85 | extent_zeroed_get(const extent_t *extent) { | |||
86 | return (bool_Bool)((extent->e_bits & EXTENT_BITS_ZEROED_MASK((((((uint64_t)0x1U) << (1)) - 1)) << ((1 + (1 + ( 1 + (12 + 0))))))) >> | |||
87 | EXTENT_BITS_ZEROED_SHIFT(1 + (1 + (1 + (12 + 0))))); | |||
88 | } | |||
89 | ||||
90 | static inline bool_Bool | |||
91 | extent_committed_get(const extent_t *extent) { | |||
92 | return (bool_Bool)((extent->e_bits & EXTENT_BITS_COMMITTED_MASK((((((uint64_t)0x1U) << (1)) - 1)) << ((1 + (12 + 0))))) >> | |||
93 | EXTENT_BITS_COMMITTED_SHIFT(1 + (12 + 0))); | |||
94 | } | |||
95 | ||||
96 | static inline bool_Bool | |||
97 | extent_dumpable_get(const extent_t *extent) { | |||
98 | return (bool_Bool)((extent->e_bits & EXTENT_BITS_DUMPABLE_MASK((((((uint64_t)0x1U) << (1)) - 1)) << ((1 + (1 + ( 12 + 0)))))) >> | |||
99 | EXTENT_BITS_DUMPABLE_SHIFT(1 + (1 + (12 + 0)))); | |||
100 | } | |||
101 | ||||
102 | static inline bool_Bool | |||
103 | extent_slab_get(const extent_t *extent) { | |||
104 | return (bool_Bool)((extent->e_bits & EXTENT_BITS_SLAB_MASK((((((uint64_t)0x1U) << (1)) - 1)) << ((12 + 0)))) >> | |||
105 | EXTENT_BITS_SLAB_SHIFT(12 + 0)); | |||
106 | } | |||
107 | ||||
108 | static inline unsigned | |||
109 | extent_nfree_get(const extent_t *extent) { | |||
110 | assert(extent_slab_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent_slab_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 110, "extent_slab_get(extent)" ); abort(); } } while (0); | |||
111 | return (unsigned)((extent->e_bits & EXTENT_BITS_NFREE_MASK((((((uint64_t)0x1U) << (((12 - 3) + 1))) - 1)) << ((8 + (2 + (1 + (1 + (1 + (1 + (12 + 0)))))))))) >> | |||
112 | EXTENT_BITS_NFREE_SHIFT(8 + (2 + (1 + (1 + (1 + (1 + (12 + 0)))))))); | |||
113 | } | |||
114 | ||||
115 | static inline void * | |||
116 | extent_base_get(const extent_t *extent) { | |||
117 | assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) ||do { if (__builtin_expect(!!(config_debug && !(extent ->e_addr == ((void *)((uintptr_t)(extent->e_addr) & ~((size_t)(((size_t)(1U << 12)) - 1)))) || !extent_slab_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 118, "extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) || !extent_slab_get(extent)" ); abort(); } } while (0) | |||
118 | !extent_slab_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent ->e_addr == ((void *)((uintptr_t)(extent->e_addr) & ~((size_t)(((size_t)(1U << 12)) - 1)))) || !extent_slab_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 118, "extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) || !extent_slab_get(extent)" ); abort(); } } while (0); | |||
119 | return PAGE_ADDR2BASE(extent->e_addr)((void *)((uintptr_t)(extent->e_addr) & ~((size_t)(((size_t )(1U << 12)) - 1)))); | |||
120 | } | |||
121 | ||||
122 | static inline void * | |||
123 | extent_addr_get(const extent_t *extent) { | |||
124 | assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) ||do { if (__builtin_expect(!!(config_debug && !(extent ->e_addr == ((void *)((uintptr_t)(extent->e_addr) & ~((size_t)(((size_t)(1U << 12)) - 1)))) || !extent_slab_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 125, "extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) || !extent_slab_get(extent)" ); abort(); } } while (0) | |||
125 | !extent_slab_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent ->e_addr == ((void *)((uintptr_t)(extent->e_addr) & ~((size_t)(((size_t)(1U << 12)) - 1)))) || !extent_slab_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 125, "extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) || !extent_slab_get(extent)" ); abort(); } } while (0); | |||
126 | return extent->e_addr; | |||
127 | } | |||
128 | ||||
129 | static inline size_t | |||
130 | extent_size_get(const extent_t *extent) { | |||
131 | return (extent->e_size_esn & EXTENT_SIZE_MASK((size_t)~(((size_t)(1U << 12))-1))); | |||
132 | } | |||
133 | ||||
134 | static inline size_t | |||
135 | extent_esn_get(const extent_t *extent) { | |||
136 | return (extent->e_size_esn & EXTENT_ESN_MASK((size_t)((size_t)(1U << 12))-1)); | |||
137 | } | |||
138 | ||||
139 | static inline size_t | |||
140 | extent_bsize_get(const extent_t *extent) { | |||
141 | return extent->e_bsize; | |||
142 | } | |||
143 | ||||
144 | static inline void * | |||
145 | extent_before_get(const extent_t *extent) { | |||
146 | return (void *)((uintptr_t)extent_base_get(extent) - PAGE((size_t)(1U << 12))); | |||
147 | } | |||
148 | ||||
149 | static inline void * | |||
150 | extent_last_get(const extent_t *extent) { | |||
151 | return (void *)((uintptr_t)extent_base_get(extent) + | |||
152 | extent_size_get(extent) - PAGE((size_t)(1U << 12))); | |||
153 | } | |||
154 | ||||
155 | static inline void * | |||
156 | extent_past_get(const extent_t *extent) { | |||
157 | return (void *)((uintptr_t)extent_base_get(extent) + | |||
158 | extent_size_get(extent)); | |||
159 | } | |||
160 | ||||
161 | static inline arena_slab_data_t * | |||
162 | extent_slab_data_get(extent_t *extent) { | |||
163 | assert(extent_slab_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent_slab_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 163, "extent_slab_get(extent)" ); abort(); } } while (0); | |||
164 | return &extent->e_slab_data; | |||
165 | } | |||
166 | ||||
167 | static inline const arena_slab_data_t * | |||
168 | extent_slab_data_get_const(const extent_t *extent) { | |||
169 | assert(extent_slab_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent_slab_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 169, "extent_slab_get(extent)" ); abort(); } } while (0); | |||
170 | return &extent->e_slab_data; | |||
171 | } | |||
172 | ||||
173 | static inline prof_tctx_t * | |||
174 | extent_prof_tctx_get(const extent_t *extent) { | |||
175 | return (prof_tctx_t *)atomic_load_p(&extent->e_prof_tctx, | |||
176 | ATOMIC_ACQUIREatomic_memory_order_acquire); | |||
177 | } | |||
178 | ||||
179 | static inline void | |||
180 | extent_arena_set(extent_t *extent, arena_t *arena) { | |||
181 | unsigned arena_ind = (arena != NULL((void*)0)) ? arena_ind_get(arena) : ((1U << | |||
182 | MALLOCX_ARENA_BITS12) - 1); | |||
183 | extent->e_bits = (extent->e_bits & ~EXTENT_BITS_ARENA_MASK((((((uint64_t)0x1U) << (12)) - 1)) << (0))) | | |||
| ||||
184 | ((uint64_t)arena_ind << EXTENT_BITS_ARENA_SHIFT0); | |||
185 | } | |||
186 | ||||
187 | static inline void | |||
188 | extent_addr_set(extent_t *extent, void *addr) { | |||
189 | extent->e_addr = addr; | |||
190 | } | |||
191 | ||||
192 | static inline void | |||
193 | extent_addr_randomize(UNUSED__attribute__((unused)) tsdn_t *tsdn, extent_t *extent, size_t alignment) { | |||
194 | assert(extent_base_get(extent) == extent_addr_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent_base_get (extent) == extent_addr_get(extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 194, "extent_base_get(extent) == extent_addr_get(extent)" ); abort(); } } while (0); | |||
195 | ||||
196 | if (alignment < PAGE((size_t)(1U << 12))) { | |||
197 | unsigned lg_range = LG_PAGE12 - | |||
198 | lg_floor(CACHELINE_CEILING(alignment)(((alignment) + (64 - 1)) & ~(64 - 1))); | |||
199 | size_t r; | |||
200 | if (!tsdn_null(tsdn)) { | |||
201 | tsd_t *tsd = tsdn_tsd(tsdn); | |||
202 | r = (size_t)prng_lg_range_u64( | |||
203 | tsd_offset_statep_get(tsd), lg_range); | |||
204 | } else { | |||
205 | r = prng_lg_range_zu( | |||
206 | &extent_arena_get(extent)->offset_state, | |||
207 | lg_range, true1); | |||
208 | } | |||
209 | uintptr_t random_offset = ((uintptr_t)r) << (LG_PAGE12 - | |||
210 | lg_range); | |||
211 | extent->e_addr = (void *)((uintptr_t)extent->e_addr + | |||
212 | random_offset); | |||
213 | assert(ALIGNMENT_ADDR2BASE(extent->e_addr, alignment) ==do { if (__builtin_expect(!!(config_debug && !(((void *)((uintptr_t)(extent->e_addr) & ((~(alignment)) + 1) )) == extent->e_addr)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 214, "ALIGNMENT_ADDR2BASE(extent->e_addr, alignment) == extent->e_addr" ); abort(); } } while (0) | |||
214 | extent->e_addr)do { if (__builtin_expect(!!(config_debug && !(((void *)((uintptr_t)(extent->e_addr) & ((~(alignment)) + 1) )) == extent->e_addr)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 214, "ALIGNMENT_ADDR2BASE(extent->e_addr, alignment) == extent->e_addr" ); abort(); } } while (0); | |||
215 | } | |||
216 | } | |||
217 | ||||
218 | static inline void | |||
219 | extent_size_set(extent_t *extent, size_t size) { | |||
220 | assert((size & ~EXTENT_SIZE_MASK) == 0)do { if (__builtin_expect(!!(config_debug && !((size & ~((size_t)~(((size_t)(1U << 12))-1))) == 0)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "include/jemalloc/internal/extent_inlines.h" , 220, "(size & ~EXTENT_SIZE_MASK) == 0"); abort(); } } while (0); | |||
221 | extent->e_size_esn = size | (extent->e_size_esn & ~EXTENT_SIZE_MASK((size_t)~(((size_t)(1U << 12))-1))); | |||
222 | } | |||
223 | ||||
224 | static inline void | |||
225 | extent_esn_set(extent_t *extent, size_t esn) { | |||
226 | extent->e_size_esn = (extent->e_size_esn & ~EXTENT_ESN_MASK((size_t)((size_t)(1U << 12))-1)) | (esn & | |||
227 | EXTENT_ESN_MASK((size_t)((size_t)(1U << 12))-1)); | |||
228 | } | |||
229 | ||||
230 | static inline void | |||
231 | extent_bsize_set(extent_t *extent, size_t bsize) { | |||
232 | extent->e_bsize = bsize; | |||
233 | } | |||
234 | ||||
235 | static inline void | |||
236 | extent_szind_set(extent_t *extent, szind_t szind) { | |||
237 | assert(szind <= NSIZES)do { if (__builtin_expect(!!(config_debug && !(szind <= 235)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 237, "szind <= NSIZES" ); abort(); } } while (0); /* NSIZES means "invalid". */ | |||
238 | extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SZIND_MASK((((((uint64_t)0x1U) << (8)) - 1)) << ((2 + (1 + ( 1 + (1 + (1 + (12 + 0))))))))) | | |||
239 | ((uint64_t)szind << EXTENT_BITS_SZIND_SHIFT(2 + (1 + (1 + (1 + (1 + (12 + 0))))))); | |||
240 | } | |||
241 | ||||
242 | static inline void | |||
243 | extent_nfree_set(extent_t *extent, unsigned nfree) { | |||
244 | assert(extent_slab_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent_slab_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 244, "extent_slab_get(extent)" ); abort(); } } while (0); | |||
245 | extent->e_bits = (extent->e_bits & ~EXTENT_BITS_NFREE_MASK((((((uint64_t)0x1U) << (((12 - 3) + 1))) - 1)) << ((8 + (2 + (1 + (1 + (1 + (1 + (12 + 0)))))))))) | | |||
246 | ((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT(8 + (2 + (1 + (1 + (1 + (1 + (12 + 0)))))))); | |||
247 | } | |||
248 | ||||
249 | static inline void | |||
250 | extent_nfree_inc(extent_t *extent) { | |||
251 | assert(extent_slab_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent_slab_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 251, "extent_slab_get(extent)" ); abort(); } } while (0); | |||
252 | extent->e_bits += ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT(8 + (2 + (1 + (1 + (1 + (1 + (12 + 0)))))))); | |||
253 | } | |||
254 | ||||
255 | static inline void | |||
256 | extent_nfree_dec(extent_t *extent) { | |||
257 | assert(extent_slab_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent_slab_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 257, "extent_slab_get(extent)" ); abort(); } } while (0); | |||
258 | extent->e_bits -= ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT(8 + (2 + (1 + (1 + (1 + (1 + (12 + 0)))))))); | |||
259 | } | |||
260 | ||||
261 | static inline void | |||
262 | extent_sn_set(extent_t *extent, size_t sn) { | |||
263 | extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SN_MASK((18446744073709551615UL) << (((12 - 3) + 1) + (8 + (2 + (1 + (1 + (1 + (1 + (12 + 0)))))))))) | | |||
264 | ((uint64_t)sn << EXTENT_BITS_SN_SHIFT(((12 - 3) + 1) + (8 + (2 + (1 + (1 + (1 + (1 + (12 + 0)))))) ))); | |||
265 | } | |||
266 | ||||
267 | static inline void | |||
268 | extent_state_set(extent_t *extent, extent_state_t state) { | |||
269 | extent->e_bits = (extent->e_bits & ~EXTENT_BITS_STATE_MASK((((((uint64_t)0x1U) << (2)) - 1)) << ((1 + (1 + ( 1 + (1 + (12 + 0)))))))) | | |||
270 | ((uint64_t)state << EXTENT_BITS_STATE_SHIFT(1 + (1 + (1 + (1 + (12 + 0)))))); | |||
271 | } | |||
272 | ||||
273 | static inline void | |||
274 | extent_zeroed_set(extent_t *extent, bool_Bool zeroed) { | |||
275 | extent->e_bits = (extent->e_bits & ~EXTENT_BITS_ZEROED_MASK((((((uint64_t)0x1U) << (1)) - 1)) << ((1 + (1 + ( 1 + (12 + 0))))))) | | |||
276 | ((uint64_t)zeroed << EXTENT_BITS_ZEROED_SHIFT(1 + (1 + (1 + (12 + 0))))); | |||
277 | } | |||
278 | ||||
279 | static inline void | |||
280 | extent_committed_set(extent_t *extent, bool_Bool committed) { | |||
281 | extent->e_bits = (extent->e_bits & ~EXTENT_BITS_COMMITTED_MASK((((((uint64_t)0x1U) << (1)) - 1)) << ((1 + (12 + 0))))) | | |||
282 | ((uint64_t)committed << EXTENT_BITS_COMMITTED_SHIFT(1 + (12 + 0))); | |||
283 | } | |||
284 | ||||
285 | static inline void | |||
286 | extent_dumpable_set(extent_t *extent, bool_Bool dumpable) { | |||
287 | extent->e_bits = (extent->e_bits & ~EXTENT_BITS_DUMPABLE_MASK((((((uint64_t)0x1U) << (1)) - 1)) << ((1 + (1 + ( 12 + 0)))))) | | |||
288 | ((uint64_t)dumpable << EXTENT_BITS_DUMPABLE_SHIFT(1 + (1 + (12 + 0)))); | |||
289 | } | |||
290 | ||||
291 | static inline void | |||
292 | extent_slab_set(extent_t *extent, bool_Bool slab) { | |||
293 | extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SLAB_MASK((((((uint64_t)0x1U) << (1)) - 1)) << ((12 + 0)))) | | |||
294 | ((uint64_t)slab << EXTENT_BITS_SLAB_SHIFT(12 + 0)); | |||
295 | } | |||
296 | ||||
297 | static inline void | |||
298 | extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx) { | |||
299 | atomic_store_p(&extent->e_prof_tctx, tctx, ATOMIC_RELEASEatomic_memory_order_release); | |||
300 | } | |||
301 | ||||
302 | static inline void | |||
303 | extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size, | |||
304 | bool_Bool slab, szind_t szind, size_t sn, extent_state_t state, bool_Bool zeroed, | |||
305 | bool_Bool committed, bool_Bool dumpable) { | |||
306 | assert(addr == PAGE_ADDR2BASE(addr) || !slab)do { if (__builtin_expect(!!(config_debug && !(addr == ((void *)((uintptr_t)(addr) & ~((size_t)(((size_t)(1U << 12)) - 1)))) || !slab)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 306, "addr == PAGE_ADDR2BASE(addr) || !slab" ); abort(); } } while (0); | |||
307 | ||||
308 | extent_arena_set(extent, arena); | |||
309 | extent_addr_set(extent, addr); | |||
310 | extent_size_set(extent, size); | |||
311 | extent_slab_set(extent, slab); | |||
312 | extent_szind_set(extent, szind); | |||
313 | extent_sn_set(extent, sn); | |||
314 | extent_state_set(extent, state); | |||
315 | extent_zeroed_set(extent, zeroed); | |||
316 | extent_committed_set(extent, committed); | |||
317 | extent_dumpable_set(extent, dumpable); | |||
318 | ql_elm_new(extent, ql_link)do { ((extent))->ql_link.qre_next = ((extent)); ((extent)) ->ql_link.qre_prev = ((extent)); } while (0); | |||
319 | if (config_prof) { | |||
320 | extent_prof_tctx_set(extent, NULL((void*)0)); | |||
321 | } | |||
322 | } | |||
323 | ||||
324 | static inline void | |||
325 | extent_binit(extent_t *extent, void *addr, size_t bsize, size_t sn) { | |||
326 | extent_arena_set(extent, NULL((void*)0)); | |||
327 | extent_addr_set(extent, addr); | |||
328 | extent_bsize_set(extent, bsize); | |||
329 | extent_slab_set(extent, false0); | |||
330 | extent_szind_set(extent, NSIZES235); | |||
331 | extent_sn_set(extent, sn); | |||
332 | extent_state_set(extent, extent_state_active); | |||
333 | extent_zeroed_set(extent, true1); | |||
334 | extent_committed_set(extent, true1); | |||
335 | extent_dumpable_set(extent, true1); | |||
336 | } | |||
337 | ||||
338 | static inline void | |||
339 | extent_list_init(extent_list_t *list) { | |||
340 | ql_new(list)do { (list)->qlh_first = ((void*)0); } while (0); | |||
341 | } | |||
342 | ||||
343 | static inline extent_t * | |||
344 | extent_list_first(const extent_list_t *list) { | |||
345 | return ql_first(list)((list)->qlh_first); | |||
346 | } | |||
347 | ||||
348 | static inline extent_t * | |||
349 | extent_list_last(const extent_list_t *list) { | |||
350 | return ql_last(list, ql_link)((((list)->qlh_first) != ((void*)0)) ? ((((list)->qlh_first ))->ql_link.qre_prev) : ((void*)0)); | |||
351 | } | |||
352 | ||||
353 | static inline void | |||
354 | extent_list_append(extent_list_t *list, extent_t *extent) { | |||
355 | ql_tail_insert(list, extent, ql_link)do { if (((list)->qlh_first) != ((void*)0)) { do { ((extent ))->ql_link.qre_prev = (((list)->qlh_first))->ql_link .qre_prev; ((extent))->ql_link.qre_next = (((list)->qlh_first )); ((extent))->ql_link.qre_prev->ql_link.qre_next = (( extent)); (((list)->qlh_first))->ql_link.qre_prev = ((extent )); } while (0); } ((list)->qlh_first) = (((extent))->ql_link .qre_next); } while (0); | |||
356 | } | |||
357 | ||||
358 | static inline void | |||
359 | extent_list_prepend(extent_list_t *list, extent_t *extent) { | |||
360 | ql_head_insert(list, extent, ql_link)do { if (((list)->qlh_first) != ((void*)0)) { do { ((extent ))->ql_link.qre_prev = (((list)->qlh_first))->ql_link .qre_prev; ((extent))->ql_link.qre_next = (((list)->qlh_first )); ((extent))->ql_link.qre_prev->ql_link.qre_next = (( extent)); (((list)->qlh_first))->ql_link.qre_prev = ((extent )); } while (0); } ((list)->qlh_first) = (extent); } while (0); | |||
361 | } | |||
362 | ||||
363 | static inline void | |||
364 | extent_list_replace(extent_list_t *list, extent_t *to_remove, | |||
365 | extent_t *to_insert) { | |||
366 | ql_after_insert(to_remove, to_insert, ql_link)do { ((to_insert))->ql_link.qre_next = ((to_remove))->ql_link .qre_next; ((to_insert))->ql_link.qre_prev = ((to_remove)) ; ((to_insert))->ql_link.qre_next->ql_link.qre_prev = ( (to_insert)); ((to_remove))->ql_link.qre_next = ((to_insert )); } while (0); | |||
367 | ql_remove(list, to_remove, ql_link)do { if (((list)->qlh_first) == (to_remove)) { ((list)-> qlh_first) = ((((list)->qlh_first))->ql_link.qre_next); } if (((list)->qlh_first) != (to_remove)) { do { ((to_remove ))->ql_link.qre_prev->ql_link.qre_next = ((to_remove))-> ql_link.qre_next; ((to_remove))->ql_link.qre_next->ql_link .qre_prev = ((to_remove))->ql_link.qre_prev; ((to_remove)) ->ql_link.qre_next = ((to_remove)); ((to_remove))->ql_link .qre_prev = ((to_remove)); } while (0); } else { ((list)-> qlh_first) = ((void*)0); } } while (0); | |||
368 | } | |||
369 | ||||
370 | static inline void | |||
371 | extent_list_remove(extent_list_t *list, extent_t *extent) { | |||
372 | ql_remove(list, extent, ql_link)do { if (((list)->qlh_first) == (extent)) { ((list)->qlh_first ) = ((((list)->qlh_first))->ql_link.qre_next); } if ((( list)->qlh_first) != (extent)) { do { ((extent))->ql_link .qre_prev->ql_link.qre_next = ((extent))->ql_link.qre_next ; ((extent))->ql_link.qre_next->ql_link.qre_prev = ((extent ))->ql_link.qre_prev; ((extent))->ql_link.qre_next = (( extent)); ((extent))->ql_link.qre_prev = ((extent)); } while (0); } else { ((list)->qlh_first) = ((void*)0); } } while (0); | |||
373 | } | |||
374 | ||||
375 | static inline int | |||
376 | extent_sn_comp(const extent_t *a, const extent_t *b) { | |||
377 | size_t a_sn = extent_sn_get(a); | |||
378 | size_t b_sn = extent_sn_get(b); | |||
379 | ||||
380 | return (a_sn > b_sn) - (a_sn < b_sn); | |||
381 | } | |||
382 | ||||
383 | static inline int | |||
384 | extent_esn_comp(const extent_t *a, const extent_t *b) { | |||
385 | size_t a_esn = extent_esn_get(a); | |||
386 | size_t b_esn = extent_esn_get(b); | |||
387 | ||||
388 | return (a_esn > b_esn) - (a_esn < b_esn); | |||
389 | } | |||
390 | ||||
391 | static inline int | |||
392 | extent_ad_comp(const extent_t *a, const extent_t *b) { | |||
393 | uintptr_t a_addr = (uintptr_t)extent_addr_get(a); | |||
394 | uintptr_t b_addr = (uintptr_t)extent_addr_get(b); | |||
395 | ||||
396 | return (a_addr > b_addr) - (a_addr < b_addr); | |||
397 | } | |||
398 | ||||
399 | static inline int | |||
400 | extent_ead_comp(const extent_t *a, const extent_t *b) { | |||
401 | uintptr_t a_eaddr = (uintptr_t)a; | |||
402 | uintptr_t b_eaddr = (uintptr_t)b; | |||
403 | ||||
404 | return (a_eaddr > b_eaddr) - (a_eaddr < b_eaddr); | |||
405 | } | |||
406 | ||||
407 | static inline int | |||
408 | extent_snad_comp(const extent_t *a, const extent_t *b) { | |||
409 | int ret; | |||
410 | ||||
411 | ret = extent_sn_comp(a, b); | |||
412 | if (ret != 0) { | |||
413 | return ret; | |||
414 | } | |||
415 | ||||
416 | ret = extent_ad_comp(a, b); | |||
417 | return ret; | |||
418 | } | |||
419 | ||||
420 | static inline int | |||
421 | extent_esnead_comp(const extent_t *a, const extent_t *b) { | |||
422 | int ret; | |||
423 | ||||
424 | ret = extent_esn_comp(a, b); | |||
425 | if (ret != 0) { | |||
426 | return ret; | |||
427 | } | |||
428 | ||||
429 | ret = extent_ead_comp(a, b); | |||
430 | return ret; | |||
431 | } | |||
432 | ||||
433 | #endif /* JEMALLOC_INTERNAL_EXTENT_INLINES_H */ |