File: | deps/jemalloc/include/jemalloc/internal/extent_inlines.h |
Warning: | line 183, column 35 The left operand of '&' is a garbage value |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | #define JEMALLOC_EXTENT_C_ | ||||
2 | #include "jemalloc/internal/jemalloc_preamble.h" | ||||
3 | #include "jemalloc/internal/jemalloc_internal_includes.h" | ||||
4 | |||||
5 | #include "jemalloc/internal/assert.h" | ||||
6 | #include "jemalloc/internal/extent_dss.h" | ||||
7 | #include "jemalloc/internal/extent_mmap.h" | ||||
8 | #include "jemalloc/internal/ph.h" | ||||
9 | #include "jemalloc/internal/rtree.h" | ||||
10 | #include "jemalloc/internal/mutex.h" | ||||
11 | #include "jemalloc/internal/mutex_pool.h" | ||||
12 | |||||
13 | /******************************************************************************/ | ||||
14 | /* Data. */ | ||||
15 | |||||
16 | rtree_t extents_rtree; | ||||
17 | /* Keyed by the address of the extent_t being protected. */ | ||||
18 | mutex_pool_t extent_mutex_pool; | ||||
19 | |||||
20 | size_t opt_lg_extent_max_active_fit = LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT6; | ||||
21 | |||||
22 | static const bitmap_info_t extents_bitmap_info = | ||||
23 | BITMAP_INFO_INITIALIZER(NPSIZES+1){ 199 +1, (((199 +1) + ((1U << (3 + 3))-1)) >> (3 + 3)) }; | ||||
24 | |||||
25 | static void *extent_alloc_default(extent_hooks_t *extent_hooks, void *new_addr, | ||||
26 | size_t size, size_t alignment, bool_Bool *zero, bool_Bool *commit, | ||||
27 | unsigned arena_ind); | ||||
28 | static bool_Bool extent_dalloc_default(extent_hooks_t *extent_hooks, void *addr, | ||||
29 | size_t size, bool_Bool committed, unsigned arena_ind); | ||||
30 | static void extent_destroy_default(extent_hooks_t *extent_hooks, void *addr, | ||||
31 | size_t size, bool_Bool committed, unsigned arena_ind); | ||||
32 | static bool_Bool extent_commit_default(extent_hooks_t *extent_hooks, void *addr, | ||||
33 | size_t size, size_t offset, size_t length, unsigned arena_ind); | ||||
34 | static bool_Bool extent_commit_impl(tsdn_t *tsdn, arena_t *arena, | ||||
35 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, | ||||
36 | size_t length, bool_Bool growing_retained); | ||||
37 | static bool_Bool extent_decommit_default(extent_hooks_t *extent_hooks, | ||||
38 | void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind); | ||||
39 | #ifdef PAGES_CAN_PURGE_LAZY | ||||
40 | static bool_Bool extent_purge_lazy_default(extent_hooks_t *extent_hooks, void *addr, | ||||
41 | size_t size, size_t offset, size_t length, unsigned arena_ind); | ||||
42 | #endif | ||||
43 | static bool_Bool extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena, | ||||
44 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, | ||||
45 | size_t length, bool_Bool growing_retained); | ||||
46 | #ifdef PAGES_CAN_PURGE_FORCED | ||||
47 | static bool_Bool extent_purge_forced_default(extent_hooks_t *extent_hooks, | ||||
48 | void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind); | ||||
49 | #endif | ||||
50 | static bool_Bool extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena, | ||||
51 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, | ||||
52 | size_t length, bool_Bool growing_retained); | ||||
53 | #ifdef JEMALLOC_MAPS_COALESCE | ||||
54 | static bool_Bool extent_split_default(extent_hooks_t *extent_hooks, void *addr, | ||||
55 | size_t size, size_t size_a, size_t size_b, bool_Bool committed, | ||||
56 | unsigned arena_ind); | ||||
57 | #endif | ||||
58 | static extent_t *extent_split_impl(tsdn_t *tsdn, arena_t *arena, | ||||
59 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a, | ||||
60 | szind_t szind_a, bool_Bool slab_a, size_t size_b, szind_t szind_b, bool_Bool slab_b, | ||||
61 | bool_Bool growing_retained); | ||||
62 | #ifdef JEMALLOC_MAPS_COALESCE | ||||
63 | static bool_Bool extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, | ||||
64 | size_t size_a, void *addr_b, size_t size_b, bool_Bool committed, | ||||
65 | unsigned arena_ind); | ||||
66 | #endif | ||||
67 | static bool_Bool extent_merge_impl(tsdn_t *tsdn, arena_t *arena, | ||||
68 | extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b, | ||||
69 | bool_Bool growing_retained); | ||||
70 | |||||
71 | const extent_hooks_t extent_hooks_default = { | ||||
72 | extent_alloc_default, | ||||
73 | extent_dalloc_default, | ||||
74 | extent_destroy_default, | ||||
75 | extent_commit_default, | ||||
76 | extent_decommit_default | ||||
77 | #ifdef PAGES_CAN_PURGE_LAZY | ||||
78 | , | ||||
79 | extent_purge_lazy_default | ||||
80 | #else | ||||
81 | , | ||||
82 | NULL((void*)0) | ||||
83 | #endif | ||||
84 | #ifdef PAGES_CAN_PURGE_FORCED | ||||
85 | , | ||||
86 | extent_purge_forced_default | ||||
87 | #else | ||||
88 | , | ||||
89 | NULL((void*)0) | ||||
90 | #endif | ||||
91 | #ifdef JEMALLOC_MAPS_COALESCE | ||||
92 | , | ||||
93 | extent_split_default, | ||||
94 | extent_merge_default | ||||
95 | #endif | ||||
96 | }; | ||||
97 | |||||
98 | /* Used exclusively for gdump triggering. */ | ||||
99 | static atomic_zu_t curpages; | ||||
100 | static atomic_zu_t highpages; | ||||
101 | |||||
102 | /******************************************************************************/ | ||||
103 | /* | ||||
104 | * Function prototypes for static functions that are referenced prior to | ||||
105 | * definition. | ||||
106 | */ | ||||
107 | |||||
108 | static void extent_deregister(tsdn_t *tsdn, extent_t *extent); | ||||
109 | static extent_t *extent_recycle(tsdn_t *tsdn, arena_t *arena, | ||||
110 | extent_hooks_t **r_extent_hooks, extents_t *extents, void *new_addr, | ||||
111 | size_t usize, size_t pad, size_t alignment, bool_Bool slab, szind_t szind, | ||||
112 | bool_Bool *zero, bool_Bool *commit, bool_Bool growing_retained); | ||||
113 | static extent_t *extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, | ||||
114 | extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, | ||||
115 | extent_t *extent, bool_Bool *coalesced, bool_Bool growing_retained); | ||||
116 | static void extent_record(tsdn_t *tsdn, arena_t *arena, | ||||
117 | extent_hooks_t **r_extent_hooks, extents_t *extents, extent_t *extent, | ||||
118 | bool_Bool growing_retained); | ||||
119 | |||||
120 | /******************************************************************************/ | ||||
121 | |||||
122 | ph_gen(UNUSED, extent_avail_, extent_tree_t, extent_t, ph_link,__attribute__((unused)) void extent_avail_new(extent_tree_t * ph) { memset(ph, 0, sizeof(struct { extent_t *ph_root; })); } __attribute__((unused)) _Bool extent_avail_empty(extent_tree_t *ph) { return (ph->ph_root == ((void*)0)); } __attribute__ ((unused)) extent_t * extent_avail_first(extent_tree_t *ph) { if (ph->ph_root == ((void*)0)) { return ((void*)0); } do { extent_t *phn = (ph->ph_root->ph_link.phn_next); if (phn != ((void*)0)) { do { ph->ph_root->ph_link.phn_prev = ( (void*)0); } while (0); do { ph->ph_root->ph_link.phn_next = ((void*)0); } while (0); do { phn->ph_link.phn_prev = ( (void*)0); } while (0); do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0); extent_t *phn0 = phn; extent_t *phn1 = ( phn0->ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void *)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while ( 0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link.phn_next = ((void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void *)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1 ) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } phn = phn0; } while (0); do { if (__builtin_expect (!!(config_debug && !((phn->ph_link.phn_next) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_next_get(extent_t, ph_link, phn) == NULL" ); abort(); } } while (0); do { if (ph->ph_root == ((void* )0)) { ph->ph_root = phn; } else if (phn == ((void*)0)) { ph ->ph_root = ph->ph_root; } else if (extent_esnead_comp( ph->ph_root, phn) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(ph->ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(phn != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (ph->ph_root, phn) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(ph->ph_root, phn) <= 0" ); abort(); } } while (0); do { phn->ph_link.phn_prev = ph ->ph_root; } while (0); phn0child = (ph->ph_root->ph_link .phn_lchild); do { phn->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = phn; } while (0); } do { ph->ph_root->ph_link .phn_lchild = phn; } while (0); } while (0); ph->ph_root = ph->ph_root; } else { do { extent_t *phn0child; do { if ( __builtin_expect(!!(config_debug && !(phn != ((void*) 0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(ph-> ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp(phn, ph->ph_root) <= 0)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "extent_esnead_comp(phn, ph->ph_root) <= 0"); abort (); } } while (0); do { ph->ph_root->ph_link.phn_prev = phn; } while (0); phn0child = (phn->ph_link.phn_lchild); do { ph->ph_root->ph_link.phn_next = phn0child; } while ( 0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = ph->ph_root; } while (0); } do { phn->ph_link .phn_lchild = ph->ph_root; } while (0); } while (0); ph-> ph_root = phn; } } while (0); } } while (0); return ph->ph_root ; } __attribute__((unused)) extent_t * extent_avail_any(extent_tree_t *ph) { if (ph->ph_root == ((void*)0)) { return ((void*)0) ; } extent_t *aux = (ph->ph_root->ph_link.phn_next); if (aux != ((void*)0)) { return aux; } return ph->ph_root; } __attribute__((unused)) void extent_avail_insert(extent_tree_t *ph, extent_t *phn) { memset(&phn->ph_link, 0, sizeof (struct { extent_t *phn_prev; extent_t *phn_next; extent_t *phn_lchild ; })); if (ph->ph_root == ((void*)0)) { ph->ph_root = phn ; } else { do { phn->ph_link.phn_next = (ph->ph_root-> ph_link.phn_next); } while (0); if ((ph->ph_root->ph_link .phn_next) != ((void*)0)) { do { (ph->ph_root->ph_link. phn_next)->ph_link.phn_prev = phn; } while (0); } do { phn ->ph_link.phn_prev = ph->ph_root; } while (0); do { ph-> ph_root->ph_link.phn_next = phn; } while (0); } } __attribute__ ((unused)) extent_t * extent_avail_remove_first(extent_tree_t *ph) { extent_t *ret; if (ph->ph_root == ((void*)0)) { return ((void*)0); } do { extent_t *phn = (ph->ph_root->ph_link .phn_next); if (phn != ((void*)0)) { do { ph->ph_root-> ph_link.phn_prev = ((void*)0); } while (0); do { ph->ph_root ->ph_link.phn_next = ((void*)0); } while (0); do { phn-> ph_link.phn_prev = ((void*)0); } while (0); do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0); extent_t *phn0 = phn ; extent_t *phn1 = (phn0->ph_link.phn_next); if (phn1 != ( (void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = (( void*)0); } while (0); do { phn1->ph_link.phn_prev = ((void *)0); } while (0); do { phn1->ph_link.phn_next = ((void*)0 ); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } phn = phn0; } while (0); do { if (__builtin_expect (!!(config_debug && !((phn->ph_link.phn_next) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_next_get(extent_t, ph_link, phn) == NULL" ); abort(); } } while (0); do { if (ph->ph_root == ((void* )0)) { ph->ph_root = phn; } else if (phn == ((void*)0)) { ph ->ph_root = ph->ph_root; } else if (extent_esnead_comp( ph->ph_root, phn) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(ph->ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(phn != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (ph->ph_root, phn) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(ph->ph_root, phn) <= 0" ); abort(); } } while (0); do { phn->ph_link.phn_prev = ph ->ph_root; } while (0); phn0child = (ph->ph_root->ph_link .phn_lchild); do { phn->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = phn; } while (0); } do { ph->ph_root->ph_link .phn_lchild = phn; } while (0); } while (0); ph->ph_root = ph->ph_root; } else { do { extent_t *phn0child; do { if ( __builtin_expect(!!(config_debug && !(phn != ((void*) 0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(ph-> ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp(phn, ph->ph_root) <= 0)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "extent_esnead_comp(phn, ph->ph_root) <= 0"); abort (); } } while (0); do { ph->ph_root->ph_link.phn_prev = phn; } while (0); phn0child = (phn->ph_link.phn_lchild); do { ph->ph_root->ph_link.phn_next = phn0child; } while ( 0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = ph->ph_root; } while (0); } do { phn->ph_link .phn_lchild = ph->ph_root; } while (0); } while (0); ph-> ph_root = phn; } } while (0); } } while (0); ret = ph->ph_root ; do { extent_t *lchild = (ph->ph_root->ph_link.phn_lchild ); if (lchild == ((void*)0)) { ph->ph_root = ((void*)0); } else { do { extent_t *head = ((void*)0); extent_t *tail = (( void*)0); extent_t *phn0 = lchild; extent_t *phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while (0); do { phn0 ->ph_link.phn_next = ((void*)0); } while (0); do { phn1-> ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link .phn_next = ((void*)0); } while (0); do { if (phn0 == ((void* )0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0 ; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } ph->ph_root = phn0; } while (0); } } while (0); return ret; } __attribute__((unused)) extent_t * extent_avail_remove_any (extent_tree_t *ph) { if (ph->ph_root == ((void*)0)) { return ((void*)0); } extent_t *ret = (ph->ph_root->ph_link.phn_next ); if (ret != ((void*)0)) { extent_t *aux = (ret->ph_link. phn_next); do { ph->ph_root->ph_link.phn_next = aux; } while (0); if (aux != ((void*)0)) { do { aux->ph_link.phn_prev = ph->ph_root; } while (0); } return ret; } ret = ph->ph_root ; do { extent_t *lchild = (ph->ph_root->ph_link.phn_lchild ); if (lchild == ((void*)0)) { ph->ph_root = ((void*)0); } else { do { extent_t *head = ((void*)0); extent_t *tail = (( void*)0); extent_t *phn0 = lchild; extent_t *phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while (0); do { phn0 ->ph_link.phn_next = ((void*)0); } while (0); do { phn1-> ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link .phn_next = ((void*)0); } while (0); do { if (phn0 == ((void* )0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0 ; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } ph->ph_root = phn0; } while (0); } } while (0); return ret; } __attribute__((unused)) void extent_avail_remove (extent_tree_t *ph, extent_t *phn) { extent_t *replace, *parent ; if (ph->ph_root == phn) { if ((phn->ph_link.phn_lchild ) == ((void*)0)) { ph->ph_root = (phn->ph_link.phn_next ); if (ph->ph_root != ((void*)0)) { do { ph->ph_root-> ph_link.phn_prev = ((void*)0); } while (0); } return; } do { extent_t *phn = (ph->ph_root->ph_link.phn_next); if (phn != ((void *)0)) { do { ph->ph_root->ph_link.phn_prev = ((void*)0) ; } while (0); do { ph->ph_root->ph_link.phn_next = ((void *)0); } while (0); do { phn->ph_link.phn_prev = ((void*)0) ; } while (0); do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0); extent_t *phn0 = phn; extent_t *phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while (0); do { phn0 ->ph_link.phn_next = ((void*)0); } while (0); do { phn1-> ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link .phn_next = ((void*)0); } while (0); do { if (phn0 == ((void* )0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0 ; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } phn = phn0; } while (0); do { if (__builtin_expect (!!(config_debug && !((phn->ph_link.phn_next) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_next_get(extent_t, ph_link, phn) == NULL" ); abort(); } } while (0); do { if (ph->ph_root == ((void* )0)) { ph->ph_root = phn; } else if (phn == ((void*)0)) { ph ->ph_root = ph->ph_root; } else if (extent_esnead_comp( ph->ph_root, phn) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(ph->ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(phn != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (ph->ph_root, phn) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(ph->ph_root, phn) <= 0" ); abort(); } } while (0); do { phn->ph_link.phn_prev = ph ->ph_root; } while (0); phn0child = (ph->ph_root->ph_link .phn_lchild); do { phn->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = phn; } while (0); } do { ph->ph_root->ph_link .phn_lchild = phn; } while (0); } while (0); ph->ph_root = ph->ph_root; } else { do { extent_t *phn0child; do { if ( __builtin_expect(!!(config_debug && !(phn != ((void*) 0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(ph-> ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp(phn, ph->ph_root) <= 0)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "extent_esnead_comp(phn, ph->ph_root) <= 0"); abort (); } } while (0); do { ph->ph_root->ph_link.phn_prev = phn; } while (0); phn0child = (phn->ph_link.phn_lchild); do { ph->ph_root->ph_link.phn_next = phn0child; } while ( 0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = ph->ph_root; } while (0); } do { phn->ph_link .phn_lchild = ph->ph_root; } while (0); } while (0); ph-> ph_root = phn; } } while (0); } } while (0); if (ph->ph_root == phn) { do { extent_t *lchild = (ph->ph_root->ph_link .phn_lchild); if (lchild == ((void*)0)) { ph->ph_root = (( void*)0); } else { do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0); extent_t *phn0 = lchild; extent_t *phn1 = (phn0->ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void *)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while ( 0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link.phn_next = ((void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void *)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1 ) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } ph->ph_root = phn0; } while (0); } } while (0); return; } } if ((parent = (phn->ph_link.phn_prev)) != ((void*)0)) { if ((parent->ph_link.phn_lchild) != phn) { parent = ((void*)0); } } do { extent_t *lchild = (phn->ph_link.phn_lchild ); if (lchild == ((void*)0)) { replace = ((void*)0); } else { do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0 ); extent_t *phn0 = lchild; extent_t *phn1 = (phn0->ph_link .phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1 ->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest ->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0-> ph_link.phn_prev = ((void*)0); } while (0); do { phn0->ph_link .phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } replace = phn0; } while (0); } } while (0); if (replace != ((void*)0)) { if (parent != ((void*)0)) { do { replace->ph_link.phn_prev = parent; } while (0); do { parent ->ph_link.phn_lchild = replace; } while (0); } else { do { replace->ph_link.phn_prev = (phn->ph_link.phn_prev); } while (0); if ((phn->ph_link.phn_prev) != ((void*)0)) { do { (phn->ph_link.phn_prev)->ph_link.phn_next = replace; } while (0); } } do { replace->ph_link.phn_next = (phn-> ph_link.phn_next); } while (0); if ((phn->ph_link.phn_next ) != ((void*)0)) { do { (phn->ph_link.phn_next)->ph_link .phn_prev = replace; } while (0); } } else { if (parent != (( void*)0)) { extent_t *next = (phn->ph_link.phn_next); do { parent->ph_link.phn_lchild = next; } while (0); if (next != ((void*)0)) { do { next->ph_link.phn_prev = parent; } while (0); } } else { do { if (__builtin_expect(!!(config_debug && !((phn->ph_link.phn_prev) != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn_prev_get(extent_t, ph_link, phn) != NULL"); abort (); } } while (0); do { (phn->ph_link.phn_prev)->ph_link .phn_next = (phn->ph_link.phn_next); } while (0); } if ((phn ->ph_link.phn_next) != ((void*)0)) { do { (phn->ph_link .phn_next)->ph_link.phn_prev = (phn->ph_link.phn_prev); } while (0); } } } | ||||
123 | extent_esnead_comp)__attribute__((unused)) void extent_avail_new(extent_tree_t * ph) { memset(ph, 0, sizeof(struct { extent_t *ph_root; })); } __attribute__((unused)) _Bool extent_avail_empty(extent_tree_t *ph) { return (ph->ph_root == ((void*)0)); } __attribute__ ((unused)) extent_t * extent_avail_first(extent_tree_t *ph) { if (ph->ph_root == ((void*)0)) { return ((void*)0); } do { extent_t *phn = (ph->ph_root->ph_link.phn_next); if (phn != ((void*)0)) { do { ph->ph_root->ph_link.phn_prev = ( (void*)0); } while (0); do { ph->ph_root->ph_link.phn_next = ((void*)0); } while (0); do { phn->ph_link.phn_prev = ( (void*)0); } while (0); do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0); extent_t *phn0 = phn; extent_t *phn1 = ( phn0->ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void *)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while ( 0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link.phn_next = ((void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void *)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1 ) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } phn = phn0; } while (0); do { if (__builtin_expect (!!(config_debug && !((phn->ph_link.phn_next) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_next_get(extent_t, ph_link, phn) == NULL" ); abort(); } } while (0); do { if (ph->ph_root == ((void* )0)) { ph->ph_root = phn; } else if (phn == ((void*)0)) { ph ->ph_root = ph->ph_root; } else if (extent_esnead_comp( ph->ph_root, phn) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(ph->ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(phn != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (ph->ph_root, phn) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(ph->ph_root, phn) <= 0" ); abort(); } } while (0); do { phn->ph_link.phn_prev = ph ->ph_root; } while (0); phn0child = (ph->ph_root->ph_link .phn_lchild); do { phn->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = phn; } while (0); } do { ph->ph_root->ph_link .phn_lchild = phn; } while (0); } while (0); ph->ph_root = ph->ph_root; } else { do { extent_t *phn0child; do { if ( __builtin_expect(!!(config_debug && !(phn != ((void*) 0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(ph-> ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp(phn, ph->ph_root) <= 0)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "extent_esnead_comp(phn, ph->ph_root) <= 0"); abort (); } } while (0); do { ph->ph_root->ph_link.phn_prev = phn; } while (0); phn0child = (phn->ph_link.phn_lchild); do { ph->ph_root->ph_link.phn_next = phn0child; } while ( 0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = ph->ph_root; } while (0); } do { phn->ph_link .phn_lchild = ph->ph_root; } while (0); } while (0); ph-> ph_root = phn; } } while (0); } } while (0); return ph->ph_root ; } __attribute__((unused)) extent_t * extent_avail_any(extent_tree_t *ph) { if (ph->ph_root == ((void*)0)) { return ((void*)0) ; } extent_t *aux = (ph->ph_root->ph_link.phn_next); if (aux != ((void*)0)) { return aux; } return ph->ph_root; } __attribute__((unused)) void extent_avail_insert(extent_tree_t *ph, extent_t *phn) { memset(&phn->ph_link, 0, sizeof (struct { extent_t *phn_prev; extent_t *phn_next; extent_t *phn_lchild ; })); if (ph->ph_root == ((void*)0)) { ph->ph_root = phn ; } else { do { phn->ph_link.phn_next = (ph->ph_root-> ph_link.phn_next); } while (0); if ((ph->ph_root->ph_link .phn_next) != ((void*)0)) { do { (ph->ph_root->ph_link. phn_next)->ph_link.phn_prev = phn; } while (0); } do { phn ->ph_link.phn_prev = ph->ph_root; } while (0); do { ph-> ph_root->ph_link.phn_next = phn; } while (0); } } __attribute__ ((unused)) extent_t * extent_avail_remove_first(extent_tree_t *ph) { extent_t *ret; if (ph->ph_root == ((void*)0)) { return ((void*)0); } do { extent_t *phn = (ph->ph_root->ph_link .phn_next); if (phn != ((void*)0)) { do { ph->ph_root-> ph_link.phn_prev = ((void*)0); } while (0); do { ph->ph_root ->ph_link.phn_next = ((void*)0); } while (0); do { phn-> ph_link.phn_prev = ((void*)0); } while (0); do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0); extent_t *phn0 = phn ; extent_t *phn1 = (phn0->ph_link.phn_next); if (phn1 != ( (void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = (( void*)0); } while (0); do { phn1->ph_link.phn_prev = ((void *)0); } while (0); do { phn1->ph_link.phn_next = ((void*)0 ); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } phn = phn0; } while (0); do { if (__builtin_expect (!!(config_debug && !((phn->ph_link.phn_next) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_next_get(extent_t, ph_link, phn) == NULL" ); abort(); } } while (0); do { if (ph->ph_root == ((void* )0)) { ph->ph_root = phn; } else if (phn == ((void*)0)) { ph ->ph_root = ph->ph_root; } else if (extent_esnead_comp( ph->ph_root, phn) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(ph->ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(phn != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (ph->ph_root, phn) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(ph->ph_root, phn) <= 0" ); abort(); } } while (0); do { phn->ph_link.phn_prev = ph ->ph_root; } while (0); phn0child = (ph->ph_root->ph_link .phn_lchild); do { phn->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = phn; } while (0); } do { ph->ph_root->ph_link .phn_lchild = phn; } while (0); } while (0); ph->ph_root = ph->ph_root; } else { do { extent_t *phn0child; do { if ( __builtin_expect(!!(config_debug && !(phn != ((void*) 0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(ph-> ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp(phn, ph->ph_root) <= 0)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "extent_esnead_comp(phn, ph->ph_root) <= 0"); abort (); } } while (0); do { ph->ph_root->ph_link.phn_prev = phn; } while (0); phn0child = (phn->ph_link.phn_lchild); do { ph->ph_root->ph_link.phn_next = phn0child; } while ( 0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = ph->ph_root; } while (0); } do { phn->ph_link .phn_lchild = ph->ph_root; } while (0); } while (0); ph-> ph_root = phn; } } while (0); } } while (0); ret = ph->ph_root ; do { extent_t *lchild = (ph->ph_root->ph_link.phn_lchild ); if (lchild == ((void*)0)) { ph->ph_root = ((void*)0); } else { do { extent_t *head = ((void*)0); extent_t *tail = (( void*)0); extent_t *phn0 = lchild; extent_t *phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while (0); do { phn0 ->ph_link.phn_next = ((void*)0); } while (0); do { phn1-> ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link .phn_next = ((void*)0); } while (0); do { if (phn0 == ((void* )0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0 ; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } ph->ph_root = phn0; } while (0); } } while (0); return ret; } __attribute__((unused)) extent_t * extent_avail_remove_any (extent_tree_t *ph) { if (ph->ph_root == ((void*)0)) { return ((void*)0); } extent_t *ret = (ph->ph_root->ph_link.phn_next ); if (ret != ((void*)0)) { extent_t *aux = (ret->ph_link. phn_next); do { ph->ph_root->ph_link.phn_next = aux; } while (0); if (aux != ((void*)0)) { do { aux->ph_link.phn_prev = ph->ph_root; } while (0); } return ret; } ret = ph->ph_root ; do { extent_t *lchild = (ph->ph_root->ph_link.phn_lchild ); if (lchild == ((void*)0)) { ph->ph_root = ((void*)0); } else { do { extent_t *head = ((void*)0); extent_t *tail = (( void*)0); extent_t *phn0 = lchild; extent_t *phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while (0); do { phn0 ->ph_link.phn_next = ((void*)0); } while (0); do { phn1-> ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link .phn_next = ((void*)0); } while (0); do { if (phn0 == ((void* )0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0 ; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } ph->ph_root = phn0; } while (0); } } while (0); return ret; } __attribute__((unused)) void extent_avail_remove (extent_tree_t *ph, extent_t *phn) { extent_t *replace, *parent ; if (ph->ph_root == phn) { if ((phn->ph_link.phn_lchild ) == ((void*)0)) { ph->ph_root = (phn->ph_link.phn_next ); if (ph->ph_root != ((void*)0)) { do { ph->ph_root-> ph_link.phn_prev = ((void*)0); } while (0); } return; } do { extent_t *phn = (ph->ph_root->ph_link.phn_next); if (phn != ((void *)0)) { do { ph->ph_root->ph_link.phn_prev = ((void*)0) ; } while (0); do { ph->ph_root->ph_link.phn_next = ((void *)0); } while (0); do { phn->ph_link.phn_prev = ((void*)0) ; } while (0); do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0); extent_t *phn0 = phn; extent_t *phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while (0); do { phn0 ->ph_link.phn_next = ((void*)0); } while (0); do { phn1-> ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link .phn_next = ((void*)0); } while (0); do { if (phn0 == ((void* )0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0 ; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } phn = phn0; } while (0); do { if (__builtin_expect (!!(config_debug && !((phn->ph_link.phn_next) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_next_get(extent_t, ph_link, phn) == NULL" ); abort(); } } while (0); do { if (ph->ph_root == ((void* )0)) { ph->ph_root = phn; } else if (phn == ((void*)0)) { ph ->ph_root = ph->ph_root; } else if (extent_esnead_comp( ph->ph_root, phn) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(ph->ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(phn != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (ph->ph_root, phn) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(ph->ph_root, phn) <= 0" ); abort(); } } while (0); do { phn->ph_link.phn_prev = ph ->ph_root; } while (0); phn0child = (ph->ph_root->ph_link .phn_lchild); do { phn->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = phn; } while (0); } do { ph->ph_root->ph_link .phn_lchild = phn; } while (0); } while (0); ph->ph_root = ph->ph_root; } else { do { extent_t *phn0child; do { if ( __builtin_expect(!!(config_debug && !(phn != ((void*) 0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(ph-> ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp(phn, ph->ph_root) <= 0)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "extent_esnead_comp(phn, ph->ph_root) <= 0"); abort (); } } while (0); do { ph->ph_root->ph_link.phn_prev = phn; } while (0); phn0child = (phn->ph_link.phn_lchild); do { ph->ph_root->ph_link.phn_next = phn0child; } while ( 0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = ph->ph_root; } while (0); } do { phn->ph_link .phn_lchild = ph->ph_root; } while (0); } while (0); ph-> ph_root = phn; } } while (0); } } while (0); if (ph->ph_root == phn) { do { extent_t *lchild = (ph->ph_root->ph_link .phn_lchild); if (lchild == ((void*)0)) { ph->ph_root = (( void*)0); } else { do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0); extent_t *phn0 = lchild; extent_t *phn1 = (phn0->ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void *)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while ( 0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link.phn_next = ((void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void *)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1 ) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } ph->ph_root = phn0; } while (0); } } while (0); return; } } if ((parent = (phn->ph_link.phn_prev)) != ((void*)0)) { if ((parent->ph_link.phn_lchild) != phn) { parent = ((void*)0); } } do { extent_t *lchild = (phn->ph_link.phn_lchild ); if (lchild == ((void*)0)) { replace = ((void*)0); } else { do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0 ); extent_t *phn0 = lchild; extent_t *phn1 = (phn0->ph_link .phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1 ->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest ->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0-> ph_link.phn_prev = ((void*)0); } while (0); do { phn0->ph_link .phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } replace = phn0; } while (0); } } while (0); if (replace != ((void*)0)) { if (parent != ((void*)0)) { do { replace->ph_link.phn_prev = parent; } while (0); do { parent ->ph_link.phn_lchild = replace; } while (0); } else { do { replace->ph_link.phn_prev = (phn->ph_link.phn_prev); } while (0); if ((phn->ph_link.phn_prev) != ((void*)0)) { do { (phn->ph_link.phn_prev)->ph_link.phn_next = replace; } while (0); } } do { replace->ph_link.phn_next = (phn-> ph_link.phn_next); } while (0); if ((phn->ph_link.phn_next ) != ((void*)0)) { do { (phn->ph_link.phn_next)->ph_link .phn_prev = replace; } while (0); } } else { if (parent != (( void*)0)) { extent_t *next = (phn->ph_link.phn_next); do { parent->ph_link.phn_lchild = next; } while (0); if (next != ((void*)0)) { do { next->ph_link.phn_prev = parent; } while (0); } } else { do { if (__builtin_expect(!!(config_debug && !((phn->ph_link.phn_prev) != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn_prev_get(extent_t, ph_link, phn) != NULL"); abort (); } } while (0); do { (phn->ph_link.phn_prev)->ph_link .phn_next = (phn->ph_link.phn_next); } while (0); } if ((phn ->ph_link.phn_next) != ((void*)0)) { do { (phn->ph_link .phn_next)->ph_link.phn_prev = (phn->ph_link.phn_prev); } while (0); } } } | ||||
124 | |||||
125 | typedef enum { | ||||
126 | lock_result_success, | ||||
127 | lock_result_failure, | ||||
128 | lock_result_no_extent | ||||
129 | } lock_result_t; | ||||
130 | |||||
131 | static lock_result_t | ||||
132 | extent_rtree_leaf_elm_try_lock(tsdn_t *tsdn, rtree_leaf_elm_t *elm, | ||||
133 | extent_t **result) { | ||||
134 | extent_t *extent1 = rtree_leaf_elm_extent_read(tsdn, &extents_rtree, | ||||
135 | elm, true1); | ||||
136 | |||||
137 | if (extent1 == NULL((void*)0)) { | ||||
138 | return lock_result_no_extent; | ||||
139 | } | ||||
140 | /* | ||||
141 | * It's possible that the extent changed out from under us, and with it | ||||
142 | * the leaf->extent mapping. We have to recheck while holding the lock. | ||||
143 | */ | ||||
144 | extent_lock(tsdn, extent1); | ||||
145 | extent_t *extent2 = rtree_leaf_elm_extent_read(tsdn, | ||||
146 | &extents_rtree, elm, true1); | ||||
147 | |||||
148 | if (extent1 == extent2) { | ||||
149 | *result = extent1; | ||||
150 | return lock_result_success; | ||||
151 | } else { | ||||
152 | extent_unlock(tsdn, extent1); | ||||
153 | return lock_result_failure; | ||||
154 | } | ||||
155 | } | ||||
156 | |||||
157 | /* | ||||
158 | * Returns a pool-locked extent_t * if there's one associated with the given | ||||
159 | * address, and NULL otherwise. | ||||
160 | */ | ||||
161 | static extent_t * | ||||
162 | extent_lock_from_addr(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, void *addr) { | ||||
163 | extent_t *ret = NULL((void*)0); | ||||
164 | rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, &extents_rtree, | ||||
165 | rtree_ctx, (uintptr_t)addr, false0, false0); | ||||
166 | if (elm == NULL((void*)0)) { | ||||
167 | return NULL((void*)0); | ||||
168 | } | ||||
169 | lock_result_t lock_result; | ||||
170 | do { | ||||
171 | lock_result = extent_rtree_leaf_elm_try_lock(tsdn, elm, &ret); | ||||
172 | } while (lock_result == lock_result_failure); | ||||
173 | return ret; | ||||
174 | } | ||||
175 | |||||
176 | extent_t * | ||||
177 | extent_alloc(tsdn_t *tsdn, arena_t *arena) { | ||||
178 | malloc_mutex_lock(tsdn, &arena->extent_avail_mtx); | ||||
179 | extent_t *extent = extent_avail_first(&arena->extent_avail); | ||||
180 | if (extent == NULL((void*)0)) { | ||||
181 | malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx); | ||||
182 | return base_alloc_extent(tsdn, arena->base); | ||||
183 | } | ||||
184 | extent_avail_remove(&arena->extent_avail, extent); | ||||
185 | malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx); | ||||
186 | return extent; | ||||
187 | } | ||||
188 | |||||
189 | void | ||||
190 | extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { | ||||
191 | malloc_mutex_lock(tsdn, &arena->extent_avail_mtx); | ||||
192 | extent_avail_insert(&arena->extent_avail, extent); | ||||
193 | malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx); | ||||
194 | } | ||||
195 | |||||
196 | extent_hooks_t * | ||||
197 | extent_hooks_get(arena_t *arena) { | ||||
198 | return base_extent_hooks_get(arena->base); | ||||
199 | } | ||||
200 | |||||
201 | extent_hooks_t * | ||||
202 | extent_hooks_set(tsd_t *tsd, arena_t *arena, extent_hooks_t *extent_hooks) { | ||||
203 | background_thread_info_t *info; | ||||
204 | if (have_background_thread) { | ||||
205 | info = arena_background_thread_info_get(arena); | ||||
206 | malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); | ||||
207 | } | ||||
208 | extent_hooks_t *ret = base_extent_hooks_set(arena->base, extent_hooks); | ||||
209 | if (have_background_thread) { | ||||
210 | malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); | ||||
211 | } | ||||
212 | |||||
213 | return ret; | ||||
214 | } | ||||
215 | |||||
216 | static void | ||||
217 | extent_hooks_assure_initialized(arena_t *arena, | ||||
218 | extent_hooks_t **r_extent_hooks) { | ||||
219 | if (*r_extent_hooks == EXTENT_HOOKS_INITIALIZER((void*)0)) { | ||||
220 | *r_extent_hooks = extent_hooks_get(arena); | ||||
221 | } | ||||
222 | } | ||||
223 | |||||
224 | #ifndef JEMALLOC_JET | ||||
225 | static | ||||
226 | #endif | ||||
227 | size_t | ||||
228 | extent_size_quantize_floor(size_t size) { | ||||
229 | size_t ret; | ||||
230 | pszind_t pind; | ||||
231 | |||||
232 | assert(size > 0)do { if (__builtin_expect(!!(config_debug && !(size > 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 232, "size > 0"); abort(); } } while (0); | ||||
233 | assert((size & PAGE_MASK) == 0)do { if (__builtin_expect(!!(config_debug && !((size & ((size_t)(((size_t)(1U << 12)) - 1))) == 0)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 233, "(size & PAGE_MASK) == 0"); abort(); } } while (0); | ||||
234 | |||||
235 | pind = sz_psz2ind(size - sz_large_pad + 1); | ||||
236 | if (pind == 0) { | ||||
237 | /* | ||||
238 | * Avoid underflow. This short-circuit would also do the right | ||||
239 | * thing for all sizes in the range for which there are | ||||
240 | * PAGE-spaced size classes, but it's simplest to just handle | ||||
241 | * the one case that would cause erroneous results. | ||||
242 | */ | ||||
243 | return size; | ||||
244 | } | ||||
245 | ret = sz_pind2sz(pind - 1) + sz_large_pad; | ||||
246 | assert(ret <= size)do { if (__builtin_expect(!!(config_debug && !(ret <= size)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 246, "ret <= size"); abort(); } } while ( 0); | ||||
247 | return ret; | ||||
248 | } | ||||
249 | |||||
250 | #ifndef JEMALLOC_JET | ||||
251 | static | ||||
252 | #endif | ||||
253 | size_t | ||||
254 | extent_size_quantize_ceil(size_t size) { | ||||
255 | size_t ret; | ||||
256 | |||||
257 | assert(size > 0)do { if (__builtin_expect(!!(config_debug && !(size > 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 257, "size > 0"); abort(); } } while (0); | ||||
258 | assert(size - sz_large_pad <= LARGE_MAXCLASS)do { if (__builtin_expect(!!(config_debug && !(size - sz_large_pad <= ((((size_t)1) << 62) + (((size_t)3) << 60)))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 258, "size - sz_large_pad <= LARGE_MAXCLASS" ); abort(); } } while (0); | ||||
259 | assert((size & PAGE_MASK) == 0)do { if (__builtin_expect(!!(config_debug && !((size & ((size_t)(((size_t)(1U << 12)) - 1))) == 0)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 259, "(size & PAGE_MASK) == 0"); abort(); } } while (0); | ||||
260 | |||||
261 | ret = extent_size_quantize_floor(size); | ||||
262 | if (ret < size) { | ||||
263 | /* | ||||
264 | * Skip a quantization that may have an adequately large extent, | ||||
265 | * because under-sized extents may be mixed in. This only | ||||
266 | * happens when an unusual size is requested, i.e. for aligned | ||||
267 | * allocation, and is just one of several places where linear | ||||
268 | * search would potentially find sufficiently aligned available | ||||
269 | * memory somewhere lower. | ||||
270 | */ | ||||
271 | ret = sz_pind2sz(sz_psz2ind(ret - sz_large_pad + 1)) + | ||||
272 | sz_large_pad; | ||||
273 | } | ||||
274 | return ret; | ||||
275 | } | ||||
276 | |||||
277 | /* Generate pairing heap functions. */ | ||||
278 | ph_gen(, extent_heap_, extent_heap_t, extent_t, ph_link, extent_snad_comp)void extent_heap_new(extent_heap_t *ph) { memset(ph, 0, sizeof (struct { extent_t *ph_root; })); } _Bool extent_heap_empty(extent_heap_t *ph) { return (ph->ph_root == ((void*)0)); } extent_t * extent_heap_first (extent_heap_t *ph) { if (ph->ph_root == ((void*)0)) { return ((void*)0); } do { extent_t *phn = (ph->ph_root->ph_link .phn_next); if (phn != ((void*)0)) { do { ph->ph_root-> ph_link.phn_prev = ((void*)0); } while (0); do { ph->ph_root ->ph_link.phn_next = ((void*)0); } while (0); do { phn-> ph_link.phn_prev = ((void*)0); } while (0); do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0); extent_t *phn0 = phn ; extent_t *phn1 = (phn0->ph_link.phn_next); if (phn1 != ( (void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = (( void*)0); } while (0); do { phn1->ph_link.phn_prev = ((void *)0); } while (0); do { phn1->ph_link.phn_next = ((void*)0 ); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_snad_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_snad_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } phn = phn0; } while (0); do { if (__builtin_expect (!!(config_debug && !((phn->ph_link.phn_next) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_next_get(extent_t, ph_link, phn) == NULL" ); abort(); } } while (0); do { if (ph->ph_root == ((void* )0)) { ph->ph_root = phn; } else if (phn == ((void*)0)) { ph ->ph_root = ph->ph_root; } else if (extent_snad_comp(ph ->ph_root, phn) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(ph->ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(phn != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (ph->ph_root, phn) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(ph->ph_root, phn) <= 0" ); abort(); } } while (0); do { phn->ph_link.phn_prev = ph ->ph_root; } while (0); phn0child = (ph->ph_root->ph_link .phn_lchild); do { phn->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = phn; } while (0); } do { ph->ph_root->ph_link .phn_lchild = phn; } while (0); } while (0); ph->ph_root = ph->ph_root; } else { do { extent_t *phn0child; do { if ( __builtin_expect(!!(config_debug && !(phn != ((void*) 0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(ph-> ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp(phn, ph->ph_root) <= 0)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "extent_snad_comp(phn, ph->ph_root) <= 0"); abort (); } } while (0); do { ph->ph_root->ph_link.phn_prev = phn; } while (0); phn0child = (phn->ph_link.phn_lchild); do { ph->ph_root->ph_link.phn_next = phn0child; } while ( 0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = ph->ph_root; } while (0); } do { phn->ph_link .phn_lchild = ph->ph_root; } while (0); } while (0); ph-> ph_root = phn; } } while (0); } } while (0); return ph->ph_root ; } extent_t * extent_heap_any(extent_heap_t *ph) { if (ph-> ph_root == ((void*)0)) { return ((void*)0); } extent_t *aux = (ph->ph_root->ph_link.phn_next); if (aux != ((void*)0) ) { return aux; } return ph->ph_root; } void extent_heap_insert (extent_heap_t *ph, extent_t *phn) { memset(&phn->ph_link , 0, sizeof(struct { extent_t *phn_prev; extent_t *phn_next; extent_t *phn_lchild; })); if (ph->ph_root == ((void*)0)) { ph-> ph_root = phn; } else { do { phn->ph_link.phn_next = (ph-> ph_root->ph_link.phn_next); } while (0); if ((ph->ph_root ->ph_link.phn_next) != ((void*)0)) { do { (ph->ph_root-> ph_link.phn_next)->ph_link.phn_prev = phn; } while (0); } do { phn->ph_link.phn_prev = ph->ph_root; } while (0); do { ph->ph_root->ph_link.phn_next = phn; } while (0); } } extent_t * extent_heap_remove_first(extent_heap_t *ph) { extent_t *ret; if (ph->ph_root == ((void*)0)) { return ((void*)0); } do { extent_t *phn = (ph->ph_root->ph_link.phn_next) ; if (phn != ((void*)0)) { do { ph->ph_root->ph_link.phn_prev = ((void*)0); } while (0); do { ph->ph_root->ph_link.phn_next = ((void*)0); } while (0); do { phn->ph_link.phn_prev = ( (void*)0); } while (0); do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0); extent_t *phn0 = phn; extent_t *phn1 = ( phn0->ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void *)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while ( 0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link.phn_next = ((void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void *)0)) { phn0 = phn0; } else if (extent_snad_comp(phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!( config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_snad_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_snad_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } phn = phn0; } while (0); do { if (__builtin_expect (!!(config_debug && !((phn->ph_link.phn_next) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_next_get(extent_t, ph_link, phn) == NULL" ); abort(); } } while (0); do { if (ph->ph_root == ((void* )0)) { ph->ph_root = phn; } else if (phn == ((void*)0)) { ph ->ph_root = ph->ph_root; } else if (extent_snad_comp(ph ->ph_root, phn) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(ph->ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(phn != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (ph->ph_root, phn) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(ph->ph_root, phn) <= 0" ); abort(); } } while (0); do { phn->ph_link.phn_prev = ph ->ph_root; } while (0); phn0child = (ph->ph_root->ph_link .phn_lchild); do { phn->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = phn; } while (0); } do { ph->ph_root->ph_link .phn_lchild = phn; } while (0); } while (0); ph->ph_root = ph->ph_root; } else { do { extent_t *phn0child; do { if ( __builtin_expect(!!(config_debug && !(phn != ((void*) 0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(ph-> ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp(phn, ph->ph_root) <= 0)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "extent_snad_comp(phn, ph->ph_root) <= 0"); abort (); } } while (0); do { ph->ph_root->ph_link.phn_prev = phn; } while (0); phn0child = (phn->ph_link.phn_lchild); do { ph->ph_root->ph_link.phn_next = phn0child; } while ( 0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = ph->ph_root; } while (0); } do { phn->ph_link .phn_lchild = ph->ph_root; } while (0); } while (0); ph-> ph_root = phn; } } while (0); } } while (0); ret = ph->ph_root ; do { extent_t *lchild = (ph->ph_root->ph_link.phn_lchild ); if (lchild == ((void*)0)) { ph->ph_root = ((void*)0); } else { do { extent_t *head = ((void*)0); extent_t *tail = (( void*)0); extent_t *phn0 = lchild; extent_t *phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while (0); do { phn0 ->ph_link.phn_next = ((void*)0); } while (0); do { phn1-> ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link .phn_next = ((void*)0); } while (0); do { if (phn0 == ((void* )0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0 ; } else if (extent_snad_comp(phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_snad_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } ph->ph_root = phn0; } while (0); } } while (0); return ret; } extent_t * extent_heap_remove_any(extent_heap_t *ph) { if (ph->ph_root == ((void*)0)) { return ((void*)0) ; } extent_t *ret = (ph->ph_root->ph_link.phn_next); if (ret != ((void*)0)) { extent_t *aux = (ret->ph_link.phn_next ); do { ph->ph_root->ph_link.phn_next = aux; } while (0 ); if (aux != ((void*)0)) { do { aux->ph_link.phn_prev = ph ->ph_root; } while (0); } return ret; } ret = ph->ph_root ; do { extent_t *lchild = (ph->ph_root->ph_link.phn_lchild ); if (lchild == ((void*)0)) { ph->ph_root = ((void*)0); } else { do { extent_t *head = ((void*)0); extent_t *tail = (( void*)0); extent_t *phn0 = lchild; extent_t *phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while (0); do { phn0 ->ph_link.phn_next = ((void*)0); } while (0); do { phn1-> ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link .phn_next = ((void*)0); } while (0); do { if (phn0 == ((void* )0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0 ; } else if (extent_snad_comp(phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_snad_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } ph->ph_root = phn0; } while (0); } } while (0); return ret; } void extent_heap_remove(extent_heap_t *ph , extent_t *phn) { extent_t *replace, *parent; if (ph->ph_root == phn) { if ((phn->ph_link.phn_lchild) == ((void*)0)) { ph ->ph_root = (phn->ph_link.phn_next); if (ph->ph_root != ((void*)0)) { do { ph->ph_root->ph_link.phn_prev = ( (void*)0); } while (0); } return; } do { extent_t *phn = (ph-> ph_root->ph_link.phn_next); if (phn != ((void*)0)) { do { ph ->ph_root->ph_link.phn_prev = ((void*)0); } while (0); do { ph->ph_root->ph_link.phn_next = ((void*)0); } while ( 0); do { phn->ph_link.phn_prev = ((void*)0); } while (0); do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0); extent_t *phn0 = phn; extent_t *phn1 = (phn0->ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link .phn_next); if (phnrest != ((void*)0)) { do { phnrest->ph_link .phn_prev = ((void*)0); } while (0); } do { phn0->ph_link. phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_snad_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_snad_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } phn = phn0; } while (0); do { if (__builtin_expect (!!(config_debug && !((phn->ph_link.phn_next) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_next_get(extent_t, ph_link, phn) == NULL" ); abort(); } } while (0); do { if (ph->ph_root == ((void* )0)) { ph->ph_root = phn; } else if (phn == ((void*)0)) { ph ->ph_root = ph->ph_root; } else if (extent_snad_comp(ph ->ph_root, phn) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(ph->ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(phn != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (ph->ph_root, phn) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(ph->ph_root, phn) <= 0" ); abort(); } } while (0); do { phn->ph_link.phn_prev = ph ->ph_root; } while (0); phn0child = (ph->ph_root->ph_link .phn_lchild); do { phn->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = phn; } while (0); } do { ph->ph_root->ph_link .phn_lchild = phn; } while (0); } while (0); ph->ph_root = ph->ph_root; } else { do { extent_t *phn0child; do { if ( __builtin_expect(!!(config_debug && !(phn != ((void*) 0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(ph-> ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp(phn, ph->ph_root) <= 0)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "extent_snad_comp(phn, ph->ph_root) <= 0"); abort (); } } while (0); do { ph->ph_root->ph_link.phn_prev = phn; } while (0); phn0child = (phn->ph_link.phn_lchild); do { ph->ph_root->ph_link.phn_next = phn0child; } while ( 0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = ph->ph_root; } while (0); } do { phn->ph_link .phn_lchild = ph->ph_root; } while (0); } while (0); ph-> ph_root = phn; } } while (0); } } while (0); if (ph->ph_root == phn) { do { extent_t *lchild = (ph->ph_root->ph_link .phn_lchild); if (lchild == ((void*)0)) { ph->ph_root = (( void*)0); } else { do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0); extent_t *phn0 = lchild; extent_t *phn1 = (phn0->ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void *)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while ( 0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link.phn_next = ((void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void *)0)) { phn0 = phn0; } else if (extent_snad_comp(phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!( config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_snad_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_snad_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } ph->ph_root = phn0; } while (0); } } while (0); return; } } if ((parent = (phn->ph_link.phn_prev)) != ((void*)0)) { if ((parent->ph_link.phn_lchild) != phn) { parent = ((void*)0); } } do { extent_t *lchild = (phn->ph_link.phn_lchild ); if (lchild == ((void*)0)) { replace = ((void*)0); } else { do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0 ); extent_t *phn0 = lchild; extent_t *phn1 = (phn0->ph_link .phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1 ->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest ->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0-> ph_link.phn_prev = ((void*)0); } while (0); do { phn0->ph_link .phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_snad_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } replace = phn0; } while (0); } } while (0); if (replace != ((void*)0)) { if (parent != ((void*)0)) { do { replace->ph_link.phn_prev = parent; } while (0); do { parent ->ph_link.phn_lchild = replace; } while (0); } else { do { replace->ph_link.phn_prev = (phn->ph_link.phn_prev); } while (0); if ((phn->ph_link.phn_prev) != ((void*)0)) { do { (phn->ph_link.phn_prev)->ph_link.phn_next = replace; } while (0); } } do { replace->ph_link.phn_next = (phn-> ph_link.phn_next); } while (0); if ((phn->ph_link.phn_next ) != ((void*)0)) { do { (phn->ph_link.phn_next)->ph_link .phn_prev = replace; } while (0); } } else { if (parent != (( void*)0)) { extent_t *next = (phn->ph_link.phn_next); do { parent->ph_link.phn_lchild = next; } while (0); if (next != ((void*)0)) { do { next->ph_link.phn_prev = parent; } while (0); } } else { do { if (__builtin_expect(!!(config_debug && !((phn->ph_link.phn_prev) != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn_prev_get(extent_t, ph_link, phn) != NULL"); abort (); } } while (0); do { (phn->ph_link.phn_prev)->ph_link .phn_next = (phn->ph_link.phn_next); } while (0); } if ((phn ->ph_link.phn_next) != ((void*)0)) { do { (phn->ph_link .phn_next)->ph_link.phn_prev = (phn->ph_link.phn_prev); } while (0); } } } | ||||
279 | |||||
280 | bool_Bool | ||||
281 | extents_init(tsdn_t *tsdn, extents_t *extents, extent_state_t state, | ||||
282 | bool_Bool delay_coalesce) { | ||||
283 | if (malloc_mutex_init(&extents->mtx, "extents", WITNESS_RANK_EXTENTS14U, | ||||
284 | malloc_mutex_rank_exclusive)) { | ||||
285 | return true1; | ||||
286 | } | ||||
287 | for (unsigned i = 0; i < NPSIZES199+1; i++) { | ||||
288 | extent_heap_new(&extents->heaps[i]); | ||||
289 | } | ||||
290 | bitmap_init(extents->bitmap, &extents_bitmap_info, true1); | ||||
291 | extent_list_init(&extents->lru); | ||||
292 | atomic_store_zu(&extents->npages, 0, ATOMIC_RELAXEDatomic_memory_order_relaxed); | ||||
293 | extents->state = state; | ||||
294 | extents->delay_coalesce = delay_coalesce; | ||||
295 | return false0; | ||||
296 | } | ||||
297 | |||||
298 | extent_state_t | ||||
299 | extents_state_get(const extents_t *extents) { | ||||
300 | return extents->state; | ||||
301 | } | ||||
302 | |||||
303 | size_t | ||||
304 | extents_npages_get(extents_t *extents) { | ||||
305 | return atomic_load_zu(&extents->npages, ATOMIC_RELAXEDatomic_memory_order_relaxed); | ||||
306 | } | ||||
307 | |||||
308 | static void | ||||
309 | extents_insert_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent) { | ||||
310 | malloc_mutex_assert_owner(tsdn, &extents->mtx); | ||||
311 | assert(extent_state_get(extent) == extents->state)do { if (__builtin_expect(!!(config_debug && !(extent_state_get (extent) == extents->state)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 311, "extent_state_get(extent) == extents->state" ); abort(); } } while (0); | ||||
312 | |||||
313 | size_t size = extent_size_get(extent); | ||||
314 | size_t psz = extent_size_quantize_floor(size); | ||||
315 | pszind_t pind = sz_psz2ind(psz); | ||||
316 | if (extent_heap_empty(&extents->heaps[pind])) { | ||||
317 | bitmap_unset(extents->bitmap, &extents_bitmap_info, | ||||
318 | (size_t)pind); | ||||
319 | } | ||||
320 | extent_heap_insert(&extents->heaps[pind], extent); | ||||
321 | extent_list_append(&extents->lru, extent); | ||||
322 | size_t npages = size >> LG_PAGE12; | ||||
323 | /* | ||||
324 | * All modifications to npages hold the mutex (as asserted above), so we | ||||
325 | * don't need an atomic fetch-add; we can get by with a load followed by | ||||
326 | * a store. | ||||
327 | */ | ||||
328 | size_t cur_extents_npages = | ||||
329 | atomic_load_zu(&extents->npages, ATOMIC_RELAXEDatomic_memory_order_relaxed); | ||||
330 | atomic_store_zu(&extents->npages, cur_extents_npages + npages, | ||||
331 | ATOMIC_RELAXEDatomic_memory_order_relaxed); | ||||
332 | } | ||||
333 | |||||
334 | static void | ||||
335 | extents_remove_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent) { | ||||
336 | malloc_mutex_assert_owner(tsdn, &extents->mtx); | ||||
337 | assert(extent_state_get(extent) == extents->state)do { if (__builtin_expect(!!(config_debug && !(extent_state_get (extent) == extents->state)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 337, "extent_state_get(extent) == extents->state" ); abort(); } } while (0); | ||||
338 | |||||
339 | size_t size = extent_size_get(extent); | ||||
340 | size_t psz = extent_size_quantize_floor(size); | ||||
341 | pszind_t pind = sz_psz2ind(psz); | ||||
342 | extent_heap_remove(&extents->heaps[pind], extent); | ||||
343 | if (extent_heap_empty(&extents->heaps[pind])) { | ||||
344 | bitmap_set(extents->bitmap, &extents_bitmap_info, | ||||
345 | (size_t)pind); | ||||
346 | } | ||||
347 | extent_list_remove(&extents->lru, extent); | ||||
348 | size_t npages = size >> LG_PAGE12; | ||||
349 | /* | ||||
350 | * As in extents_insert_locked, we hold extents->mtx and so don't need | ||||
351 | * atomic operations for updating extents->npages. | ||||
352 | */ | ||||
353 | size_t cur_extents_npages = | ||||
354 | atomic_load_zu(&extents->npages, ATOMIC_RELAXEDatomic_memory_order_relaxed); | ||||
355 | assert(cur_extents_npages >= npages)do { if (__builtin_expect(!!(config_debug && !(cur_extents_npages >= npages)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 355, "cur_extents_npages >= npages"); abort (); } } while (0); | ||||
356 | atomic_store_zu(&extents->npages, | ||||
357 | cur_extents_npages - (size >> LG_PAGE12), ATOMIC_RELAXEDatomic_memory_order_relaxed); | ||||
358 | } | ||||
359 | |||||
360 | /* | ||||
361 | * Find an extent with size [min_size, max_size) to satisfy the alignment | ||||
362 | * requirement. For each size, try only the first extent in the heap. | ||||
363 | */ | ||||
364 | static extent_t * | ||||
365 | extents_fit_alignment(extents_t *extents, size_t min_size, size_t max_size, | ||||
366 | size_t alignment) { | ||||
367 | pszind_t pind = sz_psz2ind(extent_size_quantize_ceil(min_size)); | ||||
368 | pszind_t pind_max = sz_psz2ind(extent_size_quantize_ceil(max_size)); | ||||
369 | |||||
370 | for (pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap, | ||||
371 | &extents_bitmap_info, (size_t)pind); i < pind_max; i = | ||||
372 | (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info, | ||||
373 | (size_t)i+1)) { | ||||
374 | assert(i < NPSIZES)do { if (__builtin_expect(!!(config_debug && !(i < 199)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 374, "i < NPSIZES"); abort(); } } while ( 0); | ||||
375 | assert(!extent_heap_empty(&extents->heaps[i]))do { if (__builtin_expect(!!(config_debug && !(!extent_heap_empty (&extents->heaps[i]))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 375, "!extent_heap_empty(&extents->heaps[i])" ); abort(); } } while (0); | ||||
376 | extent_t *extent = extent_heap_first(&extents->heaps[i]); | ||||
377 | uintptr_t base = (uintptr_t)extent_base_get(extent); | ||||
378 | size_t candidate_size = extent_size_get(extent); | ||||
379 | assert(candidate_size >= min_size)do { if (__builtin_expect(!!(config_debug && !(candidate_size >= min_size)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 379, "candidate_size >= min_size"); abort (); } } while (0); | ||||
380 | |||||
381 | uintptr_t next_align = ALIGNMENT_CEILING((uintptr_t)base,((((uintptr_t)base) + ((((alignment) + ((size_t)(((size_t)(1U << 12)) - 1))) & ~((size_t)(((size_t)(1U << 12 )) - 1))) - 1)) & ((~((((alignment) + ((size_t)(((size_t) (1U << 12)) - 1))) & ~((size_t)(((size_t)(1U << 12)) - 1))))) + 1)) | ||||
382 | PAGE_CEILING(alignment))((((uintptr_t)base) + ((((alignment) + ((size_t)(((size_t)(1U << 12)) - 1))) & ~((size_t)(((size_t)(1U << 12 )) - 1))) - 1)) & ((~((((alignment) + ((size_t)(((size_t) (1U << 12)) - 1))) & ~((size_t)(((size_t)(1U << 12)) - 1))))) + 1)); | ||||
383 | if (base > next_align || base + candidate_size <= next_align) { | ||||
384 | /* Overflow or not crossing the next alignment. */ | ||||
385 | continue; | ||||
386 | } | ||||
387 | |||||
388 | size_t leadsize = next_align - base; | ||||
389 | if (candidate_size - leadsize >= min_size) { | ||||
390 | return extent; | ||||
391 | } | ||||
392 | } | ||||
393 | |||||
394 | return NULL((void*)0); | ||||
395 | } | ||||
396 | |||||
397 | /* Do any-best-fit extent selection, i.e. select any extent that best fits. */ | ||||
398 | static extent_t * | ||||
399 | extents_best_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, | ||||
400 | size_t size) { | ||||
401 | pszind_t pind = sz_psz2ind(extent_size_quantize_ceil(size)); | ||||
402 | pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info, | ||||
403 | (size_t)pind); | ||||
404 | if (i < NPSIZES199+1) { | ||||
405 | /* | ||||
406 | * In order to reduce fragmentation, avoid reusing and splitting | ||||
407 | * large extents for much smaller sizes. | ||||
408 | */ | ||||
409 | if ((sz_pind2sz(i) >> opt_lg_extent_max_active_fit) > size) { | ||||
410 | return NULL((void*)0); | ||||
411 | } | ||||
412 | assert(!extent_heap_empty(&extents->heaps[i]))do { if (__builtin_expect(!!(config_debug && !(!extent_heap_empty (&extents->heaps[i]))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 412, "!extent_heap_empty(&extents->heaps[i])" ); abort(); } } while (0); | ||||
413 | extent_t *extent = extent_heap_first(&extents->heaps[i]); | ||||
414 | assert(extent_size_get(extent) >= size)do { if (__builtin_expect(!!(config_debug && !(extent_size_get (extent) >= size)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 414, "extent_size_get(extent) >= size"); abort(); } } while (0); | ||||
415 | return extent; | ||||
416 | } | ||||
417 | |||||
418 | return NULL((void*)0); | ||||
419 | } | ||||
420 | |||||
421 | /* | ||||
422 | * Do first-fit extent selection, i.e. select the oldest/lowest extent that is | ||||
423 | * large enough. | ||||
424 | */ | ||||
425 | static extent_t * | ||||
426 | extents_first_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, | ||||
427 | size_t size) { | ||||
428 | extent_t *ret = NULL((void*)0); | ||||
429 | |||||
430 | pszind_t pind = sz_psz2ind(extent_size_quantize_ceil(size)); | ||||
431 | for (pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap, | ||||
432 | &extents_bitmap_info, (size_t)pind); i < NPSIZES199+1; i = | ||||
433 | (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info, | ||||
434 | (size_t)i+1)) { | ||||
435 | assert(!extent_heap_empty(&extents->heaps[i]))do { if (__builtin_expect(!!(config_debug && !(!extent_heap_empty (&extents->heaps[i]))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 435, "!extent_heap_empty(&extents->heaps[i])" ); abort(); } } while (0); | ||||
436 | extent_t *extent = extent_heap_first(&extents->heaps[i]); | ||||
437 | assert(extent_size_get(extent) >= size)do { if (__builtin_expect(!!(config_debug && !(extent_size_get (extent) >= size)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 437, "extent_size_get(extent) >= size"); abort(); } } while (0); | ||||
438 | if (ret == NULL((void*)0) || extent_snad_comp(extent, ret) < 0) { | ||||
439 | ret = extent; | ||||
440 | } | ||||
441 | if (i == NPSIZES199) { | ||||
442 | break; | ||||
443 | } | ||||
444 | assert(i < NPSIZES)do { if (__builtin_expect(!!(config_debug && !(i < 199)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 444, "i < NPSIZES"); abort(); } } while ( 0); | ||||
445 | } | ||||
446 | |||||
447 | return ret; | ||||
448 | } | ||||
449 | |||||
450 | /* | ||||
451 | * Do {best,first}-fit extent selection, where the selection policy choice is | ||||
452 | * based on extents->delay_coalesce. Best-fit selection requires less | ||||
453 | * searching, but its layout policy is less stable and may cause higher virtual | ||||
454 | * memory fragmentation as a side effect. | ||||
455 | */ | ||||
456 | static extent_t * | ||||
457 | extents_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, | ||||
458 | size_t esize, size_t alignment) { | ||||
459 | malloc_mutex_assert_owner(tsdn, &extents->mtx); | ||||
460 | |||||
461 | size_t max_size = esize + PAGE_CEILING(alignment)(((alignment) + ((size_t)(((size_t)(1U << 12)) - 1))) & ~((size_t)(((size_t)(1U << 12)) - 1))) - PAGE((size_t)(1U << 12)); | ||||
462 | /* Beware size_t wrap-around. */ | ||||
463 | if (max_size < esize) { | ||||
464 | return NULL((void*)0); | ||||
465 | } | ||||
466 | |||||
467 | extent_t *extent = extents->delay_coalesce ? | ||||
468 | extents_best_fit_locked(tsdn, arena, extents, max_size) : | ||||
469 | extents_first_fit_locked(tsdn, arena, extents, max_size); | ||||
470 | |||||
471 | if (alignment > PAGE((size_t)(1U << 12)) && extent == NULL((void*)0)) { | ||||
472 | /* | ||||
473 | * max_size guarantees the alignment requirement but is rather | ||||
474 | * pessimistic. Next we try to satisfy the aligned allocation | ||||
475 | * with sizes in [esize, max_size). | ||||
476 | */ | ||||
477 | extent = extents_fit_alignment(extents, esize, max_size, | ||||
478 | alignment); | ||||
479 | } | ||||
480 | |||||
481 | return extent; | ||||
482 | } | ||||
483 | |||||
484 | static bool_Bool | ||||
485 | extent_try_delayed_coalesce(tsdn_t *tsdn, arena_t *arena, | ||||
486 | extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, | ||||
487 | extent_t *extent) { | ||||
488 | extent_state_set(extent, extent_state_active); | ||||
489 | bool_Bool coalesced; | ||||
490 | extent = extent_try_coalesce(tsdn, arena, r_extent_hooks, rtree_ctx, | ||||
491 | extents, extent, &coalesced, false0); | ||||
492 | extent_state_set(extent, extents_state_get(extents)); | ||||
493 | |||||
494 | if (!coalesced) { | ||||
495 | return true1; | ||||
496 | } | ||||
497 | extents_insert_locked(tsdn, extents, extent); | ||||
498 | return false0; | ||||
499 | } | ||||
500 | |||||
501 | extent_t * | ||||
502 | extents_alloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, | ||||
503 | extents_t *extents, void *new_addr, size_t size, size_t pad, | ||||
504 | size_t alignment, bool_Bool slab, szind_t szind, bool_Bool *zero, bool_Bool *commit) { | ||||
505 | assert(size + pad != 0)do { if (__builtin_expect(!!(config_debug && !(size + pad != 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 505, "size + pad != 0"); abort(); } } while (0); | ||||
506 | assert(alignment != 0)do { if (__builtin_expect(!!(config_debug && !(alignment != 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 506, "alignment != 0"); abort(); } } while ( 0); | ||||
507 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | ||||
508 | WITNESS_RANK_CORE11U, 0); | ||||
509 | |||||
510 | extent_t *extent = extent_recycle(tsdn, arena, r_extent_hooks, extents, | ||||
511 | new_addr, size, pad, alignment, slab, szind, zero, commit, false0); | ||||
512 | assert(extent == NULL || extent_dumpable_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent == ((void*)0) || extent_dumpable_get(extent))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 512, "extent == NULL || extent_dumpable_get(extent)"); abort (); } } while (0); | ||||
513 | return extent; | ||||
514 | } | ||||
515 | |||||
516 | void | ||||
517 | extents_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, | ||||
518 | extents_t *extents, extent_t *extent) { | ||||
519 | assert(extent_base_get(extent) != NULL)do { if (__builtin_expect(!!(config_debug && !(extent_base_get (extent) != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 519, "extent_base_get(extent) != NULL"); abort (); } } while (0); | ||||
520 | assert(extent_size_get(extent) != 0)do { if (__builtin_expect(!!(config_debug && !(extent_size_get (extent) != 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 520, "extent_size_get(extent) != 0"); abort (); } } while (0); | ||||
521 | assert(extent_dumpable_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent_dumpable_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 521, "extent_dumpable_get(extent)"); abort( ); } } while (0); | ||||
522 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | ||||
523 | WITNESS_RANK_CORE11U, 0); | ||||
524 | |||||
525 | extent_addr_set(extent, extent_base_get(extent)); | ||||
526 | extent_zeroed_set(extent, false0); | ||||
527 | |||||
528 | extent_record(tsdn, arena, r_extent_hooks, extents, extent, false0); | ||||
529 | } | ||||
530 | |||||
531 | extent_t * | ||||
532 | extents_evict(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, | ||||
533 | extents_t *extents, size_t npages_min) { | ||||
534 | rtree_ctx_t rtree_ctx_fallback; | ||||
535 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); | ||||
536 | |||||
537 | malloc_mutex_lock(tsdn, &extents->mtx); | ||||
538 | |||||
539 | /* | ||||
540 | * Get the LRU coalesced extent, if any. If coalescing was delayed, | ||||
541 | * the loop will iterate until the LRU extent is fully coalesced. | ||||
542 | */ | ||||
543 | extent_t *extent; | ||||
544 | while (true1) { | ||||
545 | /* Get the LRU extent, if any. */ | ||||
546 | extent = extent_list_first(&extents->lru); | ||||
547 | if (extent == NULL((void*)0)) { | ||||
548 | goto label_return; | ||||
549 | } | ||||
550 | /* Check the eviction limit. */ | ||||
551 | size_t extents_npages = atomic_load_zu(&extents->npages, | ||||
552 | ATOMIC_RELAXEDatomic_memory_order_relaxed); | ||||
553 | if (extents_npages <= npages_min) { | ||||
554 | extent = NULL((void*)0); | ||||
555 | goto label_return; | ||||
556 | } | ||||
557 | extents_remove_locked(tsdn, extents, extent); | ||||
558 | if (!extents->delay_coalesce) { | ||||
559 | break; | ||||
560 | } | ||||
561 | /* Try to coalesce. */ | ||||
562 | if (extent_try_delayed_coalesce(tsdn, arena, r_extent_hooks, | ||||
563 | rtree_ctx, extents, extent)) { | ||||
564 | break; | ||||
565 | } | ||||
566 | /* | ||||
567 | * The LRU extent was just coalesced and the result placed in | ||||
568 | * the LRU at its neighbor's position. Start over. | ||||
569 | */ | ||||
570 | } | ||||
571 | |||||
572 | /* | ||||
573 | * Either mark the extent active or deregister it to protect against | ||||
574 | * concurrent operations. | ||||
575 | */ | ||||
576 | switch (extents_state_get(extents)) { | ||||
577 | case extent_state_active: | ||||
578 | not_reached()do { if (config_debug) { malloc_printf( "<jemalloc>: %s:%d: Unreachable code reached\n" , "src/extent.c", 578); abort(); } __builtin_unreachable(); } while (0); | ||||
579 | case extent_state_dirty: | ||||
580 | case extent_state_muzzy: | ||||
581 | extent_state_set(extent, extent_state_active); | ||||
582 | break; | ||||
583 | case extent_state_retained: | ||||
584 | extent_deregister(tsdn, extent); | ||||
585 | break; | ||||
586 | default: | ||||
587 | not_reached()do { if (config_debug) { malloc_printf( "<jemalloc>: %s:%d: Unreachable code reached\n" , "src/extent.c", 587); abort(); } __builtin_unreachable(); } while (0); | ||||
588 | } | ||||
589 | |||||
590 | label_return: | ||||
591 | malloc_mutex_unlock(tsdn, &extents->mtx); | ||||
592 | return extent; | ||||
593 | } | ||||
594 | |||||
595 | static void | ||||
596 | extents_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, | ||||
597 | extents_t *extents, extent_t *extent, bool_Bool growing_retained) { | ||||
598 | /* | ||||
599 | * Leak extent after making sure its pages have already been purged, so | ||||
600 | * that this is only a virtual memory leak. | ||||
601 | */ | ||||
602 | if (extents_state_get(extents) == extent_state_dirty) { | ||||
603 | if (extent_purge_lazy_impl(tsdn, arena, r_extent_hooks, | ||||
604 | extent, 0, extent_size_get(extent), growing_retained)) { | ||||
605 | extent_purge_forced_impl(tsdn, arena, r_extent_hooks, | ||||
606 | extent, 0, extent_size_get(extent), | ||||
607 | growing_retained); | ||||
608 | } | ||||
609 | } | ||||
610 | extent_dalloc(tsdn, arena, extent); | ||||
611 | } | ||||
612 | |||||
613 | void | ||||
614 | extents_prefork(tsdn_t *tsdn, extents_t *extents) { | ||||
615 | malloc_mutex_prefork(tsdn, &extents->mtx); | ||||
616 | } | ||||
617 | |||||
618 | void | ||||
619 | extents_postfork_parent(tsdn_t *tsdn, extents_t *extents) { | ||||
620 | malloc_mutex_postfork_parent(tsdn, &extents->mtx); | ||||
621 | } | ||||
622 | |||||
623 | void | ||||
624 | extents_postfork_child(tsdn_t *tsdn, extents_t *extents) { | ||||
625 | malloc_mutex_postfork_child(tsdn, &extents->mtx); | ||||
626 | } | ||||
627 | |||||
628 | static void | ||||
629 | extent_deactivate_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, | ||||
630 | extent_t *extent) { | ||||
631 | assert(extent_arena_get(extent) == arena)do { if (__builtin_expect(!!(config_debug && !(extent_arena_get (extent) == arena)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 631, "extent_arena_get(extent) == arena"); abort (); } } while (0); | ||||
632 | assert(extent_state_get(extent) == extent_state_active)do { if (__builtin_expect(!!(config_debug && !(extent_state_get (extent) == extent_state_active)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 632, "extent_state_get(extent) == extent_state_active" ); abort(); } } while (0); | ||||
633 | |||||
634 | extent_state_set(extent, extents_state_get(extents)); | ||||
635 | extents_insert_locked(tsdn, extents, extent); | ||||
636 | } | ||||
637 | |||||
638 | static void | ||||
639 | extent_deactivate(tsdn_t *tsdn, arena_t *arena, extents_t *extents, | ||||
640 | extent_t *extent) { | ||||
641 | malloc_mutex_lock(tsdn, &extents->mtx); | ||||
642 | extent_deactivate_locked(tsdn, arena, extents, extent); | ||||
643 | malloc_mutex_unlock(tsdn, &extents->mtx); | ||||
644 | } | ||||
645 | |||||
646 | static void | ||||
647 | extent_activate_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, | ||||
648 | extent_t *extent) { | ||||
649 | assert(extent_arena_get(extent) == arena)do { if (__builtin_expect(!!(config_debug && !(extent_arena_get (extent) == arena)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 649, "extent_arena_get(extent) == arena"); abort (); } } while (0); | ||||
650 | assert(extent_state_get(extent) == extents_state_get(extents))do { if (__builtin_expect(!!(config_debug && !(extent_state_get (extent) == extents_state_get(extents))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 650, "extent_state_get(extent) == extents_state_get(extents)" ); abort(); } } while (0); | ||||
651 | |||||
652 | extents_remove_locked(tsdn, extents, extent); | ||||
653 | extent_state_set(extent, extent_state_active); | ||||
654 | } | ||||
655 | |||||
656 | static bool_Bool | ||||
657 | extent_rtree_leaf_elms_lookup(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, | ||||
658 | const extent_t *extent, bool_Bool dependent, bool_Bool init_missing, | ||||
659 | rtree_leaf_elm_t **r_elm_a, rtree_leaf_elm_t **r_elm_b) { | ||||
660 | *r_elm_a = rtree_leaf_elm_lookup(tsdn, &extents_rtree, rtree_ctx, | ||||
661 | (uintptr_t)extent_base_get(extent), dependent, init_missing); | ||||
662 | if (!dependent && *r_elm_a == NULL((void*)0)) { | ||||
663 | return true1; | ||||
664 | } | ||||
665 | assert(*r_elm_a != NULL)do { if (__builtin_expect(!!(config_debug && !(*r_elm_a != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 665, "*r_elm_a != NULL"); abort(); } } while (0); | ||||
666 | |||||
667 | *r_elm_b = rtree_leaf_elm_lookup(tsdn, &extents_rtree, rtree_ctx, | ||||
668 | (uintptr_t)extent_last_get(extent), dependent, init_missing); | ||||
669 | if (!dependent && *r_elm_b == NULL((void*)0)) { | ||||
670 | return true1; | ||||
671 | } | ||||
672 | assert(*r_elm_b != NULL)do { if (__builtin_expect(!!(config_debug && !(*r_elm_b != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 672, "*r_elm_b != NULL"); abort(); } } while (0); | ||||
673 | |||||
674 | return false0; | ||||
675 | } | ||||
676 | |||||
677 | static void | ||||
678 | extent_rtree_write_acquired(tsdn_t *tsdn, rtree_leaf_elm_t *elm_a, | ||||
679 | rtree_leaf_elm_t *elm_b, extent_t *extent, szind_t szind, bool_Bool slab) { | ||||
680 | rtree_leaf_elm_write(tsdn, &extents_rtree, elm_a, extent, szind, slab); | ||||
681 | if (elm_b != NULL((void*)0)) { | ||||
682 | rtree_leaf_elm_write(tsdn, &extents_rtree, elm_b, extent, szind, | ||||
683 | slab); | ||||
684 | } | ||||
685 | } | ||||
686 | |||||
687 | static void | ||||
688 | extent_interior_register(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, extent_t *extent, | ||||
689 | szind_t szind) { | ||||
690 | assert(extent_slab_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent_slab_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 690, "extent_slab_get(extent)"); abort(); } } while (0); | ||||
691 | |||||
692 | /* Register interior. */ | ||||
693 | for (size_t i = 1; i < (extent_size_get(extent) >> LG_PAGE12) - 1; i++) { | ||||
694 | rtree_write(tsdn, &extents_rtree, rtree_ctx, | ||||
695 | (uintptr_t)extent_base_get(extent) + (uintptr_t)(i << | ||||
696 | LG_PAGE12), extent, szind, true1); | ||||
697 | } | ||||
698 | } | ||||
699 | |||||
700 | static void | ||||
701 | extent_gdump_add(tsdn_t *tsdn, const extent_t *extent) { | ||||
702 | cassert(config_prof)do { if (__builtin_expect(!!(!(config_prof)), 0)) { do { if ( config_debug) { malloc_printf( "<jemalloc>: %s:%d: Unreachable code reached\n" , "src/extent.c", 702); abort(); } __builtin_unreachable(); } while (0); } } while (0); | ||||
703 | /* prof_gdump() requirement. */ | ||||
704 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | ||||
705 | WITNESS_RANK_CORE11U, 0); | ||||
706 | |||||
707 | if (opt_prof && extent_state_get(extent) == extent_state_active) { | ||||
708 | size_t nadd = extent_size_get(extent) >> LG_PAGE12; | ||||
709 | size_t cur = atomic_fetch_add_zu(&curpages, nadd, | ||||
710 | ATOMIC_RELAXEDatomic_memory_order_relaxed) + nadd; | ||||
711 | size_t high = atomic_load_zu(&highpages, ATOMIC_RELAXEDatomic_memory_order_relaxed); | ||||
712 | while (cur > high && !atomic_compare_exchange_weak_zu( | ||||
713 | &highpages, &high, cur, ATOMIC_RELAXEDatomic_memory_order_relaxed, ATOMIC_RELAXEDatomic_memory_order_relaxed)) { | ||||
714 | /* | ||||
715 | * Don't refresh cur, because it may have decreased | ||||
716 | * since this thread lost the highpages update race. | ||||
717 | * Note that high is updated in case of CAS failure. | ||||
718 | */ | ||||
719 | } | ||||
720 | if (cur > high && prof_gdump_get_unlocked()) { | ||||
721 | prof_gdump(tsdn); | ||||
722 | } | ||||
723 | } | ||||
724 | } | ||||
725 | |||||
726 | static void | ||||
727 | extent_gdump_sub(tsdn_t *tsdn, const extent_t *extent) { | ||||
728 | cassert(config_prof)do { if (__builtin_expect(!!(!(config_prof)), 0)) { do { if ( config_debug) { malloc_printf( "<jemalloc>: %s:%d: Unreachable code reached\n" , "src/extent.c", 728); abort(); } __builtin_unreachable(); } while (0); } } while (0); | ||||
729 | |||||
730 | if (opt_prof && extent_state_get(extent) == extent_state_active) { | ||||
731 | size_t nsub = extent_size_get(extent) >> LG_PAGE12; | ||||
732 | assert(atomic_load_zu(&curpages, ATOMIC_RELAXED) >= nsub)do { if (__builtin_expect(!!(config_debug && !(atomic_load_zu (&curpages, atomic_memory_order_relaxed) >= nsub)), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 732, "atomic_load_zu(&curpages, ATOMIC_RELAXED) >= nsub" ); abort(); } } while (0); | ||||
733 | atomic_fetch_sub_zu(&curpages, nsub, ATOMIC_RELAXEDatomic_memory_order_relaxed); | ||||
734 | } | ||||
735 | } | ||||
736 | |||||
737 | static bool_Bool | ||||
738 | extent_register_impl(tsdn_t *tsdn, extent_t *extent, bool_Bool gdump_add) { | ||||
739 | rtree_ctx_t rtree_ctx_fallback; | ||||
740 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); | ||||
741 | rtree_leaf_elm_t *elm_a, *elm_b; | ||||
742 | |||||
743 | /* | ||||
744 | * We need to hold the lock to protect against a concurrent coalesce | ||||
745 | * operation that sees us in a partial state. | ||||
746 | */ | ||||
747 | extent_lock(tsdn, extent); | ||||
748 | |||||
749 | if (extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, extent, false0, true1, | ||||
750 | &elm_a, &elm_b)) { | ||||
751 | return true1; | ||||
752 | } | ||||
753 | |||||
754 | szind_t szind = extent_szind_get_maybe_invalid(extent); | ||||
755 | bool_Bool slab = extent_slab_get(extent); | ||||
756 | extent_rtree_write_acquired(tsdn, elm_a, elm_b, extent, szind, slab); | ||||
757 | if (slab) { | ||||
758 | extent_interior_register(tsdn, rtree_ctx, extent, szind); | ||||
759 | } | ||||
760 | |||||
761 | extent_unlock(tsdn, extent); | ||||
762 | |||||
763 | if (config_prof && gdump_add) { | ||||
764 | extent_gdump_add(tsdn, extent); | ||||
765 | } | ||||
766 | |||||
767 | return false0; | ||||
768 | } | ||||
769 | |||||
770 | static bool_Bool | ||||
771 | extent_register(tsdn_t *tsdn, extent_t *extent) { | ||||
772 | return extent_register_impl(tsdn, extent, true1); | ||||
773 | } | ||||
774 | |||||
775 | static bool_Bool | ||||
776 | extent_register_no_gdump_add(tsdn_t *tsdn, extent_t *extent) { | ||||
777 | return extent_register_impl(tsdn, extent, false0); | ||||
778 | } | ||||
779 | |||||
780 | static void | ||||
781 | extent_reregister(tsdn_t *tsdn, extent_t *extent) { | ||||
782 | bool_Bool err = extent_register(tsdn, extent); | ||||
783 | assert(!err)do { if (__builtin_expect(!!(config_debug && !(!err)) , 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 783, "!err"); abort(); } } while (0); | ||||
784 | } | ||||
785 | |||||
786 | /* | ||||
787 | * Removes all pointers to the given extent from the global rtree indices for | ||||
788 | * its interior. This is relevant for slab extents, for which we need to do | ||||
789 | * metadata lookups at places other than the head of the extent. We deregister | ||||
790 | * on the interior, then, when an extent moves from being an active slab to an | ||||
791 | * inactive state. | ||||
792 | */ | ||||
793 | static void | ||||
794 | extent_interior_deregister(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, | ||||
795 | extent_t *extent) { | ||||
796 | size_t i; | ||||
797 | |||||
798 | assert(extent_slab_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent_slab_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 798, "extent_slab_get(extent)"); abort(); } } while (0); | ||||
799 | |||||
800 | for (i = 1; i < (extent_size_get(extent) >> LG_PAGE12) - 1; i++) { | ||||
801 | rtree_clear(tsdn, &extents_rtree, rtree_ctx, | ||||
802 | (uintptr_t)extent_base_get(extent) + (uintptr_t)(i << | ||||
803 | LG_PAGE12)); | ||||
804 | } | ||||
805 | } | ||||
806 | |||||
807 | /* | ||||
808 | * Removes all pointers to the given extent from the global rtree. | ||||
809 | */ | ||||
810 | static void | ||||
811 | extent_deregister_impl(tsdn_t *tsdn, extent_t *extent, bool_Bool gdump) { | ||||
812 | rtree_ctx_t rtree_ctx_fallback; | ||||
813 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); | ||||
814 | rtree_leaf_elm_t *elm_a, *elm_b; | ||||
815 | extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, extent, true1, false0, | ||||
816 | &elm_a, &elm_b); | ||||
817 | |||||
818 | extent_lock(tsdn, extent); | ||||
819 | |||||
820 | extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL((void*)0), NSIZES235, false0); | ||||
821 | if (extent_slab_get(extent)) { | ||||
822 | extent_interior_deregister(tsdn, rtree_ctx, extent); | ||||
823 | extent_slab_set(extent, false0); | ||||
824 | } | ||||
825 | |||||
826 | extent_unlock(tsdn, extent); | ||||
827 | |||||
828 | if (config_prof && gdump) { | ||||
829 | extent_gdump_sub(tsdn, extent); | ||||
830 | } | ||||
831 | } | ||||
832 | |||||
833 | static void | ||||
834 | extent_deregister(tsdn_t *tsdn, extent_t *extent) { | ||||
835 | extent_deregister_impl(tsdn, extent, true1); | ||||
836 | } | ||||
837 | |||||
838 | static void | ||||
839 | extent_deregister_no_gdump_sub(tsdn_t *tsdn, extent_t *extent) { | ||||
840 | extent_deregister_impl(tsdn, extent, false0); | ||||
841 | } | ||||
842 | |||||
843 | /* | ||||
844 | * Tries to find and remove an extent from extents that can be used for the | ||||
845 | * given allocation request. | ||||
846 | */ | ||||
847 | static extent_t * | ||||
848 | extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, | ||||
849 | extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, | ||||
850 | void *new_addr, size_t size, size_t pad, size_t alignment, bool_Bool slab, | ||||
851 | bool_Bool growing_retained) { | ||||
852 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | ||||
853 | WITNESS_RANK_CORE11U, growing_retained ? 1 : 0); | ||||
854 | assert(alignment > 0)do { if (__builtin_expect(!!(config_debug && !(alignment > 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 854, "alignment > 0"); abort(); } } while (0); | ||||
855 | if (config_debug && new_addr != NULL((void*)0)) { | ||||
856 | /* | ||||
857 | * Non-NULL new_addr has two use cases: | ||||
858 | * | ||||
859 | * 1) Recycle a known-extant extent, e.g. during purging. | ||||
860 | * 2) Perform in-place expanding reallocation. | ||||
861 | * | ||||
862 | * Regardless of use case, new_addr must either refer to a | ||||
863 | * non-existing extent, or to the base of an extant extent, | ||||
864 | * since only active slabs support interior lookups (which of | ||||
865 | * course cannot be recycled). | ||||
866 | */ | ||||
867 | assert(PAGE_ADDR2BASE(new_addr) == new_addr)do { if (__builtin_expect(!!(config_debug && !(((void *)((uintptr_t)(new_addr) & ~((size_t)(((size_t)(1U << 12)) - 1)))) == new_addr)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 867, "PAGE_ADDR2BASE(new_addr) == new_addr" ); abort(); } } while (0); | ||||
868 | assert(pad == 0)do { if (__builtin_expect(!!(config_debug && !(pad == 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 868, "pad == 0"); abort(); } } while (0); | ||||
869 | assert(alignment <= PAGE)do { if (__builtin_expect(!!(config_debug && !(alignment <= ((size_t)(1U << 12)))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 869, "alignment <= PAGE"); abort(); } } while (0); | ||||
870 | } | ||||
871 | |||||
872 | size_t esize = size + pad; | ||||
873 | malloc_mutex_lock(tsdn, &extents->mtx); | ||||
874 | extent_hooks_assure_initialized(arena, r_extent_hooks); | ||||
875 | extent_t *extent; | ||||
876 | if (new_addr != NULL((void*)0)) { | ||||
877 | extent = extent_lock_from_addr(tsdn, rtree_ctx, new_addr); | ||||
878 | if (extent != NULL((void*)0)) { | ||||
879 | /* | ||||
880 | * We might null-out extent to report an error, but we | ||||
881 | * still need to unlock the associated mutex after. | ||||
882 | */ | ||||
883 | extent_t *unlock_extent = extent; | ||||
884 | assert(extent_base_get(extent) == new_addr)do { if (__builtin_expect(!!(config_debug && !(extent_base_get (extent) == new_addr)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 884, "extent_base_get(extent) == new_addr") ; abort(); } } while (0); | ||||
885 | if (extent_arena_get(extent) != arena || | ||||
886 | extent_size_get(extent) < esize || | ||||
887 | extent_state_get(extent) != | ||||
888 | extents_state_get(extents)) { | ||||
889 | extent = NULL((void*)0); | ||||
890 | } | ||||
891 | extent_unlock(tsdn, unlock_extent); | ||||
892 | } | ||||
893 | } else { | ||||
894 | extent = extents_fit_locked(tsdn, arena, extents, esize, | ||||
895 | alignment); | ||||
896 | } | ||||
897 | if (extent == NULL((void*)0)) { | ||||
898 | malloc_mutex_unlock(tsdn, &extents->mtx); | ||||
899 | return NULL((void*)0); | ||||
900 | } | ||||
901 | |||||
902 | extent_activate_locked(tsdn, arena, extents, extent); | ||||
903 | malloc_mutex_unlock(tsdn, &extents->mtx); | ||||
904 | |||||
905 | return extent; | ||||
906 | } | ||||
907 | |||||
908 | /* | ||||
909 | * Given an allocation request and an extent guaranteed to be able to satisfy | ||||
910 | * it, this splits off lead and trail extents, leaving extent pointing to an | ||||
911 | * extent satisfying the allocation. | ||||
912 | * This function doesn't put lead or trail into any extents_t; it's the caller's | ||||
913 | * job to ensure that they can be reused. | ||||
914 | */ | ||||
915 | typedef enum { | ||||
916 | /* | ||||
917 | * Split successfully. lead, extent, and trail, are modified to extents | ||||
918 | * describing the ranges before, in, and after the given allocation. | ||||
919 | */ | ||||
920 | extent_split_interior_ok, | ||||
921 | /* | ||||
922 | * The extent can't satisfy the given allocation request. None of the | ||||
923 | * input extent_t *s are touched. | ||||
924 | */ | ||||
925 | extent_split_interior_cant_alloc, | ||||
926 | /* | ||||
927 | * In a potentially invalid state. Must leak (if *to_leak is non-NULL), | ||||
928 | * and salvage what's still salvageable (if *to_salvage is non-NULL). | ||||
929 | * None of lead, extent, or trail are valid. | ||||
930 | */ | ||||
931 | extent_split_interior_error | ||||
932 | } extent_split_interior_result_t; | ||||
933 | |||||
934 | static extent_split_interior_result_t | ||||
935 | extent_split_interior(tsdn_t *tsdn, arena_t *arena, | ||||
936 | extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, | ||||
937 | /* The result of splitting, in case of success. */ | ||||
938 | extent_t **extent, extent_t **lead, extent_t **trail, | ||||
939 | /* The mess to clean up, in case of error. */ | ||||
940 | extent_t **to_leak, extent_t **to_salvage, | ||||
941 | void *new_addr, size_t size, size_t pad, size_t alignment, bool_Bool slab, | ||||
942 | szind_t szind, bool_Bool growing_retained) { | ||||
943 | size_t esize = size + pad; | ||||
944 | size_t leadsize = ALIGNMENT_CEILING((uintptr_t)extent_base_get(*extent),((((uintptr_t)extent_base_get(*extent)) + ((((alignment) + (( size_t)(((size_t)(1U << 12)) - 1))) & ~((size_t)((( size_t)(1U << 12)) - 1))) - 1)) & ((~((((alignment) + ((size_t)(((size_t)(1U << 12)) - 1))) & ~((size_t )(((size_t)(1U << 12)) - 1))))) + 1)) | ||||
945 | PAGE_CEILING(alignment))((((uintptr_t)extent_base_get(*extent)) + ((((alignment) + (( size_t)(((size_t)(1U << 12)) - 1))) & ~((size_t)((( size_t)(1U << 12)) - 1))) - 1)) & ((~((((alignment) + ((size_t)(((size_t)(1U << 12)) - 1))) & ~((size_t )(((size_t)(1U << 12)) - 1))))) + 1)) - (uintptr_t)extent_base_get(*extent); | ||||
946 | assert(new_addr == NULL || leadsize == 0)do { if (__builtin_expect(!!(config_debug && !(new_addr == ((void*)0) || leadsize == 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 946, "new_addr == NULL || leadsize == 0"); abort (); } } while (0); | ||||
947 | if (extent_size_get(*extent) < leadsize + esize) { | ||||
948 | return extent_split_interior_cant_alloc; | ||||
949 | } | ||||
950 | size_t trailsize = extent_size_get(*extent) - leadsize - esize; | ||||
951 | |||||
952 | *lead = NULL((void*)0); | ||||
953 | *trail = NULL((void*)0); | ||||
954 | *to_leak = NULL((void*)0); | ||||
955 | *to_salvage = NULL((void*)0); | ||||
956 | |||||
957 | /* Split the lead. */ | ||||
958 | if (leadsize != 0) { | ||||
959 | *lead = *extent; | ||||
960 | *extent = extent_split_impl(tsdn, arena, r_extent_hooks, | ||||
961 | *lead, leadsize, NSIZES235, false0, esize + trailsize, szind, | ||||
962 | slab, growing_retained); | ||||
963 | if (*extent == NULL((void*)0)) { | ||||
964 | *to_leak = *lead; | ||||
965 | *lead = NULL((void*)0); | ||||
966 | return extent_split_interior_error; | ||||
967 | } | ||||
968 | } | ||||
969 | |||||
970 | /* Split the trail. */ | ||||
971 | if (trailsize != 0) { | ||||
972 | *trail = extent_split_impl(tsdn, arena, r_extent_hooks, *extent, | ||||
973 | esize, szind, slab, trailsize, NSIZES235, false0, | ||||
974 | growing_retained); | ||||
975 | if (*trail == NULL((void*)0)) { | ||||
976 | *to_leak = *extent; | ||||
977 | *to_salvage = *lead; | ||||
978 | *lead = NULL((void*)0); | ||||
979 | *extent = NULL((void*)0); | ||||
980 | return extent_split_interior_error; | ||||
981 | } | ||||
982 | } | ||||
983 | |||||
984 | if (leadsize == 0 && trailsize == 0) { | ||||
985 | /* | ||||
986 | * Splitting causes szind to be set as a side effect, but no | ||||
987 | * splitting occurred. | ||||
988 | */ | ||||
989 | extent_szind_set(*extent, szind); | ||||
990 | if (szind != NSIZES235) { | ||||
991 | rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, | ||||
992 | (uintptr_t)extent_addr_get(*extent), szind, slab); | ||||
993 | if (slab && extent_size_get(*extent) > PAGE((size_t)(1U << 12))) { | ||||
994 | rtree_szind_slab_update(tsdn, &extents_rtree, | ||||
995 | rtree_ctx, | ||||
996 | (uintptr_t)extent_past_get(*extent) - | ||||
997 | (uintptr_t)PAGE((size_t)(1U << 12)), szind, slab); | ||||
998 | } | ||||
999 | } | ||||
1000 | } | ||||
1001 | |||||
1002 | return extent_split_interior_ok; | ||||
1003 | } | ||||
1004 | |||||
1005 | /* | ||||
1006 | * This fulfills the indicated allocation request out of the given extent (which | ||||
1007 | * the caller should have ensured was big enough). If there's any unused space | ||||
1008 | * before or after the resulting allocation, that space is given its own extent | ||||
1009 | * and put back into extents. | ||||
1010 | */ | ||||
1011 | static extent_t * | ||||
1012 | extent_recycle_split(tsdn_t *tsdn, arena_t *arena, | ||||
1013 | extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, | ||||
1014 | void *new_addr, size_t size, size_t pad, size_t alignment, bool_Bool slab, | ||||
1015 | szind_t szind, extent_t *extent, bool_Bool growing_retained) { | ||||
1016 | extent_t *lead; | ||||
1017 | extent_t *trail; | ||||
1018 | extent_t *to_leak; | ||||
1019 | extent_t *to_salvage; | ||||
1020 | |||||
1021 | extent_split_interior_result_t result = extent_split_interior( | ||||
1022 | tsdn, arena, r_extent_hooks, rtree_ctx, &extent, &lead, &trail, | ||||
1023 | &to_leak, &to_salvage, new_addr, size, pad, alignment, slab, szind, | ||||
1024 | growing_retained); | ||||
1025 | |||||
1026 | if (result == extent_split_interior_ok) { | ||||
1027 | if (lead != NULL((void*)0)) { | ||||
1028 | extent_deactivate(tsdn, arena, extents, lead); | ||||
1029 | } | ||||
1030 | if (trail != NULL((void*)0)) { | ||||
1031 | extent_deactivate(tsdn, arena, extents, trail); | ||||
1032 | } | ||||
1033 | return extent; | ||||
1034 | } else { | ||||
1035 | /* | ||||
1036 | * We should have picked an extent that was large enough to | ||||
1037 | * fulfill our allocation request. | ||||
1038 | */ | ||||
1039 | assert(result == extent_split_interior_error)do { if (__builtin_expect(!!(config_debug && !(result == extent_split_interior_error)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1039, "result == extent_split_interior_error" ); abort(); } } while (0); | ||||
1040 | if (to_salvage != NULL((void*)0)) { | ||||
1041 | extent_deregister(tsdn, to_salvage); | ||||
1042 | } | ||||
1043 | if (to_leak != NULL((void*)0)) { | ||||
1044 | void *leak = extent_base_get(to_leak); | ||||
1045 | extent_deregister_no_gdump_sub(tsdn, to_leak); | ||||
1046 | extents_leak(tsdn, arena, r_extent_hooks, extents, | ||||
1047 | to_leak, growing_retained); | ||||
1048 | assert(extent_lock_from_addr(tsdn, rtree_ctx, leak)do { if (__builtin_expect(!!(config_debug && !(extent_lock_from_addr (tsdn, rtree_ctx, leak) == ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 1049, "extent_lock_from_addr(tsdn, rtree_ctx, leak) == NULL" ); abort(); } } while (0) | ||||
1049 | == NULL)do { if (__builtin_expect(!!(config_debug && !(extent_lock_from_addr (tsdn, rtree_ctx, leak) == ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 1049, "extent_lock_from_addr(tsdn, rtree_ctx, leak) == NULL" ); abort(); } } while (0); | ||||
1050 | } | ||||
1051 | return NULL((void*)0); | ||||
1052 | } | ||||
1053 | unreachable()__builtin_unreachable(); | ||||
1054 | } | ||||
1055 | |||||
1056 | /* | ||||
1057 | * Tries to satisfy the given allocation request by reusing one of the extents | ||||
1058 | * in the given extents_t. | ||||
1059 | */ | ||||
1060 | static extent_t * | ||||
1061 | extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, | ||||
1062 | extents_t *extents, void *new_addr, size_t size, size_t pad, | ||||
1063 | size_t alignment, bool_Bool slab, szind_t szind, bool_Bool *zero, bool_Bool *commit, | ||||
1064 | bool_Bool growing_retained) { | ||||
1065 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | ||||
1066 | WITNESS_RANK_CORE11U, growing_retained ? 1 : 0); | ||||
1067 | assert(new_addr == NULL || !slab)do { if (__builtin_expect(!!(config_debug && !(new_addr == ((void*)0) || !slab)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1067, "new_addr == NULL || !slab"); abort() ; } } while (0); | ||||
1068 | assert(pad == 0 || !slab)do { if (__builtin_expect(!!(config_debug && !(pad == 0 || !slab)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1068, "pad == 0 || !slab"); abort(); } } while (0); | ||||
1069 | assert(!*zero || !slab)do { if (__builtin_expect(!!(config_debug && !(!*zero || !slab)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1069, "!*zero || !slab"); abort(); } } while (0); | ||||
1070 | |||||
1071 | rtree_ctx_t rtree_ctx_fallback; | ||||
1072 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); | ||||
1073 | |||||
1074 | extent_t *extent = extent_recycle_extract(tsdn, arena, r_extent_hooks, | ||||
1075 | rtree_ctx, extents, new_addr, size, pad, alignment, slab, | ||||
1076 | growing_retained); | ||||
1077 | if (extent == NULL((void*)0)) { | ||||
1078 | return NULL((void*)0); | ||||
1079 | } | ||||
1080 | |||||
1081 | extent = extent_recycle_split(tsdn, arena, r_extent_hooks, rtree_ctx, | ||||
1082 | extents, new_addr, size, pad, alignment, slab, szind, extent, | ||||
1083 | growing_retained); | ||||
1084 | if (extent == NULL((void*)0)) { | ||||
1085 | return NULL((void*)0); | ||||
1086 | } | ||||
1087 | |||||
1088 | if (*commit && !extent_committed_get(extent)) { | ||||
1089 | if (extent_commit_impl(tsdn, arena, r_extent_hooks, extent, | ||||
1090 | 0, extent_size_get(extent), growing_retained)) { | ||||
1091 | extent_record(tsdn, arena, r_extent_hooks, extents, | ||||
1092 | extent, growing_retained); | ||||
1093 | return NULL((void*)0); | ||||
1094 | } | ||||
1095 | extent_zeroed_set(extent, true1); | ||||
1096 | } | ||||
1097 | |||||
1098 | if (extent_committed_get(extent)) { | ||||
1099 | *commit = true1; | ||||
1100 | } | ||||
1101 | if (extent_zeroed_get(extent)) { | ||||
1102 | *zero = true1; | ||||
1103 | } | ||||
1104 | |||||
1105 | if (pad != 0) { | ||||
1106 | extent_addr_randomize(tsdn, extent, alignment); | ||||
1107 | } | ||||
1108 | assert(extent_state_get(extent) == extent_state_active)do { if (__builtin_expect(!!(config_debug && !(extent_state_get (extent) == extent_state_active)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1108, "extent_state_get(extent) == extent_state_active" ); abort(); } } while (0); | ||||
1109 | if (slab) { | ||||
1110 | extent_slab_set(extent, slab); | ||||
1111 | extent_interior_register(tsdn, rtree_ctx, extent, szind); | ||||
1112 | } | ||||
1113 | |||||
1114 | if (*zero) { | ||||
1115 | void *addr = extent_base_get(extent); | ||||
1116 | size_t size = extent_size_get(extent); | ||||
1117 | if (!extent_zeroed_get(extent)) { | ||||
1118 | if (pages_purge_forced(addr, size)) { | ||||
1119 | memset(addr, 0, size); | ||||
1120 | } | ||||
1121 | } else if (config_debug) { | ||||
1122 | size_t *p = (size_t *)(uintptr_t)addr; | ||||
1123 | for (size_t i = 0; i < size / sizeof(size_t); i++) { | ||||
1124 | assert(p[i] == 0)do { if (__builtin_expect(!!(config_debug && !(p[i] == 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1124, "p[i] == 0"); abort(); } } while (0); | ||||
1125 | } | ||||
1126 | } | ||||
1127 | } | ||||
1128 | return extent; | ||||
1129 | } | ||||
1130 | |||||
1131 | /* | ||||
1132 | * If the caller specifies (!*zero), it is still possible to receive zeroed | ||||
1133 | * memory, in which case *zero is toggled to true. arena_extent_alloc() takes | ||||
1134 | * advantage of this to avoid demanding zeroed extents, but taking advantage of | ||||
1135 | * them if they are returned. | ||||
1136 | */ | ||||
1137 | static void * | ||||
1138 | extent_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, | ||||
1139 | size_t alignment, bool_Bool *zero, bool_Bool *commit, dss_prec_t dss_prec) { | ||||
1140 | void *ret; | ||||
1141 | |||||
1142 | assert(size != 0)do { if (__builtin_expect(!!(config_debug && !(size != 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1142, "size != 0"); abort(); } } while (0); | ||||
1143 | assert(alignment != 0)do { if (__builtin_expect(!!(config_debug && !(alignment != 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1143, "alignment != 0"); abort(); } } while (0); | ||||
1144 | |||||
1145 | /* "primary" dss. */ | ||||
1146 | if (have_dss && dss_prec == dss_prec_primary && (ret = | ||||
1147 | extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero, | ||||
1148 | commit)) != NULL((void*)0)) { | ||||
1149 | return ret; | ||||
1150 | } | ||||
1151 | /* mmap. */ | ||||
1152 | if ((ret = extent_alloc_mmap(new_addr, size, alignment, zero, commit)) | ||||
1153 | != NULL((void*)0)) { | ||||
1154 | return ret; | ||||
1155 | } | ||||
1156 | /* "secondary" dss. */ | ||||
1157 | if (have_dss && dss_prec == dss_prec_secondary && (ret = | ||||
1158 | extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero, | ||||
1159 | commit)) != NULL((void*)0)) { | ||||
1160 | return ret; | ||||
1161 | } | ||||
1162 | |||||
1163 | /* All strategies for allocation failed. */ | ||||
1164 | return NULL((void*)0); | ||||
1165 | } | ||||
1166 | |||||
1167 | static void * | ||||
1168 | extent_alloc_default_impl(tsdn_t *tsdn, arena_t *arena, void *new_addr, | ||||
1169 | size_t size, size_t alignment, bool_Bool *zero, bool_Bool *commit) { | ||||
1170 | void *ret = extent_alloc_core(tsdn, arena, new_addr, size, alignment, zero, | ||||
1171 | commit, (dss_prec_t)atomic_load_u(&arena->dss_prec, | ||||
1172 | ATOMIC_RELAXEDatomic_memory_order_relaxed)); | ||||
1173 | if (have_madvise_huge && ret) { | ||||
1174 | pages_set_thp_state(ret, size); | ||||
1175 | } | ||||
1176 | return ret; | ||||
1177 | } | ||||
1178 | |||||
1179 | static void * | ||||
1180 | extent_alloc_default(extent_hooks_t *extent_hooks, void *new_addr, size_t size, | ||||
1181 | size_t alignment, bool_Bool *zero, bool_Bool *commit, unsigned arena_ind) { | ||||
1182 | tsdn_t *tsdn; | ||||
1183 | arena_t *arena; | ||||
1184 | |||||
1185 | tsdn = tsdn_fetch(); | ||||
1186 | arena = arena_get(tsdn, arena_ind, false0); | ||||
1187 | /* | ||||
1188 | * The arena we're allocating on behalf of must have been initialized | ||||
1189 | * already. | ||||
1190 | */ | ||||
1191 | assert(arena != NULL)do { if (__builtin_expect(!!(config_debug && !(arena != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1191, "arena != NULL"); abort(); } } while ( 0); | ||||
1192 | |||||
1193 | return extent_alloc_default_impl(tsdn, arena, new_addr, size, | ||||
1194 | alignment, zero, commit); | ||||
1195 | } | ||||
1196 | |||||
1197 | static void | ||||
1198 | extent_hook_pre_reentrancy(tsdn_t *tsdn, arena_t *arena) { | ||||
1199 | tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn); | ||||
1200 | if (arena == arena_get(tsd_tsdn(tsd), 0, false0)) { | ||||
1201 | /* | ||||
1202 | * The only legitimate case of customized extent hooks for a0 is | ||||
1203 | * hooks with no allocation activities. One such example is to | ||||
1204 | * place metadata on pre-allocated resources such as huge pages. | ||||
1205 | * In that case, rely on reentrancy_level checks to catch | ||||
1206 | * infinite recursions. | ||||
1207 | */ | ||||
1208 | pre_reentrancy(tsd, NULL((void*)0)); | ||||
1209 | } else { | ||||
1210 | pre_reentrancy(tsd, arena); | ||||
1211 | } | ||||
1212 | } | ||||
1213 | |||||
1214 | static void | ||||
1215 | extent_hook_post_reentrancy(tsdn_t *tsdn) { | ||||
1216 | tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn); | ||||
1217 | post_reentrancy(tsd); | ||||
1218 | } | ||||
1219 | |||||
1220 | /* | ||||
1221 | * If virtual memory is retained, create increasingly larger extents from which | ||||
1222 | * to split requested extents in order to limit the total number of disjoint | ||||
1223 | * virtual memory ranges retained by each arena. | ||||
1224 | */ | ||||
1225 | static extent_t * | ||||
1226 | extent_grow_retained(tsdn_t *tsdn, arena_t *arena, | ||||
1227 | extent_hooks_t **r_extent_hooks, size_t size, size_t pad, size_t alignment, | ||||
1228 | bool_Bool slab, szind_t szind, bool_Bool *zero, bool_Bool *commit) { | ||||
1229 | malloc_mutex_assert_owner(tsdn, &arena->extent_grow_mtx); | ||||
1230 | assert(pad == 0 || !slab)do { if (__builtin_expect(!!(config_debug && !(pad == 0 || !slab)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1230, "pad == 0 || !slab"); abort(); } } while (0); | ||||
1231 | assert(!*zero || !slab)do { if (__builtin_expect(!!(config_debug && !(!*zero || !slab)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1231, "!*zero || !slab"); abort(); } } while (0); | ||||
1232 | |||||
1233 | size_t esize = size + pad; | ||||
1234 | size_t alloc_size_min = esize + PAGE_CEILING(alignment)(((alignment) + ((size_t)(((size_t)(1U << 12)) - 1))) & ~((size_t)(((size_t)(1U << 12)) - 1))) - PAGE((size_t)(1U << 12)); | ||||
1235 | /* Beware size_t wrap-around. */ | ||||
1236 | if (alloc_size_min < esize) { | ||||
1237 | goto label_err; | ||||
1238 | } | ||||
1239 | /* | ||||
1240 | * Find the next extent size in the series that would be large enough to | ||||
1241 | * satisfy this request. | ||||
1242 | */ | ||||
1243 | pszind_t egn_skip = 0; | ||||
1244 | size_t alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip); | ||||
1245 | while (alloc_size < alloc_size_min) { | ||||
1246 | egn_skip++; | ||||
1247 | if (arena->extent_grow_next + egn_skip == NPSIZES199) { | ||||
1248 | /* Outside legal range. */ | ||||
1249 | goto label_err; | ||||
1250 | } | ||||
1251 | assert(arena->extent_grow_next + egn_skip < NPSIZES)do { if (__builtin_expect(!!(config_debug && !(arena-> extent_grow_next + egn_skip < 199)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1251, "arena->extent_grow_next + egn_skip < NPSIZES" ); abort(); } } while (0); | ||||
1252 | alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip); | ||||
1253 | } | ||||
1254 | |||||
1255 | extent_t *extent = extent_alloc(tsdn, arena); | ||||
1256 | if (extent == NULL((void*)0)) { | ||||
1257 | goto label_err; | ||||
1258 | } | ||||
1259 | bool_Bool zeroed = false0; | ||||
1260 | bool_Bool committed = false0; | ||||
1261 | |||||
1262 | void *ptr; | ||||
1263 | if (*r_extent_hooks == &extent_hooks_default) { | ||||
1264 | ptr = extent_alloc_default_impl(tsdn, arena, NULL((void*)0), | ||||
1265 | alloc_size, PAGE((size_t)(1U << 12)), &zeroed, &committed); | ||||
1266 | } else { | ||||
1267 | extent_hook_pre_reentrancy(tsdn, arena); | ||||
1268 | ptr = (*r_extent_hooks)->alloc(*r_extent_hooks, NULL((void*)0), | ||||
1269 | alloc_size, PAGE((size_t)(1U << 12)), &zeroed, &committed, | ||||
1270 | arena_ind_get(arena)); | ||||
1271 | extent_hook_post_reentrancy(tsdn); | ||||
1272 | } | ||||
1273 | |||||
1274 | extent_init(extent, arena, ptr, alloc_size, false0, NSIZES235, | ||||
1275 | arena_extent_sn_next(arena), extent_state_active, zeroed, | ||||
1276 | committed, true1); | ||||
1277 | if (ptr == NULL((void*)0)) { | ||||
1278 | extent_dalloc(tsdn, arena, extent); | ||||
1279 | goto label_err; | ||||
1280 | } | ||||
1281 | |||||
1282 | if (extent_register_no_gdump_add(tsdn, extent)) { | ||||
1283 | extents_leak(tsdn, arena, r_extent_hooks, | ||||
1284 | &arena->extents_retained, extent, true1); | ||||
1285 | goto label_err; | ||||
1286 | } | ||||
1287 | |||||
1288 | if (extent_zeroed_get(extent) && extent_committed_get(extent)) { | ||||
1289 | *zero = true1; | ||||
1290 | } | ||||
1291 | if (extent_committed_get(extent)) { | ||||
1292 | *commit = true1; | ||||
1293 | } | ||||
1294 | |||||
1295 | rtree_ctx_t rtree_ctx_fallback; | ||||
1296 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); | ||||
1297 | |||||
1298 | extent_t *lead; | ||||
1299 | extent_t *trail; | ||||
1300 | extent_t *to_leak; | ||||
1301 | extent_t *to_salvage; | ||||
1302 | extent_split_interior_result_t result = extent_split_interior( | ||||
1303 | tsdn, arena, r_extent_hooks, rtree_ctx, &extent, &lead, &trail, | ||||
1304 | &to_leak, &to_salvage, NULL((void*)0), size, pad, alignment, slab, szind, | ||||
1305 | true1); | ||||
1306 | |||||
1307 | if (result == extent_split_interior_ok) { | ||||
1308 | if (lead != NULL((void*)0)) { | ||||
1309 | extent_record(tsdn, arena, r_extent_hooks, | ||||
1310 | &arena->extents_retained, lead, true1); | ||||
1311 | } | ||||
1312 | if (trail != NULL((void*)0)) { | ||||
1313 | extent_record(tsdn, arena, r_extent_hooks, | ||||
1314 | &arena->extents_retained, trail, true1); | ||||
1315 | } | ||||
1316 | } else { | ||||
1317 | /* | ||||
1318 | * We should have allocated a sufficiently large extent; the | ||||
1319 | * cant_alloc case should not occur. | ||||
1320 | */ | ||||
1321 | assert(result == extent_split_interior_error)do { if (__builtin_expect(!!(config_debug && !(result == extent_split_interior_error)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1321, "result == extent_split_interior_error" ); abort(); } } while (0); | ||||
1322 | if (to_salvage != NULL((void*)0)) { | ||||
1323 | if (config_prof) { | ||||
1324 | extent_gdump_add(tsdn, to_salvage); | ||||
1325 | } | ||||
1326 | extent_record(tsdn, arena, r_extent_hooks, | ||||
1327 | &arena->extents_retained, to_salvage, true1); | ||||
1328 | } | ||||
1329 | if (to_leak != NULL((void*)0)) { | ||||
1330 | extent_deregister_no_gdump_sub(tsdn, to_leak); | ||||
1331 | extents_leak(tsdn, arena, r_extent_hooks, | ||||
1332 | &arena->extents_retained, to_leak, true1); | ||||
1333 | } | ||||
1334 | goto label_err; | ||||
1335 | } | ||||
1336 | |||||
1337 | if (*commit && !extent_committed_get(extent)) { | ||||
1338 | if (extent_commit_impl(tsdn, arena, r_extent_hooks, extent, 0, | ||||
1339 | extent_size_get(extent), true1)) { | ||||
1340 | extent_record(tsdn, arena, r_extent_hooks, | ||||
1341 | &arena->extents_retained, extent, true1); | ||||
1342 | goto label_err; | ||||
1343 | } | ||||
1344 | extent_zeroed_set(extent, true1); | ||||
1345 | } | ||||
1346 | |||||
1347 | /* | ||||
1348 | * Increment extent_grow_next if doing so wouldn't exceed the allowed | ||||
1349 | * range. | ||||
1350 | */ | ||||
1351 | if (arena->extent_grow_next + egn_skip + 1 <= | ||||
1352 | arena->retain_grow_limit) { | ||||
1353 | arena->extent_grow_next += egn_skip + 1; | ||||
1354 | } else { | ||||
1355 | arena->extent_grow_next = arena->retain_grow_limit; | ||||
1356 | } | ||||
1357 | /* All opportunities for failure are past. */ | ||||
1358 | malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx); | ||||
1359 | |||||
1360 | if (config_prof) { | ||||
1361 | /* Adjust gdump stats now that extent is final size. */ | ||||
1362 | extent_gdump_add(tsdn, extent); | ||||
1363 | } | ||||
1364 | if (pad != 0) { | ||||
1365 | extent_addr_randomize(tsdn, extent, alignment); | ||||
1366 | } | ||||
1367 | if (slab) { | ||||
1368 | rtree_ctx_t rtree_ctx_fallback; | ||||
1369 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, | ||||
1370 | &rtree_ctx_fallback); | ||||
1371 | |||||
1372 | extent_slab_set(extent, true1); | ||||
1373 | extent_interior_register(tsdn, rtree_ctx, extent, szind); | ||||
1374 | } | ||||
1375 | if (*zero && !extent_zeroed_get(extent)) { | ||||
1376 | void *addr = extent_base_get(extent); | ||||
1377 | size_t size = extent_size_get(extent); | ||||
1378 | if (pages_purge_forced(addr, size)) { | ||||
1379 | memset(addr, 0, size); | ||||
1380 | } | ||||
1381 | } | ||||
1382 | |||||
1383 | return extent; | ||||
1384 | label_err: | ||||
1385 | malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx); | ||||
1386 | return NULL((void*)0); | ||||
1387 | } | ||||
1388 | |||||
1389 | static extent_t * | ||||
1390 | extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, | ||||
1391 | extent_hooks_t **r_extent_hooks, void *new_addr, size_t size, size_t pad, | ||||
1392 | size_t alignment, bool_Bool slab, szind_t szind, bool_Bool *zero, bool_Bool *commit) { | ||||
1393 | assert(size != 0)do { if (__builtin_expect(!!(config_debug && !(size != 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1393, "size != 0"); abort(); } } while (0); | ||||
1394 | assert(alignment != 0)do { if (__builtin_expect(!!(config_debug && !(alignment != 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1394, "alignment != 0"); abort(); } } while (0); | ||||
1395 | |||||
1396 | malloc_mutex_lock(tsdn, &arena->extent_grow_mtx); | ||||
1397 | |||||
1398 | extent_t *extent = extent_recycle(tsdn, arena, r_extent_hooks, | ||||
1399 | &arena->extents_retained, new_addr, size, pad, alignment, slab, | ||||
1400 | szind, zero, commit, true1); | ||||
1401 | if (extent != NULL((void*)0)) { | ||||
1402 | malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx); | ||||
1403 | if (config_prof) { | ||||
1404 | extent_gdump_add(tsdn, extent); | ||||
1405 | } | ||||
1406 | } else if (opt_retain && new_addr == NULL((void*)0)) { | ||||
1407 | extent = extent_grow_retained(tsdn, arena, r_extent_hooks, size, | ||||
1408 | pad, alignment, slab, szind, zero, commit); | ||||
1409 | /* extent_grow_retained() always releases extent_grow_mtx. */ | ||||
1410 | } else { | ||||
1411 | malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx); | ||||
1412 | } | ||||
1413 | malloc_mutex_assert_not_owner(tsdn, &arena->extent_grow_mtx); | ||||
1414 | |||||
1415 | return extent; | ||||
1416 | } | ||||
1417 | |||||
1418 | static extent_t * | ||||
1419 | extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, | ||||
1420 | extent_hooks_t **r_extent_hooks, void *new_addr, size_t size, size_t pad, | ||||
1421 | size_t alignment, bool_Bool slab, szind_t szind, bool_Bool *zero, bool_Bool *commit) { | ||||
1422 | size_t esize = size + pad; | ||||
1423 | extent_t *extent = extent_alloc(tsdn, arena); | ||||
1424 | if (extent == NULL((void*)0)) { | ||||
1425 | return NULL((void*)0); | ||||
1426 | } | ||||
1427 | void *addr; | ||||
1428 | if (*r_extent_hooks == &extent_hooks_default) { | ||||
1429 | /* Call directly to propagate tsdn. */ | ||||
1430 | addr = extent_alloc_default_impl(tsdn, arena, new_addr, esize, | ||||
1431 | alignment, zero, commit); | ||||
1432 | } else { | ||||
1433 | extent_hook_pre_reentrancy(tsdn, arena); | ||||
1434 | addr = (*r_extent_hooks)->alloc(*r_extent_hooks, new_addr, | ||||
1435 | esize, alignment, zero, commit, arena_ind_get(arena)); | ||||
1436 | extent_hook_post_reentrancy(tsdn); | ||||
1437 | } | ||||
1438 | if (addr == NULL((void*)0)) { | ||||
1439 | extent_dalloc(tsdn, arena, extent); | ||||
1440 | return NULL((void*)0); | ||||
1441 | } | ||||
1442 | extent_init(extent, arena, addr, esize, slab, szind, | ||||
1443 | arena_extent_sn_next(arena), extent_state_active, *zero, *commit, | ||||
1444 | true1); | ||||
1445 | if (pad != 0) { | ||||
1446 | extent_addr_randomize(tsdn, extent, alignment); | ||||
1447 | } | ||||
1448 | if (extent_register(tsdn, extent)) { | ||||
1449 | extents_leak(tsdn, arena, r_extent_hooks, | ||||
1450 | &arena->extents_retained, extent, false0); | ||||
1451 | return NULL((void*)0); | ||||
1452 | } | ||||
1453 | |||||
1454 | return extent; | ||||
1455 | } | ||||
1456 | |||||
1457 | extent_t * | ||||
1458 | extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, | ||||
1459 | extent_hooks_t **r_extent_hooks, void *new_addr, size_t size, size_t pad, | ||||
1460 | size_t alignment, bool_Bool slab, szind_t szind, bool_Bool *zero, bool_Bool *commit) { | ||||
1461 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | ||||
1462 | WITNESS_RANK_CORE11U, 0); | ||||
1463 | |||||
1464 | extent_hooks_assure_initialized(arena, r_extent_hooks); | ||||
1465 | |||||
1466 | extent_t *extent = extent_alloc_retained(tsdn, arena, r_extent_hooks, | ||||
1467 | new_addr, size, pad, alignment, slab, szind, zero, commit); | ||||
1468 | if (extent == NULL((void*)0)) { | ||||
1469 | if (opt_retain && new_addr != NULL((void*)0)) { | ||||
1470 | /* | ||||
1471 | * When retain is enabled and new_addr is set, we do not | ||||
1472 | * attempt extent_alloc_wrapper_hard which does mmap | ||||
1473 | * that is very unlikely to succeed (unless it happens | ||||
1474 | * to be at the end). | ||||
1475 | */ | ||||
1476 | return NULL((void*)0); | ||||
1477 | } | ||||
1478 | extent = extent_alloc_wrapper_hard(tsdn, arena, r_extent_hooks, | ||||
1479 | new_addr, size, pad, alignment, slab, szind, zero, commit); | ||||
1480 | } | ||||
1481 | |||||
1482 | assert(extent == NULL || extent_dumpable_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent == ((void*)0) || extent_dumpable_get(extent))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 1482, "extent == NULL || extent_dumpable_get(extent)"); abort (); } } while (0); | ||||
1483 | return extent; | ||||
1484 | } | ||||
1485 | |||||
1486 | static bool_Bool | ||||
1487 | extent_can_coalesce(arena_t *arena, extents_t *extents, const extent_t *inner, | ||||
1488 | const extent_t *outer) { | ||||
1489 | assert(extent_arena_get(inner) == arena)do { if (__builtin_expect(!!(config_debug && !(extent_arena_get (inner) == arena)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1489, "extent_arena_get(inner) == arena"); abort (); } } while (0); | ||||
1490 | if (extent_arena_get(outer) != arena) { | ||||
1491 | return false0; | ||||
1492 | } | ||||
1493 | |||||
1494 | assert(extent_state_get(inner) == extent_state_active)do { if (__builtin_expect(!!(config_debug && !(extent_state_get (inner) == extent_state_active)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1494, "extent_state_get(inner) == extent_state_active" ); abort(); } } while (0); | ||||
1495 | if (extent_state_get(outer) != extents->state) { | ||||
1496 | return false0; | ||||
1497 | } | ||||
1498 | |||||
1499 | if (extent_committed_get(inner) != extent_committed_get(outer)) { | ||||
1500 | return false0; | ||||
1501 | } | ||||
1502 | |||||
1503 | return true1; | ||||
1504 | } | ||||
1505 | |||||
1506 | static bool_Bool | ||||
1507 | extent_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, | ||||
1508 | extents_t *extents, extent_t *inner, extent_t *outer, bool_Bool forward, | ||||
1509 | bool_Bool growing_retained) { | ||||
1510 | assert(extent_can_coalesce(arena, extents, inner, outer))do { if (__builtin_expect(!!(config_debug && !(extent_can_coalesce (arena, extents, inner, outer))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1510, "extent_can_coalesce(arena, extents, inner, outer)" ); abort(); } } while (0); | ||||
1511 | |||||
1512 | extent_activate_locked(tsdn, arena, extents, outer); | ||||
1513 | |||||
1514 | malloc_mutex_unlock(tsdn, &extents->mtx); | ||||
1515 | bool_Bool err = extent_merge_impl(tsdn, arena, r_extent_hooks, | ||||
1516 | forward ? inner : outer, forward ? outer : inner, growing_retained); | ||||
1517 | malloc_mutex_lock(tsdn, &extents->mtx); | ||||
1518 | |||||
1519 | if (err) { | ||||
1520 | extent_deactivate_locked(tsdn, arena, extents, outer); | ||||
1521 | } | ||||
1522 | |||||
1523 | return err; | ||||
1524 | } | ||||
1525 | |||||
1526 | static extent_t * | ||||
1527 | extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, | ||||
1528 | extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, | ||||
1529 | extent_t *extent, bool_Bool *coalesced, bool_Bool growing_retained) { | ||||
1530 | /* | ||||
1531 | * Continue attempting to coalesce until failure, to protect against | ||||
1532 | * races with other threads that are thwarted by this one. | ||||
1533 | */ | ||||
1534 | bool_Bool again; | ||||
1535 | do { | ||||
1536 | again = false0; | ||||
1537 | |||||
1538 | /* Try to coalesce forward. */ | ||||
1539 | extent_t *next = extent_lock_from_addr(tsdn, rtree_ctx, | ||||
1540 | extent_past_get(extent)); | ||||
1541 | if (next != NULL((void*)0)) { | ||||
1542 | /* | ||||
1543 | * extents->mtx only protects against races for | ||||
1544 | * like-state extents, so call extent_can_coalesce() | ||||
1545 | * before releasing next's pool lock. | ||||
1546 | */ | ||||
1547 | bool_Bool can_coalesce = extent_can_coalesce(arena, extents, | ||||
1548 | extent, next); | ||||
1549 | |||||
1550 | extent_unlock(tsdn, next); | ||||
1551 | |||||
1552 | if (can_coalesce && !extent_coalesce(tsdn, arena, | ||||
1553 | r_extent_hooks, extents, extent, next, true1, | ||||
1554 | growing_retained)) { | ||||
1555 | if (extents->delay_coalesce) { | ||||
1556 | /* Do minimal coalescing. */ | ||||
1557 | *coalesced = true1; | ||||
1558 | return extent; | ||||
1559 | } | ||||
1560 | again = true1; | ||||
1561 | } | ||||
1562 | } | ||||
1563 | |||||
1564 | /* Try to coalesce backward. */ | ||||
1565 | extent_t *prev = extent_lock_from_addr(tsdn, rtree_ctx, | ||||
1566 | extent_before_get(extent)); | ||||
1567 | if (prev != NULL((void*)0)) { | ||||
1568 | bool_Bool can_coalesce = extent_can_coalesce(arena, extents, | ||||
1569 | extent, prev); | ||||
1570 | extent_unlock(tsdn, prev); | ||||
1571 | |||||
1572 | if (can_coalesce && !extent_coalesce(tsdn, arena, | ||||
1573 | r_extent_hooks, extents, extent, prev, false0, | ||||
1574 | growing_retained)) { | ||||
1575 | extent = prev; | ||||
1576 | if (extents->delay_coalesce) { | ||||
1577 | /* Do minimal coalescing. */ | ||||
1578 | *coalesced = true1; | ||||
1579 | return extent; | ||||
1580 | } | ||||
1581 | again = true1; | ||||
1582 | } | ||||
1583 | } | ||||
1584 | } while (again); | ||||
1585 | |||||
1586 | if (extents->delay_coalesce) { | ||||
1587 | *coalesced = false0; | ||||
1588 | } | ||||
1589 | return extent; | ||||
1590 | } | ||||
1591 | |||||
1592 | /* | ||||
1593 | * Does the metadata management portions of putting an unused extent into the | ||||
1594 | * given extents_t (coalesces, deregisters slab interiors, the heap operations). | ||||
1595 | */ | ||||
1596 | static void | ||||
1597 | extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, | ||||
1598 | extents_t *extents, extent_t *extent, bool_Bool growing_retained) { | ||||
1599 | rtree_ctx_t rtree_ctx_fallback; | ||||
1600 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); | ||||
1601 | |||||
1602 | assert((extents_state_get(extents) != extent_state_dirty &&do { if (__builtin_expect(!!(config_debug && !((extents_state_get (extents) != extent_state_dirty && extents_state_get( extents) != extent_state_muzzy) || !extent_zeroed_get(extent) )), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1604, "(extents_state_get(extents) != extent_state_dirty && extents_state_get(extents) != extent_state_muzzy) || !extent_zeroed_get(extent)" ); abort(); } } while (0) | ||||
1603 | extents_state_get(extents) != extent_state_muzzy) ||do { if (__builtin_expect(!!(config_debug && !((extents_state_get (extents) != extent_state_dirty && extents_state_get( extents) != extent_state_muzzy) || !extent_zeroed_get(extent) )), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1604, "(extents_state_get(extents) != extent_state_dirty && extents_state_get(extents) != extent_state_muzzy) || !extent_zeroed_get(extent)" ); abort(); } } while (0) | ||||
1604 | !extent_zeroed_get(extent))do { if (__builtin_expect(!!(config_debug && !((extents_state_get (extents) != extent_state_dirty && extents_state_get( extents) != extent_state_muzzy) || !extent_zeroed_get(extent) )), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1604, "(extents_state_get(extents) != extent_state_dirty && extents_state_get(extents) != extent_state_muzzy) || !extent_zeroed_get(extent)" ); abort(); } } while (0); | ||||
1605 | |||||
1606 | malloc_mutex_lock(tsdn, &extents->mtx); | ||||
1607 | extent_hooks_assure_initialized(arena, r_extent_hooks); | ||||
1608 | |||||
1609 | extent_szind_set(extent, NSIZES235); | ||||
1610 | if (extent_slab_get(extent)) { | ||||
1611 | extent_interior_deregister(tsdn, rtree_ctx, extent); | ||||
1612 | extent_slab_set(extent, false0); | ||||
1613 | } | ||||
1614 | |||||
1615 | assert(rtree_extent_read(tsdn, &extents_rtree, rtree_ctx,do { if (__builtin_expect(!!(config_debug && !(rtree_extent_read (tsdn, &extents_rtree, rtree_ctx, (uintptr_t)extent_base_get (extent), 1) == extent)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1616, "rtree_extent_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)extent_base_get(extent), true) == extent" ); abort(); } } while (0) | ||||
1616 | (uintptr_t)extent_base_get(extent), true) == extent)do { if (__builtin_expect(!!(config_debug && !(rtree_extent_read (tsdn, &extents_rtree, rtree_ctx, (uintptr_t)extent_base_get (extent), 1) == extent)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1616, "rtree_extent_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)extent_base_get(extent), true) == extent" ); abort(); } } while (0); | ||||
1617 | |||||
1618 | if (!extents->delay_coalesce) { | ||||
1619 | extent = extent_try_coalesce(tsdn, arena, r_extent_hooks, | ||||
1620 | rtree_ctx, extents, extent, NULL((void*)0), growing_retained); | ||||
1621 | } else if (extent_size_get(extent) >= LARGE_MINCLASS(((size_t)1) << 14)) { | ||||
1622 | /* Always coalesce large extents eagerly. */ | ||||
1623 | bool_Bool coalesced; | ||||
1624 | size_t prev_size; | ||||
1625 | do { | ||||
1626 | prev_size = extent_size_get(extent); | ||||
1627 | assert(extent_state_get(extent) == extent_state_active)do { if (__builtin_expect(!!(config_debug && !(extent_state_get (extent) == extent_state_active)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1627, "extent_state_get(extent) == extent_state_active" ); abort(); } } while (0); | ||||
1628 | extent = extent_try_coalesce(tsdn, arena, | ||||
1629 | r_extent_hooks, rtree_ctx, extents, extent, | ||||
1630 | &coalesced, growing_retained); | ||||
1631 | } while (coalesced && | ||||
1632 | extent_size_get(extent) >= prev_size + LARGE_MINCLASS(((size_t)1) << 14)); | ||||
1633 | } | ||||
1634 | extent_deactivate_locked(tsdn, arena, extents, extent); | ||||
1635 | |||||
1636 | malloc_mutex_unlock(tsdn, &extents->mtx); | ||||
1637 | } | ||||
1638 | |||||
1639 | void | ||||
1640 | extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { | ||||
1641 | extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER((void*)0); | ||||
1642 | |||||
1643 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | ||||
1644 | WITNESS_RANK_CORE11U, 0); | ||||
1645 | |||||
1646 | if (extent_register(tsdn, extent)) { | ||||
1647 | extents_leak(tsdn, arena, &extent_hooks, | ||||
1648 | &arena->extents_retained, extent, false0); | ||||
1649 | return; | ||||
1650 | } | ||||
1651 | extent_dalloc_wrapper(tsdn, arena, &extent_hooks, extent); | ||||
1652 | } | ||||
1653 | |||||
1654 | static bool_Bool | ||||
1655 | extent_dalloc_default_impl(void *addr, size_t size) { | ||||
1656 | if (!have_dss || !extent_in_dss(addr)) { | ||||
1657 | return extent_dalloc_mmap(addr, size); | ||||
1658 | } | ||||
1659 | return true1; | ||||
1660 | } | ||||
1661 | |||||
1662 | static bool_Bool | ||||
1663 | extent_dalloc_default(extent_hooks_t *extent_hooks, void *addr, size_t size, | ||||
1664 | bool_Bool committed, unsigned arena_ind) { | ||||
1665 | return extent_dalloc_default_impl(addr, size); | ||||
1666 | } | ||||
1667 | |||||
1668 | static bool_Bool | ||||
1669 | extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena, | ||||
1670 | extent_hooks_t **r_extent_hooks, extent_t *extent) { | ||||
1671 | bool_Bool err; | ||||
1672 | |||||
1673 | assert(extent_base_get(extent) != NULL)do { if (__builtin_expect(!!(config_debug && !(extent_base_get (extent) != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1673, "extent_base_get(extent) != NULL"); abort (); } } while (0); | ||||
1674 | assert(extent_size_get(extent) != 0)do { if (__builtin_expect(!!(config_debug && !(extent_size_get (extent) != 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1674, "extent_size_get(extent) != 0"); abort (); } } while (0); | ||||
1675 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | ||||
1676 | WITNESS_RANK_CORE11U, 0); | ||||
1677 | |||||
1678 | extent_addr_set(extent, extent_base_get(extent)); | ||||
1679 | |||||
1680 | extent_hooks_assure_initialized(arena, r_extent_hooks); | ||||
1681 | /* Try to deallocate. */ | ||||
1682 | if (*r_extent_hooks == &extent_hooks_default) { | ||||
1683 | /* Call directly to propagate tsdn. */ | ||||
1684 | err = extent_dalloc_default_impl(extent_base_get(extent), | ||||
1685 | extent_size_get(extent)); | ||||
1686 | } else { | ||||
1687 | extent_hook_pre_reentrancy(tsdn, arena); | ||||
1688 | err = ((*r_extent_hooks)->dalloc == NULL((void*)0) || | ||||
1689 | (*r_extent_hooks)->dalloc(*r_extent_hooks, | ||||
1690 | extent_base_get(extent), extent_size_get(extent), | ||||
1691 | extent_committed_get(extent), arena_ind_get(arena))); | ||||
1692 | extent_hook_post_reentrancy(tsdn); | ||||
1693 | } | ||||
1694 | |||||
1695 | if (!err) { | ||||
1696 | extent_dalloc(tsdn, arena, extent); | ||||
1697 | } | ||||
1698 | |||||
1699 | return err; | ||||
1700 | } | ||||
1701 | |||||
1702 | void | ||||
1703 | extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, | ||||
1704 | extent_hooks_t **r_extent_hooks, extent_t *extent) { | ||||
1705 | assert(extent_dumpable_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent_dumpable_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1705, "extent_dumpable_get(extent)"); abort (); } } while (0); | ||||
1706 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | ||||
1707 | WITNESS_RANK_CORE11U, 0); | ||||
1708 | |||||
1709 | /* | ||||
1710 | * Deregister first to avoid a race with other allocating threads, and | ||||
1711 | * reregister if deallocation fails. | ||||
1712 | */ | ||||
1713 | extent_deregister(tsdn, extent); | ||||
1714 | if (!extent_dalloc_wrapper_try(tsdn, arena, r_extent_hooks, extent)) { | ||||
1715 | return; | ||||
1716 | } | ||||
1717 | |||||
1718 | extent_reregister(tsdn, extent); | ||||
1719 | if (*r_extent_hooks != &extent_hooks_default) { | ||||
1720 | extent_hook_pre_reentrancy(tsdn, arena); | ||||
1721 | } | ||||
1722 | /* Try to decommit; purge if that fails. */ | ||||
1723 | bool_Bool zeroed; | ||||
1724 | if (!extent_committed_get(extent)) { | ||||
1725 | zeroed = true1; | ||||
1726 | } else if (!extent_decommit_wrapper(tsdn, arena, r_extent_hooks, extent, | ||||
1727 | 0, extent_size_get(extent))) { | ||||
1728 | zeroed = true1; | ||||
1729 | } else if ((*r_extent_hooks)->purge_forced != NULL((void*)0) && | ||||
1730 | !(*r_extent_hooks)->purge_forced(*r_extent_hooks, | ||||
1731 | extent_base_get(extent), extent_size_get(extent), 0, | ||||
1732 | extent_size_get(extent), arena_ind_get(arena))) { | ||||
1733 | zeroed = true1; | ||||
1734 | } else if (extent_state_get(extent) == extent_state_muzzy || | ||||
1735 | ((*r_extent_hooks)->purge_lazy != NULL((void*)0) && | ||||
1736 | !(*r_extent_hooks)->purge_lazy(*r_extent_hooks, | ||||
1737 | extent_base_get(extent), extent_size_get(extent), 0, | ||||
1738 | extent_size_get(extent), arena_ind_get(arena)))) { | ||||
1739 | zeroed = false0; | ||||
1740 | } else { | ||||
1741 | zeroed = false0; | ||||
1742 | } | ||||
1743 | if (*r_extent_hooks != &extent_hooks_default) { | ||||
1744 | extent_hook_post_reentrancy(tsdn); | ||||
1745 | } | ||||
1746 | extent_zeroed_set(extent, zeroed); | ||||
1747 | |||||
1748 | if (config_prof) { | ||||
1749 | extent_gdump_sub(tsdn, extent); | ||||
1750 | } | ||||
1751 | |||||
1752 | extent_record(tsdn, arena, r_extent_hooks, &arena->extents_retained, | ||||
1753 | extent, false0); | ||||
1754 | } | ||||
1755 | |||||
1756 | static void | ||||
1757 | extent_destroy_default_impl(void *addr, size_t size) { | ||||
1758 | if (!have_dss || !extent_in_dss(addr)) { | ||||
1759 | pages_unmap(addr, size); | ||||
1760 | } | ||||
1761 | } | ||||
1762 | |||||
1763 | static void | ||||
1764 | extent_destroy_default(extent_hooks_t *extent_hooks, void *addr, size_t size, | ||||
1765 | bool_Bool committed, unsigned arena_ind) { | ||||
1766 | extent_destroy_default_impl(addr, size); | ||||
1767 | } | ||||
1768 | |||||
1769 | void | ||||
1770 | extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, | ||||
1771 | extent_hooks_t **r_extent_hooks, extent_t *extent) { | ||||
1772 | assert(extent_base_get(extent) != NULL)do { if (__builtin_expect(!!(config_debug && !(extent_base_get (extent) != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1772, "extent_base_get(extent) != NULL"); abort (); } } while (0); | ||||
1773 | assert(extent_size_get(extent) != 0)do { if (__builtin_expect(!!(config_debug && !(extent_size_get (extent) != 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1773, "extent_size_get(extent) != 0"); abort (); } } while (0); | ||||
1774 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | ||||
1775 | WITNESS_RANK_CORE11U, 0); | ||||
1776 | |||||
1777 | /* Deregister first to avoid a race with other allocating threads. */ | ||||
1778 | extent_deregister(tsdn, extent); | ||||
1779 | |||||
1780 | extent_addr_set(extent, extent_base_get(extent)); | ||||
1781 | |||||
1782 | extent_hooks_assure_initialized(arena, r_extent_hooks); | ||||
1783 | /* Try to destroy; silently fail otherwise. */ | ||||
1784 | if (*r_extent_hooks == &extent_hooks_default) { | ||||
1785 | /* Call directly to propagate tsdn. */ | ||||
1786 | extent_destroy_default_impl(extent_base_get(extent), | ||||
1787 | extent_size_get(extent)); | ||||
1788 | } else if ((*r_extent_hooks)->destroy != NULL((void*)0)) { | ||||
1789 | extent_hook_pre_reentrancy(tsdn, arena); | ||||
1790 | (*r_extent_hooks)->destroy(*r_extent_hooks, | ||||
1791 | extent_base_get(extent), extent_size_get(extent), | ||||
1792 | extent_committed_get(extent), arena_ind_get(arena)); | ||||
1793 | extent_hook_post_reentrancy(tsdn); | ||||
1794 | } | ||||
1795 | |||||
1796 | extent_dalloc(tsdn, arena, extent); | ||||
1797 | } | ||||
1798 | |||||
1799 | static bool_Bool | ||||
1800 | extent_commit_default(extent_hooks_t *extent_hooks, void *addr, size_t size, | ||||
1801 | size_t offset, size_t length, unsigned arena_ind) { | ||||
1802 | return pages_commit((void *)((uintptr_t)addr + (uintptr_t)offset), | ||||
1803 | length); | ||||
1804 | } | ||||
1805 | |||||
1806 | static bool_Bool | ||||
1807 | extent_commit_impl(tsdn_t *tsdn, arena_t *arena, | ||||
1808 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, | ||||
1809 | size_t length, bool_Bool growing_retained) { | ||||
1810 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | ||||
1811 | WITNESS_RANK_CORE11U, growing_retained ? 1 : 0); | ||||
1812 | |||||
1813 | extent_hooks_assure_initialized(arena, r_extent_hooks); | ||||
1814 | if (*r_extent_hooks != &extent_hooks_default) { | ||||
1815 | extent_hook_pre_reentrancy(tsdn, arena); | ||||
1816 | } | ||||
1817 | bool_Bool err = ((*r_extent_hooks)->commit == NULL((void*)0) || | ||||
1818 | (*r_extent_hooks)->commit(*r_extent_hooks, extent_base_get(extent), | ||||
1819 | extent_size_get(extent), offset, length, arena_ind_get(arena))); | ||||
1820 | if (*r_extent_hooks != &extent_hooks_default) { | ||||
1821 | extent_hook_post_reentrancy(tsdn); | ||||
1822 | } | ||||
1823 | extent_committed_set(extent, extent_committed_get(extent) || !err); | ||||
1824 | return err; | ||||
1825 | } | ||||
1826 | |||||
1827 | bool_Bool | ||||
1828 | extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, | ||||
1829 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, | ||||
1830 | size_t length) { | ||||
1831 | return extent_commit_impl(tsdn, arena, r_extent_hooks, extent, offset, | ||||
1832 | length, false0); | ||||
1833 | } | ||||
1834 | |||||
1835 | static bool_Bool | ||||
1836 | extent_decommit_default(extent_hooks_t *extent_hooks, void *addr, size_t size, | ||||
1837 | size_t offset, size_t length, unsigned arena_ind) { | ||||
1838 | return pages_decommit((void *)((uintptr_t)addr + (uintptr_t)offset), | ||||
1839 | length); | ||||
1840 | } | ||||
1841 | |||||
1842 | bool_Bool | ||||
1843 | extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, | ||||
1844 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, | ||||
1845 | size_t length) { | ||||
1846 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | ||||
1847 | WITNESS_RANK_CORE11U, 0); | ||||
1848 | |||||
1849 | extent_hooks_assure_initialized(arena, r_extent_hooks); | ||||
1850 | |||||
1851 | if (*r_extent_hooks != &extent_hooks_default) { | ||||
1852 | extent_hook_pre_reentrancy(tsdn, arena); | ||||
1853 | } | ||||
1854 | bool_Bool err = ((*r_extent_hooks)->decommit == NULL((void*)0) || | ||||
1855 | (*r_extent_hooks)->decommit(*r_extent_hooks, | ||||
1856 | extent_base_get(extent), extent_size_get(extent), offset, length, | ||||
1857 | arena_ind_get(arena))); | ||||
1858 | if (*r_extent_hooks != &extent_hooks_default) { | ||||
1859 | extent_hook_post_reentrancy(tsdn); | ||||
1860 | } | ||||
1861 | extent_committed_set(extent, extent_committed_get(extent) && err); | ||||
1862 | return err; | ||||
1863 | } | ||||
1864 | |||||
1865 | #ifdef PAGES_CAN_PURGE_LAZY | ||||
1866 | static bool_Bool | ||||
1867 | extent_purge_lazy_default(extent_hooks_t *extent_hooks, void *addr, size_t size, | ||||
1868 | size_t offset, size_t length, unsigned arena_ind) { | ||||
1869 | assert(addr != NULL)do { if (__builtin_expect(!!(config_debug && !(addr != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1869, "addr != NULL"); abort(); } } while ( 0); | ||||
1870 | assert((offset & PAGE_MASK) == 0)do { if (__builtin_expect(!!(config_debug && !((offset & ((size_t)(((size_t)(1U << 12)) - 1))) == 0)), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1870, "(offset & PAGE_MASK) == 0"); abort (); } } while (0); | ||||
1871 | assert(length != 0)do { if (__builtin_expect(!!(config_debug && !(length != 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1871, "length != 0"); abort(); } } while (0 ); | ||||
1872 | assert((length & PAGE_MASK) == 0)do { if (__builtin_expect(!!(config_debug && !((length & ((size_t)(((size_t)(1U << 12)) - 1))) == 0)), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1872, "(length & PAGE_MASK) == 0"); abort (); } } while (0); | ||||
1873 | |||||
1874 | return pages_purge_lazy((void *)((uintptr_t)addr + (uintptr_t)offset), | ||||
1875 | length); | ||||
1876 | } | ||||
1877 | #endif | ||||
1878 | |||||
1879 | static bool_Bool | ||||
1880 | extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena, | ||||
1881 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, | ||||
1882 | size_t length, bool_Bool growing_retained) { | ||||
1883 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | ||||
1884 | WITNESS_RANK_CORE11U, growing_retained ? 1 : 0); | ||||
1885 | |||||
1886 | extent_hooks_assure_initialized(arena, r_extent_hooks); | ||||
1887 | |||||
1888 | if ((*r_extent_hooks)->purge_lazy == NULL((void*)0)) { | ||||
1889 | return true1; | ||||
1890 | } | ||||
1891 | if (*r_extent_hooks != &extent_hooks_default) { | ||||
1892 | extent_hook_pre_reentrancy(tsdn, arena); | ||||
1893 | } | ||||
1894 | bool_Bool err = (*r_extent_hooks)->purge_lazy(*r_extent_hooks, | ||||
1895 | extent_base_get(extent), extent_size_get(extent), offset, length, | ||||
1896 | arena_ind_get(arena)); | ||||
1897 | if (*r_extent_hooks != &extent_hooks_default) { | ||||
1898 | extent_hook_post_reentrancy(tsdn); | ||||
1899 | } | ||||
1900 | |||||
1901 | return err; | ||||
1902 | } | ||||
1903 | |||||
1904 | bool_Bool | ||||
1905 | extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena, | ||||
1906 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, | ||||
1907 | size_t length) { | ||||
1908 | return extent_purge_lazy_impl(tsdn, arena, r_extent_hooks, extent, | ||||
1909 | offset, length, false0); | ||||
1910 | } | ||||
1911 | |||||
1912 | #ifdef PAGES_CAN_PURGE_FORCED | ||||
1913 | static bool_Bool | ||||
1914 | extent_purge_forced_default(extent_hooks_t *extent_hooks, void *addr, | ||||
1915 | size_t size, size_t offset, size_t length, unsigned arena_ind) { | ||||
1916 | assert(addr != NULL)do { if (__builtin_expect(!!(config_debug && !(addr != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1916, "addr != NULL"); abort(); } } while ( 0); | ||||
1917 | assert((offset & PAGE_MASK) == 0)do { if (__builtin_expect(!!(config_debug && !((offset & ((size_t)(((size_t)(1U << 12)) - 1))) == 0)), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1917, "(offset & PAGE_MASK) == 0"); abort (); } } while (0); | ||||
1918 | assert(length != 0)do { if (__builtin_expect(!!(config_debug && !(length != 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1918, "length != 0"); abort(); } } while (0 ); | ||||
1919 | assert((length & PAGE_MASK) == 0)do { if (__builtin_expect(!!(config_debug && !((length & ((size_t)(((size_t)(1U << 12)) - 1))) == 0)), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1919, "(length & PAGE_MASK) == 0"); abort (); } } while (0); | ||||
1920 | |||||
1921 | return pages_purge_forced((void *)((uintptr_t)addr + | ||||
1922 | (uintptr_t)offset), length); | ||||
1923 | } | ||||
1924 | #endif | ||||
1925 | |||||
1926 | static bool_Bool | ||||
1927 | extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena, | ||||
1928 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, | ||||
1929 | size_t length, bool_Bool growing_retained) { | ||||
1930 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | ||||
1931 | WITNESS_RANK_CORE11U, growing_retained ? 1 : 0); | ||||
1932 | |||||
1933 | extent_hooks_assure_initialized(arena, r_extent_hooks); | ||||
1934 | |||||
1935 | if ((*r_extent_hooks)->purge_forced == NULL((void*)0)) { | ||||
1936 | return true1; | ||||
1937 | } | ||||
1938 | if (*r_extent_hooks != &extent_hooks_default) { | ||||
1939 | extent_hook_pre_reentrancy(tsdn, arena); | ||||
1940 | } | ||||
1941 | bool_Bool err = (*r_extent_hooks)->purge_forced(*r_extent_hooks, | ||||
1942 | extent_base_get(extent), extent_size_get(extent), offset, length, | ||||
1943 | arena_ind_get(arena)); | ||||
1944 | if (*r_extent_hooks != &extent_hooks_default) { | ||||
1945 | extent_hook_post_reentrancy(tsdn); | ||||
1946 | } | ||||
1947 | return err; | ||||
1948 | } | ||||
1949 | |||||
1950 | bool_Bool | ||||
1951 | extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena, | ||||
1952 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, | ||||
1953 | size_t length) { | ||||
1954 | return extent_purge_forced_impl(tsdn, arena, r_extent_hooks, extent, | ||||
1955 | offset, length, false0); | ||||
1956 | } | ||||
1957 | |||||
1958 | #ifdef JEMALLOC_MAPS_COALESCE | ||||
1959 | static bool_Bool | ||||
1960 | extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size, | ||||
1961 | size_t size_a, size_t size_b, bool_Bool committed, unsigned arena_ind) { | ||||
1962 | return !maps_coalesce; | ||||
1963 | } | ||||
1964 | #endif | ||||
1965 | |||||
1966 | /* | ||||
1967 | * Accepts the extent to split, and the characteristics of each side of the | ||||
1968 | * split. The 'a' parameters go with the 'lead' of the resulting pair of | ||||
1969 | * extents (the lower addressed portion of the split), and the 'b' parameters go | ||||
1970 | * with the trail (the higher addressed portion). This makes 'extent' the lead, | ||||
1971 | * and returns the trail (except in case of error). | ||||
1972 | */ | ||||
1973 | static extent_t * | ||||
1974 | extent_split_impl(tsdn_t *tsdn, arena_t *arena, | ||||
1975 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a, | ||||
1976 | szind_t szind_a, bool_Bool slab_a, size_t size_b, szind_t szind_b, bool_Bool slab_b, | ||||
1977 | bool_Bool growing_retained) { | ||||
1978 | assert(extent_size_get(extent) == size_a + size_b)do { if (__builtin_expect(!!(config_debug && !(extent_size_get (extent) == size_a + size_b)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1978, "extent_size_get(extent) == size_a + size_b" ); abort(); } } while (0); | ||||
1979 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | ||||
1980 | WITNESS_RANK_CORE11U, growing_retained
| ||||
1981 | |||||
1982 | extent_hooks_assure_initialized(arena, r_extent_hooks); | ||||
1983 | |||||
1984 | if ((*r_extent_hooks)->split == NULL((void*)0)) { | ||||
1985 | return NULL((void*)0); | ||||
1986 | } | ||||
1987 | |||||
1988 | extent_t *trail = extent_alloc(tsdn, arena); | ||||
1989 | if (trail == NULL((void*)0)) { | ||||
1990 | goto label_error_a; | ||||
1991 | } | ||||
1992 | |||||
1993 | extent_init(trail, arena, (void *)((uintptr_t)extent_base_get(extent) + | ||||
1994 | size_a), size_b, slab_b, szind_b, extent_sn_get(extent), | ||||
1995 | extent_state_get(extent), extent_zeroed_get(extent), | ||||
1996 | extent_committed_get(extent), extent_dumpable_get(extent)); | ||||
1997 | |||||
1998 | rtree_ctx_t rtree_ctx_fallback; | ||||
1999 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); | ||||
2000 | rtree_leaf_elm_t *lead_elm_a, *lead_elm_b; | ||||
2001 | { | ||||
2002 | extent_t lead; | ||||
2003 | |||||
2004 | extent_init(&lead, arena, extent_addr_get(extent), size_a, | ||||
2005 | slab_a, szind_a, extent_sn_get(extent), | ||||
2006 | extent_state_get(extent), extent_zeroed_get(extent), | ||||
2007 | extent_committed_get(extent), extent_dumpable_get(extent)); | ||||
2008 | |||||
2009 | extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, &lead, false0, | ||||
2010 | true1, &lead_elm_a, &lead_elm_b); | ||||
2011 | } | ||||
2012 | rtree_leaf_elm_t *trail_elm_a, *trail_elm_b; | ||||
2013 | extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, trail, false0, true1, | ||||
2014 | &trail_elm_a, &trail_elm_b); | ||||
2015 | |||||
2016 | if (lead_elm_a == NULL((void*)0) || lead_elm_b == NULL((void*)0) || trail_elm_a == NULL((void*)0) | ||||
2017 | || trail_elm_b == NULL((void*)0)) { | ||||
2018 | goto label_error_b; | ||||
2019 | } | ||||
2020 | |||||
2021 | extent_lock2(tsdn, extent, trail); | ||||
2022 | |||||
2023 | if (*r_extent_hooks != &extent_hooks_default) { | ||||
2024 | extent_hook_pre_reentrancy(tsdn, arena); | ||||
2025 | } | ||||
2026 | bool_Bool err = (*r_extent_hooks)->split(*r_extent_hooks, extent_base_get(extent), | ||||
2027 | size_a + size_b, size_a, size_b, extent_committed_get(extent), | ||||
2028 | arena_ind_get(arena)); | ||||
2029 | if (*r_extent_hooks != &extent_hooks_default) { | ||||
2030 | extent_hook_post_reentrancy(tsdn); | ||||
2031 | } | ||||
2032 | if (err) { | ||||
2033 | goto label_error_c; | ||||
2034 | } | ||||
2035 | |||||
2036 | extent_size_set(extent, size_a); | ||||
2037 | extent_szind_set(extent, szind_a); | ||||
2038 | |||||
2039 | extent_rtree_write_acquired(tsdn, lead_elm_a, lead_elm_b, extent, | ||||
2040 | szind_a, slab_a); | ||||
2041 | extent_rtree_write_acquired(tsdn, trail_elm_a, trail_elm_b, trail, | ||||
2042 | szind_b, slab_b); | ||||
2043 | |||||
2044 | extent_unlock2(tsdn, extent, trail); | ||||
2045 | |||||
2046 | return trail; | ||||
2047 | label_error_c: | ||||
2048 | extent_unlock2(tsdn, extent, trail); | ||||
2049 | label_error_b: | ||||
2050 | extent_dalloc(tsdn, arena, trail); | ||||
2051 | label_error_a: | ||||
2052 | return NULL((void*)0); | ||||
2053 | } | ||||
2054 | |||||
2055 | extent_t * | ||||
2056 | extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, | ||||
2057 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a, | ||||
2058 | szind_t szind_a, bool_Bool slab_a, size_t size_b, szind_t szind_b, bool_Bool slab_b) { | ||||
2059 | return extent_split_impl(tsdn, arena, r_extent_hooks, extent, size_a, | ||||
| |||||
2060 | szind_a, slab_a, size_b, szind_b, slab_b, false0); | ||||
2061 | } | ||||
2062 | |||||
2063 | static bool_Bool | ||||
2064 | extent_merge_default_impl(void *addr_a, void *addr_b) { | ||||
2065 | if (!maps_coalesce) { | ||||
2066 | return true1; | ||||
2067 | } | ||||
2068 | if (have_dss && !extent_dss_mergeable(addr_a, addr_b)) { | ||||
2069 | return true1; | ||||
2070 | } | ||||
2071 | |||||
2072 | return false0; | ||||
2073 | } | ||||
2074 | |||||
2075 | #ifdef JEMALLOC_MAPS_COALESCE | ||||
2076 | static bool_Bool | ||||
2077 | extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, | ||||
2078 | void *addr_b, size_t size_b, bool_Bool committed, unsigned arena_ind) { | ||||
2079 | return extent_merge_default_impl(addr_a, addr_b); | ||||
2080 | } | ||||
2081 | #endif | ||||
2082 | |||||
2083 | static bool_Bool | ||||
2084 | extent_merge_impl(tsdn_t *tsdn, arena_t *arena, | ||||
2085 | extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b, | ||||
2086 | bool_Bool growing_retained) { | ||||
2087 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | ||||
2088 | WITNESS_RANK_CORE11U, growing_retained ? 1 : 0); | ||||
2089 | |||||
2090 | extent_hooks_assure_initialized(arena, r_extent_hooks); | ||||
2091 | |||||
2092 | if ((*r_extent_hooks)->merge == NULL((void*)0)) { | ||||
2093 | return true1; | ||||
2094 | } | ||||
2095 | |||||
2096 | bool_Bool err; | ||||
2097 | if (*r_extent_hooks == &extent_hooks_default) { | ||||
2098 | /* Call directly to propagate tsdn. */ | ||||
2099 | err = extent_merge_default_impl(extent_base_get(a), | ||||
2100 | extent_base_get(b)); | ||||
2101 | } else { | ||||
2102 | extent_hook_pre_reentrancy(tsdn, arena); | ||||
2103 | err = (*r_extent_hooks)->merge(*r_extent_hooks, | ||||
2104 | extent_base_get(a), extent_size_get(a), extent_base_get(b), | ||||
2105 | extent_size_get(b), extent_committed_get(a), | ||||
2106 | arena_ind_get(arena)); | ||||
2107 | extent_hook_post_reentrancy(tsdn); | ||||
2108 | } | ||||
2109 | |||||
2110 | if (err) { | ||||
2111 | return true1; | ||||
2112 | } | ||||
2113 | |||||
2114 | /* | ||||
2115 | * The rtree writes must happen while all the relevant elements are | ||||
2116 | * owned, so the following code uses decomposed helper functions rather | ||||
2117 | * than extent_{,de}register() to do things in the right order. | ||||
2118 | */ | ||||
2119 | rtree_ctx_t rtree_ctx_fallback; | ||||
2120 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); | ||||
2121 | rtree_leaf_elm_t *a_elm_a, *a_elm_b, *b_elm_a, *b_elm_b; | ||||
2122 | extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, a, true1, false0, &a_elm_a, | ||||
2123 | &a_elm_b); | ||||
2124 | extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, b, true1, false0, &b_elm_a, | ||||
2125 | &b_elm_b); | ||||
2126 | |||||
2127 | extent_lock2(tsdn, a, b); | ||||
2128 | |||||
2129 | if (a_elm_b != NULL((void*)0)) { | ||||
2130 | rtree_leaf_elm_write(tsdn, &extents_rtree, a_elm_b, NULL((void*)0), | ||||
2131 | NSIZES235, false0); | ||||
2132 | } | ||||
2133 | if (b_elm_b != NULL((void*)0)) { | ||||
2134 | rtree_leaf_elm_write(tsdn, &extents_rtree, b_elm_a, NULL((void*)0), | ||||
2135 | NSIZES235, false0); | ||||
2136 | } else { | ||||
2137 | b_elm_b = b_elm_a; | ||||
2138 | } | ||||
2139 | |||||
2140 | extent_size_set(a, extent_size_get(a) + extent_size_get(b)); | ||||
2141 | extent_szind_set(a, NSIZES235); | ||||
2142 | extent_sn_set(a, (extent_sn_get(a) < extent_sn_get(b)) ? | ||||
2143 | extent_sn_get(a) : extent_sn_get(b)); | ||||
2144 | extent_zeroed_set(a, extent_zeroed_get(a) && extent_zeroed_get(b)); | ||||
2145 | |||||
2146 | extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a, NSIZES235, false0); | ||||
2147 | |||||
2148 | extent_unlock2(tsdn, a, b); | ||||
2149 | |||||
2150 | extent_dalloc(tsdn, extent_arena_get(b), b); | ||||
2151 | |||||
2152 | return false0; | ||||
2153 | } | ||||
2154 | |||||
2155 | bool_Bool | ||||
2156 | extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, | ||||
2157 | extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b) { | ||||
2158 | return extent_merge_impl(tsdn, arena, r_extent_hooks, a, b, false0); | ||||
2159 | } | ||||
2160 | |||||
2161 | bool_Bool | ||||
2162 | extent_boot(void) { | ||||
2163 | if (rtree_new(&extents_rtree, true1)) { | ||||
2164 | return true1; | ||||
2165 | } | ||||
2166 | |||||
2167 | if (mutex_pool_init(&extent_mutex_pool, "extent_mutex_pool", | ||||
2168 | WITNESS_RANK_EXTENT_POOL16U)) { | ||||
2169 | return true1; | ||||
2170 | } | ||||
2171 | |||||
2172 | if (have_dss) { | ||||
2173 | extent_dss_boot(); | ||||
2174 | } | ||||
2175 | |||||
2176 | return false0; | ||||
2177 | } |
1 | #ifndef JEMALLOC_INTERNAL_EXTENT_INLINES_H | ||||
2 | #define JEMALLOC_INTERNAL_EXTENT_INLINES_H | ||||
3 | |||||
4 | #include "jemalloc/internal/mutex.h" | ||||
5 | #include "jemalloc/internal/mutex_pool.h" | ||||
6 | #include "jemalloc/internal/pages.h" | ||||
7 | #include "jemalloc/internal/prng.h" | ||||
8 | #include "jemalloc/internal/ql.h" | ||||
9 | #include "jemalloc/internal/sz.h" | ||||
10 | |||||
11 | static inline void | ||||
12 | extent_lock(tsdn_t *tsdn, extent_t *extent) { | ||||
13 | assert(extent != NULL)do { if (__builtin_expect(!!(config_debug && !(extent != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 13, "extent != NULL" ); abort(); } } while (0); | ||||
14 | mutex_pool_lock(tsdn, &extent_mutex_pool, (uintptr_t)extent); | ||||
15 | } | ||||
16 | |||||
17 | static inline void | ||||
18 | extent_unlock(tsdn_t *tsdn, extent_t *extent) { | ||||
19 | assert(extent != NULL)do { if (__builtin_expect(!!(config_debug && !(extent != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 19, "extent != NULL" ); abort(); } } while (0); | ||||
20 | mutex_pool_unlock(tsdn, &extent_mutex_pool, (uintptr_t)extent); | ||||
21 | } | ||||
22 | |||||
23 | static inline void | ||||
24 | extent_lock2(tsdn_t *tsdn, extent_t *extent1, extent_t *extent2) { | ||||
25 | assert(extent1 != NULL && extent2 != NULL)do { if (__builtin_expect(!!(config_debug && !(extent1 != ((void*)0) && extent2 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "include/jemalloc/internal/extent_inlines.h" , 25, "extent1 != NULL && extent2 != NULL"); abort(); } } while (0); | ||||
26 | mutex_pool_lock2(tsdn, &extent_mutex_pool, (uintptr_t)extent1, | ||||
27 | (uintptr_t)extent2); | ||||
28 | } | ||||
29 | |||||
30 | static inline void | ||||
31 | extent_unlock2(tsdn_t *tsdn, extent_t *extent1, extent_t *extent2) { | ||||
32 | assert(extent1 != NULL && extent2 != NULL)do { if (__builtin_expect(!!(config_debug && !(extent1 != ((void*)0) && extent2 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "include/jemalloc/internal/extent_inlines.h" , 32, "extent1 != NULL && extent2 != NULL"); abort(); } } while (0); | ||||
33 | mutex_pool_unlock2(tsdn, &extent_mutex_pool, (uintptr_t)extent1, | ||||
34 | (uintptr_t)extent2); | ||||
35 | } | ||||
36 | |||||
37 | static inline arena_t * | ||||
38 | extent_arena_get(const extent_t *extent) { | ||||
39 | unsigned arena_ind = (unsigned)((extent->e_bits & | ||||
40 | EXTENT_BITS_ARENA_MASK((((((uint64_t)0x1U) << (12)) - 1)) << (0))) >> EXTENT_BITS_ARENA_SHIFT0); | ||||
41 | /* | ||||
42 | * The following check is omitted because we should never actually read | ||||
43 | * a NULL arena pointer. | ||||
44 | */ | ||||
45 | if (false0 && arena_ind >= MALLOCX_ARENA_LIMIT((1 << 12) - 1)) { | ||||
46 | return NULL((void*)0); | ||||
47 | } | ||||
48 | assert(arena_ind < MALLOCX_ARENA_LIMIT)do { if (__builtin_expect(!!(config_debug && !(arena_ind < ((1 << 12) - 1))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 48, "arena_ind < MALLOCX_ARENA_LIMIT" ); abort(); } } while (0); | ||||
49 | return (arena_t *)atomic_load_p(&arenas[arena_ind], ATOMIC_ACQUIREatomic_memory_order_acquire); | ||||
50 | } | ||||
51 | |||||
52 | static inline szind_t | ||||
53 | extent_szind_get_maybe_invalid(const extent_t *extent) { | ||||
54 | szind_t szind = (szind_t)((extent->e_bits & EXTENT_BITS_SZIND_MASK((((((uint64_t)0x1U) << (8)) - 1)) << ((2 + (1 + ( 1 + (1 + (1 + (12 + 0))))))))) >> | ||||
55 | EXTENT_BITS_SZIND_SHIFT(2 + (1 + (1 + (1 + (1 + (12 + 0))))))); | ||||
56 | assert(szind <= NSIZES)do { if (__builtin_expect(!!(config_debug && !(szind <= 235)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 56, "szind <= NSIZES" ); abort(); } } while (0); | ||||
57 | return szind; | ||||
58 | } | ||||
59 | |||||
60 | static inline szind_t | ||||
61 | extent_szind_get(const extent_t *extent) { | ||||
62 | szind_t szind = extent_szind_get_maybe_invalid(extent); | ||||
63 | assert(szind < NSIZES)do { if (__builtin_expect(!!(config_debug && !(szind < 235)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 63, "szind < NSIZES" ); abort(); } } while (0); /* Never call when "invalid". */ | ||||
64 | return szind; | ||||
65 | } | ||||
66 | |||||
67 | static inline size_t | ||||
68 | extent_usize_get(const extent_t *extent) { | ||||
69 | return sz_index2size(extent_szind_get(extent)); | ||||
70 | } | ||||
71 | |||||
72 | static inline size_t | ||||
73 | extent_sn_get(const extent_t *extent) { | ||||
74 | return (size_t)((extent->e_bits & EXTENT_BITS_SN_MASK((18446744073709551615UL) << (((12 - 3) + 1) + (8 + (2 + (1 + (1 + (1 + (1 + (12 + 0)))))))))) >> | ||||
75 | EXTENT_BITS_SN_SHIFT(((12 - 3) + 1) + (8 + (2 + (1 + (1 + (1 + (1 + (12 + 0)))))) ))); | ||||
76 | } | ||||
77 | |||||
78 | static inline extent_state_t | ||||
79 | extent_state_get(const extent_t *extent) { | ||||
80 | return (extent_state_t)((extent->e_bits & EXTENT_BITS_STATE_MASK((((((uint64_t)0x1U) << (2)) - 1)) << ((1 + (1 + ( 1 + (1 + (12 + 0)))))))) >> | ||||
81 | EXTENT_BITS_STATE_SHIFT(1 + (1 + (1 + (1 + (12 + 0)))))); | ||||
82 | } | ||||
83 | |||||
84 | static inline bool_Bool | ||||
85 | extent_zeroed_get(const extent_t *extent) { | ||||
86 | return (bool_Bool)((extent->e_bits & EXTENT_BITS_ZEROED_MASK((((((uint64_t)0x1U) << (1)) - 1)) << ((1 + (1 + ( 1 + (12 + 0))))))) >> | ||||
87 | EXTENT_BITS_ZEROED_SHIFT(1 + (1 + (1 + (12 + 0))))); | ||||
88 | } | ||||
89 | |||||
90 | static inline bool_Bool | ||||
91 | extent_committed_get(const extent_t *extent) { | ||||
92 | return (bool_Bool)((extent->e_bits & EXTENT_BITS_COMMITTED_MASK((((((uint64_t)0x1U) << (1)) - 1)) << ((1 + (12 + 0))))) >> | ||||
93 | EXTENT_BITS_COMMITTED_SHIFT(1 + (12 + 0))); | ||||
94 | } | ||||
95 | |||||
96 | static inline bool_Bool | ||||
97 | extent_dumpable_get(const extent_t *extent) { | ||||
98 | return (bool_Bool)((extent->e_bits & EXTENT_BITS_DUMPABLE_MASK((((((uint64_t)0x1U) << (1)) - 1)) << ((1 + (1 + ( 12 + 0)))))) >> | ||||
99 | EXTENT_BITS_DUMPABLE_SHIFT(1 + (1 + (12 + 0)))); | ||||
100 | } | ||||
101 | |||||
102 | static inline bool_Bool | ||||
103 | extent_slab_get(const extent_t *extent) { | ||||
104 | return (bool_Bool)((extent->e_bits & EXTENT_BITS_SLAB_MASK((((((uint64_t)0x1U) << (1)) - 1)) << ((12 + 0)))) >> | ||||
105 | EXTENT_BITS_SLAB_SHIFT(12 + 0)); | ||||
106 | } | ||||
107 | |||||
108 | static inline unsigned | ||||
109 | extent_nfree_get(const extent_t *extent) { | ||||
110 | assert(extent_slab_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent_slab_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 110, "extent_slab_get(extent)" ); abort(); } } while (0); | ||||
111 | return (unsigned)((extent->e_bits & EXTENT_BITS_NFREE_MASK((((((uint64_t)0x1U) << (((12 - 3) + 1))) - 1)) << ((8 + (2 + (1 + (1 + (1 + (1 + (12 + 0)))))))))) >> | ||||
112 | EXTENT_BITS_NFREE_SHIFT(8 + (2 + (1 + (1 + (1 + (1 + (12 + 0)))))))); | ||||
113 | } | ||||
114 | |||||
115 | static inline void * | ||||
116 | extent_base_get(const extent_t *extent) { | ||||
117 | assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) ||do { if (__builtin_expect(!!(config_debug && !(extent ->e_addr == ((void *)((uintptr_t)(extent->e_addr) & ~((size_t)(((size_t)(1U << 12)) - 1)))) || !extent_slab_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 118, "extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) || !extent_slab_get(extent)" ); abort(); } } while (0) | ||||
118 | !extent_slab_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent ->e_addr == ((void *)((uintptr_t)(extent->e_addr) & ~((size_t)(((size_t)(1U << 12)) - 1)))) || !extent_slab_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 118, "extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) || !extent_slab_get(extent)" ); abort(); } } while (0); | ||||
119 | return PAGE_ADDR2BASE(extent->e_addr)((void *)((uintptr_t)(extent->e_addr) & ~((size_t)(((size_t )(1U << 12)) - 1)))); | ||||
120 | } | ||||
121 | |||||
122 | static inline void * | ||||
123 | extent_addr_get(const extent_t *extent) { | ||||
124 | assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) ||do { if (__builtin_expect(!!(config_debug && !(extent ->e_addr == ((void *)((uintptr_t)(extent->e_addr) & ~((size_t)(((size_t)(1U << 12)) - 1)))) || !extent_slab_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 125, "extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) || !extent_slab_get(extent)" ); abort(); } } while (0) | ||||
125 | !extent_slab_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent ->e_addr == ((void *)((uintptr_t)(extent->e_addr) & ~((size_t)(((size_t)(1U << 12)) - 1)))) || !extent_slab_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 125, "extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) || !extent_slab_get(extent)" ); abort(); } } while (0); | ||||
126 | return extent->e_addr; | ||||
127 | } | ||||
128 | |||||
129 | static inline size_t | ||||
130 | extent_size_get(const extent_t *extent) { | ||||
131 | return (extent->e_size_esn & EXTENT_SIZE_MASK((size_t)~(((size_t)(1U << 12))-1))); | ||||
132 | } | ||||
133 | |||||
134 | static inline size_t | ||||
135 | extent_esn_get(const extent_t *extent) { | ||||
136 | return (extent->e_size_esn & EXTENT_ESN_MASK((size_t)((size_t)(1U << 12))-1)); | ||||
137 | } | ||||
138 | |||||
139 | static inline size_t | ||||
140 | extent_bsize_get(const extent_t *extent) { | ||||
141 | return extent->e_bsize; | ||||
142 | } | ||||
143 | |||||
144 | static inline void * | ||||
145 | extent_before_get(const extent_t *extent) { | ||||
146 | return (void *)((uintptr_t)extent_base_get(extent) - PAGE((size_t)(1U << 12))); | ||||
147 | } | ||||
148 | |||||
149 | static inline void * | ||||
150 | extent_last_get(const extent_t *extent) { | ||||
151 | return (void *)((uintptr_t)extent_base_get(extent) + | ||||
152 | extent_size_get(extent) - PAGE((size_t)(1U << 12))); | ||||
153 | } | ||||
154 | |||||
155 | static inline void * | ||||
156 | extent_past_get(const extent_t *extent) { | ||||
157 | return (void *)((uintptr_t)extent_base_get(extent) + | ||||
158 | extent_size_get(extent)); | ||||
159 | } | ||||
160 | |||||
161 | static inline arena_slab_data_t * | ||||
162 | extent_slab_data_get(extent_t *extent) { | ||||
163 | assert(extent_slab_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent_slab_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 163, "extent_slab_get(extent)" ); abort(); } } while (0); | ||||
164 | return &extent->e_slab_data; | ||||
165 | } | ||||
166 | |||||
167 | static inline const arena_slab_data_t * | ||||
168 | extent_slab_data_get_const(const extent_t *extent) { | ||||
169 | assert(extent_slab_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent_slab_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 169, "extent_slab_get(extent)" ); abort(); } } while (0); | ||||
170 | return &extent->e_slab_data; | ||||
171 | } | ||||
172 | |||||
173 | static inline prof_tctx_t * | ||||
174 | extent_prof_tctx_get(const extent_t *extent) { | ||||
175 | return (prof_tctx_t *)atomic_load_p(&extent->e_prof_tctx, | ||||
176 | ATOMIC_ACQUIREatomic_memory_order_acquire); | ||||
177 | } | ||||
178 | |||||
179 | static inline void | ||||
180 | extent_arena_set(extent_t *extent, arena_t *arena) { | ||||
181 | unsigned arena_ind = (arena
| ||||
182 | MALLOCX_ARENA_BITS12) - 1); | ||||
183 | extent->e_bits = (extent->e_bits & ~EXTENT_BITS_ARENA_MASK((((((uint64_t)0x1U) << (12)) - 1)) << (0))) | | ||||
| |||||
184 | ((uint64_t)arena_ind << EXTENT_BITS_ARENA_SHIFT0); | ||||
185 | } | ||||
186 | |||||
187 | static inline void | ||||
188 | extent_addr_set(extent_t *extent, void *addr) { | ||||
189 | extent->e_addr = addr; | ||||
190 | } | ||||
191 | |||||
192 | static inline void | ||||
193 | extent_addr_randomize(UNUSED__attribute__((unused)) tsdn_t *tsdn, extent_t *extent, size_t alignment) { | ||||
194 | assert(extent_base_get(extent) == extent_addr_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent_base_get (extent) == extent_addr_get(extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 194, "extent_base_get(extent) == extent_addr_get(extent)" ); abort(); } } while (0); | ||||
195 | |||||
196 | if (alignment < PAGE((size_t)(1U << 12))) { | ||||
197 | unsigned lg_range = LG_PAGE12 - | ||||
198 | lg_floor(CACHELINE_CEILING(alignment)(((alignment) + (64 - 1)) & ~(64 - 1))); | ||||
199 | size_t r; | ||||
200 | if (!tsdn_null(tsdn)) { | ||||
201 | tsd_t *tsd = tsdn_tsd(tsdn); | ||||
202 | r = (size_t)prng_lg_range_u64( | ||||
203 | tsd_offset_statep_get(tsd), lg_range); | ||||
204 | } else { | ||||
205 | r = prng_lg_range_zu( | ||||
206 | &extent_arena_get(extent)->offset_state, | ||||
207 | lg_range, true1); | ||||
208 | } | ||||
209 | uintptr_t random_offset = ((uintptr_t)r) << (LG_PAGE12 - | ||||
210 | lg_range); | ||||
211 | extent->e_addr = (void *)((uintptr_t)extent->e_addr + | ||||
212 | random_offset); | ||||
213 | assert(ALIGNMENT_ADDR2BASE(extent->e_addr, alignment) ==do { if (__builtin_expect(!!(config_debug && !(((void *)((uintptr_t)(extent->e_addr) & ((~(alignment)) + 1) )) == extent->e_addr)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 214, "ALIGNMENT_ADDR2BASE(extent->e_addr, alignment) == extent->e_addr" ); abort(); } } while (0) | ||||
214 | extent->e_addr)do { if (__builtin_expect(!!(config_debug && !(((void *)((uintptr_t)(extent->e_addr) & ((~(alignment)) + 1) )) == extent->e_addr)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 214, "ALIGNMENT_ADDR2BASE(extent->e_addr, alignment) == extent->e_addr" ); abort(); } } while (0); | ||||
215 | } | ||||
216 | } | ||||
217 | |||||
218 | static inline void | ||||
219 | extent_size_set(extent_t *extent, size_t size) { | ||||
220 | assert((size & ~EXTENT_SIZE_MASK) == 0)do { if (__builtin_expect(!!(config_debug && !((size & ~((size_t)~(((size_t)(1U << 12))-1))) == 0)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "include/jemalloc/internal/extent_inlines.h" , 220, "(size & ~EXTENT_SIZE_MASK) == 0"); abort(); } } while (0); | ||||
221 | extent->e_size_esn = size | (extent->e_size_esn & ~EXTENT_SIZE_MASK((size_t)~(((size_t)(1U << 12))-1))); | ||||
222 | } | ||||
223 | |||||
224 | static inline void | ||||
225 | extent_esn_set(extent_t *extent, size_t esn) { | ||||
226 | extent->e_size_esn = (extent->e_size_esn & ~EXTENT_ESN_MASK((size_t)((size_t)(1U << 12))-1)) | (esn & | ||||
227 | EXTENT_ESN_MASK((size_t)((size_t)(1U << 12))-1)); | ||||
228 | } | ||||
229 | |||||
230 | static inline void | ||||
231 | extent_bsize_set(extent_t *extent, size_t bsize) { | ||||
232 | extent->e_bsize = bsize; | ||||
233 | } | ||||
234 | |||||
235 | static inline void | ||||
236 | extent_szind_set(extent_t *extent, szind_t szind) { | ||||
237 | assert(szind <= NSIZES)do { if (__builtin_expect(!!(config_debug && !(szind <= 235)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 237, "szind <= NSIZES" ); abort(); } } while (0); /* NSIZES means "invalid". */ | ||||
238 | extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SZIND_MASK((((((uint64_t)0x1U) << (8)) - 1)) << ((2 + (1 + ( 1 + (1 + (1 + (12 + 0))))))))) | | ||||
239 | ((uint64_t)szind << EXTENT_BITS_SZIND_SHIFT(2 + (1 + (1 + (1 + (1 + (12 + 0))))))); | ||||
240 | } | ||||
241 | |||||
242 | static inline void | ||||
243 | extent_nfree_set(extent_t *extent, unsigned nfree) { | ||||
244 | assert(extent_slab_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent_slab_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 244, "extent_slab_get(extent)" ); abort(); } } while (0); | ||||
245 | extent->e_bits = (extent->e_bits & ~EXTENT_BITS_NFREE_MASK((((((uint64_t)0x1U) << (((12 - 3) + 1))) - 1)) << ((8 + (2 + (1 + (1 + (1 + (1 + (12 + 0)))))))))) | | ||||
246 | ((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT(8 + (2 + (1 + (1 + (1 + (1 + (12 + 0)))))))); | ||||
247 | } | ||||
248 | |||||
249 | static inline void | ||||
250 | extent_nfree_inc(extent_t *extent) { | ||||
251 | assert(extent_slab_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent_slab_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 251, "extent_slab_get(extent)" ); abort(); } } while (0); | ||||
252 | extent->e_bits += ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT(8 + (2 + (1 + (1 + (1 + (1 + (12 + 0)))))))); | ||||
253 | } | ||||
254 | |||||
255 | static inline void | ||||
256 | extent_nfree_dec(extent_t *extent) { | ||||
257 | assert(extent_slab_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent_slab_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 257, "extent_slab_get(extent)" ); abort(); } } while (0); | ||||
258 | extent->e_bits -= ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT(8 + (2 + (1 + (1 + (1 + (1 + (12 + 0)))))))); | ||||
259 | } | ||||
260 | |||||
261 | static inline void | ||||
262 | extent_sn_set(extent_t *extent, size_t sn) { | ||||
263 | extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SN_MASK((18446744073709551615UL) << (((12 - 3) + 1) + (8 + (2 + (1 + (1 + (1 + (1 + (12 + 0)))))))))) | | ||||
264 | ((uint64_t)sn << EXTENT_BITS_SN_SHIFT(((12 - 3) + 1) + (8 + (2 + (1 + (1 + (1 + (1 + (12 + 0)))))) ))); | ||||
265 | } | ||||
266 | |||||
267 | static inline void | ||||
268 | extent_state_set(extent_t *extent, extent_state_t state) { | ||||
269 | extent->e_bits = (extent->e_bits & ~EXTENT_BITS_STATE_MASK((((((uint64_t)0x1U) << (2)) - 1)) << ((1 + (1 + ( 1 + (1 + (12 + 0)))))))) | | ||||
270 | ((uint64_t)state << EXTENT_BITS_STATE_SHIFT(1 + (1 + (1 + (1 + (12 + 0)))))); | ||||
271 | } | ||||
272 | |||||
273 | static inline void | ||||
274 | extent_zeroed_set(extent_t *extent, bool_Bool zeroed) { | ||||
275 | extent->e_bits = (extent->e_bits & ~EXTENT_BITS_ZEROED_MASK((((((uint64_t)0x1U) << (1)) - 1)) << ((1 + (1 + ( 1 + (12 + 0))))))) | | ||||
276 | ((uint64_t)zeroed << EXTENT_BITS_ZEROED_SHIFT(1 + (1 + (1 + (12 + 0))))); | ||||
277 | } | ||||
278 | |||||
279 | static inline void | ||||
280 | extent_committed_set(extent_t *extent, bool_Bool committed) { | ||||
281 | extent->e_bits = (extent->e_bits & ~EXTENT_BITS_COMMITTED_MASK((((((uint64_t)0x1U) << (1)) - 1)) << ((1 + (12 + 0))))) | | ||||
282 | ((uint64_t)committed << EXTENT_BITS_COMMITTED_SHIFT(1 + (12 + 0))); | ||||
283 | } | ||||
284 | |||||
285 | static inline void | ||||
286 | extent_dumpable_set(extent_t *extent, bool_Bool dumpable) { | ||||
287 | extent->e_bits = (extent->e_bits & ~EXTENT_BITS_DUMPABLE_MASK((((((uint64_t)0x1U) << (1)) - 1)) << ((1 + (1 + ( 12 + 0)))))) | | ||||
288 | ((uint64_t)dumpable << EXTENT_BITS_DUMPABLE_SHIFT(1 + (1 + (12 + 0)))); | ||||
289 | } | ||||
290 | |||||
291 | static inline void | ||||
292 | extent_slab_set(extent_t *extent, bool_Bool slab) { | ||||
293 | extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SLAB_MASK((((((uint64_t)0x1U) << (1)) - 1)) << ((12 + 0)))) | | ||||
294 | ((uint64_t)slab << EXTENT_BITS_SLAB_SHIFT(12 + 0)); | ||||
295 | } | ||||
296 | |||||
297 | static inline void | ||||
298 | extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx) { | ||||
299 | atomic_store_p(&extent->e_prof_tctx, tctx, ATOMIC_RELEASEatomic_memory_order_release); | ||||
300 | } | ||||
301 | |||||
302 | static inline void | ||||
303 | extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size, | ||||
304 | bool_Bool slab, szind_t szind, size_t sn, extent_state_t state, bool_Bool zeroed, | ||||
305 | bool_Bool committed, bool_Bool dumpable) { | ||||
306 | assert(addr == PAGE_ADDR2BASE(addr) || !slab)do { if (__builtin_expect(!!(config_debug && !(addr == ((void *)((uintptr_t)(addr) & ~((size_t)(((size_t)(1U << 12)) - 1)))) || !slab)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "include/jemalloc/internal/extent_inlines.h", 306, "addr == PAGE_ADDR2BASE(addr) || !slab" ); abort(); } } while (0); | ||||
307 | |||||
308 | extent_arena_set(extent, arena); | ||||
309 | extent_addr_set(extent, addr); | ||||
310 | extent_size_set(extent, size); | ||||
311 | extent_slab_set(extent, slab); | ||||
312 | extent_szind_set(extent, szind); | ||||
313 | extent_sn_set(extent, sn); | ||||
314 | extent_state_set(extent, state); | ||||
315 | extent_zeroed_set(extent, zeroed); | ||||
316 | extent_committed_set(extent, committed); | ||||
317 | extent_dumpable_set(extent, dumpable); | ||||
318 | ql_elm_new(extent, ql_link)do { ((extent))->ql_link.qre_next = ((extent)); ((extent)) ->ql_link.qre_prev = ((extent)); } while (0); | ||||
319 | if (config_prof) { | ||||
320 | extent_prof_tctx_set(extent, NULL((void*)0)); | ||||
321 | } | ||||
322 | } | ||||
323 | |||||
324 | static inline void | ||||
325 | extent_binit(extent_t *extent, void *addr, size_t bsize, size_t sn) { | ||||
326 | extent_arena_set(extent, NULL((void*)0)); | ||||
327 | extent_addr_set(extent, addr); | ||||
328 | extent_bsize_set(extent, bsize); | ||||
329 | extent_slab_set(extent, false0); | ||||
330 | extent_szind_set(extent, NSIZES235); | ||||
331 | extent_sn_set(extent, sn); | ||||
332 | extent_state_set(extent, extent_state_active); | ||||
333 | extent_zeroed_set(extent, true1); | ||||
334 | extent_committed_set(extent, true1); | ||||
335 | extent_dumpable_set(extent, true1); | ||||
336 | } | ||||
337 | |||||
338 | static inline void | ||||
339 | extent_list_init(extent_list_t *list) { | ||||
340 | ql_new(list)do { (list)->qlh_first = ((void*)0); } while (0); | ||||
341 | } | ||||
342 | |||||
343 | static inline extent_t * | ||||
344 | extent_list_first(const extent_list_t *list) { | ||||
345 | return ql_first(list)((list)->qlh_first); | ||||
346 | } | ||||
347 | |||||
348 | static inline extent_t * | ||||
349 | extent_list_last(const extent_list_t *list) { | ||||
350 | return ql_last(list, ql_link)((((list)->qlh_first) != ((void*)0)) ? ((((list)->qlh_first ))->ql_link.qre_prev) : ((void*)0)); | ||||
351 | } | ||||
352 | |||||
353 | static inline void | ||||
354 | extent_list_append(extent_list_t *list, extent_t *extent) { | ||||
355 | ql_tail_insert(list, extent, ql_link)do { if (((list)->qlh_first) != ((void*)0)) { do { ((extent ))->ql_link.qre_prev = (((list)->qlh_first))->ql_link .qre_prev; ((extent))->ql_link.qre_next = (((list)->qlh_first )); ((extent))->ql_link.qre_prev->ql_link.qre_next = (( extent)); (((list)->qlh_first))->ql_link.qre_prev = ((extent )); } while (0); } ((list)->qlh_first) = (((extent))->ql_link .qre_next); } while (0); | ||||
356 | } | ||||
357 | |||||
358 | static inline void | ||||
359 | extent_list_prepend(extent_list_t *list, extent_t *extent) { | ||||
360 | ql_head_insert(list, extent, ql_link)do { if (((list)->qlh_first) != ((void*)0)) { do { ((extent ))->ql_link.qre_prev = (((list)->qlh_first))->ql_link .qre_prev; ((extent))->ql_link.qre_next = (((list)->qlh_first )); ((extent))->ql_link.qre_prev->ql_link.qre_next = (( extent)); (((list)->qlh_first))->ql_link.qre_prev = ((extent )); } while (0); } ((list)->qlh_first) = (extent); } while (0); | ||||
361 | } | ||||
362 | |||||
363 | static inline void | ||||
364 | extent_list_replace(extent_list_t *list, extent_t *to_remove, | ||||
365 | extent_t *to_insert) { | ||||
366 | ql_after_insert(to_remove, to_insert, ql_link)do { ((to_insert))->ql_link.qre_next = ((to_remove))->ql_link .qre_next; ((to_insert))->ql_link.qre_prev = ((to_remove)) ; ((to_insert))->ql_link.qre_next->ql_link.qre_prev = ( (to_insert)); ((to_remove))->ql_link.qre_next = ((to_insert )); } while (0); | ||||
367 | ql_remove(list, to_remove, ql_link)do { if (((list)->qlh_first) == (to_remove)) { ((list)-> qlh_first) = ((((list)->qlh_first))->ql_link.qre_next); } if (((list)->qlh_first) != (to_remove)) { do { ((to_remove ))->ql_link.qre_prev->ql_link.qre_next = ((to_remove))-> ql_link.qre_next; ((to_remove))->ql_link.qre_next->ql_link .qre_prev = ((to_remove))->ql_link.qre_prev; ((to_remove)) ->ql_link.qre_next = ((to_remove)); ((to_remove))->ql_link .qre_prev = ((to_remove)); } while (0); } else { ((list)-> qlh_first) = ((void*)0); } } while (0); | ||||
368 | } | ||||
369 | |||||
370 | static inline void | ||||
371 | extent_list_remove(extent_list_t *list, extent_t *extent) { | ||||
372 | ql_remove(list, extent, ql_link)do { if (((list)->qlh_first) == (extent)) { ((list)->qlh_first ) = ((((list)->qlh_first))->ql_link.qre_next); } if ((( list)->qlh_first) != (extent)) { do { ((extent))->ql_link .qre_prev->ql_link.qre_next = ((extent))->ql_link.qre_next ; ((extent))->ql_link.qre_next->ql_link.qre_prev = ((extent ))->ql_link.qre_prev; ((extent))->ql_link.qre_next = (( extent)); ((extent))->ql_link.qre_prev = ((extent)); } while (0); } else { ((list)->qlh_first) = ((void*)0); } } while (0); | ||||
373 | } | ||||
374 | |||||
375 | static inline int | ||||
376 | extent_sn_comp(const extent_t *a, const extent_t *b) { | ||||
377 | size_t a_sn = extent_sn_get(a); | ||||
378 | size_t b_sn = extent_sn_get(b); | ||||
379 | |||||
380 | return (a_sn > b_sn) - (a_sn < b_sn); | ||||
381 | } | ||||
382 | |||||
383 | static inline int | ||||
384 | extent_esn_comp(const extent_t *a, const extent_t *b) { | ||||
385 | size_t a_esn = extent_esn_get(a); | ||||
386 | size_t b_esn = extent_esn_get(b); | ||||
387 | |||||
388 | return (a_esn > b_esn) - (a_esn < b_esn); | ||||
389 | } | ||||
390 | |||||
391 | static inline int | ||||
392 | extent_ad_comp(const extent_t *a, const extent_t *b) { | ||||
393 | uintptr_t a_addr = (uintptr_t)extent_addr_get(a); | ||||
394 | uintptr_t b_addr = (uintptr_t)extent_addr_get(b); | ||||
395 | |||||
396 | return (a_addr > b_addr) - (a_addr < b_addr); | ||||
397 | } | ||||
398 | |||||
399 | static inline int | ||||
400 | extent_ead_comp(const extent_t *a, const extent_t *b) { | ||||
401 | uintptr_t a_eaddr = (uintptr_t)a; | ||||
402 | uintptr_t b_eaddr = (uintptr_t)b; | ||||
403 | |||||
404 | return (a_eaddr > b_eaddr) - (a_eaddr < b_eaddr); | ||||
405 | } | ||||
406 | |||||
407 | static inline int | ||||
408 | extent_snad_comp(const extent_t *a, const extent_t *b) { | ||||
409 | int ret; | ||||
410 | |||||
411 | ret = extent_sn_comp(a, b); | ||||
412 | if (ret != 0) { | ||||
413 | return ret; | ||||
414 | } | ||||
415 | |||||
416 | ret = extent_ad_comp(a, b); | ||||
417 | return ret; | ||||
418 | } | ||||
419 | |||||
420 | static inline int | ||||
421 | extent_esnead_comp(const extent_t *a, const extent_t *b) { | ||||
422 | int ret; | ||||
423 | |||||
424 | ret = extent_esn_comp(a, b); | ||||
425 | if (ret != 0) { | ||||
426 | return ret; | ||||
427 | } | ||||
428 | |||||
429 | ret = extent_ead_comp(a, b); | ||||
430 | return ret; | ||||
431 | } | ||||
432 | |||||
433 | #endif /* JEMALLOC_INTERNAL_EXTENT_INLINES_H */ |