| File: | deps/jemalloc/src/extent.c |
| Warning: | line 1557, column 17 Dereference of null pointer (loaded from variable 'coalesced') |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | #define JEMALLOC_EXTENT_C_ | |||
| 2 | #include "jemalloc/internal/jemalloc_preamble.h" | |||
| 3 | #include "jemalloc/internal/jemalloc_internal_includes.h" | |||
| 4 | ||||
| 5 | #include "jemalloc/internal/assert.h" | |||
| 6 | #include "jemalloc/internal/extent_dss.h" | |||
| 7 | #include "jemalloc/internal/extent_mmap.h" | |||
| 8 | #include "jemalloc/internal/ph.h" | |||
| 9 | #include "jemalloc/internal/rtree.h" | |||
| 10 | #include "jemalloc/internal/mutex.h" | |||
| 11 | #include "jemalloc/internal/mutex_pool.h" | |||
| 12 | ||||
| 13 | /******************************************************************************/ | |||
| 14 | /* Data. */ | |||
| 15 | ||||
| 16 | rtree_t extents_rtree; | |||
| 17 | /* Keyed by the address of the extent_t being protected. */ | |||
| 18 | mutex_pool_t extent_mutex_pool; | |||
| 19 | ||||
| 20 | size_t opt_lg_extent_max_active_fit = LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT6; | |||
| 21 | ||||
| 22 | static const bitmap_info_t extents_bitmap_info = | |||
| 23 | BITMAP_INFO_INITIALIZER(NPSIZES+1){ 199 +1, (((199 +1) + ((1U << (3 + 3))-1)) >> (3 + 3)) }; | |||
| 24 | ||||
| 25 | static void *extent_alloc_default(extent_hooks_t *extent_hooks, void *new_addr, | |||
| 26 | size_t size, size_t alignment, bool_Bool *zero, bool_Bool *commit, | |||
| 27 | unsigned arena_ind); | |||
| 28 | static bool_Bool extent_dalloc_default(extent_hooks_t *extent_hooks, void *addr, | |||
| 29 | size_t size, bool_Bool committed, unsigned arena_ind); | |||
| 30 | static void extent_destroy_default(extent_hooks_t *extent_hooks, void *addr, | |||
| 31 | size_t size, bool_Bool committed, unsigned arena_ind); | |||
| 32 | static bool_Bool extent_commit_default(extent_hooks_t *extent_hooks, void *addr, | |||
| 33 | size_t size, size_t offset, size_t length, unsigned arena_ind); | |||
| 34 | static bool_Bool extent_commit_impl(tsdn_t *tsdn, arena_t *arena, | |||
| 35 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, | |||
| 36 | size_t length, bool_Bool growing_retained); | |||
| 37 | static bool_Bool extent_decommit_default(extent_hooks_t *extent_hooks, | |||
| 38 | void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind); | |||
| 39 | #ifdef PAGES_CAN_PURGE_LAZY | |||
| 40 | static bool_Bool extent_purge_lazy_default(extent_hooks_t *extent_hooks, void *addr, | |||
| 41 | size_t size, size_t offset, size_t length, unsigned arena_ind); | |||
| 42 | #endif | |||
| 43 | static bool_Bool extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena, | |||
| 44 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, | |||
| 45 | size_t length, bool_Bool growing_retained); | |||
| 46 | #ifdef PAGES_CAN_PURGE_FORCED | |||
| 47 | static bool_Bool extent_purge_forced_default(extent_hooks_t *extent_hooks, | |||
| 48 | void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind); | |||
| 49 | #endif | |||
| 50 | static bool_Bool extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena, | |||
| 51 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, | |||
| 52 | size_t length, bool_Bool growing_retained); | |||
| 53 | #ifdef JEMALLOC_MAPS_COALESCE | |||
| 54 | static bool_Bool extent_split_default(extent_hooks_t *extent_hooks, void *addr, | |||
| 55 | size_t size, size_t size_a, size_t size_b, bool_Bool committed, | |||
| 56 | unsigned arena_ind); | |||
| 57 | #endif | |||
| 58 | static extent_t *extent_split_impl(tsdn_t *tsdn, arena_t *arena, | |||
| 59 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a, | |||
| 60 | szind_t szind_a, bool_Bool slab_a, size_t size_b, szind_t szind_b, bool_Bool slab_b, | |||
| 61 | bool_Bool growing_retained); | |||
| 62 | #ifdef JEMALLOC_MAPS_COALESCE | |||
| 63 | static bool_Bool extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, | |||
| 64 | size_t size_a, void *addr_b, size_t size_b, bool_Bool committed, | |||
| 65 | unsigned arena_ind); | |||
| 66 | #endif | |||
| 67 | static bool_Bool extent_merge_impl(tsdn_t *tsdn, arena_t *arena, | |||
| 68 | extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b, | |||
| 69 | bool_Bool growing_retained); | |||
| 70 | ||||
| 71 | const extent_hooks_t extent_hooks_default = { | |||
| 72 | extent_alloc_default, | |||
| 73 | extent_dalloc_default, | |||
| 74 | extent_destroy_default, | |||
| 75 | extent_commit_default, | |||
| 76 | extent_decommit_default | |||
| 77 | #ifdef PAGES_CAN_PURGE_LAZY | |||
| 78 | , | |||
| 79 | extent_purge_lazy_default | |||
| 80 | #else | |||
| 81 | , | |||
| 82 | NULL((void*)0) | |||
| 83 | #endif | |||
| 84 | #ifdef PAGES_CAN_PURGE_FORCED | |||
| 85 | , | |||
| 86 | extent_purge_forced_default | |||
| 87 | #else | |||
| 88 | , | |||
| 89 | NULL((void*)0) | |||
| 90 | #endif | |||
| 91 | #ifdef JEMALLOC_MAPS_COALESCE | |||
| 92 | , | |||
| 93 | extent_split_default, | |||
| 94 | extent_merge_default | |||
| 95 | #endif | |||
| 96 | }; | |||
| 97 | ||||
| 98 | /* Used exclusively for gdump triggering. */ | |||
| 99 | static atomic_zu_t curpages; | |||
| 100 | static atomic_zu_t highpages; | |||
| 101 | ||||
| 102 | /******************************************************************************/ | |||
| 103 | /* | |||
| 104 | * Function prototypes for static functions that are referenced prior to | |||
| 105 | * definition. | |||
| 106 | */ | |||
| 107 | ||||
| 108 | static void extent_deregister(tsdn_t *tsdn, extent_t *extent); | |||
| 109 | static extent_t *extent_recycle(tsdn_t *tsdn, arena_t *arena, | |||
| 110 | extent_hooks_t **r_extent_hooks, extents_t *extents, void *new_addr, | |||
| 111 | size_t usize, size_t pad, size_t alignment, bool_Bool slab, szind_t szind, | |||
| 112 | bool_Bool *zero, bool_Bool *commit, bool_Bool growing_retained); | |||
| 113 | static extent_t *extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, | |||
| 114 | extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, | |||
| 115 | extent_t *extent, bool_Bool *coalesced, bool_Bool growing_retained); | |||
| 116 | static void extent_record(tsdn_t *tsdn, arena_t *arena, | |||
| 117 | extent_hooks_t **r_extent_hooks, extents_t *extents, extent_t *extent, | |||
| 118 | bool_Bool growing_retained); | |||
| 119 | ||||
| 120 | /******************************************************************************/ | |||
| 121 | ||||
| 122 | ph_gen(UNUSED, extent_avail_, extent_tree_t, extent_t, ph_link,__attribute__((unused)) void extent_avail_new(extent_tree_t * ph) { memset(ph, 0, sizeof(struct { extent_t *ph_root; })); } __attribute__((unused)) _Bool extent_avail_empty(extent_tree_t *ph) { return (ph->ph_root == ((void*)0)); } __attribute__ ((unused)) extent_t * extent_avail_first(extent_tree_t *ph) { if (ph->ph_root == ((void*)0)) { return ((void*)0); } do { extent_t *phn = (ph->ph_root->ph_link.phn_next); if (phn != ((void*)0)) { do { ph->ph_root->ph_link.phn_prev = ( (void*)0); } while (0); do { ph->ph_root->ph_link.phn_next = ((void*)0); } while (0); do { phn->ph_link.phn_prev = ( (void*)0); } while (0); do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0); extent_t *phn0 = phn; extent_t *phn1 = ( phn0->ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void *)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while ( 0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link.phn_next = ((void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void *)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1 ) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } phn = phn0; } while (0); do { if (__builtin_expect (!!(config_debug && !((phn->ph_link.phn_next) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_next_get(extent_t, ph_link, phn) == NULL" ); abort(); } } while (0); do { if (ph->ph_root == ((void* )0)) { ph->ph_root = phn; } else if (phn == ((void*)0)) { ph ->ph_root = ph->ph_root; } else if (extent_esnead_comp( ph->ph_root, phn) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(ph->ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(phn != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (ph->ph_root, phn) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(ph->ph_root, phn) <= 0" ); abort(); } } while (0); do { phn->ph_link.phn_prev = ph ->ph_root; } while (0); phn0child = (ph->ph_root->ph_link .phn_lchild); do { phn->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = phn; } while (0); } do { ph->ph_root->ph_link .phn_lchild = phn; } while (0); } while (0); ph->ph_root = ph->ph_root; } else { do { extent_t *phn0child; do { if ( __builtin_expect(!!(config_debug && !(phn != ((void*) 0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(ph-> ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp(phn, ph->ph_root) <= 0)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "extent_esnead_comp(phn, ph->ph_root) <= 0"); abort (); } } while (0); do { ph->ph_root->ph_link.phn_prev = phn; } while (0); phn0child = (phn->ph_link.phn_lchild); do { ph->ph_root->ph_link.phn_next = phn0child; } while ( 0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = ph->ph_root; } while (0); } do { phn->ph_link .phn_lchild = ph->ph_root; } while (0); } while (0); ph-> ph_root = phn; } } while (0); } } while (0); return ph->ph_root ; } __attribute__((unused)) extent_t * extent_avail_any(extent_tree_t *ph) { if (ph->ph_root == ((void*)0)) { return ((void*)0) ; } extent_t *aux = (ph->ph_root->ph_link.phn_next); if (aux != ((void*)0)) { return aux; } return ph->ph_root; } __attribute__((unused)) void extent_avail_insert(extent_tree_t *ph, extent_t *phn) { memset(&phn->ph_link, 0, sizeof (struct { extent_t *phn_prev; extent_t *phn_next; extent_t *phn_lchild ; })); if (ph->ph_root == ((void*)0)) { ph->ph_root = phn ; } else { do { phn->ph_link.phn_next = (ph->ph_root-> ph_link.phn_next); } while (0); if ((ph->ph_root->ph_link .phn_next) != ((void*)0)) { do { (ph->ph_root->ph_link. phn_next)->ph_link.phn_prev = phn; } while (0); } do { phn ->ph_link.phn_prev = ph->ph_root; } while (0); do { ph-> ph_root->ph_link.phn_next = phn; } while (0); } } __attribute__ ((unused)) extent_t * extent_avail_remove_first(extent_tree_t *ph) { extent_t *ret; if (ph->ph_root == ((void*)0)) { return ((void*)0); } do { extent_t *phn = (ph->ph_root->ph_link .phn_next); if (phn != ((void*)0)) { do { ph->ph_root-> ph_link.phn_prev = ((void*)0); } while (0); do { ph->ph_root ->ph_link.phn_next = ((void*)0); } while (0); do { phn-> ph_link.phn_prev = ((void*)0); } while (0); do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0); extent_t *phn0 = phn ; extent_t *phn1 = (phn0->ph_link.phn_next); if (phn1 != ( (void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = (( void*)0); } while (0); do { phn1->ph_link.phn_prev = ((void *)0); } while (0); do { phn1->ph_link.phn_next = ((void*)0 ); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } phn = phn0; } while (0); do { if (__builtin_expect (!!(config_debug && !((phn->ph_link.phn_next) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_next_get(extent_t, ph_link, phn) == NULL" ); abort(); } } while (0); do { if (ph->ph_root == ((void* )0)) { ph->ph_root = phn; } else if (phn == ((void*)0)) { ph ->ph_root = ph->ph_root; } else if (extent_esnead_comp( ph->ph_root, phn) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(ph->ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(phn != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (ph->ph_root, phn) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(ph->ph_root, phn) <= 0" ); abort(); } } while (0); do { phn->ph_link.phn_prev = ph ->ph_root; } while (0); phn0child = (ph->ph_root->ph_link .phn_lchild); do { phn->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = phn; } while (0); } do { ph->ph_root->ph_link .phn_lchild = phn; } while (0); } while (0); ph->ph_root = ph->ph_root; } else { do { extent_t *phn0child; do { if ( __builtin_expect(!!(config_debug && !(phn != ((void*) 0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(ph-> ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp(phn, ph->ph_root) <= 0)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "extent_esnead_comp(phn, ph->ph_root) <= 0"); abort (); } } while (0); do { ph->ph_root->ph_link.phn_prev = phn; } while (0); phn0child = (phn->ph_link.phn_lchild); do { ph->ph_root->ph_link.phn_next = phn0child; } while ( 0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = ph->ph_root; } while (0); } do { phn->ph_link .phn_lchild = ph->ph_root; } while (0); } while (0); ph-> ph_root = phn; } } while (0); } } while (0); ret = ph->ph_root ; do { extent_t *lchild = (ph->ph_root->ph_link.phn_lchild ); if (lchild == ((void*)0)) { ph->ph_root = ((void*)0); } else { do { extent_t *head = ((void*)0); extent_t *tail = (( void*)0); extent_t *phn0 = lchild; extent_t *phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while (0); do { phn0 ->ph_link.phn_next = ((void*)0); } while (0); do { phn1-> ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link .phn_next = ((void*)0); } while (0); do { if (phn0 == ((void* )0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0 ; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } ph->ph_root = phn0; } while (0); } } while (0); return ret; } __attribute__((unused)) extent_t * extent_avail_remove_any (extent_tree_t *ph) { if (ph->ph_root == ((void*)0)) { return ((void*)0); } extent_t *ret = (ph->ph_root->ph_link.phn_next ); if (ret != ((void*)0)) { extent_t *aux = (ret->ph_link. phn_next); do { ph->ph_root->ph_link.phn_next = aux; } while (0); if (aux != ((void*)0)) { do { aux->ph_link.phn_prev = ph->ph_root; } while (0); } return ret; } ret = ph->ph_root ; do { extent_t *lchild = (ph->ph_root->ph_link.phn_lchild ); if (lchild == ((void*)0)) { ph->ph_root = ((void*)0); } else { do { extent_t *head = ((void*)0); extent_t *tail = (( void*)0); extent_t *phn0 = lchild; extent_t *phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while (0); do { phn0 ->ph_link.phn_next = ((void*)0); } while (0); do { phn1-> ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link .phn_next = ((void*)0); } while (0); do { if (phn0 == ((void* )0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0 ; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } ph->ph_root = phn0; } while (0); } } while (0); return ret; } __attribute__((unused)) void extent_avail_remove (extent_tree_t *ph, extent_t *phn) { extent_t *replace, *parent ; if (ph->ph_root == phn) { if ((phn->ph_link.phn_lchild ) == ((void*)0)) { ph->ph_root = (phn->ph_link.phn_next ); if (ph->ph_root != ((void*)0)) { do { ph->ph_root-> ph_link.phn_prev = ((void*)0); } while (0); } return; } do { extent_t *phn = (ph->ph_root->ph_link.phn_next); if (phn != ((void *)0)) { do { ph->ph_root->ph_link.phn_prev = ((void*)0) ; } while (0); do { ph->ph_root->ph_link.phn_next = ((void *)0); } while (0); do { phn->ph_link.phn_prev = ((void*)0) ; } while (0); do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0); extent_t *phn0 = phn; extent_t *phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while (0); do { phn0 ->ph_link.phn_next = ((void*)0); } while (0); do { phn1-> ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link .phn_next = ((void*)0); } while (0); do { if (phn0 == ((void* )0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0 ; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } phn = phn0; } while (0); do { if (__builtin_expect (!!(config_debug && !((phn->ph_link.phn_next) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_next_get(extent_t, ph_link, phn) == NULL" ); abort(); } } while (0); do { if (ph->ph_root == ((void* )0)) { ph->ph_root = phn; } else if (phn == ((void*)0)) { ph ->ph_root = ph->ph_root; } else if (extent_esnead_comp( ph->ph_root, phn) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(ph->ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(phn != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (ph->ph_root, phn) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(ph->ph_root, phn) <= 0" ); abort(); } } while (0); do { phn->ph_link.phn_prev = ph ->ph_root; } while (0); phn0child = (ph->ph_root->ph_link .phn_lchild); do { phn->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = phn; } while (0); } do { ph->ph_root->ph_link .phn_lchild = phn; } while (0); } while (0); ph->ph_root = ph->ph_root; } else { do { extent_t *phn0child; do { if ( __builtin_expect(!!(config_debug && !(phn != ((void*) 0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(ph-> ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp(phn, ph->ph_root) <= 0)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "extent_esnead_comp(phn, ph->ph_root) <= 0"); abort (); } } while (0); do { ph->ph_root->ph_link.phn_prev = phn; } while (0); phn0child = (phn->ph_link.phn_lchild); do { ph->ph_root->ph_link.phn_next = phn0child; } while ( 0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = ph->ph_root; } while (0); } do { phn->ph_link .phn_lchild = ph->ph_root; } while (0); } while (0); ph-> ph_root = phn; } } while (0); } } while (0); if (ph->ph_root == phn) { do { extent_t *lchild = (ph->ph_root->ph_link .phn_lchild); if (lchild == ((void*)0)) { ph->ph_root = (( void*)0); } else { do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0); extent_t *phn0 = lchild; extent_t *phn1 = (phn0->ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void *)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while ( 0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link.phn_next = ((void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void *)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1 ) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } ph->ph_root = phn0; } while (0); } } while (0); return; } } if ((parent = (phn->ph_link.phn_prev)) != ((void*)0)) { if ((parent->ph_link.phn_lchild) != phn) { parent = ((void*)0); } } do { extent_t *lchild = (phn->ph_link.phn_lchild ); if (lchild == ((void*)0)) { replace = ((void*)0); } else { do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0 ); extent_t *phn0 = lchild; extent_t *phn1 = (phn0->ph_link .phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1 ->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest ->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0-> ph_link.phn_prev = ((void*)0); } while (0); do { phn0->ph_link .phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } replace = phn0; } while (0); } } while (0); if (replace != ((void*)0)) { if (parent != ((void*)0)) { do { replace->ph_link.phn_prev = parent; } while (0); do { parent ->ph_link.phn_lchild = replace; } while (0); } else { do { replace->ph_link.phn_prev = (phn->ph_link.phn_prev); } while (0); if ((phn->ph_link.phn_prev) != ((void*)0)) { do { (phn->ph_link.phn_prev)->ph_link.phn_next = replace; } while (0); } } do { replace->ph_link.phn_next = (phn-> ph_link.phn_next); } while (0); if ((phn->ph_link.phn_next ) != ((void*)0)) { do { (phn->ph_link.phn_next)->ph_link .phn_prev = replace; } while (0); } } else { if (parent != (( void*)0)) { extent_t *next = (phn->ph_link.phn_next); do { parent->ph_link.phn_lchild = next; } while (0); if (next != ((void*)0)) { do { next->ph_link.phn_prev = parent; } while (0); } } else { do { if (__builtin_expect(!!(config_debug && !((phn->ph_link.phn_prev) != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn_prev_get(extent_t, ph_link, phn) != NULL"); abort (); } } while (0); do { (phn->ph_link.phn_prev)->ph_link .phn_next = (phn->ph_link.phn_next); } while (0); } if ((phn ->ph_link.phn_next) != ((void*)0)) { do { (phn->ph_link .phn_next)->ph_link.phn_prev = (phn->ph_link.phn_prev); } while (0); } } } | |||
| 123 | extent_esnead_comp)__attribute__((unused)) void extent_avail_new(extent_tree_t * ph) { memset(ph, 0, sizeof(struct { extent_t *ph_root; })); } __attribute__((unused)) _Bool extent_avail_empty(extent_tree_t *ph) { return (ph->ph_root == ((void*)0)); } __attribute__ ((unused)) extent_t * extent_avail_first(extent_tree_t *ph) { if (ph->ph_root == ((void*)0)) { return ((void*)0); } do { extent_t *phn = (ph->ph_root->ph_link.phn_next); if (phn != ((void*)0)) { do { ph->ph_root->ph_link.phn_prev = ( (void*)0); } while (0); do { ph->ph_root->ph_link.phn_next = ((void*)0); } while (0); do { phn->ph_link.phn_prev = ( (void*)0); } while (0); do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0); extent_t *phn0 = phn; extent_t *phn1 = ( phn0->ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void *)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while ( 0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link.phn_next = ((void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void *)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1 ) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } phn = phn0; } while (0); do { if (__builtin_expect (!!(config_debug && !((phn->ph_link.phn_next) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_next_get(extent_t, ph_link, phn) == NULL" ); abort(); } } while (0); do { if (ph->ph_root == ((void* )0)) { ph->ph_root = phn; } else if (phn == ((void*)0)) { ph ->ph_root = ph->ph_root; } else if (extent_esnead_comp( ph->ph_root, phn) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(ph->ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(phn != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (ph->ph_root, phn) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(ph->ph_root, phn) <= 0" ); abort(); } } while (0); do { phn->ph_link.phn_prev = ph ->ph_root; } while (0); phn0child = (ph->ph_root->ph_link .phn_lchild); do { phn->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = phn; } while (0); } do { ph->ph_root->ph_link .phn_lchild = phn; } while (0); } while (0); ph->ph_root = ph->ph_root; } else { do { extent_t *phn0child; do { if ( __builtin_expect(!!(config_debug && !(phn != ((void*) 0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(ph-> ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp(phn, ph->ph_root) <= 0)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "extent_esnead_comp(phn, ph->ph_root) <= 0"); abort (); } } while (0); do { ph->ph_root->ph_link.phn_prev = phn; } while (0); phn0child = (phn->ph_link.phn_lchild); do { ph->ph_root->ph_link.phn_next = phn0child; } while ( 0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = ph->ph_root; } while (0); } do { phn->ph_link .phn_lchild = ph->ph_root; } while (0); } while (0); ph-> ph_root = phn; } } while (0); } } while (0); return ph->ph_root ; } __attribute__((unused)) extent_t * extent_avail_any(extent_tree_t *ph) { if (ph->ph_root == ((void*)0)) { return ((void*)0) ; } extent_t *aux = (ph->ph_root->ph_link.phn_next); if (aux != ((void*)0)) { return aux; } return ph->ph_root; } __attribute__((unused)) void extent_avail_insert(extent_tree_t *ph, extent_t *phn) { memset(&phn->ph_link, 0, sizeof (struct { extent_t *phn_prev; extent_t *phn_next; extent_t *phn_lchild ; })); if (ph->ph_root == ((void*)0)) { ph->ph_root = phn ; } else { do { phn->ph_link.phn_next = (ph->ph_root-> ph_link.phn_next); } while (0); if ((ph->ph_root->ph_link .phn_next) != ((void*)0)) { do { (ph->ph_root->ph_link. phn_next)->ph_link.phn_prev = phn; } while (0); } do { phn ->ph_link.phn_prev = ph->ph_root; } while (0); do { ph-> ph_root->ph_link.phn_next = phn; } while (0); } } __attribute__ ((unused)) extent_t * extent_avail_remove_first(extent_tree_t *ph) { extent_t *ret; if (ph->ph_root == ((void*)0)) { return ((void*)0); } do { extent_t *phn = (ph->ph_root->ph_link .phn_next); if (phn != ((void*)0)) { do { ph->ph_root-> ph_link.phn_prev = ((void*)0); } while (0); do { ph->ph_root ->ph_link.phn_next = ((void*)0); } while (0); do { phn-> ph_link.phn_prev = ((void*)0); } while (0); do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0); extent_t *phn0 = phn ; extent_t *phn1 = (phn0->ph_link.phn_next); if (phn1 != ( (void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = (( void*)0); } while (0); do { phn1->ph_link.phn_prev = ((void *)0); } while (0); do { phn1->ph_link.phn_next = ((void*)0 ); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } phn = phn0; } while (0); do { if (__builtin_expect (!!(config_debug && !((phn->ph_link.phn_next) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_next_get(extent_t, ph_link, phn) == NULL" ); abort(); } } while (0); do { if (ph->ph_root == ((void* )0)) { ph->ph_root = phn; } else if (phn == ((void*)0)) { ph ->ph_root = ph->ph_root; } else if (extent_esnead_comp( ph->ph_root, phn) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(ph->ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(phn != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (ph->ph_root, phn) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(ph->ph_root, phn) <= 0" ); abort(); } } while (0); do { phn->ph_link.phn_prev = ph ->ph_root; } while (0); phn0child = (ph->ph_root->ph_link .phn_lchild); do { phn->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = phn; } while (0); } do { ph->ph_root->ph_link .phn_lchild = phn; } while (0); } while (0); ph->ph_root = ph->ph_root; } else { do { extent_t *phn0child; do { if ( __builtin_expect(!!(config_debug && !(phn != ((void*) 0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(ph-> ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp(phn, ph->ph_root) <= 0)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "extent_esnead_comp(phn, ph->ph_root) <= 0"); abort (); } } while (0); do { ph->ph_root->ph_link.phn_prev = phn; } while (0); phn0child = (phn->ph_link.phn_lchild); do { ph->ph_root->ph_link.phn_next = phn0child; } while ( 0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = ph->ph_root; } while (0); } do { phn->ph_link .phn_lchild = ph->ph_root; } while (0); } while (0); ph-> ph_root = phn; } } while (0); } } while (0); ret = ph->ph_root ; do { extent_t *lchild = (ph->ph_root->ph_link.phn_lchild ); if (lchild == ((void*)0)) { ph->ph_root = ((void*)0); } else { do { extent_t *head = ((void*)0); extent_t *tail = (( void*)0); extent_t *phn0 = lchild; extent_t *phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while (0); do { phn0 ->ph_link.phn_next = ((void*)0); } while (0); do { phn1-> ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link .phn_next = ((void*)0); } while (0); do { if (phn0 == ((void* )0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0 ; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } ph->ph_root = phn0; } while (0); } } while (0); return ret; } __attribute__((unused)) extent_t * extent_avail_remove_any (extent_tree_t *ph) { if (ph->ph_root == ((void*)0)) { return ((void*)0); } extent_t *ret = (ph->ph_root->ph_link.phn_next ); if (ret != ((void*)0)) { extent_t *aux = (ret->ph_link. phn_next); do { ph->ph_root->ph_link.phn_next = aux; } while (0); if (aux != ((void*)0)) { do { aux->ph_link.phn_prev = ph->ph_root; } while (0); } return ret; } ret = ph->ph_root ; do { extent_t *lchild = (ph->ph_root->ph_link.phn_lchild ); if (lchild == ((void*)0)) { ph->ph_root = ((void*)0); } else { do { extent_t *head = ((void*)0); extent_t *tail = (( void*)0); extent_t *phn0 = lchild; extent_t *phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while (0); do { phn0 ->ph_link.phn_next = ((void*)0); } while (0); do { phn1-> ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link .phn_next = ((void*)0); } while (0); do { if (phn0 == ((void* )0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0 ; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } ph->ph_root = phn0; } while (0); } } while (0); return ret; } __attribute__((unused)) void extent_avail_remove (extent_tree_t *ph, extent_t *phn) { extent_t *replace, *parent ; if (ph->ph_root == phn) { if ((phn->ph_link.phn_lchild ) == ((void*)0)) { ph->ph_root = (phn->ph_link.phn_next ); if (ph->ph_root != ((void*)0)) { do { ph->ph_root-> ph_link.phn_prev = ((void*)0); } while (0); } return; } do { extent_t *phn = (ph->ph_root->ph_link.phn_next); if (phn != ((void *)0)) { do { ph->ph_root->ph_link.phn_prev = ((void*)0) ; } while (0); do { ph->ph_root->ph_link.phn_next = ((void *)0); } while (0); do { phn->ph_link.phn_prev = ((void*)0) ; } while (0); do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0); extent_t *phn0 = phn; extent_t *phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while (0); do { phn0 ->ph_link.phn_next = ((void*)0); } while (0); do { phn1-> ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link .phn_next = ((void*)0); } while (0); do { if (phn0 == ((void* )0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0 ; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } phn = phn0; } while (0); do { if (__builtin_expect (!!(config_debug && !((phn->ph_link.phn_next) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_next_get(extent_t, ph_link, phn) == NULL" ); abort(); } } while (0); do { if (ph->ph_root == ((void* )0)) { ph->ph_root = phn; } else if (phn == ((void*)0)) { ph ->ph_root = ph->ph_root; } else if (extent_esnead_comp( ph->ph_root, phn) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(ph->ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(phn != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (ph->ph_root, phn) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(ph->ph_root, phn) <= 0" ); abort(); } } while (0); do { phn->ph_link.phn_prev = ph ->ph_root; } while (0); phn0child = (ph->ph_root->ph_link .phn_lchild); do { phn->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = phn; } while (0); } do { ph->ph_root->ph_link .phn_lchild = phn; } while (0); } while (0); ph->ph_root = ph->ph_root; } else { do { extent_t *phn0child; do { if ( __builtin_expect(!!(config_debug && !(phn != ((void*) 0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(ph-> ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp(phn, ph->ph_root) <= 0)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "extent_esnead_comp(phn, ph->ph_root) <= 0"); abort (); } } while (0); do { ph->ph_root->ph_link.phn_prev = phn; } while (0); phn0child = (phn->ph_link.phn_lchild); do { ph->ph_root->ph_link.phn_next = phn0child; } while ( 0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = ph->ph_root; } while (0); } do { phn->ph_link .phn_lchild = ph->ph_root; } while (0); } while (0); ph-> ph_root = phn; } } while (0); } } while (0); if (ph->ph_root == phn) { do { extent_t *lchild = (ph->ph_root->ph_link .phn_lchild); if (lchild == ((void*)0)) { ph->ph_root = (( void*)0); } else { do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0); extent_t *phn0 = lchild; extent_t *phn1 = (phn0->ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void *)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while ( 0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link.phn_next = ((void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void *)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1 ) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } ph->ph_root = phn0; } while (0); } } while (0); return; } } if ((parent = (phn->ph_link.phn_prev)) != ((void*)0)) { if ((parent->ph_link.phn_lchild) != phn) { parent = ((void*)0); } } do { extent_t *lchild = (phn->ph_link.phn_lchild ); if (lchild == ((void*)0)) { replace = ((void*)0); } else { do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0 ); extent_t *phn0 = lchild; extent_t *phn1 = (phn0->ph_link .phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1 ->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest ->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0-> ph_link.phn_prev = ((void*)0); } while (0); do { phn0->ph_link .phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_esnead_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_esnead_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_esnead_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 123, "extent_esnead_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } replace = phn0; } while (0); } } while (0); if (replace != ((void*)0)) { if (parent != ((void*)0)) { do { replace->ph_link.phn_prev = parent; } while (0); do { parent ->ph_link.phn_lchild = replace; } while (0); } else { do { replace->ph_link.phn_prev = (phn->ph_link.phn_prev); } while (0); if ((phn->ph_link.phn_prev) != ((void*)0)) { do { (phn->ph_link.phn_prev)->ph_link.phn_next = replace; } while (0); } } do { replace->ph_link.phn_next = (phn-> ph_link.phn_next); } while (0); if ((phn->ph_link.phn_next ) != ((void*)0)) { do { (phn->ph_link.phn_next)->ph_link .phn_prev = replace; } while (0); } } else { if (parent != (( void*)0)) { extent_t *next = (phn->ph_link.phn_next); do { parent->ph_link.phn_lchild = next; } while (0); if (next != ((void*)0)) { do { next->ph_link.phn_prev = parent; } while (0); } } else { do { if (__builtin_expect(!!(config_debug && !((phn->ph_link.phn_prev) != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 123, "phn_prev_get(extent_t, ph_link, phn) != NULL"); abort (); } } while (0); do { (phn->ph_link.phn_prev)->ph_link .phn_next = (phn->ph_link.phn_next); } while (0); } if ((phn ->ph_link.phn_next) != ((void*)0)) { do { (phn->ph_link .phn_next)->ph_link.phn_prev = (phn->ph_link.phn_prev); } while (0); } } } | |||
| 124 | ||||
| 125 | typedef enum { | |||
| 126 | lock_result_success, | |||
| 127 | lock_result_failure, | |||
| 128 | lock_result_no_extent | |||
| 129 | } lock_result_t; | |||
| 130 | ||||
| 131 | static lock_result_t | |||
| 132 | extent_rtree_leaf_elm_try_lock(tsdn_t *tsdn, rtree_leaf_elm_t *elm, | |||
| 133 | extent_t **result) { | |||
| 134 | extent_t *extent1 = rtree_leaf_elm_extent_read(tsdn, &extents_rtree, | |||
| 135 | elm, true1); | |||
| 136 | ||||
| 137 | if (extent1 == NULL((void*)0)) { | |||
| 138 | return lock_result_no_extent; | |||
| 139 | } | |||
| 140 | /* | |||
| 141 | * It's possible that the extent changed out from under us, and with it | |||
| 142 | * the leaf->extent mapping. We have to recheck while holding the lock. | |||
| 143 | */ | |||
| 144 | extent_lock(tsdn, extent1); | |||
| 145 | extent_t *extent2 = rtree_leaf_elm_extent_read(tsdn, | |||
| 146 | &extents_rtree, elm, true1); | |||
| 147 | ||||
| 148 | if (extent1 == extent2) { | |||
| 149 | *result = extent1; | |||
| 150 | return lock_result_success; | |||
| 151 | } else { | |||
| 152 | extent_unlock(tsdn, extent1); | |||
| 153 | return lock_result_failure; | |||
| 154 | } | |||
| 155 | } | |||
| 156 | ||||
| 157 | /* | |||
| 158 | * Returns a pool-locked extent_t * if there's one associated with the given | |||
| 159 | * address, and NULL otherwise. | |||
| 160 | */ | |||
| 161 | static extent_t * | |||
| 162 | extent_lock_from_addr(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, void *addr) { | |||
| 163 | extent_t *ret = NULL((void*)0); | |||
| 164 | rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, &extents_rtree, | |||
| 165 | rtree_ctx, (uintptr_t)addr, false0, false0); | |||
| 166 | if (elm == NULL((void*)0)) { | |||
| 167 | return NULL((void*)0); | |||
| 168 | } | |||
| 169 | lock_result_t lock_result; | |||
| 170 | do { | |||
| 171 | lock_result = extent_rtree_leaf_elm_try_lock(tsdn, elm, &ret); | |||
| 172 | } while (lock_result == lock_result_failure); | |||
| 173 | return ret; | |||
| 174 | } | |||
| 175 | ||||
| 176 | extent_t * | |||
| 177 | extent_alloc(tsdn_t *tsdn, arena_t *arena) { | |||
| 178 | malloc_mutex_lock(tsdn, &arena->extent_avail_mtx); | |||
| 179 | extent_t *extent = extent_avail_first(&arena->extent_avail); | |||
| 180 | if (extent == NULL((void*)0)) { | |||
| 181 | malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx); | |||
| 182 | return base_alloc_extent(tsdn, arena->base); | |||
| 183 | } | |||
| 184 | extent_avail_remove(&arena->extent_avail, extent); | |||
| 185 | malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx); | |||
| 186 | return extent; | |||
| 187 | } | |||
| 188 | ||||
| 189 | void | |||
| 190 | extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { | |||
| 191 | malloc_mutex_lock(tsdn, &arena->extent_avail_mtx); | |||
| 192 | extent_avail_insert(&arena->extent_avail, extent); | |||
| 193 | malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx); | |||
| 194 | } | |||
| 195 | ||||
| 196 | extent_hooks_t * | |||
| 197 | extent_hooks_get(arena_t *arena) { | |||
| 198 | return base_extent_hooks_get(arena->base); | |||
| 199 | } | |||
| 200 | ||||
| 201 | extent_hooks_t * | |||
| 202 | extent_hooks_set(tsd_t *tsd, arena_t *arena, extent_hooks_t *extent_hooks) { | |||
| 203 | background_thread_info_t *info; | |||
| 204 | if (have_background_thread) { | |||
| 205 | info = arena_background_thread_info_get(arena); | |||
| 206 | malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); | |||
| 207 | } | |||
| 208 | extent_hooks_t *ret = base_extent_hooks_set(arena->base, extent_hooks); | |||
| 209 | if (have_background_thread) { | |||
| 210 | malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); | |||
| 211 | } | |||
| 212 | ||||
| 213 | return ret; | |||
| 214 | } | |||
| 215 | ||||
| 216 | static void | |||
| 217 | extent_hooks_assure_initialized(arena_t *arena, | |||
| 218 | extent_hooks_t **r_extent_hooks) { | |||
| 219 | if (*r_extent_hooks == EXTENT_HOOKS_INITIALIZER((void*)0)) { | |||
| 220 | *r_extent_hooks = extent_hooks_get(arena); | |||
| 221 | } | |||
| 222 | } | |||
| 223 | ||||
| 224 | #ifndef JEMALLOC_JET | |||
| 225 | static | |||
| 226 | #endif | |||
| 227 | size_t | |||
| 228 | extent_size_quantize_floor(size_t size) { | |||
| 229 | size_t ret; | |||
| 230 | pszind_t pind; | |||
| 231 | ||||
| 232 | assert(size > 0)do { if (__builtin_expect(!!(config_debug && !(size > 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 232, "size > 0"); abort(); } } while (0); | |||
| 233 | assert((size & PAGE_MASK) == 0)do { if (__builtin_expect(!!(config_debug && !((size & ((size_t)(((size_t)(1U << 12)) - 1))) == 0)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 233, "(size & PAGE_MASK) == 0"); abort(); } } while (0); | |||
| 234 | ||||
| 235 | pind = sz_psz2ind(size - sz_large_pad + 1); | |||
| 236 | if (pind == 0) { | |||
| 237 | /* | |||
| 238 | * Avoid underflow. This short-circuit would also do the right | |||
| 239 | * thing for all sizes in the range for which there are | |||
| 240 | * PAGE-spaced size classes, but it's simplest to just handle | |||
| 241 | * the one case that would cause erroneous results. | |||
| 242 | */ | |||
| 243 | return size; | |||
| 244 | } | |||
| 245 | ret = sz_pind2sz(pind - 1) + sz_large_pad; | |||
| 246 | assert(ret <= size)do { if (__builtin_expect(!!(config_debug && !(ret <= size)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 246, "ret <= size"); abort(); } } while ( 0); | |||
| 247 | return ret; | |||
| 248 | } | |||
| 249 | ||||
| 250 | #ifndef JEMALLOC_JET | |||
| 251 | static | |||
| 252 | #endif | |||
| 253 | size_t | |||
| 254 | extent_size_quantize_ceil(size_t size) { | |||
| 255 | size_t ret; | |||
| 256 | ||||
| 257 | assert(size > 0)do { if (__builtin_expect(!!(config_debug && !(size > 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 257, "size > 0"); abort(); } } while (0); | |||
| 258 | assert(size - sz_large_pad <= LARGE_MAXCLASS)do { if (__builtin_expect(!!(config_debug && !(size - sz_large_pad <= ((((size_t)1) << 62) + (((size_t)3) << 60)))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 258, "size - sz_large_pad <= LARGE_MAXCLASS" ); abort(); } } while (0); | |||
| 259 | assert((size & PAGE_MASK) == 0)do { if (__builtin_expect(!!(config_debug && !((size & ((size_t)(((size_t)(1U << 12)) - 1))) == 0)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 259, "(size & PAGE_MASK) == 0"); abort(); } } while (0); | |||
| 260 | ||||
| 261 | ret = extent_size_quantize_floor(size); | |||
| 262 | if (ret < size) { | |||
| 263 | /* | |||
| 264 | * Skip a quantization that may have an adequately large extent, | |||
| 265 | * because under-sized extents may be mixed in. This only | |||
| 266 | * happens when an unusual size is requested, i.e. for aligned | |||
| 267 | * allocation, and is just one of several places where linear | |||
| 268 | * search would potentially find sufficiently aligned available | |||
| 269 | * memory somewhere lower. | |||
| 270 | */ | |||
| 271 | ret = sz_pind2sz(sz_psz2ind(ret - sz_large_pad + 1)) + | |||
| 272 | sz_large_pad; | |||
| 273 | } | |||
| 274 | return ret; | |||
| 275 | } | |||
| 276 | ||||
| 277 | /* Generate pairing heap functions. */ | |||
| 278 | ph_gen(, extent_heap_, extent_heap_t, extent_t, ph_link, extent_snad_comp)void extent_heap_new(extent_heap_t *ph) { memset(ph, 0, sizeof (struct { extent_t *ph_root; })); } _Bool extent_heap_empty(extent_heap_t *ph) { return (ph->ph_root == ((void*)0)); } extent_t * extent_heap_first (extent_heap_t *ph) { if (ph->ph_root == ((void*)0)) { return ((void*)0); } do { extent_t *phn = (ph->ph_root->ph_link .phn_next); if (phn != ((void*)0)) { do { ph->ph_root-> ph_link.phn_prev = ((void*)0); } while (0); do { ph->ph_root ->ph_link.phn_next = ((void*)0); } while (0); do { phn-> ph_link.phn_prev = ((void*)0); } while (0); do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0); extent_t *phn0 = phn ; extent_t *phn1 = (phn0->ph_link.phn_next); if (phn1 != ( (void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = (( void*)0); } while (0); do { phn1->ph_link.phn_prev = ((void *)0); } while (0); do { phn1->ph_link.phn_next = ((void*)0 ); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_snad_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_snad_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } phn = phn0; } while (0); do { if (__builtin_expect (!!(config_debug && !((phn->ph_link.phn_next) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_next_get(extent_t, ph_link, phn) == NULL" ); abort(); } } while (0); do { if (ph->ph_root == ((void* )0)) { ph->ph_root = phn; } else if (phn == ((void*)0)) { ph ->ph_root = ph->ph_root; } else if (extent_snad_comp(ph ->ph_root, phn) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(ph->ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(phn != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (ph->ph_root, phn) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(ph->ph_root, phn) <= 0" ); abort(); } } while (0); do { phn->ph_link.phn_prev = ph ->ph_root; } while (0); phn0child = (ph->ph_root->ph_link .phn_lchild); do { phn->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = phn; } while (0); } do { ph->ph_root->ph_link .phn_lchild = phn; } while (0); } while (0); ph->ph_root = ph->ph_root; } else { do { extent_t *phn0child; do { if ( __builtin_expect(!!(config_debug && !(phn != ((void*) 0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(ph-> ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp(phn, ph->ph_root) <= 0)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "extent_snad_comp(phn, ph->ph_root) <= 0"); abort (); } } while (0); do { ph->ph_root->ph_link.phn_prev = phn; } while (0); phn0child = (phn->ph_link.phn_lchild); do { ph->ph_root->ph_link.phn_next = phn0child; } while ( 0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = ph->ph_root; } while (0); } do { phn->ph_link .phn_lchild = ph->ph_root; } while (0); } while (0); ph-> ph_root = phn; } } while (0); } } while (0); return ph->ph_root ; } extent_t * extent_heap_any(extent_heap_t *ph) { if (ph-> ph_root == ((void*)0)) { return ((void*)0); } extent_t *aux = (ph->ph_root->ph_link.phn_next); if (aux != ((void*)0) ) { return aux; } return ph->ph_root; } void extent_heap_insert (extent_heap_t *ph, extent_t *phn) { memset(&phn->ph_link , 0, sizeof(struct { extent_t *phn_prev; extent_t *phn_next; extent_t *phn_lchild; })); if (ph->ph_root == ((void*)0)) { ph-> ph_root = phn; } else { do { phn->ph_link.phn_next = (ph-> ph_root->ph_link.phn_next); } while (0); if ((ph->ph_root ->ph_link.phn_next) != ((void*)0)) { do { (ph->ph_root-> ph_link.phn_next)->ph_link.phn_prev = phn; } while (0); } do { phn->ph_link.phn_prev = ph->ph_root; } while (0); do { ph->ph_root->ph_link.phn_next = phn; } while (0); } } extent_t * extent_heap_remove_first(extent_heap_t *ph) { extent_t *ret; if (ph->ph_root == ((void*)0)) { return ((void*)0); } do { extent_t *phn = (ph->ph_root->ph_link.phn_next) ; if (phn != ((void*)0)) { do { ph->ph_root->ph_link.phn_prev = ((void*)0); } while (0); do { ph->ph_root->ph_link.phn_next = ((void*)0); } while (0); do { phn->ph_link.phn_prev = ( (void*)0); } while (0); do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0); extent_t *phn0 = phn; extent_t *phn1 = ( phn0->ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void *)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while ( 0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link.phn_next = ((void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void *)0)) { phn0 = phn0; } else if (extent_snad_comp(phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!( config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_snad_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_snad_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } phn = phn0; } while (0); do { if (__builtin_expect (!!(config_debug && !((phn->ph_link.phn_next) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_next_get(extent_t, ph_link, phn) == NULL" ); abort(); } } while (0); do { if (ph->ph_root == ((void* )0)) { ph->ph_root = phn; } else if (phn == ((void*)0)) { ph ->ph_root = ph->ph_root; } else if (extent_snad_comp(ph ->ph_root, phn) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(ph->ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(phn != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (ph->ph_root, phn) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(ph->ph_root, phn) <= 0" ); abort(); } } while (0); do { phn->ph_link.phn_prev = ph ->ph_root; } while (0); phn0child = (ph->ph_root->ph_link .phn_lchild); do { phn->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = phn; } while (0); } do { ph->ph_root->ph_link .phn_lchild = phn; } while (0); } while (0); ph->ph_root = ph->ph_root; } else { do { extent_t *phn0child; do { if ( __builtin_expect(!!(config_debug && !(phn != ((void*) 0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(ph-> ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp(phn, ph->ph_root) <= 0)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "extent_snad_comp(phn, ph->ph_root) <= 0"); abort (); } } while (0); do { ph->ph_root->ph_link.phn_prev = phn; } while (0); phn0child = (phn->ph_link.phn_lchild); do { ph->ph_root->ph_link.phn_next = phn0child; } while ( 0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = ph->ph_root; } while (0); } do { phn->ph_link .phn_lchild = ph->ph_root; } while (0); } while (0); ph-> ph_root = phn; } } while (0); } } while (0); ret = ph->ph_root ; do { extent_t *lchild = (ph->ph_root->ph_link.phn_lchild ); if (lchild == ((void*)0)) { ph->ph_root = ((void*)0); } else { do { extent_t *head = ((void*)0); extent_t *tail = (( void*)0); extent_t *phn0 = lchild; extent_t *phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while (0); do { phn0 ->ph_link.phn_next = ((void*)0); } while (0); do { phn1-> ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link .phn_next = ((void*)0); } while (0); do { if (phn0 == ((void* )0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0 ; } else if (extent_snad_comp(phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_snad_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } ph->ph_root = phn0; } while (0); } } while (0); return ret; } extent_t * extent_heap_remove_any(extent_heap_t *ph) { if (ph->ph_root == ((void*)0)) { return ((void*)0) ; } extent_t *ret = (ph->ph_root->ph_link.phn_next); if (ret != ((void*)0)) { extent_t *aux = (ret->ph_link.phn_next ); do { ph->ph_root->ph_link.phn_next = aux; } while (0 ); if (aux != ((void*)0)) { do { aux->ph_link.phn_prev = ph ->ph_root; } while (0); } return ret; } ret = ph->ph_root ; do { extent_t *lchild = (ph->ph_root->ph_link.phn_lchild ); if (lchild == ((void*)0)) { ph->ph_root = ((void*)0); } else { do { extent_t *head = ((void*)0); extent_t *tail = (( void*)0); extent_t *phn0 = lchild; extent_t *phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while (0); do { phn0 ->ph_link.phn_next = ((void*)0); } while (0); do { phn1-> ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link .phn_next = ((void*)0); } while (0); do { if (phn0 == ((void* )0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0 ; } else if (extent_snad_comp(phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_snad_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } ph->ph_root = phn0; } while (0); } } while (0); return ret; } void extent_heap_remove(extent_heap_t *ph , extent_t *phn) { extent_t *replace, *parent; if (ph->ph_root == phn) { if ((phn->ph_link.phn_lchild) == ((void*)0)) { ph ->ph_root = (phn->ph_link.phn_next); if (ph->ph_root != ((void*)0)) { do { ph->ph_root->ph_link.phn_prev = ( (void*)0); } while (0); } return; } do { extent_t *phn = (ph-> ph_root->ph_link.phn_next); if (phn != ((void*)0)) { do { ph ->ph_root->ph_link.phn_prev = ((void*)0); } while (0); do { ph->ph_root->ph_link.phn_next = ((void*)0); } while ( 0); do { phn->ph_link.phn_prev = ((void*)0); } while (0); do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0); extent_t *phn0 = phn; extent_t *phn1 = (phn0->ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link .phn_next); if (phnrest != ((void*)0)) { do { phnrest->ph_link .phn_prev = ((void*)0); } while (0); } do { phn0->ph_link. phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_snad_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_snad_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } phn = phn0; } while (0); do { if (__builtin_expect (!!(config_debug && !((phn->ph_link.phn_next) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_next_get(extent_t, ph_link, phn) == NULL" ); abort(); } } while (0); do { if (ph->ph_root == ((void* )0)) { ph->ph_root = phn; } else if (phn == ((void*)0)) { ph ->ph_root = ph->ph_root; } else if (extent_snad_comp(ph ->ph_root, phn) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!(config_debug && !(ph->ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(phn != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (ph->ph_root, phn) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(ph->ph_root, phn) <= 0" ); abort(); } } while (0); do { phn->ph_link.phn_prev = ph ->ph_root; } while (0); phn0child = (ph->ph_root->ph_link .phn_lchild); do { phn->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = phn; } while (0); } do { ph->ph_root->ph_link .phn_lchild = phn; } while (0); } while (0); ph->ph_root = ph->ph_root; } else { do { extent_t *phn0child; do { if ( __builtin_expect(!!(config_debug && !(phn != ((void*) 0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn != NULL"); abort(); } } while (0) ; do { if (__builtin_expect(!!(config_debug && !(ph-> ph_root != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "ph->ph_root != NULL"); abort(); } } while (0); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp(phn, ph->ph_root) <= 0)), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "extent_snad_comp(phn, ph->ph_root) <= 0"); abort (); } } while (0); do { ph->ph_root->ph_link.phn_prev = phn; } while (0); phn0child = (phn->ph_link.phn_lchild); do { ph->ph_root->ph_link.phn_next = phn0child; } while ( 0); if (phn0child != ((void*)0)) { do { phn0child->ph_link .phn_prev = ph->ph_root; } while (0); } do { phn->ph_link .phn_lchild = ph->ph_root; } while (0); } while (0); ph-> ph_root = phn; } } while (0); } } while (0); if (ph->ph_root == phn) { do { extent_t *lchild = (ph->ph_root->ph_link .phn_lchild); if (lchild == ((void*)0)) { ph->ph_root = (( void*)0); } else { do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0); extent_t *phn0 = lchild; extent_t *phn1 = (phn0->ph_link.phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1->ph_link.phn_next); if (phnrest != ((void *)0)) { do { phnrest->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link.phn_prev = ((void*)0); } while ( 0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link.phn_next = ((void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void *)0)) { phn0 = phn0; } else if (extent_snad_comp(phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect(!!( config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_snad_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_snad_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } ph->ph_root = phn0; } while (0); } } while (0); return; } } if ((parent = (phn->ph_link.phn_prev)) != ((void*)0)) { if ((parent->ph_link.phn_lchild) != phn) { parent = ((void*)0); } } do { extent_t *lchild = (phn->ph_link.phn_lchild ); if (lchild == ((void*)0)) { replace = ((void*)0); } else { do { extent_t *head = ((void*)0); extent_t *tail = ((void*)0 ); extent_t *phn0 = lchild; extent_t *phn1 = (phn0->ph_link .phn_next); if (phn1 != ((void*)0)) { extent_t *phnrest = (phn1 ->ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest ->ph_link.phn_prev = ((void*)0); } while (0); } do { phn0-> ph_link.phn_prev = ((void*)0); } while (0); do { phn0->ph_link .phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ((void*)0); } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); head = tail = phn0 ; phn0 = phnrest; while (phn0 != ((void*)0)) { phn1 = (phn0-> ph_link.phn_next); if (phn1 != ((void*)0)) { phnrest = (phn1-> ph_link.phn_next); if (phnrest != ((void*)0)) { do { phnrest-> ph_link.phn_prev = ((void*)0); } while (0); } do { phn0->ph_link .phn_prev = ((void*)0); } while (0); do { phn0->ph_link.phn_next = ((void*)0); } while (0); do { phn1->ph_link.phn_prev = ( (void*)0); } while (0); do { phn1->ph_link.phn_next = ((void *)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1 ; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp (phn0, phn1) < 0) { do { extent_t *phn0child; do { if (__builtin_expect (!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn0 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn1 != NULL"); abort(); } } while (0); do { if (__builtin_expect (!!(config_debug && !(extent_snad_comp(phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); do { tail-> ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = phnrest ; } else { do { tail->ph_link.phn_next = phn0; } while (0) ; tail = phn0; phn0 = ((void*)0); } } phn0 = head; phn1 = (phn0 ->ph_link.phn_next); if (phn1 != ((void*)0)) { while (1) { head = (phn1->ph_link.phn_next); do { if (__builtin_expect (!!(config_debug && !((phn0->ph_link.phn_prev) == ( (void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn0) == NULL" ); abort(); } } while (0); do { phn0->ph_link.phn_next = ( (void*)0); } while (0); do { if (__builtin_expect(!!(config_debug && !((phn1->ph_link.phn_prev) == ((void*)0))), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn_prev_get(extent_t, ph_link, phn1) == NULL" ); abort(); } } while (0); do { phn1->ph_link.phn_next = ( (void*)0); } while (0); do { if (phn0 == ((void*)0)) { phn0 = phn1; } else if (phn1 == ((void*)0)) { phn0 = phn0; } else if (extent_snad_comp(phn0, phn1) < 0) { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn0, phn1) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn0, phn1) <= 0" ); abort(); } } while (0); do { phn1->ph_link.phn_prev = phn0 ; } while (0); phn0child = (phn0->ph_link.phn_lchild); do { phn1->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn1; } while (0); } do { phn0->ph_link.phn_lchild = phn1; } while (0); } while (0); phn0 = phn0; } else { do { extent_t *phn0child ; do { if (__builtin_expect(!!(config_debug && !(phn1 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn1 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(phn0 != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "phn0 != NULL"); abort(); } } while (0 ); do { if (__builtin_expect(!!(config_debug && !(extent_snad_comp (phn1, phn0) <= 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 278, "extent_snad_comp(phn1, phn0) <= 0" ); abort(); } } while (0); do { phn0->ph_link.phn_prev = phn1 ; } while (0); phn0child = (phn1->ph_link.phn_lchild); do { phn0->ph_link.phn_next = phn0child; } while (0); if (phn0child != ((void*)0)) { do { phn0child->ph_link.phn_prev = phn0; } while (0); } do { phn1->ph_link.phn_lchild = phn0; } while (0); } while (0); phn0 = phn1; } } while (0); if (head == (( void*)0)) { break; } do { tail->ph_link.phn_next = phn0; } while (0); tail = phn0; phn0 = head; phn1 = (phn0->ph_link .phn_next); } } } replace = phn0; } while (0); } } while (0); if (replace != ((void*)0)) { if (parent != ((void*)0)) { do { replace->ph_link.phn_prev = parent; } while (0); do { parent ->ph_link.phn_lchild = replace; } while (0); } else { do { replace->ph_link.phn_prev = (phn->ph_link.phn_prev); } while (0); if ((phn->ph_link.phn_prev) != ((void*)0)) { do { (phn->ph_link.phn_prev)->ph_link.phn_next = replace; } while (0); } } do { replace->ph_link.phn_next = (phn-> ph_link.phn_next); } while (0); if ((phn->ph_link.phn_next ) != ((void*)0)) { do { (phn->ph_link.phn_next)->ph_link .phn_prev = replace; } while (0); } } else { if (parent != (( void*)0)) { extent_t *next = (phn->ph_link.phn_next); do { parent->ph_link.phn_lchild = next; } while (0); if (next != ((void*)0)) { do { next->ph_link.phn_prev = parent; } while (0); } } else { do { if (__builtin_expect(!!(config_debug && !((phn->ph_link.phn_prev) != ((void*)0))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 278, "phn_prev_get(extent_t, ph_link, phn) != NULL"); abort (); } } while (0); do { (phn->ph_link.phn_prev)->ph_link .phn_next = (phn->ph_link.phn_next); } while (0); } if ((phn ->ph_link.phn_next) != ((void*)0)) { do { (phn->ph_link .phn_next)->ph_link.phn_prev = (phn->ph_link.phn_prev); } while (0); } } } | |||
| 279 | ||||
| 280 | bool_Bool | |||
| 281 | extents_init(tsdn_t *tsdn, extents_t *extents, extent_state_t state, | |||
| 282 | bool_Bool delay_coalesce) { | |||
| 283 | if (malloc_mutex_init(&extents->mtx, "extents", WITNESS_RANK_EXTENTS14U, | |||
| 284 | malloc_mutex_rank_exclusive)) { | |||
| 285 | return true1; | |||
| 286 | } | |||
| 287 | for (unsigned i = 0; i < NPSIZES199+1; i++) { | |||
| 288 | extent_heap_new(&extents->heaps[i]); | |||
| 289 | } | |||
| 290 | bitmap_init(extents->bitmap, &extents_bitmap_info, true1); | |||
| 291 | extent_list_init(&extents->lru); | |||
| 292 | atomic_store_zu(&extents->npages, 0, ATOMIC_RELAXEDatomic_memory_order_relaxed); | |||
| 293 | extents->state = state; | |||
| 294 | extents->delay_coalesce = delay_coalesce; | |||
| 295 | return false0; | |||
| 296 | } | |||
| 297 | ||||
| 298 | extent_state_t | |||
| 299 | extents_state_get(const extents_t *extents) { | |||
| 300 | return extents->state; | |||
| 301 | } | |||
| 302 | ||||
| 303 | size_t | |||
| 304 | extents_npages_get(extents_t *extents) { | |||
| 305 | return atomic_load_zu(&extents->npages, ATOMIC_RELAXEDatomic_memory_order_relaxed); | |||
| 306 | } | |||
| 307 | ||||
| 308 | static void | |||
| 309 | extents_insert_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent) { | |||
| 310 | malloc_mutex_assert_owner(tsdn, &extents->mtx); | |||
| 311 | assert(extent_state_get(extent) == extents->state)do { if (__builtin_expect(!!(config_debug && !(extent_state_get (extent) == extents->state)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 311, "extent_state_get(extent) == extents->state" ); abort(); } } while (0); | |||
| 312 | ||||
| 313 | size_t size = extent_size_get(extent); | |||
| 314 | size_t psz = extent_size_quantize_floor(size); | |||
| 315 | pszind_t pind = sz_psz2ind(psz); | |||
| 316 | if (extent_heap_empty(&extents->heaps[pind])) { | |||
| 317 | bitmap_unset(extents->bitmap, &extents_bitmap_info, | |||
| 318 | (size_t)pind); | |||
| 319 | } | |||
| 320 | extent_heap_insert(&extents->heaps[pind], extent); | |||
| 321 | extent_list_append(&extents->lru, extent); | |||
| 322 | size_t npages = size >> LG_PAGE12; | |||
| 323 | /* | |||
| 324 | * All modifications to npages hold the mutex (as asserted above), so we | |||
| 325 | * don't need an atomic fetch-add; we can get by with a load followed by | |||
| 326 | * a store. | |||
| 327 | */ | |||
| 328 | size_t cur_extents_npages = | |||
| 329 | atomic_load_zu(&extents->npages, ATOMIC_RELAXEDatomic_memory_order_relaxed); | |||
| 330 | atomic_store_zu(&extents->npages, cur_extents_npages + npages, | |||
| 331 | ATOMIC_RELAXEDatomic_memory_order_relaxed); | |||
| 332 | } | |||
| 333 | ||||
| 334 | static void | |||
| 335 | extents_remove_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent) { | |||
| 336 | malloc_mutex_assert_owner(tsdn, &extents->mtx); | |||
| 337 | assert(extent_state_get(extent) == extents->state)do { if (__builtin_expect(!!(config_debug && !(extent_state_get (extent) == extents->state)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 337, "extent_state_get(extent) == extents->state" ); abort(); } } while (0); | |||
| 338 | ||||
| 339 | size_t size = extent_size_get(extent); | |||
| 340 | size_t psz = extent_size_quantize_floor(size); | |||
| 341 | pszind_t pind = sz_psz2ind(psz); | |||
| 342 | extent_heap_remove(&extents->heaps[pind], extent); | |||
| 343 | if (extent_heap_empty(&extents->heaps[pind])) { | |||
| 344 | bitmap_set(extents->bitmap, &extents_bitmap_info, | |||
| 345 | (size_t)pind); | |||
| 346 | } | |||
| 347 | extent_list_remove(&extents->lru, extent); | |||
| 348 | size_t npages = size >> LG_PAGE12; | |||
| 349 | /* | |||
| 350 | * As in extents_insert_locked, we hold extents->mtx and so don't need | |||
| 351 | * atomic operations for updating extents->npages. | |||
| 352 | */ | |||
| 353 | size_t cur_extents_npages = | |||
| 354 | atomic_load_zu(&extents->npages, ATOMIC_RELAXEDatomic_memory_order_relaxed); | |||
| 355 | assert(cur_extents_npages >= npages)do { if (__builtin_expect(!!(config_debug && !(cur_extents_npages >= npages)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 355, "cur_extents_npages >= npages"); abort (); } } while (0); | |||
| 356 | atomic_store_zu(&extents->npages, | |||
| 357 | cur_extents_npages - (size >> LG_PAGE12), ATOMIC_RELAXEDatomic_memory_order_relaxed); | |||
| 358 | } | |||
| 359 | ||||
| 360 | /* | |||
| 361 | * Find an extent with size [min_size, max_size) to satisfy the alignment | |||
| 362 | * requirement. For each size, try only the first extent in the heap. | |||
| 363 | */ | |||
| 364 | static extent_t * | |||
| 365 | extents_fit_alignment(extents_t *extents, size_t min_size, size_t max_size, | |||
| 366 | size_t alignment) { | |||
| 367 | pszind_t pind = sz_psz2ind(extent_size_quantize_ceil(min_size)); | |||
| 368 | pszind_t pind_max = sz_psz2ind(extent_size_quantize_ceil(max_size)); | |||
| 369 | ||||
| 370 | for (pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap, | |||
| 371 | &extents_bitmap_info, (size_t)pind); i < pind_max; i = | |||
| 372 | (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info, | |||
| 373 | (size_t)i+1)) { | |||
| 374 | assert(i < NPSIZES)do { if (__builtin_expect(!!(config_debug && !(i < 199)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 374, "i < NPSIZES"); abort(); } } while ( 0); | |||
| 375 | assert(!extent_heap_empty(&extents->heaps[i]))do { if (__builtin_expect(!!(config_debug && !(!extent_heap_empty (&extents->heaps[i]))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 375, "!extent_heap_empty(&extents->heaps[i])" ); abort(); } } while (0); | |||
| 376 | extent_t *extent = extent_heap_first(&extents->heaps[i]); | |||
| 377 | uintptr_t base = (uintptr_t)extent_base_get(extent); | |||
| 378 | size_t candidate_size = extent_size_get(extent); | |||
| 379 | assert(candidate_size >= min_size)do { if (__builtin_expect(!!(config_debug && !(candidate_size >= min_size)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 379, "candidate_size >= min_size"); abort (); } } while (0); | |||
| 380 | ||||
| 381 | uintptr_t next_align = ALIGNMENT_CEILING((uintptr_t)base,((((uintptr_t)base) + ((((alignment) + ((size_t)(((size_t)(1U << 12)) - 1))) & ~((size_t)(((size_t)(1U << 12 )) - 1))) - 1)) & ((~((((alignment) + ((size_t)(((size_t) (1U << 12)) - 1))) & ~((size_t)(((size_t)(1U << 12)) - 1))))) + 1)) | |||
| 382 | PAGE_CEILING(alignment))((((uintptr_t)base) + ((((alignment) + ((size_t)(((size_t)(1U << 12)) - 1))) & ~((size_t)(((size_t)(1U << 12 )) - 1))) - 1)) & ((~((((alignment) + ((size_t)(((size_t) (1U << 12)) - 1))) & ~((size_t)(((size_t)(1U << 12)) - 1))))) + 1)); | |||
| 383 | if (base > next_align || base + candidate_size <= next_align) { | |||
| 384 | /* Overflow or not crossing the next alignment. */ | |||
| 385 | continue; | |||
| 386 | } | |||
| 387 | ||||
| 388 | size_t leadsize = next_align - base; | |||
| 389 | if (candidate_size - leadsize >= min_size) { | |||
| 390 | return extent; | |||
| 391 | } | |||
| 392 | } | |||
| 393 | ||||
| 394 | return NULL((void*)0); | |||
| 395 | } | |||
| 396 | ||||
| 397 | /* Do any-best-fit extent selection, i.e. select any extent that best fits. */ | |||
| 398 | static extent_t * | |||
| 399 | extents_best_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, | |||
| 400 | size_t size) { | |||
| 401 | pszind_t pind = sz_psz2ind(extent_size_quantize_ceil(size)); | |||
| 402 | pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info, | |||
| 403 | (size_t)pind); | |||
| 404 | if (i < NPSIZES199+1) { | |||
| 405 | /* | |||
| 406 | * In order to reduce fragmentation, avoid reusing and splitting | |||
| 407 | * large extents for much smaller sizes. | |||
| 408 | */ | |||
| 409 | if ((sz_pind2sz(i) >> opt_lg_extent_max_active_fit) > size) { | |||
| 410 | return NULL((void*)0); | |||
| 411 | } | |||
| 412 | assert(!extent_heap_empty(&extents->heaps[i]))do { if (__builtin_expect(!!(config_debug && !(!extent_heap_empty (&extents->heaps[i]))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 412, "!extent_heap_empty(&extents->heaps[i])" ); abort(); } } while (0); | |||
| 413 | extent_t *extent = extent_heap_first(&extents->heaps[i]); | |||
| 414 | assert(extent_size_get(extent) >= size)do { if (__builtin_expect(!!(config_debug && !(extent_size_get (extent) >= size)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 414, "extent_size_get(extent) >= size"); abort(); } } while (0); | |||
| 415 | return extent; | |||
| 416 | } | |||
| 417 | ||||
| 418 | return NULL((void*)0); | |||
| 419 | } | |||
| 420 | ||||
| 421 | /* | |||
| 422 | * Do first-fit extent selection, i.e. select the oldest/lowest extent that is | |||
| 423 | * large enough. | |||
| 424 | */ | |||
| 425 | static extent_t * | |||
| 426 | extents_first_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, | |||
| 427 | size_t size) { | |||
| 428 | extent_t *ret = NULL((void*)0); | |||
| 429 | ||||
| 430 | pszind_t pind = sz_psz2ind(extent_size_quantize_ceil(size)); | |||
| 431 | for (pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap, | |||
| 432 | &extents_bitmap_info, (size_t)pind); i < NPSIZES199+1; i = | |||
| 433 | (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info, | |||
| 434 | (size_t)i+1)) { | |||
| 435 | assert(!extent_heap_empty(&extents->heaps[i]))do { if (__builtin_expect(!!(config_debug && !(!extent_heap_empty (&extents->heaps[i]))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 435, "!extent_heap_empty(&extents->heaps[i])" ); abort(); } } while (0); | |||
| 436 | extent_t *extent = extent_heap_first(&extents->heaps[i]); | |||
| 437 | assert(extent_size_get(extent) >= size)do { if (__builtin_expect(!!(config_debug && !(extent_size_get (extent) >= size)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 437, "extent_size_get(extent) >= size"); abort(); } } while (0); | |||
| 438 | if (ret == NULL((void*)0) || extent_snad_comp(extent, ret) < 0) { | |||
| 439 | ret = extent; | |||
| 440 | } | |||
| 441 | if (i == NPSIZES199) { | |||
| 442 | break; | |||
| 443 | } | |||
| 444 | assert(i < NPSIZES)do { if (__builtin_expect(!!(config_debug && !(i < 199)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 444, "i < NPSIZES"); abort(); } } while ( 0); | |||
| 445 | } | |||
| 446 | ||||
| 447 | return ret; | |||
| 448 | } | |||
| 449 | ||||
| 450 | /* | |||
| 451 | * Do {best,first}-fit extent selection, where the selection policy choice is | |||
| 452 | * based on extents->delay_coalesce. Best-fit selection requires less | |||
| 453 | * searching, but its layout policy is less stable and may cause higher virtual | |||
| 454 | * memory fragmentation as a side effect. | |||
| 455 | */ | |||
| 456 | static extent_t * | |||
| 457 | extents_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, | |||
| 458 | size_t esize, size_t alignment) { | |||
| 459 | malloc_mutex_assert_owner(tsdn, &extents->mtx); | |||
| 460 | ||||
| 461 | size_t max_size = esize + PAGE_CEILING(alignment)(((alignment) + ((size_t)(((size_t)(1U << 12)) - 1))) & ~((size_t)(((size_t)(1U << 12)) - 1))) - PAGE((size_t)(1U << 12)); | |||
| 462 | /* Beware size_t wrap-around. */ | |||
| 463 | if (max_size < esize) { | |||
| 464 | return NULL((void*)0); | |||
| 465 | } | |||
| 466 | ||||
| 467 | extent_t *extent = extents->delay_coalesce ? | |||
| 468 | extents_best_fit_locked(tsdn, arena, extents, max_size) : | |||
| 469 | extents_first_fit_locked(tsdn, arena, extents, max_size); | |||
| 470 | ||||
| 471 | if (alignment > PAGE((size_t)(1U << 12)) && extent == NULL((void*)0)) { | |||
| 472 | /* | |||
| 473 | * max_size guarantees the alignment requirement but is rather | |||
| 474 | * pessimistic. Next we try to satisfy the aligned allocation | |||
| 475 | * with sizes in [esize, max_size). | |||
| 476 | */ | |||
| 477 | extent = extents_fit_alignment(extents, esize, max_size, | |||
| 478 | alignment); | |||
| 479 | } | |||
| 480 | ||||
| 481 | return extent; | |||
| 482 | } | |||
| 483 | ||||
| 484 | static bool_Bool | |||
| 485 | extent_try_delayed_coalesce(tsdn_t *tsdn, arena_t *arena, | |||
| 486 | extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, | |||
| 487 | extent_t *extent) { | |||
| 488 | extent_state_set(extent, extent_state_active); | |||
| 489 | bool_Bool coalesced; | |||
| 490 | extent = extent_try_coalesce(tsdn, arena, r_extent_hooks, rtree_ctx, | |||
| 491 | extents, extent, &coalesced, false0); | |||
| 492 | extent_state_set(extent, extents_state_get(extents)); | |||
| 493 | ||||
| 494 | if (!coalesced) { | |||
| 495 | return true1; | |||
| 496 | } | |||
| 497 | extents_insert_locked(tsdn, extents, extent); | |||
| 498 | return false0; | |||
| 499 | } | |||
| 500 | ||||
| 501 | extent_t * | |||
| 502 | extents_alloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, | |||
| 503 | extents_t *extents, void *new_addr, size_t size, size_t pad, | |||
| 504 | size_t alignment, bool_Bool slab, szind_t szind, bool_Bool *zero, bool_Bool *commit) { | |||
| 505 | assert(size + pad != 0)do { if (__builtin_expect(!!(config_debug && !(size + pad != 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 505, "size + pad != 0"); abort(); } } while (0); | |||
| 506 | assert(alignment != 0)do { if (__builtin_expect(!!(config_debug && !(alignment != 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 506, "alignment != 0"); abort(); } } while ( 0); | |||
| 507 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | |||
| 508 | WITNESS_RANK_CORE11U, 0); | |||
| 509 | ||||
| 510 | extent_t *extent = extent_recycle(tsdn, arena, r_extent_hooks, extents, | |||
| 511 | new_addr, size, pad, alignment, slab, szind, zero, commit, false0); | |||
| 512 | assert(extent == NULL || extent_dumpable_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent == ((void*)0) || extent_dumpable_get(extent))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 512, "extent == NULL || extent_dumpable_get(extent)"); abort (); } } while (0); | |||
| 513 | return extent; | |||
| 514 | } | |||
| 515 | ||||
| 516 | void | |||
| 517 | extents_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, | |||
| 518 | extents_t *extents, extent_t *extent) { | |||
| 519 | assert(extent_base_get(extent) != NULL)do { if (__builtin_expect(!!(config_debug && !(extent_base_get (extent) != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 519, "extent_base_get(extent) != NULL"); abort (); } } while (0); | |||
| 520 | assert(extent_size_get(extent) != 0)do { if (__builtin_expect(!!(config_debug && !(extent_size_get (extent) != 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 520, "extent_size_get(extent) != 0"); abort (); } } while (0); | |||
| 521 | assert(extent_dumpable_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent_dumpable_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 521, "extent_dumpable_get(extent)"); abort( ); } } while (0); | |||
| 522 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | |||
| 523 | WITNESS_RANK_CORE11U, 0); | |||
| 524 | ||||
| 525 | extent_addr_set(extent, extent_base_get(extent)); | |||
| 526 | extent_zeroed_set(extent, false0); | |||
| 527 | ||||
| 528 | extent_record(tsdn, arena, r_extent_hooks, extents, extent, false0); | |||
| 529 | } | |||
| 530 | ||||
| 531 | extent_t * | |||
| 532 | extents_evict(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, | |||
| 533 | extents_t *extents, size_t npages_min) { | |||
| 534 | rtree_ctx_t rtree_ctx_fallback; | |||
| 535 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); | |||
| 536 | ||||
| 537 | malloc_mutex_lock(tsdn, &extents->mtx); | |||
| 538 | ||||
| 539 | /* | |||
| 540 | * Get the LRU coalesced extent, if any. If coalescing was delayed, | |||
| 541 | * the loop will iterate until the LRU extent is fully coalesced. | |||
| 542 | */ | |||
| 543 | extent_t *extent; | |||
| 544 | while (true1) { | |||
| 545 | /* Get the LRU extent, if any. */ | |||
| 546 | extent = extent_list_first(&extents->lru); | |||
| 547 | if (extent == NULL((void*)0)) { | |||
| 548 | goto label_return; | |||
| 549 | } | |||
| 550 | /* Check the eviction limit. */ | |||
| 551 | size_t extents_npages = atomic_load_zu(&extents->npages, | |||
| 552 | ATOMIC_RELAXEDatomic_memory_order_relaxed); | |||
| 553 | if (extents_npages <= npages_min) { | |||
| 554 | extent = NULL((void*)0); | |||
| 555 | goto label_return; | |||
| 556 | } | |||
| 557 | extents_remove_locked(tsdn, extents, extent); | |||
| 558 | if (!extents->delay_coalesce) { | |||
| 559 | break; | |||
| 560 | } | |||
| 561 | /* Try to coalesce. */ | |||
| 562 | if (extent_try_delayed_coalesce(tsdn, arena, r_extent_hooks, | |||
| 563 | rtree_ctx, extents, extent)) { | |||
| 564 | break; | |||
| 565 | } | |||
| 566 | /* | |||
| 567 | * The LRU extent was just coalesced and the result placed in | |||
| 568 | * the LRU at its neighbor's position. Start over. | |||
| 569 | */ | |||
| 570 | } | |||
| 571 | ||||
| 572 | /* | |||
| 573 | * Either mark the extent active or deregister it to protect against | |||
| 574 | * concurrent operations. | |||
| 575 | */ | |||
| 576 | switch (extents_state_get(extents)) { | |||
| 577 | case extent_state_active: | |||
| 578 | not_reached()do { if (config_debug) { malloc_printf( "<jemalloc>: %s:%d: Unreachable code reached\n" , "src/extent.c", 578); abort(); } __builtin_unreachable(); } while (0); | |||
| 579 | case extent_state_dirty: | |||
| 580 | case extent_state_muzzy: | |||
| 581 | extent_state_set(extent, extent_state_active); | |||
| 582 | break; | |||
| 583 | case extent_state_retained: | |||
| 584 | extent_deregister(tsdn, extent); | |||
| 585 | break; | |||
| 586 | default: | |||
| 587 | not_reached()do { if (config_debug) { malloc_printf( "<jemalloc>: %s:%d: Unreachable code reached\n" , "src/extent.c", 587); abort(); } __builtin_unreachable(); } while (0); | |||
| 588 | } | |||
| 589 | ||||
| 590 | label_return: | |||
| 591 | malloc_mutex_unlock(tsdn, &extents->mtx); | |||
| 592 | return extent; | |||
| 593 | } | |||
| 594 | ||||
| 595 | static void | |||
| 596 | extents_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, | |||
| 597 | extents_t *extents, extent_t *extent, bool_Bool growing_retained) { | |||
| 598 | /* | |||
| 599 | * Leak extent after making sure its pages have already been purged, so | |||
| 600 | * that this is only a virtual memory leak. | |||
| 601 | */ | |||
| 602 | if (extents_state_get(extents) == extent_state_dirty) { | |||
| 603 | if (extent_purge_lazy_impl(tsdn, arena, r_extent_hooks, | |||
| 604 | extent, 0, extent_size_get(extent), growing_retained)) { | |||
| 605 | extent_purge_forced_impl(tsdn, arena, r_extent_hooks, | |||
| 606 | extent, 0, extent_size_get(extent), | |||
| 607 | growing_retained); | |||
| 608 | } | |||
| 609 | } | |||
| 610 | extent_dalloc(tsdn, arena, extent); | |||
| 611 | } | |||
| 612 | ||||
| 613 | void | |||
| 614 | extents_prefork(tsdn_t *tsdn, extents_t *extents) { | |||
| 615 | malloc_mutex_prefork(tsdn, &extents->mtx); | |||
| 616 | } | |||
| 617 | ||||
| 618 | void | |||
| 619 | extents_postfork_parent(tsdn_t *tsdn, extents_t *extents) { | |||
| 620 | malloc_mutex_postfork_parent(tsdn, &extents->mtx); | |||
| 621 | } | |||
| 622 | ||||
| 623 | void | |||
| 624 | extents_postfork_child(tsdn_t *tsdn, extents_t *extents) { | |||
| 625 | malloc_mutex_postfork_child(tsdn, &extents->mtx); | |||
| 626 | } | |||
| 627 | ||||
| 628 | static void | |||
| 629 | extent_deactivate_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, | |||
| 630 | extent_t *extent) { | |||
| 631 | assert(extent_arena_get(extent) == arena)do { if (__builtin_expect(!!(config_debug && !(extent_arena_get (extent) == arena)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 631, "extent_arena_get(extent) == arena"); abort (); } } while (0); | |||
| 632 | assert(extent_state_get(extent) == extent_state_active)do { if (__builtin_expect(!!(config_debug && !(extent_state_get (extent) == extent_state_active)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 632, "extent_state_get(extent) == extent_state_active" ); abort(); } } while (0); | |||
| 633 | ||||
| 634 | extent_state_set(extent, extents_state_get(extents)); | |||
| 635 | extents_insert_locked(tsdn, extents, extent); | |||
| 636 | } | |||
| 637 | ||||
| 638 | static void | |||
| 639 | extent_deactivate(tsdn_t *tsdn, arena_t *arena, extents_t *extents, | |||
| 640 | extent_t *extent) { | |||
| 641 | malloc_mutex_lock(tsdn, &extents->mtx); | |||
| 642 | extent_deactivate_locked(tsdn, arena, extents, extent); | |||
| 643 | malloc_mutex_unlock(tsdn, &extents->mtx); | |||
| 644 | } | |||
| 645 | ||||
| 646 | static void | |||
| 647 | extent_activate_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, | |||
| 648 | extent_t *extent) { | |||
| 649 | assert(extent_arena_get(extent) == arena)do { if (__builtin_expect(!!(config_debug && !(extent_arena_get (extent) == arena)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 649, "extent_arena_get(extent) == arena"); abort (); } } while (0); | |||
| 650 | assert(extent_state_get(extent) == extents_state_get(extents))do { if (__builtin_expect(!!(config_debug && !(extent_state_get (extent) == extents_state_get(extents))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 650, "extent_state_get(extent) == extents_state_get(extents)" ); abort(); } } while (0); | |||
| 651 | ||||
| 652 | extents_remove_locked(tsdn, extents, extent); | |||
| 653 | extent_state_set(extent, extent_state_active); | |||
| 654 | } | |||
| 655 | ||||
| 656 | static bool_Bool | |||
| 657 | extent_rtree_leaf_elms_lookup(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, | |||
| 658 | const extent_t *extent, bool_Bool dependent, bool_Bool init_missing, | |||
| 659 | rtree_leaf_elm_t **r_elm_a, rtree_leaf_elm_t **r_elm_b) { | |||
| 660 | *r_elm_a = rtree_leaf_elm_lookup(tsdn, &extents_rtree, rtree_ctx, | |||
| 661 | (uintptr_t)extent_base_get(extent), dependent, init_missing); | |||
| 662 | if (!dependent && *r_elm_a == NULL((void*)0)) { | |||
| 663 | return true1; | |||
| 664 | } | |||
| 665 | assert(*r_elm_a != NULL)do { if (__builtin_expect(!!(config_debug && !(*r_elm_a != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 665, "*r_elm_a != NULL"); abort(); } } while (0); | |||
| 666 | ||||
| 667 | *r_elm_b = rtree_leaf_elm_lookup(tsdn, &extents_rtree, rtree_ctx, | |||
| 668 | (uintptr_t)extent_last_get(extent), dependent, init_missing); | |||
| 669 | if (!dependent && *r_elm_b == NULL((void*)0)) { | |||
| 670 | return true1; | |||
| 671 | } | |||
| 672 | assert(*r_elm_b != NULL)do { if (__builtin_expect(!!(config_debug && !(*r_elm_b != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 672, "*r_elm_b != NULL"); abort(); } } while (0); | |||
| 673 | ||||
| 674 | return false0; | |||
| 675 | } | |||
| 676 | ||||
| 677 | static void | |||
| 678 | extent_rtree_write_acquired(tsdn_t *tsdn, rtree_leaf_elm_t *elm_a, | |||
| 679 | rtree_leaf_elm_t *elm_b, extent_t *extent, szind_t szind, bool_Bool slab) { | |||
| 680 | rtree_leaf_elm_write(tsdn, &extents_rtree, elm_a, extent, szind, slab); | |||
| 681 | if (elm_b != NULL((void*)0)) { | |||
| 682 | rtree_leaf_elm_write(tsdn, &extents_rtree, elm_b, extent, szind, | |||
| 683 | slab); | |||
| 684 | } | |||
| 685 | } | |||
| 686 | ||||
| 687 | static void | |||
| 688 | extent_interior_register(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, extent_t *extent, | |||
| 689 | szind_t szind) { | |||
| 690 | assert(extent_slab_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent_slab_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 690, "extent_slab_get(extent)"); abort(); } } while (0); | |||
| 691 | ||||
| 692 | /* Register interior. */ | |||
| 693 | for (size_t i = 1; i < (extent_size_get(extent) >> LG_PAGE12) - 1; i++) { | |||
| 694 | rtree_write(tsdn, &extents_rtree, rtree_ctx, | |||
| 695 | (uintptr_t)extent_base_get(extent) + (uintptr_t)(i << | |||
| 696 | LG_PAGE12), extent, szind, true1); | |||
| 697 | } | |||
| 698 | } | |||
| 699 | ||||
| 700 | static void | |||
| 701 | extent_gdump_add(tsdn_t *tsdn, const extent_t *extent) { | |||
| 702 | cassert(config_prof)do { if (__builtin_expect(!!(!(config_prof)), 0)) { do { if ( config_debug) { malloc_printf( "<jemalloc>: %s:%d: Unreachable code reached\n" , "src/extent.c", 702); abort(); } __builtin_unreachable(); } while (0); } } while (0); | |||
| 703 | /* prof_gdump() requirement. */ | |||
| 704 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | |||
| 705 | WITNESS_RANK_CORE11U, 0); | |||
| 706 | ||||
| 707 | if (opt_prof && extent_state_get(extent) == extent_state_active) { | |||
| 708 | size_t nadd = extent_size_get(extent) >> LG_PAGE12; | |||
| 709 | size_t cur = atomic_fetch_add_zu(&curpages, nadd, | |||
| 710 | ATOMIC_RELAXEDatomic_memory_order_relaxed) + nadd; | |||
| 711 | size_t high = atomic_load_zu(&highpages, ATOMIC_RELAXEDatomic_memory_order_relaxed); | |||
| 712 | while (cur > high && !atomic_compare_exchange_weak_zu( | |||
| 713 | &highpages, &high, cur, ATOMIC_RELAXEDatomic_memory_order_relaxed, ATOMIC_RELAXEDatomic_memory_order_relaxed)) { | |||
| 714 | /* | |||
| 715 | * Don't refresh cur, because it may have decreased | |||
| 716 | * since this thread lost the highpages update race. | |||
| 717 | * Note that high is updated in case of CAS failure. | |||
| 718 | */ | |||
| 719 | } | |||
| 720 | if (cur > high && prof_gdump_get_unlocked()) { | |||
| 721 | prof_gdump(tsdn); | |||
| 722 | } | |||
| 723 | } | |||
| 724 | } | |||
| 725 | ||||
| 726 | static void | |||
| 727 | extent_gdump_sub(tsdn_t *tsdn, const extent_t *extent) { | |||
| 728 | cassert(config_prof)do { if (__builtin_expect(!!(!(config_prof)), 0)) { do { if ( config_debug) { malloc_printf( "<jemalloc>: %s:%d: Unreachable code reached\n" , "src/extent.c", 728); abort(); } __builtin_unreachable(); } while (0); } } while (0); | |||
| 729 | ||||
| 730 | if (opt_prof && extent_state_get(extent) == extent_state_active) { | |||
| 731 | size_t nsub = extent_size_get(extent) >> LG_PAGE12; | |||
| 732 | assert(atomic_load_zu(&curpages, ATOMIC_RELAXED) >= nsub)do { if (__builtin_expect(!!(config_debug && !(atomic_load_zu (&curpages, atomic_memory_order_relaxed) >= nsub)), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 732, "atomic_load_zu(&curpages, ATOMIC_RELAXED) >= nsub" ); abort(); } } while (0); | |||
| 733 | atomic_fetch_sub_zu(&curpages, nsub, ATOMIC_RELAXEDatomic_memory_order_relaxed); | |||
| 734 | } | |||
| 735 | } | |||
| 736 | ||||
| 737 | static bool_Bool | |||
| 738 | extent_register_impl(tsdn_t *tsdn, extent_t *extent, bool_Bool gdump_add) { | |||
| 739 | rtree_ctx_t rtree_ctx_fallback; | |||
| 740 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); | |||
| 741 | rtree_leaf_elm_t *elm_a, *elm_b; | |||
| 742 | ||||
| 743 | /* | |||
| 744 | * We need to hold the lock to protect against a concurrent coalesce | |||
| 745 | * operation that sees us in a partial state. | |||
| 746 | */ | |||
| 747 | extent_lock(tsdn, extent); | |||
| 748 | ||||
| 749 | if (extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, extent, false0, true1, | |||
| 750 | &elm_a, &elm_b)) { | |||
| 751 | return true1; | |||
| 752 | } | |||
| 753 | ||||
| 754 | szind_t szind = extent_szind_get_maybe_invalid(extent); | |||
| 755 | bool_Bool slab = extent_slab_get(extent); | |||
| 756 | extent_rtree_write_acquired(tsdn, elm_a, elm_b, extent, szind, slab); | |||
| 757 | if (slab) { | |||
| 758 | extent_interior_register(tsdn, rtree_ctx, extent, szind); | |||
| 759 | } | |||
| 760 | ||||
| 761 | extent_unlock(tsdn, extent); | |||
| 762 | ||||
| 763 | if (config_prof && gdump_add) { | |||
| 764 | extent_gdump_add(tsdn, extent); | |||
| 765 | } | |||
| 766 | ||||
| 767 | return false0; | |||
| 768 | } | |||
| 769 | ||||
| 770 | static bool_Bool | |||
| 771 | extent_register(tsdn_t *tsdn, extent_t *extent) { | |||
| 772 | return extent_register_impl(tsdn, extent, true1); | |||
| 773 | } | |||
| 774 | ||||
| 775 | static bool_Bool | |||
| 776 | extent_register_no_gdump_add(tsdn_t *tsdn, extent_t *extent) { | |||
| 777 | return extent_register_impl(tsdn, extent, false0); | |||
| 778 | } | |||
| 779 | ||||
| 780 | static void | |||
| 781 | extent_reregister(tsdn_t *tsdn, extent_t *extent) { | |||
| 782 | bool_Bool err = extent_register(tsdn, extent); | |||
| 783 | assert(!err)do { if (__builtin_expect(!!(config_debug && !(!err)) , 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 783, "!err"); abort(); } } while (0); | |||
| 784 | } | |||
| 785 | ||||
| 786 | /* | |||
| 787 | * Removes all pointers to the given extent from the global rtree indices for | |||
| 788 | * its interior. This is relevant for slab extents, for which we need to do | |||
| 789 | * metadata lookups at places other than the head of the extent. We deregister | |||
| 790 | * on the interior, then, when an extent moves from being an active slab to an | |||
| 791 | * inactive state. | |||
| 792 | */ | |||
| 793 | static void | |||
| 794 | extent_interior_deregister(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, | |||
| 795 | extent_t *extent) { | |||
| 796 | size_t i; | |||
| 797 | ||||
| 798 | assert(extent_slab_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent_slab_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 798, "extent_slab_get(extent)"); abort(); } } while (0); | |||
| 799 | ||||
| 800 | for (i = 1; i < (extent_size_get(extent) >> LG_PAGE12) - 1; i++) { | |||
| 801 | rtree_clear(tsdn, &extents_rtree, rtree_ctx, | |||
| 802 | (uintptr_t)extent_base_get(extent) + (uintptr_t)(i << | |||
| 803 | LG_PAGE12)); | |||
| 804 | } | |||
| 805 | } | |||
| 806 | ||||
| 807 | /* | |||
| 808 | * Removes all pointers to the given extent from the global rtree. | |||
| 809 | */ | |||
| 810 | static void | |||
| 811 | extent_deregister_impl(tsdn_t *tsdn, extent_t *extent, bool_Bool gdump) { | |||
| 812 | rtree_ctx_t rtree_ctx_fallback; | |||
| 813 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); | |||
| 814 | rtree_leaf_elm_t *elm_a, *elm_b; | |||
| 815 | extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, extent, true1, false0, | |||
| 816 | &elm_a, &elm_b); | |||
| 817 | ||||
| 818 | extent_lock(tsdn, extent); | |||
| 819 | ||||
| 820 | extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL((void*)0), NSIZES235, false0); | |||
| 821 | if (extent_slab_get(extent)) { | |||
| 822 | extent_interior_deregister(tsdn, rtree_ctx, extent); | |||
| 823 | extent_slab_set(extent, false0); | |||
| 824 | } | |||
| 825 | ||||
| 826 | extent_unlock(tsdn, extent); | |||
| 827 | ||||
| 828 | if (config_prof && gdump) { | |||
| 829 | extent_gdump_sub(tsdn, extent); | |||
| 830 | } | |||
| 831 | } | |||
| 832 | ||||
| 833 | static void | |||
| 834 | extent_deregister(tsdn_t *tsdn, extent_t *extent) { | |||
| 835 | extent_deregister_impl(tsdn, extent, true1); | |||
| 836 | } | |||
| 837 | ||||
| 838 | static void | |||
| 839 | extent_deregister_no_gdump_sub(tsdn_t *tsdn, extent_t *extent) { | |||
| 840 | extent_deregister_impl(tsdn, extent, false0); | |||
| 841 | } | |||
| 842 | ||||
| 843 | /* | |||
| 844 | * Tries to find and remove an extent from extents that can be used for the | |||
| 845 | * given allocation request. | |||
| 846 | */ | |||
| 847 | static extent_t * | |||
| 848 | extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, | |||
| 849 | extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, | |||
| 850 | void *new_addr, size_t size, size_t pad, size_t alignment, bool_Bool slab, | |||
| 851 | bool_Bool growing_retained) { | |||
| 852 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | |||
| 853 | WITNESS_RANK_CORE11U, growing_retained ? 1 : 0); | |||
| 854 | assert(alignment > 0)do { if (__builtin_expect(!!(config_debug && !(alignment > 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 854, "alignment > 0"); abort(); } } while (0); | |||
| 855 | if (config_debug && new_addr != NULL((void*)0)) { | |||
| 856 | /* | |||
| 857 | * Non-NULL new_addr has two use cases: | |||
| 858 | * | |||
| 859 | * 1) Recycle a known-extant extent, e.g. during purging. | |||
| 860 | * 2) Perform in-place expanding reallocation. | |||
| 861 | * | |||
| 862 | * Regardless of use case, new_addr must either refer to a | |||
| 863 | * non-existing extent, or to the base of an extant extent, | |||
| 864 | * since only active slabs support interior lookups (which of | |||
| 865 | * course cannot be recycled). | |||
| 866 | */ | |||
| 867 | assert(PAGE_ADDR2BASE(new_addr) == new_addr)do { if (__builtin_expect(!!(config_debug && !(((void *)((uintptr_t)(new_addr) & ~((size_t)(((size_t)(1U << 12)) - 1)))) == new_addr)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 867, "PAGE_ADDR2BASE(new_addr) == new_addr" ); abort(); } } while (0); | |||
| 868 | assert(pad == 0)do { if (__builtin_expect(!!(config_debug && !(pad == 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 868, "pad == 0"); abort(); } } while (0); | |||
| 869 | assert(alignment <= PAGE)do { if (__builtin_expect(!!(config_debug && !(alignment <= ((size_t)(1U << 12)))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 869, "alignment <= PAGE"); abort(); } } while (0); | |||
| 870 | } | |||
| 871 | ||||
| 872 | size_t esize = size + pad; | |||
| 873 | malloc_mutex_lock(tsdn, &extents->mtx); | |||
| 874 | extent_hooks_assure_initialized(arena, r_extent_hooks); | |||
| 875 | extent_t *extent; | |||
| 876 | if (new_addr != NULL((void*)0)) { | |||
| 877 | extent = extent_lock_from_addr(tsdn, rtree_ctx, new_addr); | |||
| 878 | if (extent != NULL((void*)0)) { | |||
| 879 | /* | |||
| 880 | * We might null-out extent to report an error, but we | |||
| 881 | * still need to unlock the associated mutex after. | |||
| 882 | */ | |||
| 883 | extent_t *unlock_extent = extent; | |||
| 884 | assert(extent_base_get(extent) == new_addr)do { if (__builtin_expect(!!(config_debug && !(extent_base_get (extent) == new_addr)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 884, "extent_base_get(extent) == new_addr") ; abort(); } } while (0); | |||
| 885 | if (extent_arena_get(extent) != arena || | |||
| 886 | extent_size_get(extent) < esize || | |||
| 887 | extent_state_get(extent) != | |||
| 888 | extents_state_get(extents)) { | |||
| 889 | extent = NULL((void*)0); | |||
| 890 | } | |||
| 891 | extent_unlock(tsdn, unlock_extent); | |||
| 892 | } | |||
| 893 | } else { | |||
| 894 | extent = extents_fit_locked(tsdn, arena, extents, esize, | |||
| 895 | alignment); | |||
| 896 | } | |||
| 897 | if (extent == NULL((void*)0)) { | |||
| 898 | malloc_mutex_unlock(tsdn, &extents->mtx); | |||
| 899 | return NULL((void*)0); | |||
| 900 | } | |||
| 901 | ||||
| 902 | extent_activate_locked(tsdn, arena, extents, extent); | |||
| 903 | malloc_mutex_unlock(tsdn, &extents->mtx); | |||
| 904 | ||||
| 905 | return extent; | |||
| 906 | } | |||
| 907 | ||||
| 908 | /* | |||
| 909 | * Given an allocation request and an extent guaranteed to be able to satisfy | |||
| 910 | * it, this splits off lead and trail extents, leaving extent pointing to an | |||
| 911 | * extent satisfying the allocation. | |||
| 912 | * This function doesn't put lead or trail into any extents_t; it's the caller's | |||
| 913 | * job to ensure that they can be reused. | |||
| 914 | */ | |||
| 915 | typedef enum { | |||
| 916 | /* | |||
| 917 | * Split successfully. lead, extent, and trail, are modified to extents | |||
| 918 | * describing the ranges before, in, and after the given allocation. | |||
| 919 | */ | |||
| 920 | extent_split_interior_ok, | |||
| 921 | /* | |||
| 922 | * The extent can't satisfy the given allocation request. None of the | |||
| 923 | * input extent_t *s are touched. | |||
| 924 | */ | |||
| 925 | extent_split_interior_cant_alloc, | |||
| 926 | /* | |||
| 927 | * In a potentially invalid state. Must leak (if *to_leak is non-NULL), | |||
| 928 | * and salvage what's still salvageable (if *to_salvage is non-NULL). | |||
| 929 | * None of lead, extent, or trail are valid. | |||
| 930 | */ | |||
| 931 | extent_split_interior_error | |||
| 932 | } extent_split_interior_result_t; | |||
| 933 | ||||
| 934 | static extent_split_interior_result_t | |||
| 935 | extent_split_interior(tsdn_t *tsdn, arena_t *arena, | |||
| 936 | extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, | |||
| 937 | /* The result of splitting, in case of success. */ | |||
| 938 | extent_t **extent, extent_t **lead, extent_t **trail, | |||
| 939 | /* The mess to clean up, in case of error. */ | |||
| 940 | extent_t **to_leak, extent_t **to_salvage, | |||
| 941 | void *new_addr, size_t size, size_t pad, size_t alignment, bool_Bool slab, | |||
| 942 | szind_t szind, bool_Bool growing_retained) { | |||
| 943 | size_t esize = size + pad; | |||
| 944 | size_t leadsize = ALIGNMENT_CEILING((uintptr_t)extent_base_get(*extent),((((uintptr_t)extent_base_get(*extent)) + ((((alignment) + (( size_t)(((size_t)(1U << 12)) - 1))) & ~((size_t)((( size_t)(1U << 12)) - 1))) - 1)) & ((~((((alignment) + ((size_t)(((size_t)(1U << 12)) - 1))) & ~((size_t )(((size_t)(1U << 12)) - 1))))) + 1)) | |||
| 945 | PAGE_CEILING(alignment))((((uintptr_t)extent_base_get(*extent)) + ((((alignment) + (( size_t)(((size_t)(1U << 12)) - 1))) & ~((size_t)((( size_t)(1U << 12)) - 1))) - 1)) & ((~((((alignment) + ((size_t)(((size_t)(1U << 12)) - 1))) & ~((size_t )(((size_t)(1U << 12)) - 1))))) + 1)) - (uintptr_t)extent_base_get(*extent); | |||
| 946 | assert(new_addr == NULL || leadsize == 0)do { if (__builtin_expect(!!(config_debug && !(new_addr == ((void*)0) || leadsize == 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 946, "new_addr == NULL || leadsize == 0"); abort (); } } while (0); | |||
| 947 | if (extent_size_get(*extent) < leadsize + esize) { | |||
| 948 | return extent_split_interior_cant_alloc; | |||
| 949 | } | |||
| 950 | size_t trailsize = extent_size_get(*extent) - leadsize - esize; | |||
| 951 | ||||
| 952 | *lead = NULL((void*)0); | |||
| 953 | *trail = NULL((void*)0); | |||
| 954 | *to_leak = NULL((void*)0); | |||
| 955 | *to_salvage = NULL((void*)0); | |||
| 956 | ||||
| 957 | /* Split the lead. */ | |||
| 958 | if (leadsize != 0) { | |||
| 959 | *lead = *extent; | |||
| 960 | *extent = extent_split_impl(tsdn, arena, r_extent_hooks, | |||
| 961 | *lead, leadsize, NSIZES235, false0, esize + trailsize, szind, | |||
| 962 | slab, growing_retained); | |||
| 963 | if (*extent == NULL((void*)0)) { | |||
| 964 | *to_leak = *lead; | |||
| 965 | *lead = NULL((void*)0); | |||
| 966 | return extent_split_interior_error; | |||
| 967 | } | |||
| 968 | } | |||
| 969 | ||||
| 970 | /* Split the trail. */ | |||
| 971 | if (trailsize != 0) { | |||
| 972 | *trail = extent_split_impl(tsdn, arena, r_extent_hooks, *extent, | |||
| 973 | esize, szind, slab, trailsize, NSIZES235, false0, | |||
| 974 | growing_retained); | |||
| 975 | if (*trail == NULL((void*)0)) { | |||
| 976 | *to_leak = *extent; | |||
| 977 | *to_salvage = *lead; | |||
| 978 | *lead = NULL((void*)0); | |||
| 979 | *extent = NULL((void*)0); | |||
| 980 | return extent_split_interior_error; | |||
| 981 | } | |||
| 982 | } | |||
| 983 | ||||
| 984 | if (leadsize == 0 && trailsize == 0) { | |||
| 985 | /* | |||
| 986 | * Splitting causes szind to be set as a side effect, but no | |||
| 987 | * splitting occurred. | |||
| 988 | */ | |||
| 989 | extent_szind_set(*extent, szind); | |||
| 990 | if (szind != NSIZES235) { | |||
| 991 | rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, | |||
| 992 | (uintptr_t)extent_addr_get(*extent), szind, slab); | |||
| 993 | if (slab && extent_size_get(*extent) > PAGE((size_t)(1U << 12))) { | |||
| 994 | rtree_szind_slab_update(tsdn, &extents_rtree, | |||
| 995 | rtree_ctx, | |||
| 996 | (uintptr_t)extent_past_get(*extent) - | |||
| 997 | (uintptr_t)PAGE((size_t)(1U << 12)), szind, slab); | |||
| 998 | } | |||
| 999 | } | |||
| 1000 | } | |||
| 1001 | ||||
| 1002 | return extent_split_interior_ok; | |||
| 1003 | } | |||
| 1004 | ||||
| 1005 | /* | |||
| 1006 | * This fulfills the indicated allocation request out of the given extent (which | |||
| 1007 | * the caller should have ensured was big enough). If there's any unused space | |||
| 1008 | * before or after the resulting allocation, that space is given its own extent | |||
| 1009 | * and put back into extents. | |||
| 1010 | */ | |||
| 1011 | static extent_t * | |||
| 1012 | extent_recycle_split(tsdn_t *tsdn, arena_t *arena, | |||
| 1013 | extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, | |||
| 1014 | void *new_addr, size_t size, size_t pad, size_t alignment, bool_Bool slab, | |||
| 1015 | szind_t szind, extent_t *extent, bool_Bool growing_retained) { | |||
| 1016 | extent_t *lead; | |||
| 1017 | extent_t *trail; | |||
| 1018 | extent_t *to_leak; | |||
| 1019 | extent_t *to_salvage; | |||
| 1020 | ||||
| 1021 | extent_split_interior_result_t result = extent_split_interior( | |||
| 1022 | tsdn, arena, r_extent_hooks, rtree_ctx, &extent, &lead, &trail, | |||
| 1023 | &to_leak, &to_salvage, new_addr, size, pad, alignment, slab, szind, | |||
| 1024 | growing_retained); | |||
| 1025 | ||||
| 1026 | if (result == extent_split_interior_ok) { | |||
| 1027 | if (lead != NULL((void*)0)) { | |||
| 1028 | extent_deactivate(tsdn, arena, extents, lead); | |||
| 1029 | } | |||
| 1030 | if (trail != NULL((void*)0)) { | |||
| 1031 | extent_deactivate(tsdn, arena, extents, trail); | |||
| 1032 | } | |||
| 1033 | return extent; | |||
| 1034 | } else { | |||
| 1035 | /* | |||
| 1036 | * We should have picked an extent that was large enough to | |||
| 1037 | * fulfill our allocation request. | |||
| 1038 | */ | |||
| 1039 | assert(result == extent_split_interior_error)do { if (__builtin_expect(!!(config_debug && !(result == extent_split_interior_error)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1039, "result == extent_split_interior_error" ); abort(); } } while (0); | |||
| 1040 | if (to_salvage != NULL((void*)0)) { | |||
| 1041 | extent_deregister(tsdn, to_salvage); | |||
| 1042 | } | |||
| 1043 | if (to_leak != NULL((void*)0)) { | |||
| 1044 | void *leak = extent_base_get(to_leak); | |||
| 1045 | extent_deregister_no_gdump_sub(tsdn, to_leak); | |||
| 1046 | extents_leak(tsdn, arena, r_extent_hooks, extents, | |||
| 1047 | to_leak, growing_retained); | |||
| 1048 | assert(extent_lock_from_addr(tsdn, rtree_ctx, leak)do { if (__builtin_expect(!!(config_debug && !(extent_lock_from_addr (tsdn, rtree_ctx, leak) == ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 1049, "extent_lock_from_addr(tsdn, rtree_ctx, leak) == NULL" ); abort(); } } while (0) | |||
| 1049 | == NULL)do { if (__builtin_expect(!!(config_debug && !(extent_lock_from_addr (tsdn, rtree_ctx, leak) == ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 1049, "extent_lock_from_addr(tsdn, rtree_ctx, leak) == NULL" ); abort(); } } while (0); | |||
| 1050 | } | |||
| 1051 | return NULL((void*)0); | |||
| 1052 | } | |||
| 1053 | unreachable()__builtin_unreachable(); | |||
| 1054 | } | |||
| 1055 | ||||
| 1056 | /* | |||
| 1057 | * Tries to satisfy the given allocation request by reusing one of the extents | |||
| 1058 | * in the given extents_t. | |||
| 1059 | */ | |||
| 1060 | static extent_t * | |||
| 1061 | extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, | |||
| 1062 | extents_t *extents, void *new_addr, size_t size, size_t pad, | |||
| 1063 | size_t alignment, bool_Bool slab, szind_t szind, bool_Bool *zero, bool_Bool *commit, | |||
| 1064 | bool_Bool growing_retained) { | |||
| 1065 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | |||
| 1066 | WITNESS_RANK_CORE11U, growing_retained ? 1 : 0); | |||
| 1067 | assert(new_addr == NULL || !slab)do { if (__builtin_expect(!!(config_debug && !(new_addr == ((void*)0) || !slab)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1067, "new_addr == NULL || !slab"); abort() ; } } while (0); | |||
| 1068 | assert(pad == 0 || !slab)do { if (__builtin_expect(!!(config_debug && !(pad == 0 || !slab)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1068, "pad == 0 || !slab"); abort(); } } while (0); | |||
| 1069 | assert(!*zero || !slab)do { if (__builtin_expect(!!(config_debug && !(!*zero || !slab)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1069, "!*zero || !slab"); abort(); } } while (0); | |||
| 1070 | ||||
| 1071 | rtree_ctx_t rtree_ctx_fallback; | |||
| 1072 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); | |||
| 1073 | ||||
| 1074 | extent_t *extent = extent_recycle_extract(tsdn, arena, r_extent_hooks, | |||
| 1075 | rtree_ctx, extents, new_addr, size, pad, alignment, slab, | |||
| 1076 | growing_retained); | |||
| 1077 | if (extent == NULL((void*)0)) { | |||
| 1078 | return NULL((void*)0); | |||
| 1079 | } | |||
| 1080 | ||||
| 1081 | extent = extent_recycle_split(tsdn, arena, r_extent_hooks, rtree_ctx, | |||
| 1082 | extents, new_addr, size, pad, alignment, slab, szind, extent, | |||
| 1083 | growing_retained); | |||
| 1084 | if (extent == NULL((void*)0)) { | |||
| 1085 | return NULL((void*)0); | |||
| 1086 | } | |||
| 1087 | ||||
| 1088 | if (*commit && !extent_committed_get(extent)) { | |||
| 1089 | if (extent_commit_impl(tsdn, arena, r_extent_hooks, extent, | |||
| 1090 | 0, extent_size_get(extent), growing_retained)) { | |||
| 1091 | extent_record(tsdn, arena, r_extent_hooks, extents, | |||
| 1092 | extent, growing_retained); | |||
| 1093 | return NULL((void*)0); | |||
| 1094 | } | |||
| 1095 | extent_zeroed_set(extent, true1); | |||
| 1096 | } | |||
| 1097 | ||||
| 1098 | if (extent_committed_get(extent)) { | |||
| 1099 | *commit = true1; | |||
| 1100 | } | |||
| 1101 | if (extent_zeroed_get(extent)) { | |||
| 1102 | *zero = true1; | |||
| 1103 | } | |||
| 1104 | ||||
| 1105 | if (pad != 0) { | |||
| 1106 | extent_addr_randomize(tsdn, extent, alignment); | |||
| 1107 | } | |||
| 1108 | assert(extent_state_get(extent) == extent_state_active)do { if (__builtin_expect(!!(config_debug && !(extent_state_get (extent) == extent_state_active)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1108, "extent_state_get(extent) == extent_state_active" ); abort(); } } while (0); | |||
| 1109 | if (slab) { | |||
| 1110 | extent_slab_set(extent, slab); | |||
| 1111 | extent_interior_register(tsdn, rtree_ctx, extent, szind); | |||
| 1112 | } | |||
| 1113 | ||||
| 1114 | if (*zero) { | |||
| 1115 | void *addr = extent_base_get(extent); | |||
| 1116 | size_t size = extent_size_get(extent); | |||
| 1117 | if (!extent_zeroed_get(extent)) { | |||
| 1118 | if (pages_purge_forced(addr, size)) { | |||
| 1119 | memset(addr, 0, size); | |||
| 1120 | } | |||
| 1121 | } else if (config_debug) { | |||
| 1122 | size_t *p = (size_t *)(uintptr_t)addr; | |||
| 1123 | for (size_t i = 0; i < size / sizeof(size_t); i++) { | |||
| 1124 | assert(p[i] == 0)do { if (__builtin_expect(!!(config_debug && !(p[i] == 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1124, "p[i] == 0"); abort(); } } while (0); | |||
| 1125 | } | |||
| 1126 | } | |||
| 1127 | } | |||
| 1128 | return extent; | |||
| 1129 | } | |||
| 1130 | ||||
| 1131 | /* | |||
| 1132 | * If the caller specifies (!*zero), it is still possible to receive zeroed | |||
| 1133 | * memory, in which case *zero is toggled to true. arena_extent_alloc() takes | |||
| 1134 | * advantage of this to avoid demanding zeroed extents, but taking advantage of | |||
| 1135 | * them if they are returned. | |||
| 1136 | */ | |||
| 1137 | static void * | |||
| 1138 | extent_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, | |||
| 1139 | size_t alignment, bool_Bool *zero, bool_Bool *commit, dss_prec_t dss_prec) { | |||
| 1140 | void *ret; | |||
| 1141 | ||||
| 1142 | assert(size != 0)do { if (__builtin_expect(!!(config_debug && !(size != 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1142, "size != 0"); abort(); } } while (0); | |||
| 1143 | assert(alignment != 0)do { if (__builtin_expect(!!(config_debug && !(alignment != 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1143, "alignment != 0"); abort(); } } while (0); | |||
| 1144 | ||||
| 1145 | /* "primary" dss. */ | |||
| 1146 | if (have_dss && dss_prec == dss_prec_primary && (ret = | |||
| 1147 | extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero, | |||
| 1148 | commit)) != NULL((void*)0)) { | |||
| 1149 | return ret; | |||
| 1150 | } | |||
| 1151 | /* mmap. */ | |||
| 1152 | if ((ret = extent_alloc_mmap(new_addr, size, alignment, zero, commit)) | |||
| 1153 | != NULL((void*)0)) { | |||
| 1154 | return ret; | |||
| 1155 | } | |||
| 1156 | /* "secondary" dss. */ | |||
| 1157 | if (have_dss && dss_prec == dss_prec_secondary && (ret = | |||
| 1158 | extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero, | |||
| 1159 | commit)) != NULL((void*)0)) { | |||
| 1160 | return ret; | |||
| 1161 | } | |||
| 1162 | ||||
| 1163 | /* All strategies for allocation failed. */ | |||
| 1164 | return NULL((void*)0); | |||
| 1165 | } | |||
| 1166 | ||||
| 1167 | static void * | |||
| 1168 | extent_alloc_default_impl(tsdn_t *tsdn, arena_t *arena, void *new_addr, | |||
| 1169 | size_t size, size_t alignment, bool_Bool *zero, bool_Bool *commit) { | |||
| 1170 | void *ret = extent_alloc_core(tsdn, arena, new_addr, size, alignment, zero, | |||
| 1171 | commit, (dss_prec_t)atomic_load_u(&arena->dss_prec, | |||
| 1172 | ATOMIC_RELAXEDatomic_memory_order_relaxed)); | |||
| 1173 | if (have_madvise_huge && ret) { | |||
| 1174 | pages_set_thp_state(ret, size); | |||
| 1175 | } | |||
| 1176 | return ret; | |||
| 1177 | } | |||
| 1178 | ||||
| 1179 | static void * | |||
| 1180 | extent_alloc_default(extent_hooks_t *extent_hooks, void *new_addr, size_t size, | |||
| 1181 | size_t alignment, bool_Bool *zero, bool_Bool *commit, unsigned arena_ind) { | |||
| 1182 | tsdn_t *tsdn; | |||
| 1183 | arena_t *arena; | |||
| 1184 | ||||
| 1185 | tsdn = tsdn_fetch(); | |||
| 1186 | arena = arena_get(tsdn, arena_ind, false0); | |||
| 1187 | /* | |||
| 1188 | * The arena we're allocating on behalf of must have been initialized | |||
| 1189 | * already. | |||
| 1190 | */ | |||
| 1191 | assert(arena != NULL)do { if (__builtin_expect(!!(config_debug && !(arena != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1191, "arena != NULL"); abort(); } } while ( 0); | |||
| 1192 | ||||
| 1193 | return extent_alloc_default_impl(tsdn, arena, new_addr, size, | |||
| 1194 | alignment, zero, commit); | |||
| 1195 | } | |||
| 1196 | ||||
| 1197 | static void | |||
| 1198 | extent_hook_pre_reentrancy(tsdn_t *tsdn, arena_t *arena) { | |||
| 1199 | tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn); | |||
| 1200 | if (arena == arena_get(tsd_tsdn(tsd), 0, false0)) { | |||
| 1201 | /* | |||
| 1202 | * The only legitimate case of customized extent hooks for a0 is | |||
| 1203 | * hooks with no allocation activities. One such example is to | |||
| 1204 | * place metadata on pre-allocated resources such as huge pages. | |||
| 1205 | * In that case, rely on reentrancy_level checks to catch | |||
| 1206 | * infinite recursions. | |||
| 1207 | */ | |||
| 1208 | pre_reentrancy(tsd, NULL((void*)0)); | |||
| 1209 | } else { | |||
| 1210 | pre_reentrancy(tsd, arena); | |||
| 1211 | } | |||
| 1212 | } | |||
| 1213 | ||||
| 1214 | static void | |||
| 1215 | extent_hook_post_reentrancy(tsdn_t *tsdn) { | |||
| 1216 | tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn); | |||
| 1217 | post_reentrancy(tsd); | |||
| 1218 | } | |||
| 1219 | ||||
| 1220 | /* | |||
| 1221 | * If virtual memory is retained, create increasingly larger extents from which | |||
| 1222 | * to split requested extents in order to limit the total number of disjoint | |||
| 1223 | * virtual memory ranges retained by each arena. | |||
| 1224 | */ | |||
| 1225 | static extent_t * | |||
| 1226 | extent_grow_retained(tsdn_t *tsdn, arena_t *arena, | |||
| 1227 | extent_hooks_t **r_extent_hooks, size_t size, size_t pad, size_t alignment, | |||
| 1228 | bool_Bool slab, szind_t szind, bool_Bool *zero, bool_Bool *commit) { | |||
| 1229 | malloc_mutex_assert_owner(tsdn, &arena->extent_grow_mtx); | |||
| 1230 | assert(pad == 0 || !slab)do { if (__builtin_expect(!!(config_debug && !(pad == 0 || !slab)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1230, "pad == 0 || !slab"); abort(); } } while (0); | |||
| 1231 | assert(!*zero || !slab)do { if (__builtin_expect(!!(config_debug && !(!*zero || !slab)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1231, "!*zero || !slab"); abort(); } } while (0); | |||
| 1232 | ||||
| 1233 | size_t esize = size + pad; | |||
| 1234 | size_t alloc_size_min = esize + PAGE_CEILING(alignment)(((alignment) + ((size_t)(((size_t)(1U << 12)) - 1))) & ~((size_t)(((size_t)(1U << 12)) - 1))) - PAGE((size_t)(1U << 12)); | |||
| 1235 | /* Beware size_t wrap-around. */ | |||
| 1236 | if (alloc_size_min < esize) { | |||
| 1237 | goto label_err; | |||
| 1238 | } | |||
| 1239 | /* | |||
| 1240 | * Find the next extent size in the series that would be large enough to | |||
| 1241 | * satisfy this request. | |||
| 1242 | */ | |||
| 1243 | pszind_t egn_skip = 0; | |||
| 1244 | size_t alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip); | |||
| 1245 | while (alloc_size < alloc_size_min) { | |||
| 1246 | egn_skip++; | |||
| 1247 | if (arena->extent_grow_next + egn_skip == NPSIZES199) { | |||
| 1248 | /* Outside legal range. */ | |||
| 1249 | goto label_err; | |||
| 1250 | } | |||
| 1251 | assert(arena->extent_grow_next + egn_skip < NPSIZES)do { if (__builtin_expect(!!(config_debug && !(arena-> extent_grow_next + egn_skip < 199)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1251, "arena->extent_grow_next + egn_skip < NPSIZES" ); abort(); } } while (0); | |||
| 1252 | alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip); | |||
| 1253 | } | |||
| 1254 | ||||
| 1255 | extent_t *extent = extent_alloc(tsdn, arena); | |||
| 1256 | if (extent == NULL((void*)0)) { | |||
| 1257 | goto label_err; | |||
| 1258 | } | |||
| 1259 | bool_Bool zeroed = false0; | |||
| 1260 | bool_Bool committed = false0; | |||
| 1261 | ||||
| 1262 | void *ptr; | |||
| 1263 | if (*r_extent_hooks == &extent_hooks_default) { | |||
| 1264 | ptr = extent_alloc_default_impl(tsdn, arena, NULL((void*)0), | |||
| 1265 | alloc_size, PAGE((size_t)(1U << 12)), &zeroed, &committed); | |||
| 1266 | } else { | |||
| 1267 | extent_hook_pre_reentrancy(tsdn, arena); | |||
| 1268 | ptr = (*r_extent_hooks)->alloc(*r_extent_hooks, NULL((void*)0), | |||
| 1269 | alloc_size, PAGE((size_t)(1U << 12)), &zeroed, &committed, | |||
| 1270 | arena_ind_get(arena)); | |||
| 1271 | extent_hook_post_reentrancy(tsdn); | |||
| 1272 | } | |||
| 1273 | ||||
| 1274 | extent_init(extent, arena, ptr, alloc_size, false0, NSIZES235, | |||
| 1275 | arena_extent_sn_next(arena), extent_state_active, zeroed, | |||
| 1276 | committed, true1); | |||
| 1277 | if (ptr == NULL((void*)0)) { | |||
| 1278 | extent_dalloc(tsdn, arena, extent); | |||
| 1279 | goto label_err; | |||
| 1280 | } | |||
| 1281 | ||||
| 1282 | if (extent_register_no_gdump_add(tsdn, extent)) { | |||
| 1283 | extents_leak(tsdn, arena, r_extent_hooks, | |||
| 1284 | &arena->extents_retained, extent, true1); | |||
| 1285 | goto label_err; | |||
| 1286 | } | |||
| 1287 | ||||
| 1288 | if (extent_zeroed_get(extent) && extent_committed_get(extent)) { | |||
| 1289 | *zero = true1; | |||
| 1290 | } | |||
| 1291 | if (extent_committed_get(extent)) { | |||
| 1292 | *commit = true1; | |||
| 1293 | } | |||
| 1294 | ||||
| 1295 | rtree_ctx_t rtree_ctx_fallback; | |||
| 1296 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); | |||
| 1297 | ||||
| 1298 | extent_t *lead; | |||
| 1299 | extent_t *trail; | |||
| 1300 | extent_t *to_leak; | |||
| 1301 | extent_t *to_salvage; | |||
| 1302 | extent_split_interior_result_t result = extent_split_interior( | |||
| 1303 | tsdn, arena, r_extent_hooks, rtree_ctx, &extent, &lead, &trail, | |||
| 1304 | &to_leak, &to_salvage, NULL((void*)0), size, pad, alignment, slab, szind, | |||
| 1305 | true1); | |||
| 1306 | ||||
| 1307 | if (result == extent_split_interior_ok) { | |||
| 1308 | if (lead != NULL((void*)0)) { | |||
| 1309 | extent_record(tsdn, arena, r_extent_hooks, | |||
| 1310 | &arena->extents_retained, lead, true1); | |||
| 1311 | } | |||
| 1312 | if (trail != NULL((void*)0)) { | |||
| 1313 | extent_record(tsdn, arena, r_extent_hooks, | |||
| 1314 | &arena->extents_retained, trail, true1); | |||
| 1315 | } | |||
| 1316 | } else { | |||
| 1317 | /* | |||
| 1318 | * We should have allocated a sufficiently large extent; the | |||
| 1319 | * cant_alloc case should not occur. | |||
| 1320 | */ | |||
| 1321 | assert(result == extent_split_interior_error)do { if (__builtin_expect(!!(config_debug && !(result == extent_split_interior_error)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1321, "result == extent_split_interior_error" ); abort(); } } while (0); | |||
| 1322 | if (to_salvage != NULL((void*)0)) { | |||
| 1323 | if (config_prof) { | |||
| 1324 | extent_gdump_add(tsdn, to_salvage); | |||
| 1325 | } | |||
| 1326 | extent_record(tsdn, arena, r_extent_hooks, | |||
| 1327 | &arena->extents_retained, to_salvage, true1); | |||
| 1328 | } | |||
| 1329 | if (to_leak != NULL((void*)0)) { | |||
| 1330 | extent_deregister_no_gdump_sub(tsdn, to_leak); | |||
| 1331 | extents_leak(tsdn, arena, r_extent_hooks, | |||
| 1332 | &arena->extents_retained, to_leak, true1); | |||
| 1333 | } | |||
| 1334 | goto label_err; | |||
| 1335 | } | |||
| 1336 | ||||
| 1337 | if (*commit && !extent_committed_get(extent)) { | |||
| 1338 | if (extent_commit_impl(tsdn, arena, r_extent_hooks, extent, 0, | |||
| 1339 | extent_size_get(extent), true1)) { | |||
| 1340 | extent_record(tsdn, arena, r_extent_hooks, | |||
| 1341 | &arena->extents_retained, extent, true1); | |||
| 1342 | goto label_err; | |||
| 1343 | } | |||
| 1344 | extent_zeroed_set(extent, true1); | |||
| 1345 | } | |||
| 1346 | ||||
| 1347 | /* | |||
| 1348 | * Increment extent_grow_next if doing so wouldn't exceed the allowed | |||
| 1349 | * range. | |||
| 1350 | */ | |||
| 1351 | if (arena->extent_grow_next + egn_skip + 1 <= | |||
| 1352 | arena->retain_grow_limit) { | |||
| 1353 | arena->extent_grow_next += egn_skip + 1; | |||
| 1354 | } else { | |||
| 1355 | arena->extent_grow_next = arena->retain_grow_limit; | |||
| 1356 | } | |||
| 1357 | /* All opportunities for failure are past. */ | |||
| 1358 | malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx); | |||
| 1359 | ||||
| 1360 | if (config_prof) { | |||
| 1361 | /* Adjust gdump stats now that extent is final size. */ | |||
| 1362 | extent_gdump_add(tsdn, extent); | |||
| 1363 | } | |||
| 1364 | if (pad != 0) { | |||
| 1365 | extent_addr_randomize(tsdn, extent, alignment); | |||
| 1366 | } | |||
| 1367 | if (slab) { | |||
| 1368 | rtree_ctx_t rtree_ctx_fallback; | |||
| 1369 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, | |||
| 1370 | &rtree_ctx_fallback); | |||
| 1371 | ||||
| 1372 | extent_slab_set(extent, true1); | |||
| 1373 | extent_interior_register(tsdn, rtree_ctx, extent, szind); | |||
| 1374 | } | |||
| 1375 | if (*zero && !extent_zeroed_get(extent)) { | |||
| 1376 | void *addr = extent_base_get(extent); | |||
| 1377 | size_t size = extent_size_get(extent); | |||
| 1378 | if (pages_purge_forced(addr, size)) { | |||
| 1379 | memset(addr, 0, size); | |||
| 1380 | } | |||
| 1381 | } | |||
| 1382 | ||||
| 1383 | return extent; | |||
| 1384 | label_err: | |||
| 1385 | malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx); | |||
| 1386 | return NULL((void*)0); | |||
| 1387 | } | |||
| 1388 | ||||
| 1389 | static extent_t * | |||
| 1390 | extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, | |||
| 1391 | extent_hooks_t **r_extent_hooks, void *new_addr, size_t size, size_t pad, | |||
| 1392 | size_t alignment, bool_Bool slab, szind_t szind, bool_Bool *zero, bool_Bool *commit) { | |||
| 1393 | assert(size != 0)do { if (__builtin_expect(!!(config_debug && !(size != 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1393, "size != 0"); abort(); } } while (0); | |||
| 1394 | assert(alignment != 0)do { if (__builtin_expect(!!(config_debug && !(alignment != 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1394, "alignment != 0"); abort(); } } while (0); | |||
| 1395 | ||||
| 1396 | malloc_mutex_lock(tsdn, &arena->extent_grow_mtx); | |||
| 1397 | ||||
| 1398 | extent_t *extent = extent_recycle(tsdn, arena, r_extent_hooks, | |||
| 1399 | &arena->extents_retained, new_addr, size, pad, alignment, slab, | |||
| 1400 | szind, zero, commit, true1); | |||
| 1401 | if (extent != NULL((void*)0)) { | |||
| 1402 | malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx); | |||
| 1403 | if (config_prof) { | |||
| 1404 | extent_gdump_add(tsdn, extent); | |||
| 1405 | } | |||
| 1406 | } else if (opt_retain && new_addr == NULL((void*)0)) { | |||
| 1407 | extent = extent_grow_retained(tsdn, arena, r_extent_hooks, size, | |||
| 1408 | pad, alignment, slab, szind, zero, commit); | |||
| 1409 | /* extent_grow_retained() always releases extent_grow_mtx. */ | |||
| 1410 | } else { | |||
| 1411 | malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx); | |||
| 1412 | } | |||
| 1413 | malloc_mutex_assert_not_owner(tsdn, &arena->extent_grow_mtx); | |||
| 1414 | ||||
| 1415 | return extent; | |||
| 1416 | } | |||
| 1417 | ||||
| 1418 | static extent_t * | |||
| 1419 | extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, | |||
| 1420 | extent_hooks_t **r_extent_hooks, void *new_addr, size_t size, size_t pad, | |||
| 1421 | size_t alignment, bool_Bool slab, szind_t szind, bool_Bool *zero, bool_Bool *commit) { | |||
| 1422 | size_t esize = size + pad; | |||
| 1423 | extent_t *extent = extent_alloc(tsdn, arena); | |||
| 1424 | if (extent == NULL((void*)0)) { | |||
| 1425 | return NULL((void*)0); | |||
| 1426 | } | |||
| 1427 | void *addr; | |||
| 1428 | if (*r_extent_hooks == &extent_hooks_default) { | |||
| 1429 | /* Call directly to propagate tsdn. */ | |||
| 1430 | addr = extent_alloc_default_impl(tsdn, arena, new_addr, esize, | |||
| 1431 | alignment, zero, commit); | |||
| 1432 | } else { | |||
| 1433 | extent_hook_pre_reentrancy(tsdn, arena); | |||
| 1434 | addr = (*r_extent_hooks)->alloc(*r_extent_hooks, new_addr, | |||
| 1435 | esize, alignment, zero, commit, arena_ind_get(arena)); | |||
| 1436 | extent_hook_post_reentrancy(tsdn); | |||
| 1437 | } | |||
| 1438 | if (addr == NULL((void*)0)) { | |||
| 1439 | extent_dalloc(tsdn, arena, extent); | |||
| 1440 | return NULL((void*)0); | |||
| 1441 | } | |||
| 1442 | extent_init(extent, arena, addr, esize, slab, szind, | |||
| 1443 | arena_extent_sn_next(arena), extent_state_active, *zero, *commit, | |||
| 1444 | true1); | |||
| 1445 | if (pad != 0) { | |||
| 1446 | extent_addr_randomize(tsdn, extent, alignment); | |||
| 1447 | } | |||
| 1448 | if (extent_register(tsdn, extent)) { | |||
| 1449 | extents_leak(tsdn, arena, r_extent_hooks, | |||
| 1450 | &arena->extents_retained, extent, false0); | |||
| 1451 | return NULL((void*)0); | |||
| 1452 | } | |||
| 1453 | ||||
| 1454 | return extent; | |||
| 1455 | } | |||
| 1456 | ||||
| 1457 | extent_t * | |||
| 1458 | extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, | |||
| 1459 | extent_hooks_t **r_extent_hooks, void *new_addr, size_t size, size_t pad, | |||
| 1460 | size_t alignment, bool_Bool slab, szind_t szind, bool_Bool *zero, bool_Bool *commit) { | |||
| 1461 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | |||
| 1462 | WITNESS_RANK_CORE11U, 0); | |||
| 1463 | ||||
| 1464 | extent_hooks_assure_initialized(arena, r_extent_hooks); | |||
| 1465 | ||||
| 1466 | extent_t *extent = extent_alloc_retained(tsdn, arena, r_extent_hooks, | |||
| 1467 | new_addr, size, pad, alignment, slab, szind, zero, commit); | |||
| 1468 | if (extent == NULL((void*)0)) { | |||
| 1469 | if (opt_retain && new_addr != NULL((void*)0)) { | |||
| 1470 | /* | |||
| 1471 | * When retain is enabled and new_addr is set, we do not | |||
| 1472 | * attempt extent_alloc_wrapper_hard which does mmap | |||
| 1473 | * that is very unlikely to succeed (unless it happens | |||
| 1474 | * to be at the end). | |||
| 1475 | */ | |||
| 1476 | return NULL((void*)0); | |||
| 1477 | } | |||
| 1478 | extent = extent_alloc_wrapper_hard(tsdn, arena, r_extent_hooks, | |||
| 1479 | new_addr, size, pad, alignment, slab, szind, zero, commit); | |||
| 1480 | } | |||
| 1481 | ||||
| 1482 | assert(extent == NULL || extent_dumpable_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent == ((void*)0) || extent_dumpable_get(extent))), 0)) { malloc_printf ( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", "src/extent.c" , 1482, "extent == NULL || extent_dumpable_get(extent)"); abort (); } } while (0); | |||
| 1483 | return extent; | |||
| 1484 | } | |||
| 1485 | ||||
| 1486 | static bool_Bool | |||
| 1487 | extent_can_coalesce(arena_t *arena, extents_t *extents, const extent_t *inner, | |||
| 1488 | const extent_t *outer) { | |||
| 1489 | assert(extent_arena_get(inner) == arena)do { if (__builtin_expect(!!(config_debug && !(extent_arena_get (inner) == arena)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1489, "extent_arena_get(inner) == arena"); abort (); } } while (0); | |||
| 1490 | if (extent_arena_get(outer) != arena) { | |||
| 1491 | return false0; | |||
| 1492 | } | |||
| 1493 | ||||
| 1494 | assert(extent_state_get(inner) == extent_state_active)do { if (__builtin_expect(!!(config_debug && !(extent_state_get (inner) == extent_state_active)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1494, "extent_state_get(inner) == extent_state_active" ); abort(); } } while (0); | |||
| 1495 | if (extent_state_get(outer) != extents->state) { | |||
| 1496 | return false0; | |||
| 1497 | } | |||
| 1498 | ||||
| 1499 | if (extent_committed_get(inner) != extent_committed_get(outer)) { | |||
| 1500 | return false0; | |||
| 1501 | } | |||
| 1502 | ||||
| 1503 | return true1; | |||
| 1504 | } | |||
| 1505 | ||||
| 1506 | static bool_Bool | |||
| 1507 | extent_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, | |||
| 1508 | extents_t *extents, extent_t *inner, extent_t *outer, bool_Bool forward, | |||
| 1509 | bool_Bool growing_retained) { | |||
| 1510 | assert(extent_can_coalesce(arena, extents, inner, outer))do { if (__builtin_expect(!!(config_debug && !(extent_can_coalesce (arena, extents, inner, outer))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1510, "extent_can_coalesce(arena, extents, inner, outer)" ); abort(); } } while (0); | |||
| 1511 | ||||
| 1512 | extent_activate_locked(tsdn, arena, extents, outer); | |||
| 1513 | ||||
| 1514 | malloc_mutex_unlock(tsdn, &extents->mtx); | |||
| 1515 | bool_Bool err = extent_merge_impl(tsdn, arena, r_extent_hooks, | |||
| 1516 | forward ? inner : outer, forward ? outer : inner, growing_retained); | |||
| 1517 | malloc_mutex_lock(tsdn, &extents->mtx); | |||
| 1518 | ||||
| 1519 | if (err) { | |||
| 1520 | extent_deactivate_locked(tsdn, arena, extents, outer); | |||
| 1521 | } | |||
| 1522 | ||||
| 1523 | return err; | |||
| 1524 | } | |||
| 1525 | ||||
| 1526 | static extent_t * | |||
| 1527 | extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, | |||
| 1528 | extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, | |||
| 1529 | extent_t *extent, bool_Bool *coalesced, bool_Bool growing_retained) { | |||
| 1530 | /* | |||
| 1531 | * Continue attempting to coalesce until failure, to protect against | |||
| 1532 | * races with other threads that are thwarted by this one. | |||
| 1533 | */ | |||
| 1534 | bool_Bool again; | |||
| 1535 | do { | |||
| 1536 | again = false0; | |||
| 1537 | ||||
| 1538 | /* Try to coalesce forward. */ | |||
| 1539 | extent_t *next = extent_lock_from_addr(tsdn, rtree_ctx, | |||
| 1540 | extent_past_get(extent)); | |||
| 1541 | if (next != NULL((void*)0)) { | |||
| 1542 | /* | |||
| 1543 | * extents->mtx only protects against races for | |||
| 1544 | * like-state extents, so call extent_can_coalesce() | |||
| 1545 | * before releasing next's pool lock. | |||
| 1546 | */ | |||
| 1547 | bool_Bool can_coalesce = extent_can_coalesce(arena, extents, | |||
| 1548 | extent, next); | |||
| 1549 | ||||
| 1550 | extent_unlock(tsdn, next); | |||
| 1551 | ||||
| 1552 | if (can_coalesce && !extent_coalesce(tsdn, arena, | |||
| 1553 | r_extent_hooks, extents, extent, next, true1, | |||
| 1554 | growing_retained)) { | |||
| 1555 | if (extents->delay_coalesce) { | |||
| 1556 | /* Do minimal coalescing. */ | |||
| 1557 | *coalesced = true1; | |||
| ||||
| 1558 | return extent; | |||
| 1559 | } | |||
| 1560 | again = true1; | |||
| 1561 | } | |||
| 1562 | } | |||
| 1563 | ||||
| 1564 | /* Try to coalesce backward. */ | |||
| 1565 | extent_t *prev = extent_lock_from_addr(tsdn, rtree_ctx, | |||
| 1566 | extent_before_get(extent)); | |||
| 1567 | if (prev != NULL((void*)0)) { | |||
| 1568 | bool_Bool can_coalesce = extent_can_coalesce(arena, extents, | |||
| 1569 | extent, prev); | |||
| 1570 | extent_unlock(tsdn, prev); | |||
| 1571 | ||||
| 1572 | if (can_coalesce && !extent_coalesce(tsdn, arena, | |||
| 1573 | r_extent_hooks, extents, extent, prev, false0, | |||
| 1574 | growing_retained)) { | |||
| 1575 | extent = prev; | |||
| 1576 | if (extents->delay_coalesce) { | |||
| 1577 | /* Do minimal coalescing. */ | |||
| 1578 | *coalesced = true1; | |||
| 1579 | return extent; | |||
| 1580 | } | |||
| 1581 | again = true1; | |||
| 1582 | } | |||
| 1583 | } | |||
| 1584 | } while (again); | |||
| 1585 | ||||
| 1586 | if (extents->delay_coalesce) { | |||
| 1587 | *coalesced = false0; | |||
| 1588 | } | |||
| 1589 | return extent; | |||
| 1590 | } | |||
| 1591 | ||||
| 1592 | /* | |||
| 1593 | * Does the metadata management portions of putting an unused extent into the | |||
| 1594 | * given extents_t (coalesces, deregisters slab interiors, the heap operations). | |||
| 1595 | */ | |||
| 1596 | static void | |||
| 1597 | extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, | |||
| 1598 | extents_t *extents, extent_t *extent, bool_Bool growing_retained) { | |||
| 1599 | rtree_ctx_t rtree_ctx_fallback; | |||
| 1600 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); | |||
| 1601 | ||||
| 1602 | assert((extents_state_get(extents) != extent_state_dirty &&do { if (__builtin_expect(!!(config_debug && !((extents_state_get (extents) != extent_state_dirty && extents_state_get( extents) != extent_state_muzzy) || !extent_zeroed_get(extent) )), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1604, "(extents_state_get(extents) != extent_state_dirty && extents_state_get(extents) != extent_state_muzzy) || !extent_zeroed_get(extent)" ); abort(); } } while (0) | |||
| 1603 | extents_state_get(extents) != extent_state_muzzy) ||do { if (__builtin_expect(!!(config_debug && !((extents_state_get (extents) != extent_state_dirty && extents_state_get( extents) != extent_state_muzzy) || !extent_zeroed_get(extent) )), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1604, "(extents_state_get(extents) != extent_state_dirty && extents_state_get(extents) != extent_state_muzzy) || !extent_zeroed_get(extent)" ); abort(); } } while (0) | |||
| 1604 | !extent_zeroed_get(extent))do { if (__builtin_expect(!!(config_debug && !((extents_state_get (extents) != extent_state_dirty && extents_state_get( extents) != extent_state_muzzy) || !extent_zeroed_get(extent) )), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1604, "(extents_state_get(extents) != extent_state_dirty && extents_state_get(extents) != extent_state_muzzy) || !extent_zeroed_get(extent)" ); abort(); } } while (0); | |||
| 1605 | ||||
| 1606 | malloc_mutex_lock(tsdn, &extents->mtx); | |||
| 1607 | extent_hooks_assure_initialized(arena, r_extent_hooks); | |||
| 1608 | ||||
| 1609 | extent_szind_set(extent, NSIZES235); | |||
| 1610 | if (extent_slab_get(extent)) { | |||
| 1611 | extent_interior_deregister(tsdn, rtree_ctx, extent); | |||
| 1612 | extent_slab_set(extent, false0); | |||
| 1613 | } | |||
| 1614 | ||||
| 1615 | assert(rtree_extent_read(tsdn, &extents_rtree, rtree_ctx,do { if (__builtin_expect(!!(config_debug && !(rtree_extent_read (tsdn, &extents_rtree, rtree_ctx, (uintptr_t)extent_base_get (extent), 1) == extent)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1616, "rtree_extent_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)extent_base_get(extent), true) == extent" ); abort(); } } while (0) | |||
| 1616 | (uintptr_t)extent_base_get(extent), true) == extent)do { if (__builtin_expect(!!(config_debug && !(rtree_extent_read (tsdn, &extents_rtree, rtree_ctx, (uintptr_t)extent_base_get (extent), 1) == extent)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1616, "rtree_extent_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)extent_base_get(extent), true) == extent" ); abort(); } } while (0); | |||
| 1617 | ||||
| 1618 | if (!extents->delay_coalesce) { | |||
| 1619 | extent = extent_try_coalesce(tsdn, arena, r_extent_hooks, | |||
| 1620 | rtree_ctx, extents, extent, NULL((void*)0), growing_retained); | |||
| 1621 | } else if (extent_size_get(extent) >= LARGE_MINCLASS(((size_t)1) << 14)) { | |||
| 1622 | /* Always coalesce large extents eagerly. */ | |||
| 1623 | bool_Bool coalesced; | |||
| 1624 | size_t prev_size; | |||
| 1625 | do { | |||
| 1626 | prev_size = extent_size_get(extent); | |||
| 1627 | assert(extent_state_get(extent) == extent_state_active)do { if (__builtin_expect(!!(config_debug && !(extent_state_get (extent) == extent_state_active)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1627, "extent_state_get(extent) == extent_state_active" ); abort(); } } while (0); | |||
| 1628 | extent = extent_try_coalesce(tsdn, arena, | |||
| 1629 | r_extent_hooks, rtree_ctx, extents, extent, | |||
| 1630 | &coalesced, growing_retained); | |||
| 1631 | } while (coalesced && | |||
| 1632 | extent_size_get(extent) >= prev_size + LARGE_MINCLASS(((size_t)1) << 14)); | |||
| 1633 | } | |||
| 1634 | extent_deactivate_locked(tsdn, arena, extents, extent); | |||
| 1635 | ||||
| 1636 | malloc_mutex_unlock(tsdn, &extents->mtx); | |||
| 1637 | } | |||
| 1638 | ||||
| 1639 | void | |||
| 1640 | extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { | |||
| 1641 | extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER((void*)0); | |||
| 1642 | ||||
| 1643 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | |||
| 1644 | WITNESS_RANK_CORE11U, 0); | |||
| 1645 | ||||
| 1646 | if (extent_register(tsdn, extent)) { | |||
| ||||
| 1647 | extents_leak(tsdn, arena, &extent_hooks, | |||
| 1648 | &arena->extents_retained, extent, false0); | |||
| 1649 | return; | |||
| 1650 | } | |||
| 1651 | extent_dalloc_wrapper(tsdn, arena, &extent_hooks, extent); | |||
| 1652 | } | |||
| 1653 | ||||
| 1654 | static bool_Bool | |||
| 1655 | extent_dalloc_default_impl(void *addr, size_t size) { | |||
| 1656 | if (!have_dss || !extent_in_dss(addr)) { | |||
| 1657 | return extent_dalloc_mmap(addr, size); | |||
| 1658 | } | |||
| 1659 | return true1; | |||
| 1660 | } | |||
| 1661 | ||||
| 1662 | static bool_Bool | |||
| 1663 | extent_dalloc_default(extent_hooks_t *extent_hooks, void *addr, size_t size, | |||
| 1664 | bool_Bool committed, unsigned arena_ind) { | |||
| 1665 | return extent_dalloc_default_impl(addr, size); | |||
| 1666 | } | |||
| 1667 | ||||
| 1668 | static bool_Bool | |||
| 1669 | extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena, | |||
| 1670 | extent_hooks_t **r_extent_hooks, extent_t *extent) { | |||
| 1671 | bool_Bool err; | |||
| 1672 | ||||
| 1673 | assert(extent_base_get(extent) != NULL)do { if (__builtin_expect(!!(config_debug && !(extent_base_get (extent) != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1673, "extent_base_get(extent) != NULL"); abort (); } } while (0); | |||
| 1674 | assert(extent_size_get(extent) != 0)do { if (__builtin_expect(!!(config_debug && !(extent_size_get (extent) != 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1674, "extent_size_get(extent) != 0"); abort (); } } while (0); | |||
| 1675 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | |||
| 1676 | WITNESS_RANK_CORE11U, 0); | |||
| 1677 | ||||
| 1678 | extent_addr_set(extent, extent_base_get(extent)); | |||
| 1679 | ||||
| 1680 | extent_hooks_assure_initialized(arena, r_extent_hooks); | |||
| 1681 | /* Try to deallocate. */ | |||
| 1682 | if (*r_extent_hooks == &extent_hooks_default) { | |||
| 1683 | /* Call directly to propagate tsdn. */ | |||
| 1684 | err = extent_dalloc_default_impl(extent_base_get(extent), | |||
| 1685 | extent_size_get(extent)); | |||
| 1686 | } else { | |||
| 1687 | extent_hook_pre_reentrancy(tsdn, arena); | |||
| 1688 | err = ((*r_extent_hooks)->dalloc == NULL((void*)0) || | |||
| 1689 | (*r_extent_hooks)->dalloc(*r_extent_hooks, | |||
| 1690 | extent_base_get(extent), extent_size_get(extent), | |||
| 1691 | extent_committed_get(extent), arena_ind_get(arena))); | |||
| 1692 | extent_hook_post_reentrancy(tsdn); | |||
| 1693 | } | |||
| 1694 | ||||
| 1695 | if (!err) { | |||
| 1696 | extent_dalloc(tsdn, arena, extent); | |||
| 1697 | } | |||
| 1698 | ||||
| 1699 | return err; | |||
| 1700 | } | |||
| 1701 | ||||
| 1702 | void | |||
| 1703 | extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, | |||
| 1704 | extent_hooks_t **r_extent_hooks, extent_t *extent) { | |||
| 1705 | assert(extent_dumpable_get(extent))do { if (__builtin_expect(!!(config_debug && !(extent_dumpable_get (extent))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1705, "extent_dumpable_get(extent)"); abort (); } } while (0); | |||
| 1706 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | |||
| 1707 | WITNESS_RANK_CORE11U, 0); | |||
| 1708 | ||||
| 1709 | /* | |||
| 1710 | * Deregister first to avoid a race with other allocating threads, and | |||
| 1711 | * reregister if deallocation fails. | |||
| 1712 | */ | |||
| 1713 | extent_deregister(tsdn, extent); | |||
| 1714 | if (!extent_dalloc_wrapper_try(tsdn, arena, r_extent_hooks, extent)) { | |||
| 1715 | return; | |||
| 1716 | } | |||
| 1717 | ||||
| 1718 | extent_reregister(tsdn, extent); | |||
| 1719 | if (*r_extent_hooks != &extent_hooks_default) { | |||
| 1720 | extent_hook_pre_reentrancy(tsdn, arena); | |||
| 1721 | } | |||
| 1722 | /* Try to decommit; purge if that fails. */ | |||
| 1723 | bool_Bool zeroed; | |||
| 1724 | if (!extent_committed_get(extent)) { | |||
| 1725 | zeroed = true1; | |||
| 1726 | } else if (!extent_decommit_wrapper(tsdn, arena, r_extent_hooks, extent, | |||
| 1727 | 0, extent_size_get(extent))) { | |||
| 1728 | zeroed = true1; | |||
| 1729 | } else if ((*r_extent_hooks)->purge_forced != NULL((void*)0) && | |||
| 1730 | !(*r_extent_hooks)->purge_forced(*r_extent_hooks, | |||
| 1731 | extent_base_get(extent), extent_size_get(extent), 0, | |||
| 1732 | extent_size_get(extent), arena_ind_get(arena))) { | |||
| 1733 | zeroed = true1; | |||
| 1734 | } else if (extent_state_get(extent) == extent_state_muzzy || | |||
| 1735 | ((*r_extent_hooks)->purge_lazy != NULL((void*)0) && | |||
| 1736 | !(*r_extent_hooks)->purge_lazy(*r_extent_hooks, | |||
| 1737 | extent_base_get(extent), extent_size_get(extent), 0, | |||
| 1738 | extent_size_get(extent), arena_ind_get(arena)))) { | |||
| 1739 | zeroed = false0; | |||
| 1740 | } else { | |||
| 1741 | zeroed = false0; | |||
| 1742 | } | |||
| 1743 | if (*r_extent_hooks != &extent_hooks_default) { | |||
| 1744 | extent_hook_post_reentrancy(tsdn); | |||
| 1745 | } | |||
| 1746 | extent_zeroed_set(extent, zeroed); | |||
| 1747 | ||||
| 1748 | if (config_prof
| |||
| 1749 | extent_gdump_sub(tsdn, extent); | |||
| 1750 | } | |||
| 1751 | ||||
| 1752 | extent_record(tsdn, arena, r_extent_hooks, &arena->extents_retained, | |||
| 1753 | extent, false0); | |||
| 1754 | } | |||
| 1755 | ||||
| 1756 | static void | |||
| 1757 | extent_destroy_default_impl(void *addr, size_t size) { | |||
| 1758 | if (!have_dss || !extent_in_dss(addr)) { | |||
| 1759 | pages_unmap(addr, size); | |||
| 1760 | } | |||
| 1761 | } | |||
| 1762 | ||||
| 1763 | static void | |||
| 1764 | extent_destroy_default(extent_hooks_t *extent_hooks, void *addr, size_t size, | |||
| 1765 | bool_Bool committed, unsigned arena_ind) { | |||
| 1766 | extent_destroy_default_impl(addr, size); | |||
| 1767 | } | |||
| 1768 | ||||
| 1769 | void | |||
| 1770 | extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, | |||
| 1771 | extent_hooks_t **r_extent_hooks, extent_t *extent) { | |||
| 1772 | assert(extent_base_get(extent) != NULL)do { if (__builtin_expect(!!(config_debug && !(extent_base_get (extent) != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1772, "extent_base_get(extent) != NULL"); abort (); } } while (0); | |||
| 1773 | assert(extent_size_get(extent) != 0)do { if (__builtin_expect(!!(config_debug && !(extent_size_get (extent) != 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1773, "extent_size_get(extent) != 0"); abort (); } } while (0); | |||
| 1774 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | |||
| 1775 | WITNESS_RANK_CORE11U, 0); | |||
| 1776 | ||||
| 1777 | /* Deregister first to avoid a race with other allocating threads. */ | |||
| 1778 | extent_deregister(tsdn, extent); | |||
| 1779 | ||||
| 1780 | extent_addr_set(extent, extent_base_get(extent)); | |||
| 1781 | ||||
| 1782 | extent_hooks_assure_initialized(arena, r_extent_hooks); | |||
| 1783 | /* Try to destroy; silently fail otherwise. */ | |||
| 1784 | if (*r_extent_hooks == &extent_hooks_default) { | |||
| 1785 | /* Call directly to propagate tsdn. */ | |||
| 1786 | extent_destroy_default_impl(extent_base_get(extent), | |||
| 1787 | extent_size_get(extent)); | |||
| 1788 | } else if ((*r_extent_hooks)->destroy != NULL((void*)0)) { | |||
| 1789 | extent_hook_pre_reentrancy(tsdn, arena); | |||
| 1790 | (*r_extent_hooks)->destroy(*r_extent_hooks, | |||
| 1791 | extent_base_get(extent), extent_size_get(extent), | |||
| 1792 | extent_committed_get(extent), arena_ind_get(arena)); | |||
| 1793 | extent_hook_post_reentrancy(tsdn); | |||
| 1794 | } | |||
| 1795 | ||||
| 1796 | extent_dalloc(tsdn, arena, extent); | |||
| 1797 | } | |||
| 1798 | ||||
| 1799 | static bool_Bool | |||
| 1800 | extent_commit_default(extent_hooks_t *extent_hooks, void *addr, size_t size, | |||
| 1801 | size_t offset, size_t length, unsigned arena_ind) { | |||
| 1802 | return pages_commit((void *)((uintptr_t)addr + (uintptr_t)offset), | |||
| 1803 | length); | |||
| 1804 | } | |||
| 1805 | ||||
| 1806 | static bool_Bool | |||
| 1807 | extent_commit_impl(tsdn_t *tsdn, arena_t *arena, | |||
| 1808 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, | |||
| 1809 | size_t length, bool_Bool growing_retained) { | |||
| 1810 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | |||
| 1811 | WITNESS_RANK_CORE11U, growing_retained ? 1 : 0); | |||
| 1812 | ||||
| 1813 | extent_hooks_assure_initialized(arena, r_extent_hooks); | |||
| 1814 | if (*r_extent_hooks != &extent_hooks_default) { | |||
| 1815 | extent_hook_pre_reentrancy(tsdn, arena); | |||
| 1816 | } | |||
| 1817 | bool_Bool err = ((*r_extent_hooks)->commit == NULL((void*)0) || | |||
| 1818 | (*r_extent_hooks)->commit(*r_extent_hooks, extent_base_get(extent), | |||
| 1819 | extent_size_get(extent), offset, length, arena_ind_get(arena))); | |||
| 1820 | if (*r_extent_hooks != &extent_hooks_default) { | |||
| 1821 | extent_hook_post_reentrancy(tsdn); | |||
| 1822 | } | |||
| 1823 | extent_committed_set(extent, extent_committed_get(extent) || !err); | |||
| 1824 | return err; | |||
| 1825 | } | |||
| 1826 | ||||
| 1827 | bool_Bool | |||
| 1828 | extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, | |||
| 1829 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, | |||
| 1830 | size_t length) { | |||
| 1831 | return extent_commit_impl(tsdn, arena, r_extent_hooks, extent, offset, | |||
| 1832 | length, false0); | |||
| 1833 | } | |||
| 1834 | ||||
| 1835 | static bool_Bool | |||
| 1836 | extent_decommit_default(extent_hooks_t *extent_hooks, void *addr, size_t size, | |||
| 1837 | size_t offset, size_t length, unsigned arena_ind) { | |||
| 1838 | return pages_decommit((void *)((uintptr_t)addr + (uintptr_t)offset), | |||
| 1839 | length); | |||
| 1840 | } | |||
| 1841 | ||||
| 1842 | bool_Bool | |||
| 1843 | extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, | |||
| 1844 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, | |||
| 1845 | size_t length) { | |||
| 1846 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | |||
| 1847 | WITNESS_RANK_CORE11U, 0); | |||
| 1848 | ||||
| 1849 | extent_hooks_assure_initialized(arena, r_extent_hooks); | |||
| 1850 | ||||
| 1851 | if (*r_extent_hooks != &extent_hooks_default) { | |||
| 1852 | extent_hook_pre_reentrancy(tsdn, arena); | |||
| 1853 | } | |||
| 1854 | bool_Bool err = ((*r_extent_hooks)->decommit == NULL((void*)0) || | |||
| 1855 | (*r_extent_hooks)->decommit(*r_extent_hooks, | |||
| 1856 | extent_base_get(extent), extent_size_get(extent), offset, length, | |||
| 1857 | arena_ind_get(arena))); | |||
| 1858 | if (*r_extent_hooks != &extent_hooks_default) { | |||
| 1859 | extent_hook_post_reentrancy(tsdn); | |||
| 1860 | } | |||
| 1861 | extent_committed_set(extent, extent_committed_get(extent) && err); | |||
| 1862 | return err; | |||
| 1863 | } | |||
| 1864 | ||||
| 1865 | #ifdef PAGES_CAN_PURGE_LAZY | |||
| 1866 | static bool_Bool | |||
| 1867 | extent_purge_lazy_default(extent_hooks_t *extent_hooks, void *addr, size_t size, | |||
| 1868 | size_t offset, size_t length, unsigned arena_ind) { | |||
| 1869 | assert(addr != NULL)do { if (__builtin_expect(!!(config_debug && !(addr != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1869, "addr != NULL"); abort(); } } while ( 0); | |||
| 1870 | assert((offset & PAGE_MASK) == 0)do { if (__builtin_expect(!!(config_debug && !((offset & ((size_t)(((size_t)(1U << 12)) - 1))) == 0)), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1870, "(offset & PAGE_MASK) == 0"); abort (); } } while (0); | |||
| 1871 | assert(length != 0)do { if (__builtin_expect(!!(config_debug && !(length != 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1871, "length != 0"); abort(); } } while (0 ); | |||
| 1872 | assert((length & PAGE_MASK) == 0)do { if (__builtin_expect(!!(config_debug && !((length & ((size_t)(((size_t)(1U << 12)) - 1))) == 0)), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1872, "(length & PAGE_MASK) == 0"); abort (); } } while (0); | |||
| 1873 | ||||
| 1874 | return pages_purge_lazy((void *)((uintptr_t)addr + (uintptr_t)offset), | |||
| 1875 | length); | |||
| 1876 | } | |||
| 1877 | #endif | |||
| 1878 | ||||
| 1879 | static bool_Bool | |||
| 1880 | extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena, | |||
| 1881 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, | |||
| 1882 | size_t length, bool_Bool growing_retained) { | |||
| 1883 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | |||
| 1884 | WITNESS_RANK_CORE11U, growing_retained ? 1 : 0); | |||
| 1885 | ||||
| 1886 | extent_hooks_assure_initialized(arena, r_extent_hooks); | |||
| 1887 | ||||
| 1888 | if ((*r_extent_hooks)->purge_lazy == NULL((void*)0)) { | |||
| 1889 | return true1; | |||
| 1890 | } | |||
| 1891 | if (*r_extent_hooks != &extent_hooks_default) { | |||
| 1892 | extent_hook_pre_reentrancy(tsdn, arena); | |||
| 1893 | } | |||
| 1894 | bool_Bool err = (*r_extent_hooks)->purge_lazy(*r_extent_hooks, | |||
| 1895 | extent_base_get(extent), extent_size_get(extent), offset, length, | |||
| 1896 | arena_ind_get(arena)); | |||
| 1897 | if (*r_extent_hooks != &extent_hooks_default) { | |||
| 1898 | extent_hook_post_reentrancy(tsdn); | |||
| 1899 | } | |||
| 1900 | ||||
| 1901 | return err; | |||
| 1902 | } | |||
| 1903 | ||||
| 1904 | bool_Bool | |||
| 1905 | extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena, | |||
| 1906 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, | |||
| 1907 | size_t length) { | |||
| 1908 | return extent_purge_lazy_impl(tsdn, arena, r_extent_hooks, extent, | |||
| 1909 | offset, length, false0); | |||
| 1910 | } | |||
| 1911 | ||||
| 1912 | #ifdef PAGES_CAN_PURGE_FORCED | |||
| 1913 | static bool_Bool | |||
| 1914 | extent_purge_forced_default(extent_hooks_t *extent_hooks, void *addr, | |||
| 1915 | size_t size, size_t offset, size_t length, unsigned arena_ind) { | |||
| 1916 | assert(addr != NULL)do { if (__builtin_expect(!!(config_debug && !(addr != ((void*)0))), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1916, "addr != NULL"); abort(); } } while ( 0); | |||
| 1917 | assert((offset & PAGE_MASK) == 0)do { if (__builtin_expect(!!(config_debug && !((offset & ((size_t)(((size_t)(1U << 12)) - 1))) == 0)), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1917, "(offset & PAGE_MASK) == 0"); abort (); } } while (0); | |||
| 1918 | assert(length != 0)do { if (__builtin_expect(!!(config_debug && !(length != 0)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1918, "length != 0"); abort(); } } while (0 ); | |||
| 1919 | assert((length & PAGE_MASK) == 0)do { if (__builtin_expect(!!(config_debug && !((length & ((size_t)(((size_t)(1U << 12)) - 1))) == 0)), 0) ) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1919, "(length & PAGE_MASK) == 0"); abort (); } } while (0); | |||
| 1920 | ||||
| 1921 | return pages_purge_forced((void *)((uintptr_t)addr + | |||
| 1922 | (uintptr_t)offset), length); | |||
| 1923 | } | |||
| 1924 | #endif | |||
| 1925 | ||||
| 1926 | static bool_Bool | |||
| 1927 | extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena, | |||
| 1928 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, | |||
| 1929 | size_t length, bool_Bool growing_retained) { | |||
| 1930 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | |||
| 1931 | WITNESS_RANK_CORE11U, growing_retained ? 1 : 0); | |||
| 1932 | ||||
| 1933 | extent_hooks_assure_initialized(arena, r_extent_hooks); | |||
| 1934 | ||||
| 1935 | if ((*r_extent_hooks)->purge_forced == NULL((void*)0)) { | |||
| 1936 | return true1; | |||
| 1937 | } | |||
| 1938 | if (*r_extent_hooks != &extent_hooks_default) { | |||
| 1939 | extent_hook_pre_reentrancy(tsdn, arena); | |||
| 1940 | } | |||
| 1941 | bool_Bool err = (*r_extent_hooks)->purge_forced(*r_extent_hooks, | |||
| 1942 | extent_base_get(extent), extent_size_get(extent), offset, length, | |||
| 1943 | arena_ind_get(arena)); | |||
| 1944 | if (*r_extent_hooks != &extent_hooks_default) { | |||
| 1945 | extent_hook_post_reentrancy(tsdn); | |||
| 1946 | } | |||
| 1947 | return err; | |||
| 1948 | } | |||
| 1949 | ||||
| 1950 | bool_Bool | |||
| 1951 | extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena, | |||
| 1952 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, | |||
| 1953 | size_t length) { | |||
| 1954 | return extent_purge_forced_impl(tsdn, arena, r_extent_hooks, extent, | |||
| 1955 | offset, length, false0); | |||
| 1956 | } | |||
| 1957 | ||||
| 1958 | #ifdef JEMALLOC_MAPS_COALESCE | |||
| 1959 | static bool_Bool | |||
| 1960 | extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size, | |||
| 1961 | size_t size_a, size_t size_b, bool_Bool committed, unsigned arena_ind) { | |||
| 1962 | return !maps_coalesce; | |||
| 1963 | } | |||
| 1964 | #endif | |||
| 1965 | ||||
| 1966 | /* | |||
| 1967 | * Accepts the extent to split, and the characteristics of each side of the | |||
| 1968 | * split. The 'a' parameters go with the 'lead' of the resulting pair of | |||
| 1969 | * extents (the lower addressed portion of the split), and the 'b' parameters go | |||
| 1970 | * with the trail (the higher addressed portion). This makes 'extent' the lead, | |||
| 1971 | * and returns the trail (except in case of error). | |||
| 1972 | */ | |||
| 1973 | static extent_t * | |||
| 1974 | extent_split_impl(tsdn_t *tsdn, arena_t *arena, | |||
| 1975 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a, | |||
| 1976 | szind_t szind_a, bool_Bool slab_a, size_t size_b, szind_t szind_b, bool_Bool slab_b, | |||
| 1977 | bool_Bool growing_retained) { | |||
| 1978 | assert(extent_size_get(extent) == size_a + size_b)do { if (__builtin_expect(!!(config_debug && !(extent_size_get (extent) == size_a + size_b)), 0)) { malloc_printf( "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n" , "src/extent.c", 1978, "extent_size_get(extent) == size_a + size_b" ); abort(); } } while (0); | |||
| 1979 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | |||
| 1980 | WITNESS_RANK_CORE11U, growing_retained ? 1 : 0); | |||
| 1981 | ||||
| 1982 | extent_hooks_assure_initialized(arena, r_extent_hooks); | |||
| 1983 | ||||
| 1984 | if ((*r_extent_hooks)->split == NULL((void*)0)) { | |||
| 1985 | return NULL((void*)0); | |||
| 1986 | } | |||
| 1987 | ||||
| 1988 | extent_t *trail = extent_alloc(tsdn, arena); | |||
| 1989 | if (trail == NULL((void*)0)) { | |||
| 1990 | goto label_error_a; | |||
| 1991 | } | |||
| 1992 | ||||
| 1993 | extent_init(trail, arena, (void *)((uintptr_t)extent_base_get(extent) + | |||
| 1994 | size_a), size_b, slab_b, szind_b, extent_sn_get(extent), | |||
| 1995 | extent_state_get(extent), extent_zeroed_get(extent), | |||
| 1996 | extent_committed_get(extent), extent_dumpable_get(extent)); | |||
| 1997 | ||||
| 1998 | rtree_ctx_t rtree_ctx_fallback; | |||
| 1999 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); | |||
| 2000 | rtree_leaf_elm_t *lead_elm_a, *lead_elm_b; | |||
| 2001 | { | |||
| 2002 | extent_t lead; | |||
| 2003 | ||||
| 2004 | extent_init(&lead, arena, extent_addr_get(extent), size_a, | |||
| 2005 | slab_a, szind_a, extent_sn_get(extent), | |||
| 2006 | extent_state_get(extent), extent_zeroed_get(extent), | |||
| 2007 | extent_committed_get(extent), extent_dumpable_get(extent)); | |||
| 2008 | ||||
| 2009 | extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, &lead, false0, | |||
| 2010 | true1, &lead_elm_a, &lead_elm_b); | |||
| 2011 | } | |||
| 2012 | rtree_leaf_elm_t *trail_elm_a, *trail_elm_b; | |||
| 2013 | extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, trail, false0, true1, | |||
| 2014 | &trail_elm_a, &trail_elm_b); | |||
| 2015 | ||||
| 2016 | if (lead_elm_a == NULL((void*)0) || lead_elm_b == NULL((void*)0) || trail_elm_a == NULL((void*)0) | |||
| 2017 | || trail_elm_b == NULL((void*)0)) { | |||
| 2018 | goto label_error_b; | |||
| 2019 | } | |||
| 2020 | ||||
| 2021 | extent_lock2(tsdn, extent, trail); | |||
| 2022 | ||||
| 2023 | if (*r_extent_hooks != &extent_hooks_default) { | |||
| 2024 | extent_hook_pre_reentrancy(tsdn, arena); | |||
| 2025 | } | |||
| 2026 | bool_Bool err = (*r_extent_hooks)->split(*r_extent_hooks, extent_base_get(extent), | |||
| 2027 | size_a + size_b, size_a, size_b, extent_committed_get(extent), | |||
| 2028 | arena_ind_get(arena)); | |||
| 2029 | if (*r_extent_hooks != &extent_hooks_default) { | |||
| 2030 | extent_hook_post_reentrancy(tsdn); | |||
| 2031 | } | |||
| 2032 | if (err) { | |||
| 2033 | goto label_error_c; | |||
| 2034 | } | |||
| 2035 | ||||
| 2036 | extent_size_set(extent, size_a); | |||
| 2037 | extent_szind_set(extent, szind_a); | |||
| 2038 | ||||
| 2039 | extent_rtree_write_acquired(tsdn, lead_elm_a, lead_elm_b, extent, | |||
| 2040 | szind_a, slab_a); | |||
| 2041 | extent_rtree_write_acquired(tsdn, trail_elm_a, trail_elm_b, trail, | |||
| 2042 | szind_b, slab_b); | |||
| 2043 | ||||
| 2044 | extent_unlock2(tsdn, extent, trail); | |||
| 2045 | ||||
| 2046 | return trail; | |||
| 2047 | label_error_c: | |||
| 2048 | extent_unlock2(tsdn, extent, trail); | |||
| 2049 | label_error_b: | |||
| 2050 | extent_dalloc(tsdn, arena, trail); | |||
| 2051 | label_error_a: | |||
| 2052 | return NULL((void*)0); | |||
| 2053 | } | |||
| 2054 | ||||
| 2055 | extent_t * | |||
| 2056 | extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, | |||
| 2057 | extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a, | |||
| 2058 | szind_t szind_a, bool_Bool slab_a, size_t size_b, szind_t szind_b, bool_Bool slab_b) { | |||
| 2059 | return extent_split_impl(tsdn, arena, r_extent_hooks, extent, size_a, | |||
| 2060 | szind_a, slab_a, size_b, szind_b, slab_b, false0); | |||
| 2061 | } | |||
| 2062 | ||||
| 2063 | static bool_Bool | |||
| 2064 | extent_merge_default_impl(void *addr_a, void *addr_b) { | |||
| 2065 | if (!maps_coalesce) { | |||
| 2066 | return true1; | |||
| 2067 | } | |||
| 2068 | if (have_dss && !extent_dss_mergeable(addr_a, addr_b)) { | |||
| 2069 | return true1; | |||
| 2070 | } | |||
| 2071 | ||||
| 2072 | return false0; | |||
| 2073 | } | |||
| 2074 | ||||
| 2075 | #ifdef JEMALLOC_MAPS_COALESCE | |||
| 2076 | static bool_Bool | |||
| 2077 | extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, | |||
| 2078 | void *addr_b, size_t size_b, bool_Bool committed, unsigned arena_ind) { | |||
| 2079 | return extent_merge_default_impl(addr_a, addr_b); | |||
| 2080 | } | |||
| 2081 | #endif | |||
| 2082 | ||||
| 2083 | static bool_Bool | |||
| 2084 | extent_merge_impl(tsdn_t *tsdn, arena_t *arena, | |||
| 2085 | extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b, | |||
| 2086 | bool_Bool growing_retained) { | |||
| 2087 | witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), | |||
| 2088 | WITNESS_RANK_CORE11U, growing_retained ? 1 : 0); | |||
| 2089 | ||||
| 2090 | extent_hooks_assure_initialized(arena, r_extent_hooks); | |||
| 2091 | ||||
| 2092 | if ((*r_extent_hooks)->merge == NULL((void*)0)) { | |||
| 2093 | return true1; | |||
| 2094 | } | |||
| 2095 | ||||
| 2096 | bool_Bool err; | |||
| 2097 | if (*r_extent_hooks == &extent_hooks_default) { | |||
| 2098 | /* Call directly to propagate tsdn. */ | |||
| 2099 | err = extent_merge_default_impl(extent_base_get(a), | |||
| 2100 | extent_base_get(b)); | |||
| 2101 | } else { | |||
| 2102 | extent_hook_pre_reentrancy(tsdn, arena); | |||
| 2103 | err = (*r_extent_hooks)->merge(*r_extent_hooks, | |||
| 2104 | extent_base_get(a), extent_size_get(a), extent_base_get(b), | |||
| 2105 | extent_size_get(b), extent_committed_get(a), | |||
| 2106 | arena_ind_get(arena)); | |||
| 2107 | extent_hook_post_reentrancy(tsdn); | |||
| 2108 | } | |||
| 2109 | ||||
| 2110 | if (err) { | |||
| 2111 | return true1; | |||
| 2112 | } | |||
| 2113 | ||||
| 2114 | /* | |||
| 2115 | * The rtree writes must happen while all the relevant elements are | |||
| 2116 | * owned, so the following code uses decomposed helper functions rather | |||
| 2117 | * than extent_{,de}register() to do things in the right order. | |||
| 2118 | */ | |||
| 2119 | rtree_ctx_t rtree_ctx_fallback; | |||
| 2120 | rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); | |||
| 2121 | rtree_leaf_elm_t *a_elm_a, *a_elm_b, *b_elm_a, *b_elm_b; | |||
| 2122 | extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, a, true1, false0, &a_elm_a, | |||
| 2123 | &a_elm_b); | |||
| 2124 | extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, b, true1, false0, &b_elm_a, | |||
| 2125 | &b_elm_b); | |||
| 2126 | ||||
| 2127 | extent_lock2(tsdn, a, b); | |||
| 2128 | ||||
| 2129 | if (a_elm_b != NULL((void*)0)) { | |||
| 2130 | rtree_leaf_elm_write(tsdn, &extents_rtree, a_elm_b, NULL((void*)0), | |||
| 2131 | NSIZES235, false0); | |||
| 2132 | } | |||
| 2133 | if (b_elm_b != NULL((void*)0)) { | |||
| 2134 | rtree_leaf_elm_write(tsdn, &extents_rtree, b_elm_a, NULL((void*)0), | |||
| 2135 | NSIZES235, false0); | |||
| 2136 | } else { | |||
| 2137 | b_elm_b = b_elm_a; | |||
| 2138 | } | |||
| 2139 | ||||
| 2140 | extent_size_set(a, extent_size_get(a) + extent_size_get(b)); | |||
| 2141 | extent_szind_set(a, NSIZES235); | |||
| 2142 | extent_sn_set(a, (extent_sn_get(a) < extent_sn_get(b)) ? | |||
| 2143 | extent_sn_get(a) : extent_sn_get(b)); | |||
| 2144 | extent_zeroed_set(a, extent_zeroed_get(a) && extent_zeroed_get(b)); | |||
| 2145 | ||||
| 2146 | extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a, NSIZES235, false0); | |||
| 2147 | ||||
| 2148 | extent_unlock2(tsdn, a, b); | |||
| 2149 | ||||
| 2150 | extent_dalloc(tsdn, extent_arena_get(b), b); | |||
| 2151 | ||||
| 2152 | return false0; | |||
| 2153 | } | |||
| 2154 | ||||
| 2155 | bool_Bool | |||
| 2156 | extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, | |||
| 2157 | extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b) { | |||
| 2158 | return extent_merge_impl(tsdn, arena, r_extent_hooks, a, b, false0); | |||
| 2159 | } | |||
| 2160 | ||||
| 2161 | bool_Bool | |||
| 2162 | extent_boot(void) { | |||
| 2163 | if (rtree_new(&extents_rtree, true1)) { | |||
| 2164 | return true1; | |||
| 2165 | } | |||
| 2166 | ||||
| 2167 | if (mutex_pool_init(&extent_mutex_pool, "extent_mutex_pool", | |||
| 2168 | WITNESS_RANK_EXTENT_POOL16U)) { | |||
| 2169 | return true1; | |||
| 2170 | } | |||
| 2171 | ||||
| 2172 | if (have_dss) { | |||
| 2173 | extent_dss_boot(); | |||
| 2174 | } | |||
| 2175 | ||||
| 2176 | return false0; | |||
| 2177 | } |