performance and bug fixes

Author: struct

Makefile

@@ -361,6 +361,19 @@ library_less_strict: clean
 	$(CC) $(CFLAGS) $(LIBRARY) $(OPTIMIZE) $(OS_FLAGS) $(C_SRCS) -o $(BUILD_DIR)/$(LIBNAME)
 	$(STRIP)
 
+## Build a performance-optimized library with the most expensive security
+## features disabled. Intended for benchmarking and performance measurement.
+## All other flags inherit from the top-level defaults.
+library_benchmark: DISABLE_CANARY = -DDISABLE_CANARY=1
+library_benchmark: PRE_POPULATE_PAGES = -DPRE_POPULATE_PAGES=1
+library_benchmark: RANDOMIZE_FREELIST = -DRANDOMIZE_FREELIST=0
+library_benchmark: MASK_PTRS = -DMASK_PTRS=0
+library_benchmark: ABORT_ON_UNOWNED_PTR = -DABORT_ON_UNOWNED_PTR=0
+library_benchmark: clean
+	@echo "make library_benchmark"
+	$(CC) $(CFLAGS) $(LIBRARY) $(OPTIMIZE) $(OS_FLAGS) $(C_SRCS) -o $(BUILD_DIR)/$(LIBNAME)
+	$(STRIP)
+
 ## Build a debug version of the library
 library_debug: clean
 	@echo "make library debug"

PERFORMANCE.md

@@ -153,35 +153,37 @@ iso_realloc/iso_free 1834784 tests completed in 0.901481 seconds
 The following benchmarks were collected from [mimalloc-bench](https://github.com/daanx/mimalloc-bench) with the default configuration of IsoAlloc. As you can see from the data IsoAlloc is competitive with other allocators for some benchmarks but clearly falls behind on others. For any benchmark that IsoAlloc scores poorly on I was able to tweak its build to improve the CPU time and memory consumption. It's worth noting that IsoAlloc was able to stay competitive even with performing many security checks not present in other allocators. Please note these are 'best case' measurements, not averages.
 
 ```
+make library_benchmark
+
 #------------------------------------------------------------------
 # test    alloc   time  rss    user  sys  page-faults page-reclaims
-cfrac       je    03.07 4560 3.06 0.00 0 455
-cfrac       mi    02.92 2676 2.92 0.00 0 348
-cfrac       iso   05.16 30764 5.08 0.08 0 7497
+cfrac       je    03.07 4552 3.06 0.00 0 454
+cfrac       mi    02.97 2484 2.96 0.00 0 347
+cfrac       iso   04.78 30612 4.69 0.09 0 7503
 
-espresso    je    02.49 5032 2.48 0.00 0 550
-espresso    mi    02.47 3004 2.45 0.01 0 3636
-espresso    iso   03.25 69124 3.16 0.09 0 30105
+espresso    je    02.51 4872 2.50 0.01 0 540
+espresso    mi    02.43 3032 2.42 0.01 0 3630
+espresso    iso   03.16 69608 3.07 0.07 0 30334
 
-barnes      je    01.71 59916 1.68 0.02 0 16684
-barnes      mi    01.64 57864 1.61 0.02 0 16550
-barnes      iso   01.65 74968 1.61 0.03 0 20851
+barnes      je    01.71 59900 1.67 0.03 0 16686
+barnes      mi    01.65 57672 1.62 0.02 0 16550
+barnes      iso   01.65 74812 1.62 0.03 0 20849
 
-gs          je    00.15 37756 0.13 0.01 0 5812
-gs          mi    00.15 33668 0.14 0.01 0 5110
-gs          iso   00.23 67960 0.16 0.06 0 18846
+gs          je    00.17 37748 0.15 0.01 0 5814
+gs          mi    00.16 33888 0.14 0.01 0 5109
+gs          iso   00.22 68136 0.15 0.06 0 18916
 
-larsonN     je    1.153 269184 98.81 1.00 0 419378
-larsonN     mi    1.037 301044 99.34 0.41 0 83267
-larsonN     iso   1304.061 121072 6.10 70.16 0 30031
+larsonN     je    1.188 261884 98.91 0.92 0 421848
+larsonN     mi    1.016 299752 99.53 0.38 0 80202
+larsonN     iso   1328.904 121096 6.15 69.78 0 30219
 
-rocksdb     je    02.49 162976 2.09 0.60 0 38215
-rocksdb     mi    02.22 160392 1.86 0.54 0 37563
-rocksdb     iso   02.87 197548 2.58 0.59 0 46899
+rocksdb     je    02.46 162340 2.05 0.63 0 38383
+rocksdb     mi    02.33 160156 1.92 0.63 0 37585
+rocksdb     iso   02.96 195948 2.64 0.66 0 46584
 
-redis       je    3.319 9484 0.14 0.02 0 1540
-redis       mi    2.840 7124 0.12 0.02 0 1254
-redis       iso   7.340 49712 0.34 0.04 0 14959
+redis       je    3.160 9492 0.13 0.02 0 1528
+redis       mi    2.780 7084 0.12 0.02 0 1257
+redis       iso   7.579 50516 0.35 0.05 0 15187
 ```
 
 IsoAlloc isn't quite ready for performance sensitive server workloads. However it's more than fast enough for client side mobile/desktop applications with risky C/C++ attack surfaces. These environments have threat models similar to what IsoAlloc was designed for.

include/iso_alloc_internal.h

@@ -376,7 +376,7 @@ INTERNAL_HIDDEN INLINE void populate_zone_cache(iso_alloc_zone_t *zone);
 INTERNAL_HIDDEN INLINE void flush_chunk_quarantine(void);
 INTERNAL_HIDDEN INLINE void clear_zone_cache(void);
 INTERNAL_HIDDEN iso_alloc_big_zone_t *iso_find_big_zone(void *p, bool remove);
-INTERNAL_HIDDEN iso_alloc_zone_t *is_zone_usable(iso_alloc_zone_t *zone, size_t size);
+INTERNAL_HIDDEN FLATTEN iso_alloc_zone_t *is_zone_usable(iso_alloc_zone_t *zone, size_t size);
 INTERNAL_HIDDEN iso_alloc_zone_t *find_suitable_zone(size_t size);
 INTERNAL_HIDDEN iso_alloc_zone_t *iso_new_zone(size_t size, bool internal);
 INTERNAL_HIDDEN iso_alloc_zone_t *_iso_new_zone(size_t size, bool internal, int32_t index);
@@ -385,7 +385,7 @@ INTERNAL_HIDDEN iso_alloc_zone_t *iso_find_zone_range(void *p);
 INTERNAL_HIDDEN iso_alloc_zone_t *search_chunk_lookup_table(const void *p);
 INTERNAL_HIDDEN bit_slot_t iso_scan_zone_free_slot_slow(iso_alloc_zone_t *zone);
 INTERNAL_HIDDEN bit_slot_t iso_scan_zone_free_slot(iso_alloc_zone_t *zone);
-INTERNAL_HIDDEN bit_slot_t get_next_free_bit_slot(iso_alloc_zone_t *zone);
+INTERNAL_HIDDEN INLINE bit_slot_t get_next_free_bit_slot(iso_alloc_zone_t *zone);
 INTERNAL_HIDDEN iso_alloc_root *iso_alloc_new_root(void);
 INTERNAL_HIDDEN bool is_pow2(uint64_t sz);
 INTERNAL_HIDDEN bool _is_zone_retired(iso_alloc_zone_t *zone);
@@ -416,7 +416,7 @@ INTERNAL_HIDDEN void *_untag_ptr(void *p, iso_alloc_zone_t *zone);
 INTERNAL_HIDDEN void _free_big_zone_list(iso_alloc_big_zone_t *head);
 INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_big_alloc(size_t size);
 INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_alloc(iso_alloc_zone_t *zone, size_t size);
-INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_alloc_bitslot_from_zone(bit_slot_t bitslot, iso_alloc_zone_t *zone);
+INTERNAL_HIDDEN INLINE ASSUME_ALIGNED void *_iso_alloc_bitslot_from_zone(bit_slot_t bitslot, iso_alloc_zone_t *zone);
 INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_calloc(size_t nmemb, size_t size);
 INTERNAL_HIDDEN void *_iso_alloc_ptr_search(void *n, bool poison);
 INTERNAL_HIDDEN INLINE uint64_t us_rand_uint64(uint64_t *seed);

src/iso_alloc.c

@@ -577,7 +577,7 @@ INTERNAL_HIDDEN void fill_free_bit_slots(iso_alloc_zone_t *zone) {
     }
 
     bit_slot_t *free_bit_slots = zone->free_bit_slots;
-    __iso_memset(free_bit_slots, BAD_BIT_SLOT, ZONE_FREE_LIST_SZ);
+    __iso_memset(free_bit_slots, BAD_BIT_SLOT, sizeof(zone->free_bit_slots));
     zone->free_bit_slots_usable = 0;
     free_bit_slot_t free_bit_slots_index;
 
@@ -668,9 +668,9 @@ INTERNAL_HIDDEN INLINE void insert_free_bit_slot(iso_alloc_zone_t *zone, int64_t
     zone->is_full = false;
 }
 
-INTERNAL_HIDDEN bit_slot_t get_next_free_bit_slot(iso_alloc_zone_t *zone) {
+INTERNAL_HIDDEN INLINE bit_slot_t get_next_free_bit_slot(iso_alloc_zone_t *zone) {
     if(zone->free_bit_slots_usable >= ZONE_FREE_LIST_SZ ||
-       zone->free_bit_slots_usable > zone->free_bit_slots_index) {
+       zone->free_bit_slots_usable >= zone->free_bit_slots_index) {
         return BAD_BIT_SLOT;
     }
 
@@ -783,7 +783,7 @@ INTERNAL_HIDDEN bit_slot_t iso_scan_zone_free_slot_slow(iso_alloc_zone_t *zone)
     return BAD_BIT_SLOT;
 }
 
-INTERNAL_HIDDEN iso_alloc_zone_t *is_zone_usable(iso_alloc_zone_t *zone, size_t size) {
+INTERNAL_HIDDEN FLATTEN iso_alloc_zone_t *is_zone_usable(iso_alloc_zone_t *zone, size_t size) {
 #if CPU_PIN
     if(zone->cpu_core != _iso_getcpu()) {
         return false;
@@ -947,7 +947,7 @@ INTERNAL_HIDDEN iso_alloc_zone_t *find_suitable_zone(size_t size) {
     return NULL;
 }
 
-INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_alloc_bitslot_from_zone(bit_slot_t bitslot, iso_alloc_zone_t *zone) {
+INTERNAL_HIDDEN INLINE ASSUME_ALIGNED void *_iso_alloc_bitslot_from_zone(bit_slot_t bitslot, iso_alloc_zone_t *zone) {
     const bitmap_index_t dwords_to_bit_slot = (bitslot >> BITS_PER_QWORD_SHIFT);
     const int64_t which_bit = WHICH_BIT(bitslot);
 
@@ -1019,7 +1019,7 @@ INTERNAL_HIDDEN INLINE void populate_zone_cache(iso_alloc_zone_t *zone) {
         _zone_cache_count++;
     } else {
         /* Evict oldest entry (index 0) via FIFO: shift all entries down by one */
-        memmove(&tzc[0], &tzc[1], (ZONE_CACHE_SZ - 1) * sizeof(_tzc));
+        __iso_memmove(&tzc[0], &tzc[1], (ZONE_CACHE_SZ - 1) * sizeof(_tzc));
         _zone_cache_count = ZONE_CACHE_SZ - 1;
         tzc[_zone_cache_count].zone = zone;
         tzc[_zone_cache_count].chunk_size = zone->chunk_size;
@@ -1030,32 +1030,30 @@ INTERNAL_HIDDEN INLINE void populate_zone_cache(iso_alloc_zone_t *zone) {
 }
 
 INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_calloc(size_t nmemb, size_t size) {
-    unsigned int res;
+    size_t res;
 
     if(size < SMALLEST_CHUNK_SZ) {
         size = SMALLEST_CHUNK_SZ;
     } else if((size % SZ_ALIGNMENT) != 0) {
         size = ALIGN_SZ_UP(size);
     }
 
-    size_t sz = nmemb * size;
-
-    if(sz < size || UNLIKELY(__builtin_mul_overflow(nmemb, size, &res))) {
-        LOG("Call to calloc() will overflow nmemb=%d size=%d = %u", nmemb, size, nmemb * size);
+    if(UNLIKELY(__builtin_mul_overflow(nmemb, size, &res)) || UNLIKELY(res > BIG_SZ_MAX)) {
+        LOG("Call to calloc() will overflow nmemb=%zu size=%zu", nmemb, size);
         return NULL;
     }
 
-    void *p = _iso_alloc(NULL, sz);
+    void *p = _iso_alloc(NULL, res);
 
 #if NO_ZERO_ALLOCATIONS
     /* Without this check we would immediately segfault in
      * the call to __iso_memset() to zeroize the chunk */
-    if(UNLIKELY(sz == 0)) {
+    if(UNLIKELY(res == 0)) {
         return p;
     }
 #endif
 
-    __iso_memset(p, 0x0, sz);
+    __iso_memset(p, 0x0, res);
     return p;
 }
 
@@ -1120,8 +1118,8 @@ INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_alloc(iso_alloc_zone_t *zone, size_t s
     }
 
 #if ALLOC_SANITY
-    /* We don't sample if we are allocating from a private zone */
-    if(zone != NULL && zone->internal == true) {
+    /* We only sample if a zone was not directly passed */
+    if(zone != NULL) {
         if(size < g_page_size && _sane_sampled < MAX_SANE_SAMPLES) {
             /* If we chose to sample this allocation then
              * _iso_alloc_sample will call UNLOCK_ROOT() */
@@ -1760,7 +1758,8 @@ INTERNAL_HIDDEN iso_alloc_big_zone_t *iso_find_big_zone(void *p, bool remove) {
     LOCK_BIG_ZONE_USED();
 
     if(_root->big_zone_used == NULL) {
-        LOG_AND_ABORT("There are no big zones allocated");
+        UNLOCK_BIG_ZONE_USED();
+        return NULL;
     }
 
     iso_alloc_big_zone_t *big_zone = _root->big_zone_used;

src/iso_alloc_interfaces.c

@@ -95,13 +95,13 @@ EXTERNAL_API FLATTEN NO_DISCARD REALLOC_SIZE ASSUME_ALIGNED void *iso_realloc(vo
 }
 
 EXTERNAL_API FLATTEN NO_DISCARD MALLOC_ATTR REALLOC_SIZE ASSUME_ALIGNED void *iso_reallocarray(void *p, size_t nmemb, size_t size) {
-    unsigned int res;
+    size_t res;
 
-    if(__builtin_mul_overflow(nmemb, size, &res)) {
+    if(__builtin_mul_overflow(nmemb, size, &res) || res > BIG_SZ_MAX) {
         return NULL;
     }
 
-    return iso_realloc(p, nmemb * size);
+    return iso_realloc(p, res);
 }
 
 EXTERNAL_API FLATTEN NO_DISCARD ASSUME_ALIGNED char *iso_strdup(const char *str) {

src/iso_alloc_util.c

@@ -153,6 +153,22 @@ void *mmap_pages(size_t size, bool populate, const char *name, int32_t prot) {
 
     p = mmap(p, sz, prot, flags, fd, 0);
 
+#if __linux__ && MAP_HUGETLB && HUGE_PAGES
+    /* If the huge page allocation failed, retry with regular pages.
+     * This can happen when /proc/sys/vm/nr_hugepages is 0 or
+     * exhausted, which is common in LD_PRELOAD environments. */
+    if(p == MAP_FAILED && (flags & MAP_HUGETLB)) {
+        flags &= ~MAP_HUGETLB;
+        p = mmap(p, sz, prot, flags, fd, 0);
+    }
+#elif __APPLE__ && VM_FLAGS_SUPERPAGE_SIZE_2MB && HUGE_PAGES
+    /* Same fallback for macOS superpage allocations */
+    if(p == MAP_FAILED && fd == VM_FLAGS_SUPERPAGE_SIZE_2MB) {
+        fd = -1;
+        p = mmap(p, sz, prot, flags, fd, 0);
+    }
+#endif
+
     if(p == MAP_FAILED) {
         LOG_AND_ABORT("Failed to mmap rw pages");
     }