2 * Copyright © 2010 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
31 #include "util/macros.h"
32 #include "util/u_math.h"
34 /* Some versions of MinGW are missing _vscprintf's declaration, although they
35 * still provide the symbol in the import library. */
37 _CRTIMP
int _vscprintf(const char *format
, va_list argptr
);
44 #define va_copy(dest, src) __va_copy((dest), (src))
46 #define va_copy(dest, src) (dest) = (src)
50 #define CANARY 0x5A1106
52 /* Align the header's size so that ralloc() allocations will return with the
53 * same alignment as a libc malloc would have (8 on 32-bit GLIBC, 16 on
54 * 64-bit), avoiding performance penalities on x86 and alignment faults on
64 #elif defined(__LP64__)
65 __attribute__((aligned(16)))
67 __attribute__((aligned(8)))
72 /* A canary value used to determine whether a pointer is ralloc'd. */
76 struct ralloc_header
*parent
;
78 /* The first child (head of a linked list) */
79 struct ralloc_header
*child
;
81 /* Linked list of siblings */
82 struct ralloc_header
*prev
;
83 struct ralloc_header
*next
;
85 void (*destructor
)(void *);
88 typedef struct ralloc_header ralloc_header
;
90 static void unlink_block(ralloc_header
*info
);
91 static void unsafe_free(ralloc_header
*info
);
93 static ralloc_header
*
94 get_header(const void *ptr
)
96 ralloc_header
*info
= (ralloc_header
*) (((char *) ptr
) -
97 sizeof(ralloc_header
));
98 assert(info
->canary
== CANARY
);
102 #define PTR_FROM_HEADER(info) (((char *) info) + sizeof(ralloc_header))
105 add_child(ralloc_header
*parent
, ralloc_header
*info
)
107 if (parent
!= NULL
) {
108 info
->parent
= parent
;
109 info
->next
= parent
->child
;
110 parent
->child
= info
;
112 if (info
->next
!= NULL
)
113 info
->next
->prev
= info
;
118 ralloc_context(const void *ctx
)
120 return ralloc_size(ctx
, 0);
124 ralloc_size(const void *ctx
, size_t size
)
126 /* Some malloc allocation doesn't always align to 16 bytes even on 64 bits
127 * system, from Android bionic/tests/malloc_test.cpp:
128 * - Allocations of a size that rounds up to a multiple of 16 bytes
129 * must have at least 16 byte alignment.
130 * - Allocations of a size that rounds up to a multiple of 8 bytes and
131 * not 16 bytes, are only required to have at least 8 byte alignment.
133 void *block
= malloc(align64(size
+ sizeof(ralloc_header
),
134 alignof(ralloc_header
)));
136 ralloc_header
*parent
;
138 if (unlikely(block
== NULL
))
141 info
= (ralloc_header
*) block
;
142 /* measurements have shown that calloc is slower (because of
143 * the multiplication overflow checking?), so clear things
150 info
->destructor
= NULL
;
152 parent
= ctx
!= NULL
? get_header(ctx
) : NULL
;
154 add_child(parent
, info
);
157 info
->canary
= CANARY
;
160 return PTR_FROM_HEADER(info
);
164 rzalloc_size(const void *ctx
, size_t size
)
166 void *ptr
= ralloc_size(ctx
, size
);
169 memset(ptr
, 0, size
);
174 /* helper function - assumes ptr != NULL */
176 resize(void *ptr
, size_t size
)
178 ralloc_header
*child
, *old
, *info
;
180 old
= get_header(ptr
);
181 info
= realloc(old
, align64(size
+ sizeof(ralloc_header
),
182 alignof(ralloc_header
)));
187 /* Update parent and sibling's links to the reallocated node. */
188 if (info
!= old
&& info
->parent
!= NULL
) {
189 if (info
->parent
->child
== old
)
190 info
->parent
->child
= info
;
192 if (info
->prev
!= NULL
)
193 info
->prev
->next
= info
;
195 if (info
->next
!= NULL
)
196 info
->next
->prev
= info
;
199 /* Update child->parent links for all children */
200 for (child
= info
->child
; child
!= NULL
; child
= child
->next
)
201 child
->parent
= info
;
203 return PTR_FROM_HEADER(info
);
207 reralloc_size(const void *ctx
, void *ptr
, size_t size
)
209 if (unlikely(ptr
== NULL
))
210 return ralloc_size(ctx
, size
);
212 assert(ralloc_parent(ptr
) == ctx
);
213 return resize(ptr
, size
);
217 rerzalloc_size(const void *ctx
, void *ptr
, size_t old_size
, size_t new_size
)
219 if (unlikely(ptr
== NULL
))
220 return rzalloc_size(ctx
, new_size
);
222 assert(ralloc_parent(ptr
) == ctx
);
223 ptr
= resize(ptr
, new_size
);
225 if (new_size
> old_size
)
226 memset((char *)ptr
+ old_size
, 0, new_size
- old_size
);
232 ralloc_array_size(const void *ctx
, size_t size
, unsigned count
)
234 if (count
> SIZE_MAX
/size
)
237 return ralloc_size(ctx
, size
* count
);
241 rzalloc_array_size(const void *ctx
, size_t size
, unsigned count
)
243 if (count
> SIZE_MAX
/size
)
246 return rzalloc_size(ctx
, size
* count
);
250 reralloc_array_size(const void *ctx
, void *ptr
, size_t size
, unsigned count
)
252 if (count
> SIZE_MAX
/size
)
255 return reralloc_size(ctx
, ptr
, size
* count
);
259 rerzalloc_array_size(const void *ctx
, void *ptr
, size_t size
,
260 unsigned old_count
, unsigned new_count
)
262 if (new_count
> SIZE_MAX
/size
)
265 return rerzalloc_size(ctx
, ptr
, size
* old_count
, size
* new_count
);
269 ralloc_free(void *ptr
)
276 info
= get_header(ptr
);
282 unlink_block(ralloc_header
*info
)
284 /* Unlink from parent & siblings */
285 if (info
->parent
!= NULL
) {
286 if (info
->parent
->child
== info
)
287 info
->parent
->child
= info
->next
;
289 if (info
->prev
!= NULL
)
290 info
->prev
->next
= info
->next
;
292 if (info
->next
!= NULL
)
293 info
->next
->prev
= info
->prev
;
301 unsafe_free(ralloc_header
*info
)
303 /* Recursively free any children...don't waste time unlinking them. */
305 while (info
->child
!= NULL
) {
307 info
->child
= temp
->next
;
311 /* Free the block itself. Call the destructor first, if any. */
312 if (info
->destructor
!= NULL
)
313 info
->destructor(PTR_FROM_HEADER(info
));
319 ralloc_steal(const void *new_ctx
, void *ptr
)
321 ralloc_header
*info
, *parent
;
323 if (unlikely(ptr
== NULL
))
326 info
= get_header(ptr
);
327 parent
= new_ctx
? get_header(new_ctx
) : NULL
;
331 add_child(parent
, info
);
335 ralloc_adopt(const void *new_ctx
, void *old_ctx
)
337 ralloc_header
*new_info
, *old_info
, *child
;
339 if (unlikely(old_ctx
== NULL
))
342 old_info
= get_header(old_ctx
);
343 new_info
= get_header(new_ctx
);
345 /* If there are no children, bail. */
346 if (unlikely(old_info
->child
== NULL
))
349 /* Set all the children's parent to new_ctx; get a pointer to the last child. */
350 for (child
= old_info
->child
; child
->next
!= NULL
; child
= child
->next
) {
351 child
->parent
= new_info
;
353 child
->parent
= new_info
;
355 /* Connect the two lists together; parent them to new_ctx; make old_ctx empty. */
356 child
->next
= new_info
->child
;
358 child
->next
->prev
= child
;
359 new_info
->child
= old_info
->child
;
360 old_info
->child
= NULL
;
364 ralloc_parent(const void *ptr
)
368 if (unlikely(ptr
== NULL
))
371 info
= get_header(ptr
);
372 return info
->parent
? PTR_FROM_HEADER(info
->parent
) : NULL
;
376 ralloc_set_destructor(const void *ptr
, void(*destructor
)(void *))
378 ralloc_header
*info
= get_header(ptr
);
379 info
->destructor
= destructor
;
383 ralloc_strdup(const void *ctx
, const char *str
)
388 if (unlikely(str
== NULL
))
392 ptr
= ralloc_array(ctx
, char, n
+ 1);
399 ralloc_strndup(const void *ctx
, const char *str
, size_t max
)
404 if (unlikely(str
== NULL
))
407 n
= strnlen(str
, max
);
408 ptr
= ralloc_array(ctx
, char, n
+ 1);
414 /* helper routine for strcat/strncat - n is the exact amount to copy */
416 cat(char **dest
, const char *str
, size_t n
)
419 size_t existing_length
;
420 assert(dest
!= NULL
&& *dest
!= NULL
);
422 existing_length
= strlen(*dest
);
423 both
= resize(*dest
, existing_length
+ n
+ 1);
424 if (unlikely(both
== NULL
))
427 memcpy(both
+ existing_length
, str
, n
);
428 both
[existing_length
+ n
] = '\0';
436 ralloc_strcat(char **dest
, const char *str
)
438 return cat(dest
, str
, strlen(str
));
442 ralloc_strncat(char **dest
, const char *str
, size_t n
)
444 return cat(dest
, str
, strnlen(str
, n
));
448 ralloc_str_append(char **dest
, const char *str
,
449 size_t existing_length
, size_t str_size
)
452 assert(dest
!= NULL
&& *dest
!= NULL
);
454 both
= resize(*dest
, existing_length
+ str_size
+ 1);
455 if (unlikely(both
== NULL
))
458 memcpy(both
+ existing_length
, str
, str_size
);
459 both
[existing_length
+ str_size
] = '\0';
467 ralloc_asprintf(const void *ctx
, const char *fmt
, ...)
472 ptr
= ralloc_vasprintf(ctx
, fmt
, args
);
477 /* Return the length of the string that would be generated by a printf-style
478 * format and argument list, not including the \0 byte.
481 printf_length(const char *fmt
, va_list untouched_args
)
486 /* Make a copy of the va_list so the original caller can still use it */
488 va_copy(args
, untouched_args
);
491 /* We need to use _vcsprintf to calculate the size as vsnprintf returns -1
492 * if the number of characters to write is greater than count.
494 size
= _vscprintf(fmt
, args
);
497 size
= vsnprintf(&junk
, 1, fmt
, args
);
507 ralloc_vasprintf(const void *ctx
, const char *fmt
, va_list args
)
509 size_t size
= printf_length(fmt
, args
) + 1;
511 char *ptr
= ralloc_size(ctx
, size
);
513 vsnprintf(ptr
, size
, fmt
, args
);
519 ralloc_asprintf_append(char **str
, const char *fmt
, ...)
524 success
= ralloc_vasprintf_append(str
, fmt
, args
);
530 ralloc_vasprintf_append(char **str
, const char *fmt
, va_list args
)
532 size_t existing_length
;
534 existing_length
= *str
? strlen(*str
) : 0;
535 return ralloc_vasprintf_rewrite_tail(str
, &existing_length
, fmt
, args
);
539 ralloc_asprintf_rewrite_tail(char **str
, size_t *start
, const char *fmt
, ...)
544 success
= ralloc_vasprintf_rewrite_tail(str
, start
, fmt
, args
);
550 ralloc_vasprintf_rewrite_tail(char **str
, size_t *start
, const char *fmt
,
558 if (unlikely(*str
== NULL
)) {
559 // Assuming a NULL context is probably bad, but it's expected behavior.
560 *str
= ralloc_vasprintf(NULL
, fmt
, args
);
561 *start
= strlen(*str
);
565 new_length
= printf_length(fmt
, args
);
567 ptr
= resize(*str
, *start
+ new_length
+ 1);
568 if (unlikely(ptr
== NULL
))
571 vsnprintf(ptr
+ *start
, new_length
+ 1, fmt
, args
);
573 *start
+= new_length
;
577 /***************************************************************************
578 * Linear allocator for short-lived allocations.
579 ***************************************************************************
581 * The allocator consists of a parent node (2K buffer), which requires
582 * a ralloc parent, and child nodes (allocations). Child nodes can't be freed
583 * directly, because the parent doesn't track them. You have to release
584 * the parent node in order to release all its children.
586 * The allocator uses a fixed-sized buffer with a monotonically increasing
587 * offset after each allocation. If the buffer is all used, another buffer
588 * is allocated, sharing the same ralloc parent, so all buffers are at
589 * the same level in the ralloc hierarchy.
591 * The linear parent node is always the first buffer and keeps track of all
595 #define MIN_LINEAR_BUFSIZE 2048
596 #define SUBALLOC_ALIGNMENT 8
597 #define LMAGIC 0x87b9c7d3
602 #elif defined(__LP64__)
603 __attribute__((aligned(16)))
605 __attribute__((aligned(8)))
609 unsigned magic
; /* for debugging */
611 unsigned offset
; /* points to the first unused byte in the buffer */
612 unsigned size
; /* size of the buffer */
613 void *ralloc_parent
; /* new buffers will use this */
614 struct linear_header
*next
; /* next buffer if we have more */
615 struct linear_header
*latest
; /* the only buffer that has free space */
617 /* After this structure, the buffer begins.
618 * Each suballocation consists of linear_size_chunk as its header followed
619 * by the suballocation, so it goes:
621 * - linear_size_chunk
623 * - linear_size_chunk
627 * linear_size_chunk is only needed by linear_realloc.
631 struct linear_size_chunk
{
632 unsigned size
; /* for realloc */
636 typedef struct linear_header linear_header
;
637 typedef struct linear_size_chunk linear_size_chunk
;
639 #define LINEAR_PARENT_TO_HEADER(parent) \
641 ((char*)(parent) - sizeof(linear_size_chunk) - sizeof(linear_header))
643 /* Allocate the linear buffer with its header. */
644 static linear_header
*
645 create_linear_node(void *ralloc_ctx
, unsigned min_size
)
649 min_size
+= sizeof(linear_size_chunk
);
651 if (likely(min_size
< MIN_LINEAR_BUFSIZE
))
652 min_size
= MIN_LINEAR_BUFSIZE
;
654 node
= ralloc_size(ralloc_ctx
, sizeof(linear_header
) + min_size
);
659 node
->magic
= LMAGIC
;
662 node
->size
= min_size
;
663 node
->ralloc_parent
= ralloc_ctx
;
670 linear_alloc_child(void *parent
, unsigned size
)
672 linear_header
*first
= LINEAR_PARENT_TO_HEADER(parent
);
673 linear_header
*latest
= first
->latest
;
674 linear_header
*new_node
;
675 linear_size_chunk
*ptr
;
678 assert(first
->magic
== LMAGIC
);
679 assert(!latest
->next
);
681 size
= ALIGN_POT(size
, SUBALLOC_ALIGNMENT
);
682 full_size
= sizeof(linear_size_chunk
) + size
;
684 if (unlikely(latest
->offset
+ full_size
> latest
->size
)) {
685 /* allocate a new node */
686 new_node
= create_linear_node(latest
->ralloc_parent
, size
);
687 if (unlikely(!new_node
))
690 first
->latest
= new_node
;
691 latest
->latest
= new_node
;
692 latest
->next
= new_node
;
696 ptr
= (linear_size_chunk
*)((char*)&latest
[1] + latest
->offset
);
698 latest
->offset
+= full_size
;
700 assert((uintptr_t)&ptr
[1] % SUBALLOC_ALIGNMENT
== 0);
705 linear_alloc_parent(void *ralloc_ctx
, unsigned size
)
709 if (unlikely(!ralloc_ctx
))
712 size
= ALIGN_POT(size
, SUBALLOC_ALIGNMENT
);
714 node
= create_linear_node(ralloc_ctx
, size
);
718 return linear_alloc_child((char*)node
+
719 sizeof(linear_header
) +
720 sizeof(linear_size_chunk
), size
);
724 linear_zalloc_child(void *parent
, unsigned size
)
726 void *ptr
= linear_alloc_child(parent
, size
);
729 memset(ptr
, 0, size
);
734 linear_zalloc_parent(void *parent
, unsigned size
)
736 void *ptr
= linear_alloc_parent(parent
, size
);
739 memset(ptr
, 0, size
);
744 linear_free_parent(void *ptr
)
751 node
= LINEAR_PARENT_TO_HEADER(ptr
);
752 assert(node
->magic
== LMAGIC
);
763 ralloc_steal_linear_parent(void *new_ralloc_ctx
, void *ptr
)
770 node
= LINEAR_PARENT_TO_HEADER(ptr
);
771 assert(node
->magic
== LMAGIC
);
774 ralloc_steal(new_ralloc_ctx
, node
);
775 node
->ralloc_parent
= new_ralloc_ctx
;
781 ralloc_parent_of_linear_parent(void *ptr
)
783 linear_header
*node
= LINEAR_PARENT_TO_HEADER(ptr
);
784 assert(node
->magic
== LMAGIC
);
785 return node
->ralloc_parent
;
789 linear_realloc(void *parent
, void *old
, unsigned new_size
)
791 unsigned old_size
= 0;
792 ralloc_header
*new_ptr
;
794 new_ptr
= linear_alloc_child(parent
, new_size
);
799 old_size
= ((linear_size_chunk
*)old
)[-1].size
;
801 if (likely(new_ptr
&& old_size
))
802 memcpy(new_ptr
, old
, MIN2(old_size
, new_size
));
807 /* All code below is pretty much copied from ralloc and only the alloc
808 * calls are different.
812 linear_strdup(void *parent
, const char *str
)
821 ptr
= linear_alloc_child(parent
, n
+ 1);
831 linear_asprintf(void *parent
, const char *fmt
, ...)
836 ptr
= linear_vasprintf(parent
, fmt
, args
);
842 linear_vasprintf(void *parent
, const char *fmt
, va_list args
)
844 unsigned size
= printf_length(fmt
, args
) + 1;
846 char *ptr
= linear_alloc_child(parent
, size
);
848 vsnprintf(ptr
, size
, fmt
, args
);
854 linear_asprintf_append(void *parent
, char **str
, const char *fmt
, ...)
859 success
= linear_vasprintf_append(parent
, str
, fmt
, args
);
865 linear_vasprintf_append(void *parent
, char **str
, const char *fmt
, va_list args
)
867 size_t existing_length
;
869 existing_length
= *str
? strlen(*str
) : 0;
870 return linear_vasprintf_rewrite_tail(parent
, str
, &existing_length
, fmt
, args
);
874 linear_asprintf_rewrite_tail(void *parent
, char **str
, size_t *start
,
875 const char *fmt
, ...)
880 success
= linear_vasprintf_rewrite_tail(parent
, str
, start
, fmt
, args
);
886 linear_vasprintf_rewrite_tail(void *parent
, char **str
, size_t *start
,
887 const char *fmt
, va_list args
)
894 if (unlikely(*str
== NULL
)) {
895 *str
= linear_vasprintf(parent
, fmt
, args
);
896 *start
= strlen(*str
);
900 new_length
= printf_length(fmt
, args
);
902 ptr
= linear_realloc(parent
, *str
, *start
+ new_length
+ 1);
903 if (unlikely(ptr
== NULL
))
906 vsnprintf(ptr
+ *start
, new_length
+ 1, fmt
, args
);
908 *start
+= new_length
;
912 /* helper routine for strcat/strncat - n is the exact amount to copy */
914 linear_cat(void *parent
, char **dest
, const char *str
, unsigned n
)
917 unsigned existing_length
;
918 assert(dest
!= NULL
&& *dest
!= NULL
);
920 existing_length
= strlen(*dest
);
921 both
= linear_realloc(parent
, *dest
, existing_length
+ n
+ 1);
922 if (unlikely(both
== NULL
))
925 memcpy(both
+ existing_length
, str
, n
);
926 both
[existing_length
+ n
] = '\0';
933 linear_strcat(void *parent
, char **dest
, const char *str
)
935 return linear_cat(parent
, dest
, str
, strlen(str
));