sys.path.insert(0, os.path.join(os.path.dirname(sys.argv[0]), '../../auxiliary/util'))
-from u_format_access import *
+from u_format_pack import *
-def generate_format_read(format, dst_type, dst_native_type, dst_suffix):
+def is_format_supported(format):
+ '''Determines whether we actually have the plumbing necessary to generate the
+ to read/write to/from this format.'''
+
+ # FIXME: Ideally we would support any format combination here.
+
+ if format.layout != PLAIN:
+ return False
+
+ for i in range(4):
+ channel = format.channels[i]
+ if channel.type not in (VOID, UNSIGNED, SIGNED, FLOAT):
+ return False
+ if channel.type == FLOAT and channel.size not in (16, 32 ,64):
+ return False
+
+ if format.colorspace not in ('rgb', 'srgb'):
+ return False
+
+ return True
+
+
+def generate_format_read(format, dst_channel, dst_native_type, dst_suffix):
'''Generate the function to read pixels from a particular format'''
- name = short_name(format)
+ name = format.short_name()
src_native_type = native_type(format)
print 'static void'
- print 'lp_tile_%s_read_%s(%s *dst, const uint8_t *src, unsigned src_stride, unsigned x0, unsigned y0, unsigned w, unsigned h)' % (name, dst_suffix, dst_native_type)
+ print 'lp_tile_%s_swizzle_%s(%s *dst, const uint8_t *src, unsigned src_stride, unsigned x0, unsigned y0)' % (name, dst_suffix, dst_native_type)
print '{'
print ' unsigned x, y;'
print ' const uint8_t *src_row = src + y0*src_stride;'
- print ' for (y = 0; y < h; ++y) {'
+ print ' for (y = 0; y < TILE_SIZE; ++y) {'
print ' const %s *src_pixel = (const %s *)(src_row + x0*%u);' % (src_native_type, src_native_type, format.stride())
- print ' for (x = 0; x < w; ++x) {'
+ print ' for (x = 0; x < TILE_SIZE; ++x) {'
names = ['']*4
- if format.colorspace == 'rgb':
+ if format.colorspace in ('rgb', 'srgb'):
for i in range(4):
- swizzle = format.out_swizzle[i]
+ swizzle = format.swizzles[i]
if swizzle < 4:
names[swizzle] += 'rgba'[i]
elif format.colorspace == 'zs':
- swizzle = format.out_swizzle[0]
+ swizzle = format.swizzles[0]
if swizzle < 4:
names[swizzle] = 'z'
else:
else:
assert False
- if format.layout == ARITH:
- print ' %s pixel = *src_pixel++;' % src_native_type
- shift = 0;
- for i in range(4):
- src_type = format.in_types[i]
- width = src_type.size
- if names[i]:
- value = 'pixel'
- mask = (1 << width) - 1
- if shift:
- value = '(%s >> %u)' % (value, shift)
- if shift + width < format.block_size():
- value = '(%s & 0x%x)' % (value, mask)
- value = conversion_expr(src_type, dst_type, dst_native_type, value)
- print ' %s %s = %s;' % (dst_native_type, names[i], value)
- shift += width
- elif format.layout == ARRAY:
- for i in range(4):
- src_type = format.in_types[i]
- if names[i]:
- value = '(*src_pixel++)'
- value = conversion_expr(src_type, dst_type, dst_native_type, value)
- print ' %s %s = %s;' % (dst_native_type, names[i], value)
+ if format.layout == PLAIN:
+ if not format.is_array():
+ print ' %s pixel = *src_pixel++;' % src_native_type
+ shift = 0;
+ for i in range(4):
+ src_channel = format.channels[i]
+ width = src_channel.size
+ if names[i]:
+ value = 'pixel'
+ mask = (1 << width) - 1
+ if shift:
+ value = '(%s >> %u)' % (value, shift)
+ if shift + width < format.block_size():
+ value = '(%s & 0x%x)' % (value, mask)
+ value = conversion_expr(src_channel, dst_channel, dst_native_type, value, clamp=False)
+ print ' %s %s = %s;' % (dst_native_type, names[i], value)
+ shift += width
+ else:
+ for i in range(4):
+ if names[i]:
+ print ' %s %s;' % (dst_native_type, names[i])
+ for i in range(4):
+ src_channel = format.channels[i]
+ if names[i]:
+ value = '(*src_pixel++)'
+ value = conversion_expr(src_channel, dst_channel, dst_native_type, value, clamp=False)
+ print ' %s = %s;' % (names[i], value)
+ elif src_channel.size:
+ print ' ++src_pixel;'
else:
assert False
for i in range(4):
- if format.colorspace == 'rgb':
- swizzle = format.out_swizzle[i]
+ if format.colorspace in ('rgb', 'srgb'):
+ swizzle = format.swizzles[i]
if swizzle < 4:
value = names[swizzle]
elif swizzle == SWIZZLE_0:
value = '0'
elif swizzle == SWIZZLE_1:
- value = '1'
+ value = get_one(dst_channel)
else:
assert False
elif format.colorspace == 'zs':
if i < 3:
value = 'z'
else:
- value = '1'
+ value = get_one(dst_channel)
else:
assert False
print ' TILE_PIXEL(dst, x, y, %u) = %s; /* %s */' % (i, value, 'rgba'[i])
print
-def generate_format_write(format, src_type, src_native_type, src_suffix):
- '''Generate the function to write pixels to a particular format'''
+def pack_rgba(format, src_channel, r, g, b, a):
+ """Return an expression for packing r, g, b, a into a pixel of the
+ given format. Ex: '(b << 24) | (g << 16) | (r << 8) | (a << 0)'
+ """
+ assert format.colorspace in ('rgb', 'srgb')
+ inv_swizzle = format.inv_swizzles()
+ shift = 0
+ expr = None
+ for i in range(4):
+ # choose r, g, b, or a depending on the inverse swizzle term
+ if inv_swizzle[i] == 0:
+ value = r
+ elif inv_swizzle[i] == 1:
+ value = g
+ elif inv_swizzle[i] == 2:
+ value = b
+ elif inv_swizzle[i] == 3:
+ value = a
+ else:
+ value = None
+
+ if value:
+ dst_channel = format.channels[i]
+ dst_native_type = native_type(format)
+ value = conversion_expr(src_channel, dst_channel, dst_native_type, value, clamp=False)
+ term = "((%s) << %d)" % (value, shift)
+ if expr:
+ expr = expr + " | " + term
+ else:
+ expr = term
+
+ width = format.channels[i].size
+ shift = shift + width
+ return expr
+
+
+def emit_unrolled_unswizzle_code(format, src_channel):
+ '''Emit code for writing a block based on unrolled loops.
+ This is considerably faster than the TILE_PIXEL-based code below.
+ '''
+ dst_native_type = 'uint%u_t' % format.block_size()
+ print ' const unsigned dstpix_stride = dst_stride / %d;' % format.stride()
+ print ' %s *dstpix = (%s *) dst;' % (dst_native_type, dst_native_type)
+ print ' unsigned int qx, qy, i;'
+ print
+ print ' for (qy = 0; qy < TILE_SIZE; qy += TILE_VECTOR_HEIGHT) {'
+ print ' const unsigned py = y0 + qy;'
+ print ' for (qx = 0; qx < TILE_SIZE; qx += TILE_VECTOR_WIDTH) {'
+ print ' const unsigned px = x0 + qx;'
+ print ' const uint8_t *r = src + 0 * TILE_C_STRIDE;'
+ print ' const uint8_t *g = src + 1 * TILE_C_STRIDE;'
+ print ' const uint8_t *b = src + 2 * TILE_C_STRIDE;'
+ print ' const uint8_t *a = src + 3 * TILE_C_STRIDE;'
+ print ' (void) r; (void) g; (void) b; (void) a; /* silence warnings */'
+ print ' for (i = 0; i < TILE_C_STRIDE; i += 2) {'
+ print ' const uint32_t pixel0 = %s;' % pack_rgba(format, src_channel, "r[i+0]", "g[i+0]", "b[i+0]", "a[i+0]")
+ print ' const uint32_t pixel1 = %s;' % pack_rgba(format, src_channel, "r[i+1]", "g[i+1]", "b[i+1]", "a[i+1]")
+ print ' const unsigned offset = (py + tile_y_offset[i]) * dstpix_stride + (px + tile_x_offset[i]);'
+ print ' dstpix[offset + 0] = pixel0;'
+ print ' dstpix[offset + 1] = pixel1;'
+ print ' }'
+ print ' src += TILE_X_STRIDE;'
+ print ' }'
+ print ' }'
- name = short_name(format)
+def emit_tile_pixel_unswizzle_code(format, src_channel):
+ '''Emit code for writing a block based on the TILE_PIXEL macro.'''
dst_native_type = native_type(format)
- print 'static void'
- print 'lp_tile_%s_write_%s(const %s *src, uint8_t *dst, unsigned dst_stride, unsigned x0, unsigned y0, unsigned w, unsigned h)' % (name, src_suffix, src_native_type)
- print '{'
+ inv_swizzle = format.inv_swizzles()
+
print ' unsigned x, y;'
print ' uint8_t *dst_row = dst + y0*dst_stride;'
- print ' for (y = 0; y < h; ++y) {'
+ print ' for (y = 0; y < TILE_SIZE; ++y) {'
print ' %s *dst_pixel = (%s *)(dst_row + x0*%u);' % (dst_native_type, dst_native_type, format.stride())
- print ' for (x = 0; x < w; ++x) {'
-
- inv_swizzle = [None]*4
- if format.colorspace == 'rgb':
- for i in range(4):
- swizzle = format.out_swizzle[i]
- if swizzle < 4:
- inv_swizzle[swizzle] = i
- elif format.colorspace == 'zs':
- swizzle = format.out_swizzle[0]
- if swizzle < 4:
- inv_swizzle[swizzle] = 0
- else:
- assert False
-
- if format.layout == ARITH:
- print ' %s pixel = 0;' % dst_native_type
- shift = 0;
- for i in range(4):
- dst_type = format.in_types[i]
- width = dst_type.size
- if inv_swizzle[i] is not None:
- value = 'TILE_PIXEL(src, x, y, %u)' % inv_swizzle[i]
- value = conversion_expr(src_type, dst_type, dst_native_type, value)
- if shift:
- value = '(%s << %u)' % (value, shift)
- print ' pixel |= %s;' % value
- shift += width
- print ' *dst_pixel++ = pixel;'
- elif format.layout == ARRAY:
- for i in range(4):
- dst_type = format.in_types[i]
- if inv_swizzle[i] is not None:
- value = 'TILE_PIXEL(src, x, y, %u)' % inv_swizzle[i]
- value = conversion_expr(src_type, dst_type, dst_native_type, value)
- print ' *dst_pixel++ = %s;' % value
+ print ' for (x = 0; x < TILE_SIZE; ++x) {'
+
+ if format.layout == PLAIN:
+ if not format.is_array():
+ print ' %s pixel = 0;' % dst_native_type
+ shift = 0;
+ for i in range(4):
+ dst_channel = format.channels[i]
+ width = dst_channel.size
+ if inv_swizzle[i] is not None:
+ value = 'TILE_PIXEL(src, x, y, %u)' % inv_swizzle[i]
+ value = conversion_expr(src_channel, dst_channel, dst_native_type, value, clamp=False)
+ if shift:
+ value = '(%s << %u)' % (value, shift)
+ print ' pixel |= %s;' % value
+ shift += width
+ print ' *dst_pixel++ = pixel;'
+ else:
+ for i in range(4):
+ dst_channel = format.channels[i]
+ if inv_swizzle[i] is not None:
+ value = 'TILE_PIXEL(src, x, y, %u)' % inv_swizzle[i]
+ value = conversion_expr(src_channel, dst_channel, dst_native_type, value, clamp=False)
+ print ' *dst_pixel++ = %s;' % value
+ elif dst_channel.size:
+ print ' ++dst_pixel;'
else:
assert False
print ' }'
print ' dst_row += dst_stride;'
print ' }'
+
+
+def generate_format_write(format, src_channel, src_native_type, src_suffix):
+ '''Generate the function to write pixels to a particular format'''
+
+ name = format.short_name()
+
+ print 'static void'
+ print 'lp_tile_%s_unswizzle_%s(const %s *src, uint8_t *dst, unsigned dst_stride, unsigned x0, unsigned y0)' % (name, src_suffix, src_native_type)
+ print '{'
+ if format.layout == PLAIN \
+ and format.colorspace == 'rgb' \
+ and format.block_size() <= 32 \
+ and format.is_pot() \
+ and not format.is_mixed() \
+ and (format.channels[0].type == UNSIGNED \
+ or format.channels[1].type == UNSIGNED):
+ emit_unrolled_unswizzle_code(format, src_channel)
+ else:
+ emit_tile_pixel_unswizzle_code(format, src_channel)
print '}'
print
-def generate_read(formats, dst_type, dst_native_type, dst_suffix):
+def generate_sse2():
+ print '''
+#if defined(PIPE_ARCH_SSE)
+
+#include "util/u_sse.h"
+
+static ALWAYS_INLINE void
+swz4( const __m128i * restrict x,
+ const __m128i * restrict y,
+ const __m128i * restrict z,
+ const __m128i * restrict w,
+ __m128i * restrict a,
+ __m128i * restrict b,
+ __m128i * restrict c,
+ __m128i * restrict d)
+{
+ __m128i i, j, k, l;
+ __m128i m, n, o, p;
+ __m128i e, f, g, h;
+
+ m = _mm_unpacklo_epi8(*x,*y);
+ n = _mm_unpackhi_epi8(*x,*y);
+ o = _mm_unpacklo_epi8(*z,*w);
+ p = _mm_unpackhi_epi8(*z,*w);
+
+ i = _mm_unpacklo_epi16(m,n);
+ j = _mm_unpackhi_epi16(m,n);
+ k = _mm_unpacklo_epi16(o,p);
+ l = _mm_unpackhi_epi16(o,p);
+
+ e = _mm_unpacklo_epi8(i,j);
+ f = _mm_unpackhi_epi8(i,j);
+ g = _mm_unpacklo_epi8(k,l);
+ h = _mm_unpackhi_epi8(k,l);
+
+ *a = _mm_unpacklo_epi64(e,g);
+ *b = _mm_unpackhi_epi64(e,g);
+ *c = _mm_unpacklo_epi64(f,h);
+ *d = _mm_unpackhi_epi64(f,h);
+}
+
+static ALWAYS_INLINE void
+unswz4( const __m128i * restrict a,
+ const __m128i * restrict b,
+ const __m128i * restrict c,
+ const __m128i * restrict d,
+ __m128i * restrict x,
+ __m128i * restrict y,
+ __m128i * restrict z,
+ __m128i * restrict w)
+{
+ __m128i i, j, k, l;
+ __m128i m, n, o, p;
+
+ i = _mm_unpacklo_epi8(*a,*b);
+ j = _mm_unpackhi_epi8(*a,*b);
+ k = _mm_unpacklo_epi8(*c,*d);
+ l = _mm_unpackhi_epi8(*c,*d);
+
+ m = _mm_unpacklo_epi16(i,k);
+ n = _mm_unpackhi_epi16(i,k);
+ o = _mm_unpacklo_epi16(j,l);
+ p = _mm_unpackhi_epi16(j,l);
+
+ *x = _mm_unpacklo_epi64(m,n);
+ *y = _mm_unpackhi_epi64(m,n);
+ *z = _mm_unpacklo_epi64(o,p);
+ *w = _mm_unpackhi_epi64(o,p);
+}
+
+static void
+lp_tile_b8g8r8a8_unorm_swizzle_4ub_sse2(uint8_t * restrict dst,
+ const uint8_t * restrict src, unsigned src_stride,
+ unsigned x0, unsigned y0)
+{
+ __m128i *dst128 = (__m128i *) dst;
+ unsigned x, y;
+
+ src += y0 * src_stride;
+ src += x0 * sizeof(uint32_t);
+
+ for (y = 0; y < TILE_SIZE; y += 4) {
+ const uint8_t *src_row = src;
+
+ for (x = 0; x < TILE_SIZE; x += 4) {
+ swz4((const __m128i *) (src_row + 0 * src_stride),
+ (const __m128i *) (src_row + 1 * src_stride),
+ (const __m128i *) (src_row + 2 * src_stride),
+ (const __m128i *) (src_row + 3 * src_stride),
+ dst128 + 2, /* b */
+ dst128 + 1, /* g */
+ dst128 + 0, /* r */
+ dst128 + 3); /* a */
+
+ dst128 += 4;
+ src_row += sizeof(__m128i);
+ }
+
+ src += 4 * src_stride;
+ }
+}
+
+static void
+lp_tile_b8g8r8a8_unorm_unswizzle_4ub_sse2(const uint8_t * restrict src,
+ uint8_t * restrict dst, unsigned dst_stride,
+ unsigned x0, unsigned y0)
+{
+ unsigned int x, y;
+ const __m128i *src128 = (const __m128i *) src;
+
+ dst += y0 * dst_stride;
+ dst += x0 * sizeof(uint32_t);
+
+ for (y = 0; y < TILE_SIZE; y += 4) {
+ const uint8_t *dst_row = dst;
+
+ for (x = 0; x < TILE_SIZE; x += 4) {
+ unswz4( &src128[2], /* b */
+ &src128[1], /* g */
+ &src128[0], /* r */
+ &src128[3], /* a */
+ (__m128i *) (dst_row + 0 * dst_stride),
+ (__m128i *) (dst_row + 1 * dst_stride),
+ (__m128i *) (dst_row + 2 * dst_stride),
+ (__m128i *) (dst_row + 3 * dst_stride));
+
+ src128 += 4;
+ dst_row += sizeof(__m128i);;
+ }
+
+ dst += 4 * dst_stride;
+ }
+}
+
+#endif /* PIPE_ARCH_SSE */
+'''
+
+
+def generate_swizzle(formats, dst_channel, dst_native_type, dst_suffix):
'''Generate the dispatch function to read pixels from any format'''
for format in formats:
if is_format_supported(format):
- generate_format_read(format, dst_type, dst_native_type, dst_suffix)
+ generate_format_read(format, dst_channel, dst_native_type, dst_suffix)
print 'void'
- print 'lp_tile_read_%s(enum pipe_format format, %s *dst, const void *src, unsigned src_stride, unsigned x, unsigned y, unsigned w, unsigned h)' % (dst_suffix, dst_native_type)
+ print 'lp_tile_swizzle_%s(enum pipe_format format, %s *dst, const void *src, unsigned src_stride, unsigned x, unsigned y)' % (dst_suffix, dst_native_type)
print '{'
- print ' void (*func)(%s *dst, const uint8_t *src, unsigned src_stride, unsigned x0, unsigned y0, unsigned w, unsigned h);' % dst_native_type
+ print ' void (*func)(%s *dst, const uint8_t *src, unsigned src_stride, unsigned x0, unsigned y0);' % dst_native_type
+ print '#ifdef DEBUG'
+ print ' lp_tile_swizzle_count += 1;'
+ print '#endif'
print ' switch(format) {'
for format in formats:
if is_format_supported(format):
print ' case %s:' % format.name
- print ' func = &lp_tile_%s_read_%s;' % (short_name(format), dst_suffix)
+ func_name = 'lp_tile_%s_swizzle_%s' % (format.short_name(), dst_suffix)
+ if format.name == 'PIPE_FORMAT_B8G8R8A8_UNORM':
+ print '#ifdef PIPE_ARCH_SSE'
+ print ' func = util_cpu_caps.has_sse2 ? %s_sse2 : %s;' % (func_name, func_name)
+ print '#else'
+ print ' func = %s;' % (func_name,)
+ print '#endif'
+ else:
+ print ' func = %s;' % (func_name,)
print ' break;'
print ' default:'
- print ' debug_printf("unsupported format\\n");'
+ print ' debug_printf("%s: unsupported format %s\\n", __FUNCTION__, util_format_name(format));'
print ' return;'
print ' }'
- print ' func(dst, (const uint8_t *)src, src_stride, x, y, w, h);'
+ print ' func(dst, (const uint8_t *)src, src_stride, x, y);'
print '}'
print
-def generate_write(formats, src_type, src_native_type, src_suffix):
+def generate_unswizzle(formats, src_channel, src_native_type, src_suffix):
'''Generate the dispatch function to write pixels to any format'''
for format in formats:
if is_format_supported(format):
- generate_format_write(format, src_type, src_native_type, src_suffix)
+ generate_format_write(format, src_channel, src_native_type, src_suffix)
print 'void'
- print 'lp_tile_write_%s(enum pipe_format format, const %s *src, void *dst, unsigned dst_stride, unsigned x, unsigned y, unsigned w, unsigned h)' % (src_suffix, src_native_type)
+ print 'lp_tile_unswizzle_%s(enum pipe_format format, const %s *src, void *dst, unsigned dst_stride, unsigned x, unsigned y)' % (src_suffix, src_native_type)
print '{'
- print ' void (*func)(const %s *src, uint8_t *dst, unsigned dst_stride, unsigned x0, unsigned y0, unsigned w, unsigned h);' % src_native_type
+ print ' void (*func)(const %s *src, uint8_t *dst, unsigned dst_stride, unsigned x0, unsigned y0);' % src_native_type
+ print '#ifdef DEBUG'
+ print ' lp_tile_unswizzle_count += 1;'
+ print '#endif'
print ' switch(format) {'
for format in formats:
if is_format_supported(format):
print ' case %s:' % format.name
- print ' func = &lp_tile_%s_write_%s;' % (short_name(format), src_suffix)
+ func_name = 'lp_tile_%s_unswizzle_%s' % (format.short_name(), src_suffix)
+ if format.name == 'PIPE_FORMAT_B8G8R8A8_UNORM':
+ print '#ifdef PIPE_ARCH_SSE'
+ print ' func = util_cpu_caps.has_sse2 ? %s_sse2 : %s;' % (func_name, func_name)
+ print '#else'
+ print ' func = %s;' % (func_name,)
+ print '#endif'
+ else:
+ print ' func = %s;' % (func_name,)
print ' break;'
print ' default:'
- print ' debug_printf("unsupported format\\n");'
+ print ' debug_printf("%s: unsupported format %s\\n", __FUNCTION__, util_format_name(format));'
print ' return;'
print ' }'
- print ' func(src, (uint8_t *)dst, dst_stride, x, y, w, h);'
+ print ' func(src, (uint8_t *)dst, dst_stride, x, y);'
print '}'
print
print '#include "pipe/p_compiler.h"'
print '#include "util/u_format.h"'
print '#include "util/u_math.h"'
+ print '#include "util/u_half.h"'
+ print '#include "util/u_cpu_detect.h"'
print '#include "lp_tile_soa.h"'
print
+ print '#ifdef DEBUG'
+ print 'unsigned lp_tile_unswizzle_count = 0;'
+ print 'unsigned lp_tile_swizzle_count = 0;'
+ print '#endif'
+ print
print 'const unsigned char'
print 'tile_offset[TILE_VECTOR_HEIGHT][TILE_VECTOR_WIDTH] = {'
- print ' { 0, 1, 4, 5, 8, 9, 12, 13},'
- print ' { 2, 3, 6, 7, 10, 11, 14, 15}'
+ print ' { 0, 1, 4, 5},'
+ print ' { 2, 3, 6, 7},'
+ print ' { 8, 9, 12, 13},'
+ print ' { 10, 11, 14, 15}'
+ print '};'
+ print
+ print '/* Note: these lookup tables could be replaced with some'
+ print ' * bit-twiddling code, but this is a little faster.'
+ print ' */'
+ print 'static unsigned tile_x_offset[TILE_VECTOR_WIDTH * TILE_VECTOR_HEIGHT] = {'
+ print ' 0, 1, 0, 1, 2, 3, 2, 3,'
+ print ' 0, 1, 0, 1, 2, 3, 2, 3'
+ print '};'
+ print
+ print 'static unsigned tile_y_offset[TILE_VECTOR_WIDTH * TILE_VECTOR_HEIGHT] = {'
+ print ' 0, 0, 1, 1, 0, 0, 1, 1,'
+ print ' 2, 2, 3, 3, 2, 2, 3, 3'
print '};'
print
- generate_clamp()
+ generate_sse2()
- type = Type(UNSIGNED, True, 8)
+ channel = Channel(UNSIGNED, True, 8)
native_type = 'uint8_t'
suffix = '4ub'
- generate_read(formats, type, native_type, suffix)
- generate_write(formats, type, native_type, suffix)
+ generate_swizzle(formats, channel, native_type, suffix)
+ generate_unswizzle(formats, channel, native_type, suffix)
if __name__ == '__main__':
projects / mesa.git / blobdiff