* IN THE SOFTWARE.
*/
-#include "compiler/nir/nir_builder.h"
+#include "blorp_nir_builder.h"
+#include "compiler/nir/nir_format_convert.h"
#include "blorp_priv.h"
+#include "util/format_rgb9e5.h"
/* header-only include needed for _mesa_unorm_to_float and friends. */
#include "mesa/main/format_utils.h"
blorp_nir_tex(nir_builder *b, struct brw_blorp_blit_vars *v,
const struct brw_blorp_blit_prog_key *key, nir_ssa_def *pos)
{
+ if (key->need_src_offset)
+ pos = nir_fadd(b, pos, nir_i2f32(b, nir_load_var(b, v->v_src_offset)));
+
/* If the sampler requires normalized coordinates, we need to compensate. */
if (key->src_coords_normalized)
pos = nir_fmul(b, pos, nir_load_var(b, v->v_src_inv_size));
}
static nir_ssa_def *
-blorp_nir_txf_ms_mcs(nir_builder *b, struct brw_blorp_blit_vars *v, nir_ssa_def *pos)
+blorp_blit_txf_ms_mcs(nir_builder *b, struct brw_blorp_blit_vars *v,
+ nir_ssa_def *pos)
{
nir_tex_instr *tex =
blorp_create_nir_tex_instr(b, v, nir_texop_txf_ms_mcs,
return &tex->dest.ssa;
}
-static nir_ssa_def *
-nir_mask_shift_or(struct nir_builder *b, nir_ssa_def *dst, nir_ssa_def *src,
- uint32_t src_mask, int src_left_shift)
-{
- nir_ssa_def *masked = nir_iand(b, src, nir_imm_int(b, src_mask));
-
- nir_ssa_def *shifted;
- if (src_left_shift > 0) {
- shifted = nir_ishl(b, masked, nir_imm_int(b, src_left_shift));
- } else if (src_left_shift < 0) {
- shifted = nir_ushr(b, masked, nir_imm_int(b, -src_left_shift));
- } else {
- assert(src_left_shift == 0);
- shifted = masked;
- }
-
- return nir_ior(b, dst, shifted);
-}
-
/**
* Emit code to compensate for the difference between Y and W tiling.
*
nir_ssa_def *mcs = NULL;
if (tex_aux_usage == ISL_AUX_USAGE_MCS)
- mcs = blorp_nir_txf_ms_mcs(b, v, pos);
+ mcs = blorp_blit_txf_ms_mcs(b, v, pos);
/* We add together samples using a binary tree structure, e.g. for 4x MSAA:
*
* ld2dms are equivalent (since all samples are on sample slice 0).
* Since we have already sampled from sample 0, all we need to do is
* skip the remaining fetches and averaging if MCS is zero.
+ *
+ * It's also trivial to detect when the MCS has the magic clear color
+ * value. In this case, the txf we did on sample 0 will return the
+ * clear color and we can skip the remaining fetches just like we do
+ * when MCS == 0.
*/
nir_ssa_def *mcs_zero =
nir_ieq(b, nir_channel(b, mcs, 0), nir_imm_int(b, 0));
mcs_zero = nir_iand(b, mcs_zero,
nir_ieq(b, nir_channel(b, mcs, 1), nir_imm_int(b, 0)));
}
+ nir_ssa_def *mcs_clear =
+ blorp_nir_mcs_is_clear_color(b, mcs, tex_samples);
nir_if *if_stmt = nir_if_create(b->shader);
- if_stmt->condition = nir_src_for_ssa(mcs_zero);
+ if_stmt->condition = nir_src_for_ssa(nir_ior(b, mcs_zero, mcs_clear));
nir_cf_node_insert(b->cursor, &if_stmt->cf_node);
b->cursor = nir_after_cf_list(&if_stmt->then_list);
*/
nir_ssa_def *mcs = NULL;
if (key->tex_aux_usage == ISL_AUX_USAGE_MCS)
- mcs = blorp_nir_txf_ms_mcs(b, v, sample_coords_int);
+ mcs = blorp_blit_txf_ms_mcs(b, v, sample_coords_int);
/* Compute sample index and map the sample index to a sample number.
* Sample index layout shows the numbering of slots in a rectangular
}
}
+static nir_ssa_def *
+select_color_channel(struct nir_builder *b, nir_ssa_def *color,
+ nir_alu_type data_type,
+ enum isl_channel_select chan)
+{
+ if (chan == ISL_CHANNEL_SELECT_ZERO) {
+ return nir_imm_int(b, 0);
+ } else if (chan == ISL_CHANNEL_SELECT_ONE) {
+ switch (data_type) {
+ case nir_type_int:
+ case nir_type_uint:
+ return nir_imm_int(b, 1);
+ case nir_type_float:
+ return nir_imm_float(b, 1);
+ default:
+ unreachable("Invalid data type");
+ }
+ } else {
+ assert((unsigned)(chan - ISL_CHANNEL_SELECT_RED) < 4);
+ return nir_channel(b, color, chan - ISL_CHANNEL_SELECT_RED);
+ }
+}
+
+static nir_ssa_def *
+swizzle_color(struct nir_builder *b, nir_ssa_def *color,
+ struct isl_swizzle swizzle, nir_alu_type data_type)
+{
+ return nir_vec4(b,
+ select_color_channel(b, color, data_type, swizzle.r),
+ select_color_channel(b, color, data_type, swizzle.g),
+ select_color_channel(b, color, data_type, swizzle.b),
+ select_color_channel(b, color, data_type, swizzle.a));
+}
+
+static nir_ssa_def *
+convert_color(struct nir_builder *b, nir_ssa_def *color,
+ const struct brw_blorp_blit_prog_key *key)
+{
+ /* All of our color conversions end up generating a single-channel color
+ * value that we need to write out.
+ */
+ nir_ssa_def *value;
+
+ if (key->dst_format == ISL_FORMAT_R24_UNORM_X8_TYPELESS) {
+ /* The destination image is bound as R32_UNORM but the data needs to be
+ * in R24_UNORM_X8_TYPELESS. The bottom 24 are the actual data and the
+ * top 8 need to be zero. We can accomplish this by simply multiplying
+ * by a factor to scale things down.
+ */
+ float factor = (float)((1 << 24) - 1) / (float)UINT32_MAX;
+ value = nir_fmul(b, nir_fsat(b, nir_channel(b, color, 0)),
+ nir_imm_float(b, factor));
+ } else if (key->dst_format == ISL_FORMAT_L8_UNORM_SRGB) {
+ value = nir_format_linear_to_srgb(b, color);
+ } else if (key->dst_format == ISL_FORMAT_R9G9B9E5_SHAREDEXP) {
+ value = nir_format_pack_r9g9b9e5(b, color);
+ } else {
+ unreachable("Unsupported format conversion");
+ }
+
+ nir_ssa_def *u = nir_ssa_undef(b, 1, 32);
+ return nir_vec4(b, value, u, u, u);
+}
+
/**
* Generator for WM programs used in BLORP blits.
*
key->dst_layout);
}
+ nir_ssa_def *comp = NULL;
+ if (key->dst_rgb) {
+ /* The destination image is bound as a red texture three times as wide
+ * as the actual image. Our shader is effectively running one color
+ * component at a time. We need to save off the component and adjust
+ * the destination position.
+ */
+ assert(dst_pos->num_components == 2);
+ nir_ssa_def *dst_x = nir_channel(&b, dst_pos, 0);
+ comp = nir_umod(&b, dst_x, nir_imm_int(&b, 3));
+ dst_pos = nir_vec2(&b, nir_idiv(&b, dst_x, nir_imm_int(&b, 3)),
+ nir_channel(&b, dst_pos, 1));
+ }
+
/* Now (X, Y, S) = decode_msaa(dst_samples, detile(dst_tiling, offset)).
*
* That is: X, Y and S now contain the true coordinates and sample index of
* representing the four samples that maxe up a pixel. So we need
* to multiply our X and Y coordinates each by 2 and then add 1.
*/
- src_pos = nir_ishl(&b, src_pos, nir_imm_int(&b, 1));
- src_pos = nir_iadd(&b, src_pos, nir_imm_int(&b, 1));
- src_pos = nir_i2f32(&b, src_pos);
+ assert(key->src_coords_normalized);
+ src_pos = nir_fadd(&b,
+ nir_i2f32(&b, src_pos),
+ nir_imm_float(&b, 0.5f));
color = blorp_nir_tex(&b, &v, key, src_pos);
} else {
/* Gen7+ hardware doesn't automaticaly blend. */
} else {
nir_ssa_def *mcs = NULL;
if (key->tex_aux_usage == ISL_AUX_USAGE_MCS)
- mcs = blorp_nir_txf_ms_mcs(&b, &v, src_pos);
+ mcs = blorp_blit_txf_ms_mcs(&b, &v, src_pos);
color = blorp_nir_txf_ms(&b, &v, src_pos, mcs, key->texture_data_type);
}
}
}
- if (key->dst_bpc != key->src_bpc)
+ if (!isl_swizzle_is_identity(key->src_swizzle)) {
+ color = swizzle_color(&b, color, key->src_swizzle,
+ key->texture_data_type);
+ }
+
+ if (!isl_swizzle_is_identity(key->dst_swizzle)) {
+ color = swizzle_color(&b, color, isl_swizzle_invert(key->dst_swizzle),
+ nir_type_int);
+ }
+
+ if (key->dst_bpc != key->src_bpc) {
+ assert(isl_swizzle_is_identity(key->src_swizzle));
+ assert(isl_swizzle_is_identity(key->dst_swizzle));
color = bit_cast_color(&b, color, key);
+ }
+
+ if (key->dst_format)
+ color = convert_color(&b, color, key);
if (key->dst_rgb) {
/* The destination image is bound as a red texture three times as wide
* from the source color and write that to destination red.
*/
assert(dst_pos->num_components == 2);
- nir_ssa_def *comp =
- nir_umod(&b, nir_channel(&b, dst_pos, 0), nir_imm_int(&b, 3));
nir_ssa_def *color_component =
nir_bcsel(&b, nir_ieq(&b, comp, nir_imm_int(&b, 0)),
void *mem_ctx = ralloc_context(NULL);
const unsigned *program;
- unsigned program_size;
struct brw_wm_prog_data prog_data;
nir_shader *nir = brw_blorp_build_nir_shader(blorp, mem_ctx, prog_key);
wm_key.multisample_fbo = prog_key->rt_samples > 1;
program = blorp_compile_fs(blorp, mem_ctx, nir, &wm_key, false,
- &prog_data, &program_size);
+ &prog_data);
bool result =
blorp->upload_shader(blorp, prog_key, sizeof(*prog_key),
- program, program_size,
+ program, prog_data.base.program_size,
&prog_data.base, sizeof(prog_data),
¶ms->wm_prog_kernel, ¶ms->wm_prog_data);
else
layer = info->view.base_array_layer;
- uint32_t x_offset_sa, y_offset_sa;
- isl_surf_get_image_offset_sa(&info->surf, info->view.base_level,
- layer, z, &x_offset_sa, &y_offset_sa);
-
uint32_t byte_offset;
- isl_tiling_get_intratile_offset_sa(isl_dev, info->surf.tiling,
- info->surf.format, info->surf.row_pitch,
- x_offset_sa, y_offset_sa,
- &byte_offset,
- &info->tile_x_sa, &info->tile_y_sa);
+ isl_surf_get_image_surf(isl_dev, &info->surf,
+ info->view.base_level, layer, z,
+ &info->surf,
+ &byte_offset, &info->tile_x_sa, &info->tile_y_sa);
info->addr.offset += byte_offset;
- const uint32_t slice_width_px =
- minify(info->surf.logical_level0_px.width, info->view.base_level);
- const uint32_t slice_height_px =
- minify(info->surf.logical_level0_px.height, info->view.base_level);
-
uint32_t tile_x_px, tile_y_px;
surf_get_intratile_offset_px(info, &tile_x_px, &tile_y_px);
- struct isl_surf_init_info init_info = {
- .dim = ISL_SURF_DIM_2D,
- .format = info->surf.format,
- .width = slice_width_px + tile_x_px,
- .height = slice_height_px + tile_y_px,
- .depth = 1,
- .levels = 1,
- .array_len = 1,
- .samples = info->surf.samples,
- .row_pitch = info->surf.row_pitch,
- .usage = info->surf.usage,
- .tiling_flags = 1 << info->surf.tiling,
- };
-
- ok = isl_surf_init_s(isl_dev, &info->surf, &init_info);
- assert(ok);
+ /* Instead of using the X/Y Offset fields in RENDER_SURFACE_STATE, we place
+ * the image at the tile boundary and offset our sampling or rendering.
+ * For this reason, we need to grow the image by the offset to ensure that
+ * the hardware doesn't think we've gone past the edge.
+ */
+ info->surf.logical_level0_px.w += tile_x_px;
+ info->surf.logical_level0_px.h += tile_y_px;
+ info->surf.phys_level0_sa.w += info->tile_x_sa;
+ info->surf.phys_level0_sa.h += info->tile_y_sa;
/* The view is also different now. */
info->view.base_level = 0;
struct blt_axis x, y;
};
+static enum isl_format
+get_red_format_for_rgb_format(enum isl_format format)
+{
+ const struct isl_format_layout *fmtl = isl_format_get_layout(format);
+
+ switch (fmtl->channels.r.bits) {
+ case 8:
+ switch (fmtl->channels.r.type) {
+ case ISL_UNORM:
+ return ISL_FORMAT_R8_UNORM;
+ case ISL_SNORM:
+ return ISL_FORMAT_R8_SNORM;
+ case ISL_UINT:
+ return ISL_FORMAT_R8_UINT;
+ case ISL_SINT:
+ return ISL_FORMAT_R8_SINT;
+ default:
+ unreachable("Invalid 8-bit RGB channel type");
+ }
+ case 16:
+ switch (fmtl->channels.r.type) {
+ case ISL_UNORM:
+ return ISL_FORMAT_R16_UNORM;
+ case ISL_SNORM:
+ return ISL_FORMAT_R16_SNORM;
+ case ISL_SFLOAT:
+ return ISL_FORMAT_R16_FLOAT;
+ case ISL_UINT:
+ return ISL_FORMAT_R16_UINT;
+ case ISL_SINT:
+ return ISL_FORMAT_R16_SINT;
+ default:
+ unreachable("Invalid 8-bit RGB channel type");
+ }
+ case 32:
+ switch (fmtl->channels.r.type) {
+ case ISL_SFLOAT:
+ return ISL_FORMAT_R32_FLOAT;
+ case ISL_UINT:
+ return ISL_FORMAT_R32_UINT;
+ case ISL_SINT:
+ return ISL_FORMAT_R32_SINT;
+ default:
+ unreachable("Invalid 8-bit RGB channel type");
+ }
+ default:
+ unreachable("Invalid number of red channel bits");
+ }
+}
+
static void
surf_fake_rgb_with_red(const struct isl_device *isl_dev,
- struct brw_blorp_surface_info *info,
- uint32_t *x, uint32_t *width)
+ struct brw_blorp_surface_info *info)
{
blorp_surf_convert_to_single_slice(isl_dev, info);
info->surf.logical_level0_px.width *= 3;
info->surf.phys_level0_sa.width *= 3;
- *x *= 3;
- *width *= 3;
+ info->tile_x_sa *= 3;
+
+ enum isl_format red_format =
+ get_red_format_for_rgb_format(info->view.format);
- enum isl_format red_format;
- switch (info->view.format) {
- case ISL_FORMAT_R8G8B8_UNORM:
- red_format = ISL_FORMAT_R8_UNORM;
- break;
- case ISL_FORMAT_R8G8B8_UINT:
- red_format = ISL_FORMAT_R8_UINT;
- break;
- case ISL_FORMAT_R16G16B16_UNORM:
- red_format = ISL_FORMAT_R16_UNORM;
- break;
- case ISL_FORMAT_R16G16B16_UINT:
- red_format = ISL_FORMAT_R16_UINT;
- break;
- case ISL_FORMAT_R32G32B32_UINT:
- red_format = ISL_FORMAT_R32_UINT;
- break;
- default:
- unreachable("Invalid RGB copy destination format");
- }
assert(isl_format_get_layout(red_format)->channels.r.type ==
isl_format_get_layout(info->view.format)->channels.r.type);
assert(isl_format_get_layout(red_format)->channels.r.bits ==
info->surf.format = info->view.format = red_format;
}
-static void
-fake_dest_rgb_with_red(const struct isl_device *dev,
- struct blorp_params *params,
- struct brw_blorp_blit_prog_key *wm_prog_key,
- struct blt_coords *coords)
-{
- /* Handle RGB destinations for blorp_copy */
- const struct isl_format_layout *dst_fmtl =
- isl_format_get_layout(params->dst.surf.format);
-
- if (dst_fmtl->bpb % 3 == 0) {
- uint32_t dst_x = coords->x.dst0;
- uint32_t dst_width = coords->x.dst1 - dst_x;
- surf_fake_rgb_with_red(dev, ¶ms->dst,
- &dst_x, &dst_width);
- coords->x.dst0 = dst_x;
- coords->x.dst1 = dst_x + dst_width;
- wm_prog_key->dst_rgb = true;
- wm_prog_key->need_dst_offset = true;
- }
-}
-
enum blit_shrink_status {
BLIT_NO_SHRINK = 0,
BLIT_WIDTH_SHRINK = 1,
{
const struct gen_device_info *devinfo = batch->blorp->isl_dev->info;
- fake_dest_rgb_with_red(batch->blorp->isl_dev, params, wm_prog_key, coords);
-
if (isl_format_has_sint_channel(params->src.view.format)) {
wm_prog_key->texture_data_type = nir_type_int;
} else if (isl_format_has_uint_channel(params->src.view.format)) {
/* The MinLOD and MinimumArrayElement don't work properly for cube maps.
* Convert them to a single slice on gen4.
*/
- if (params->dst.surf.usage & ISL_SURF_USAGE_CUBE_BIT)
+ if (params->dst.surf.usage & ISL_SURF_USAGE_CUBE_BIT) {
blorp_surf_convert_to_single_slice(batch->blorp->isl_dev, ¶ms->dst);
+ wm_prog_key->need_dst_offset = true;
+ }
- if (params->src.surf.usage & ISL_SURF_USAGE_CUBE_BIT)
+ if (params->src.surf.usage & ISL_SURF_USAGE_CUBE_BIT) {
blorp_surf_convert_to_single_slice(batch->blorp->isl_dev, ¶ms->src);
+ wm_prog_key->need_src_offset = true;
+ }
}
if (devinfo->gen > 6 &&
params->num_samples = params->dst.surf.samples;
+ if ((wm_prog_key->bilinear_filter ||
+ (wm_prog_key->blend && !wm_prog_key->blit_scaled)) &&
+ batch->blorp->isl_dev->info->gen <= 6) {
+ /* Gen4-5 don't support non-normalized texture coordinates */
+ wm_prog_key->src_coords_normalized = true;
+ params->wm_inputs.src_inv_size[0] =
+ 1.0f / minify(params->src.surf.logical_level0_px.width,
+ params->src.view.base_level);
+ params->wm_inputs.src_inv_size[1] =
+ 1.0f / minify(params->src.surf.logical_level0_px.height,
+ params->src.view.base_level);
+ }
+
+ if (isl_format_get_layout(params->dst.view.format)->bpb % 3 == 0) {
+ /* We can't render to RGB formats natively because they aren't a
+ * power-of-two size. Instead, we fake them by using a red format
+ * with the same channel type and size and emitting shader code to
+ * only write one channel at a time.
+ */
+ params->x0 *= 3;
+ params->x1 *= 3;
+
+ surf_fake_rgb_with_red(batch->blorp->isl_dev, ¶ms->dst);
+
+ wm_prog_key->dst_rgb = true;
+ wm_prog_key->need_dst_offset = true;
+ } else if (isl_format_is_rgbx(params->dst.view.format)) {
+ /* We can handle RGBX formats easily enough by treating them as RGBA */
+ params->dst.view.format =
+ isl_format_rgbx_to_rgba(params->dst.view.format);
+ } else if (params->dst.view.format == ISL_FORMAT_R24_UNORM_X8_TYPELESS) {
+ wm_prog_key->dst_format = params->dst.view.format;
+ params->dst.view.format = ISL_FORMAT_R32_UNORM;
+ } else if (params->dst.view.format == ISL_FORMAT_A4B4G4R4_UNORM) {
+ params->dst.view.swizzle =
+ isl_swizzle_compose(params->dst.view.swizzle,
+ ISL_SWIZZLE(ALPHA, RED, GREEN, BLUE));
+ params->dst.view.format = ISL_FORMAT_B4G4R4A4_UNORM;
+ } else if (params->dst.view.format == ISL_FORMAT_L8_UNORM_SRGB) {
+ wm_prog_key->dst_format = params->dst.view.format;
+ params->dst.view.format = ISL_FORMAT_R8_UNORM;
+ } else if (params->dst.view.format == ISL_FORMAT_R9G9B9E5_SHAREDEXP) {
+ wm_prog_key->dst_format = params->dst.view.format;
+ params->dst.view.format = ISL_FORMAT_R32_UINT;
+ }
+
+ if (devinfo->gen <= 7 && !devinfo->is_haswell &&
+ !isl_swizzle_is_identity(params->src.view.swizzle)) {
+ wm_prog_key->src_swizzle = params->src.view.swizzle;
+ params->src.view.swizzle = ISL_SWIZZLE_IDENTITY;
+ } else {
+ wm_prog_key->src_swizzle = ISL_SWIZZLE_IDENTITY;
+ }
+
+ if (!isl_swizzle_supports_rendering(devinfo, params->dst.view.swizzle)) {
+ wm_prog_key->dst_swizzle = params->dst.view.swizzle;
+ params->dst.view.swizzle = ISL_SWIZZLE_IDENTITY;
+ } else {
+ wm_prog_key->dst_swizzle = ISL_SWIZZLE_IDENTITY;
+ }
+
if (params->src.tile_x_sa || params->src.tile_y_sa) {
assert(wm_prog_key->need_src_offset);
surf_get_intratile_offset_px(¶ms->src,
*/
x_offset_sa = (uint32_t)*x0 * px_size_sa.w + info->tile_x_sa;
y_offset_sa = (uint32_t)*y0 * px_size_sa.h + info->tile_y_sa;
- isl_tiling_get_intratile_offset_sa(dev, info->surf.tiling,
+ isl_tiling_get_intratile_offset_sa(info->surf.tiling,
info->surf.format, info->surf.row_pitch,
x_offset_sa, y_offset_sa,
&byte_offset,
struct blorp_params params;
blorp_params_init(¶ms);
+ /* We cannot handle combined depth and stencil. */
+ if (src_surf->surf->usage & ISL_SURF_USAGE_STENCIL_BIT)
+ assert(src_surf->surf->format == ISL_FORMAT_R8_UINT);
+ if (dst_surf->surf->usage & ISL_SURF_USAGE_STENCIL_BIT)
+ assert(dst_surf->surf->format == ISL_FORMAT_R8_UINT);
+
if (dst_surf->surf->usage & ISL_SURF_USAGE_STENCIL_BIT) {
assert(src_surf->surf->usage & ISL_SURF_USAGE_STENCIL_BIT);
/* Prior to Broadwell, we can't render to R8_UINT */
if (filter == GL_LINEAR &&
params.src.surf.samples <= 1 && params.dst.surf.samples <= 1) {
wm_prog_key.bilinear_filter = true;
-
- if (batch->blorp->isl_dev->info->gen < 6) {
- /* Gen4-5 don't support non-normalized texture coordinates */
- wm_prog_key.src_coords_normalized = true;
- params.wm_inputs.src_inv_size[0] =
- 1.0f / minify(params.src.surf.logical_level0_px.width, src_level);
- params.wm_inputs.src_inv_size[1] =
- 1.0f / minify(params.src.surf.logical_level0_px.height, src_level);
- }
}
if ((params.src.surf.usage & ISL_SURF_USAGE_DEPTH_BIT) == 0 &&
case ISL_FORMAT_R32G32B32A32_UINT:
case ISL_FORMAT_R32G32B32A32_UNORM:
case ISL_FORMAT_R32G32B32A32_SNORM:
+ case ISL_FORMAT_R32G32B32X32_FLOAT:
return ISL_FORMAT_R32G32B32A32_UINT;
case ISL_FORMAT_R16G16B16A16_UNORM:
return bits;
}
-static void
-surf_convert_to_uncompressed(const struct isl_device *isl_dev,
- struct brw_blorp_surface_info *info,
- uint32_t *x, uint32_t *y,
- uint32_t *width, uint32_t *height)
+void
+blorp_surf_convert_to_uncompressed(const struct isl_device *isl_dev,
+ struct brw_blorp_surface_info *info,
+ uint32_t *x, uint32_t *y,
+ uint32_t *width, uint32_t *height)
{
const struct isl_format_layout *fmtl =
isl_format_get_layout(info->surf.format);
*/
blorp_surf_convert_to_single_slice(isl_dev, info);
- if (width || height) {
+ if (width && height) {
#ifndef NDEBUG
uint32_t right_edge_px = info->tile_x_sa + *x + *width;
uint32_t bottom_edge_px = info->tile_y_sa + *y + *height;
*height = DIV_ROUND_UP(*height, fmtl->bh);
}
- assert(*x % fmtl->bw == 0);
- assert(*y % fmtl->bh == 0);
- *x /= fmtl->bw;
- *y /= fmtl->bh;
+ if (x && y) {
+ assert(*x % fmtl->bw == 0);
+ assert(*y % fmtl->bh == 0);
+ *x /= fmtl->bw;
+ *y /= fmtl->bh;
+ }
info->surf.logical_level0_px.width =
DIV_ROUND_UP(info->surf.logical_level0_px.width, fmtl->bw);
}
if (params.src.aux_usage == ISL_AUX_USAGE_CCS_E) {
+ /* It's safe to do a blorp_copy between things which are sRGB with CCS_E
+ * enabled even though CCS_E doesn't technically do sRGB on SKL because
+ * we stomp everything to UINT anyway. The one thing we have to be
+ * careful of is clear colors. Because fast clear colors for sRGB on
+ * gen9 are encoded as the float values between format conversion and
+ * sRGB curve application, a given clear color float will convert to the
+ * same bits regardless of whether the format is UNORM or sRGB.
+ * Therefore, we can handle sRGB without any special cases.
+ */
+ UNUSED enum isl_format linear_src_format =
+ isl_format_srgb_to_linear(src_surf->surf->format);
assert(isl_formats_are_ccs_e_compatible(batch->blorp->isl_dev->info,
- src_surf->surf->format,
+ linear_src_format,
params.src.view.format));
params.src.clear_color =
bitcast_color_value_to_uint(params.src.clear_color, src_fmtl);
}
if (params.dst.aux_usage == ISL_AUX_USAGE_CCS_E) {
+ /* See above where we handle linear_src_format */
+ UNUSED enum isl_format linear_dst_format =
+ isl_format_srgb_to_linear(dst_surf->surf->format);
assert(isl_formats_are_ccs_e_compatible(batch->blorp->isl_dev->info,
- dst_surf->surf->format,
+ linear_dst_format,
params.dst.view.format));
params.dst.clear_color =
bitcast_color_value_to_uint(params.dst.clear_color, dst_fmtl);
isl_format_get_layout(params.dst.view.format)->channels.r.bits;
if (src_fmtl->bw > 1 || src_fmtl->bh > 1) {
- surf_convert_to_uncompressed(batch->blorp->isl_dev, ¶ms.src,
- &src_x, &src_y, &src_width, &src_height);
+ blorp_surf_convert_to_uncompressed(batch->blorp->isl_dev, ¶ms.src,
+ &src_x, &src_y,
+ &src_width, &src_height);
wm_prog_key.need_src_offset = true;
}
if (dst_fmtl->bw > 1 || dst_fmtl->bh > 1) {
- surf_convert_to_uncompressed(batch->blorp->isl_dev, ¶ms.dst,
- &dst_x, &dst_y, NULL, NULL);
+ blorp_surf_convert_to_uncompressed(batch->blorp->isl_dev, ¶ms.dst,
+ &dst_x, &dst_y, NULL, NULL);
wm_prog_key.need_dst_offset = true;
}
do_blorp_blit(batch, ¶ms, &wm_prog_key, &coords);
}
+
+static enum isl_format
+isl_format_for_size(unsigned size_B)
+{
+ switch (size_B) {
+ case 1: return ISL_FORMAT_R8_UINT;
+ case 2: return ISL_FORMAT_R8G8_UINT;
+ case 4: return ISL_FORMAT_R8G8B8A8_UINT;
+ case 8: return ISL_FORMAT_R16G16B16A16_UINT;
+ case 16: return ISL_FORMAT_R32G32B32A32_UINT;
+ default:
+ unreachable("Not a power-of-two format size");
+ }
+}
+
+/**
+ * Returns the greatest common divisor of a and b that is a power of two.
+ */
+static uint64_t
+gcd_pow2_u64(uint64_t a, uint64_t b)
+{
+ assert(a > 0 || b > 0);
+
+ unsigned a_log2 = ffsll(a) - 1;
+ unsigned b_log2 = ffsll(b) - 1;
+
+ /* If either a or b is 0, then a_log2 or b_log2 till be UINT_MAX in which
+ * case, the MIN2() will take the other one. If both are 0 then we will
+ * hit the assert above.
+ */
+ return 1 << MIN2(a_log2, b_log2);
+}
+
+static void
+do_buffer_copy(struct blorp_batch *batch,
+ struct blorp_address *src,
+ struct blorp_address *dst,
+ int width, int height, int block_size)
+{
+ /* The actual format we pick doesn't matter as blorp will throw it away.
+ * The only thing that actually matters is the size.
+ */
+ enum isl_format format = isl_format_for_size(block_size);
+
+ UNUSED bool ok;
+ struct isl_surf surf;
+ ok = isl_surf_init(batch->blorp->isl_dev, &surf,
+ .dim = ISL_SURF_DIM_2D,
+ .format = format,
+ .width = width,
+ .height = height,
+ .depth = 1,
+ .levels = 1,
+ .array_len = 1,
+ .samples = 1,
+ .row_pitch = width * block_size,
+ .usage = ISL_SURF_USAGE_TEXTURE_BIT |
+ ISL_SURF_USAGE_RENDER_TARGET_BIT,
+ .tiling_flags = ISL_TILING_LINEAR_BIT);
+ assert(ok);
+
+ struct blorp_surf src_blorp_surf = {
+ .surf = &surf,
+ .addr = *src,
+ };
+
+ struct blorp_surf dst_blorp_surf = {
+ .surf = &surf,
+ .addr = *dst,
+ };
+
+ blorp_copy(batch, &src_blorp_surf, 0, 0, &dst_blorp_surf, 0, 0,
+ 0, 0, 0, 0, width, height);
+}
+
+void
+blorp_buffer_copy(struct blorp_batch *batch,
+ struct blorp_address src,
+ struct blorp_address dst,
+ uint64_t size)
+{
+ const struct gen_device_info *devinfo = batch->blorp->isl_dev->info;
+ uint64_t copy_size = size;
+
+ /* This is maximum possible width/height our HW can handle */
+ uint64_t max_surface_dim = 1 << (devinfo->gen >= 7 ? 14 : 13);
+
+ /* First, we compute the biggest format that can be used with the
+ * given offsets and size.
+ */
+ int bs = 16;
+ bs = gcd_pow2_u64(bs, src.offset);
+ bs = gcd_pow2_u64(bs, dst.offset);
+ bs = gcd_pow2_u64(bs, size);
+
+ /* First, we make a bunch of max-sized copies */
+ uint64_t max_copy_size = max_surface_dim * max_surface_dim * bs;
+ while (copy_size >= max_copy_size) {
+ do_buffer_copy(batch, &src, &dst, max_surface_dim, max_surface_dim, bs);
+ copy_size -= max_copy_size;
+ src.offset += max_copy_size;
+ dst.offset += max_copy_size;
+ }
+
+ /* Now make a max-width copy */
+ uint64_t height = copy_size / (max_surface_dim * bs);
+ assert(height < max_surface_dim);
+ if (height != 0) {
+ uint64_t rect_copy_size = height * max_surface_dim * bs;
+ do_buffer_copy(batch, &src, &dst, max_surface_dim, height, bs);
+ copy_size -= rect_copy_size;
+ src.offset += rect_copy_size;
+ dst.offset += rect_copy_size;
+ }
+
+ /* Finally, make a small copy to finish it off */
+ if (copy_size != 0) {
+ do_buffer_copy(batch, &src, &dst, copy_size / bs, 1, bs);
+ }
+}
projects / mesa.git / blobdiff