2 * Copyright © 2012 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
25 * \file lower_ubo_reference.cpp
27 * IR lower pass to replace dereferences of variables in a uniform
28 * buffer object with usage of ir_binop_ubo_load expressions, each of
29 * which can read data up to the size of a vec4.
31 * This relieves drivers of the responsibility to deal with tricky UBO
32 * layout issues like std140 structures and row_major matrices on
36 #include "lower_buffer_access.h"
37 #include "ir_builder.h"
38 #include "main/macros.h"
39 #include "glsl_parser_extras.h"
41 using namespace ir_builder
;
44 * Determine if a thing being dereferenced is row-major
46 * There is some trickery here.
48 * If the thing being dereferenced is a member of uniform block \b without an
49 * instance name, then the name of the \c ir_variable is the field name of an
50 * interface type. If this field is row-major, then the thing referenced is
53 * If the thing being dereferenced is a member of uniform block \b with an
54 * instance name, then the last dereference in the tree will be an
55 * \c ir_dereference_record. If that record field is row-major, then the
56 * thing referenced is row-major.
59 is_dereferenced_thing_row_major(const ir_rvalue
*deref
)
62 const ir_rvalue
*ir
= deref
;
65 matrix
= matrix
|| ir
->type
->without_array()->is_matrix();
67 switch (ir
->ir_type
) {
68 case ir_type_dereference_array
: {
69 const ir_dereference_array
*const array_deref
=
70 (const ir_dereference_array
*) ir
;
72 ir
= array_deref
->array
;
76 case ir_type_dereference_record
: {
77 const ir_dereference_record
*const record_deref
=
78 (const ir_dereference_record
*) ir
;
80 ir
= record_deref
->record
;
82 const int idx
= ir
->type
->field_index(record_deref
->field
);
85 const enum glsl_matrix_layout matrix_layout
=
86 glsl_matrix_layout(ir
->type
->fields
.structure
[idx
].matrix_layout
);
88 switch (matrix_layout
) {
89 case GLSL_MATRIX_LAYOUT_INHERITED
:
91 case GLSL_MATRIX_LAYOUT_COLUMN_MAJOR
:
93 case GLSL_MATRIX_LAYOUT_ROW_MAJOR
:
94 return matrix
|| deref
->type
->without_array()->is_record();
100 case ir_type_dereference_variable
: {
101 const ir_dereference_variable
*const var_deref
=
102 (const ir_dereference_variable
*) ir
;
104 const enum glsl_matrix_layout matrix_layout
=
105 glsl_matrix_layout(var_deref
->var
->data
.matrix_layout
);
107 switch (matrix_layout
) {
108 case GLSL_MATRIX_LAYOUT_INHERITED
:
111 case GLSL_MATRIX_LAYOUT_COLUMN_MAJOR
:
113 case GLSL_MATRIX_LAYOUT_ROW_MAJOR
:
114 return matrix
|| deref
->type
->without_array()->is_record();
117 unreachable("invalid matrix layout");
126 /* The tree must have ended with a dereference that wasn't an
127 * ir_dereference_variable. That is invalid, and it should be impossible.
129 unreachable("invalid dereference tree");
134 class lower_ubo_reference_visitor
:
135 public lower_buffer_access::lower_buffer_access
{
137 lower_ubo_reference_visitor(struct gl_shader
*shader
)
142 void handle_rvalue(ir_rvalue
**rvalue
);
143 ir_visitor_status
visit_enter(ir_assignment
*ir
);
145 void setup_for_load_or_store(ir_variable
*var
,
148 unsigned *const_offset
,
152 ir_expression
*ubo_load(const struct glsl_type
*type
,
154 ir_call
*ssbo_load(const struct glsl_type
*type
,
157 bool check_for_buffer_array_copy(ir_assignment
*ir
);
158 bool check_for_buffer_struct_copy(ir_assignment
*ir
);
159 void check_for_ssbo_store(ir_assignment
*ir
);
160 void write_to_memory(ir_dereference
*deref
,
162 ir_variable
*write_var
,
163 unsigned write_mask
);
164 ir_call
*ssbo_store(ir_rvalue
*deref
, ir_rvalue
*offset
,
165 unsigned write_mask
);
171 ssbo_unsized_array_length_access
,
173 } buffer_access_type
;
175 void insert_buffer_access(void *mem_ctx
, ir_dereference
*deref
,
176 const glsl_type
*type
, ir_rvalue
*offset
,
177 unsigned mask
, int channel
);
179 ir_visitor_status
visit_enter(class ir_expression
*);
180 ir_expression
*calculate_ssbo_unsized_array_length(ir_expression
*expr
);
181 void check_ssbo_unsized_array_length_expression(class ir_expression
*);
182 void check_ssbo_unsized_array_length_assignment(ir_assignment
*ir
);
184 ir_expression
*process_ssbo_unsized_array_length(ir_rvalue
**,
187 ir_expression
*emit_ssbo_get_buffer_size();
189 unsigned calculate_unsized_array_stride(ir_dereference
*deref
,
192 ir_call
*lower_ssbo_atomic_intrinsic(ir_call
*ir
);
193 ir_call
*check_for_ssbo_atomic_intrinsic(ir_call
*ir
);
194 ir_visitor_status
visit_enter(ir_call
*ir
);
197 struct gl_shader
*shader
;
198 struct gl_uniform_buffer_variable
*ubo_var
;
199 ir_rvalue
*uniform_block
;
204 * Determine the name of the interface block field
206 * This is the name of the specific member as it would appear in the
207 * \c gl_uniform_buffer_variable::Name field in the shader's
208 * \c UniformBlocks array.
211 interface_field_name(void *mem_ctx
, char *base_name
, ir_rvalue
*d
,
212 ir_rvalue
**nonconst_block_index
)
214 *nonconst_block_index
= NULL
;
215 char *name_copy
= NULL
;
216 size_t base_length
= 0;
218 /* Loop back through the IR until we find the uniform block */
221 switch (ir
->ir_type
) {
222 case ir_type_dereference_variable
: {
228 case ir_type_dereference_record
: {
229 ir_dereference_record
*r
= (ir_dereference_record
*) ir
;
230 ir
= r
->record
->as_dereference();
232 /* If we got here it means any previous array subscripts belong to
233 * block members and not the block itself so skip over them in the
240 case ir_type_dereference_array
: {
241 ir_dereference_array
*a
= (ir_dereference_array
*) ir
;
242 ir
= a
->array
->as_dereference();
246 case ir_type_swizzle
: {
247 ir_swizzle
*s
= (ir_swizzle
*) ir
;
248 ir
= s
->val
->as_dereference();
249 /* Skip swizzle in the next pass */
255 assert(!"Should not get here.");
261 switch (d
->ir_type
) {
262 case ir_type_dereference_variable
: {
263 ir_dereference_variable
*v
= (ir_dereference_variable
*) d
;
264 if (name_copy
!= NULL
&&
265 v
->var
->is_interface_instance() &&
266 v
->var
->type
->is_array()) {
269 *nonconst_block_index
= NULL
;
276 case ir_type_dereference_array
: {
277 ir_dereference_array
*a
= (ir_dereference_array
*) d
;
280 if (name_copy
== NULL
) {
281 name_copy
= ralloc_strdup(mem_ctx
, base_name
);
282 base_length
= strlen(name_copy
);
285 /* For arrays of arrays we start at the innermost array and work our
286 * way out so we need to insert the subscript at the base of the
287 * name string rather than just attaching it to the end.
289 new_length
= base_length
;
290 ir_constant
*const_index
= a
->array_index
->as_constant();
291 char *end
= ralloc_strdup(NULL
, &name_copy
[new_length
]);
293 ir_rvalue
*array_index
= a
->array_index
;
294 if (array_index
->type
!= glsl_type::uint_type
)
295 array_index
= i2u(array_index
);
297 if (a
->array
->type
->is_array() &&
298 a
->array
->type
->fields
.array
->is_array()) {
299 ir_constant
*base_size
= new(mem_ctx
)
300 ir_constant(a
->array
->type
->fields
.array
->arrays_of_arrays_size());
301 array_index
= mul(array_index
, base_size
);
304 if (*nonconst_block_index
) {
305 *nonconst_block_index
= add(*nonconst_block_index
, array_index
);
307 *nonconst_block_index
= array_index
;
310 ralloc_asprintf_rewrite_tail(&name_copy
, &new_length
, "[0]%s",
313 ralloc_asprintf_rewrite_tail(&name_copy
, &new_length
, "[%d]%s",
314 const_index
->get_uint_component(0),
319 d
= a
->array
->as_dereference();
325 assert(!"Should not get here.");
330 assert(!"Should not get here.");
335 lower_ubo_reference_visitor::setup_for_load_or_store(ir_variable
*var
,
338 unsigned *const_offset
,
343 /* Determine the name of the interface block */
344 ir_rvalue
*nonconst_block_index
;
345 const char *const field_name
=
346 interface_field_name(mem_ctx
, (char *) var
->get_interface_type()->name
,
347 deref
, &nonconst_block_index
);
349 /* Locate the block by interface name */
351 struct gl_uniform_block
**blocks
;
352 if (this->buffer_access_type
!= ubo_load_access
) {
353 num_blocks
= shader
->NumShaderStorageBlocks
;
354 blocks
= shader
->ShaderStorageBlocks
;
356 num_blocks
= shader
->NumUniformBlocks
;
357 blocks
= shader
->UniformBlocks
;
359 this->uniform_block
= NULL
;
360 for (unsigned i
= 0; i
< num_blocks
; i
++) {
361 if (strcmp(field_name
, blocks
[i
]->Name
) == 0) {
363 ir_constant
*index
= new(mem_ctx
) ir_constant(i
);
365 if (nonconst_block_index
) {
366 this->uniform_block
= add(nonconst_block_index
, index
);
368 this->uniform_block
= index
;
371 this->ubo_var
= var
->is_interface_instance()
372 ? &blocks
[i
]->Uniforms
[0] : &blocks
[i
]->Uniforms
[var
->data
.location
];
378 assert(this->uniform_block
);
380 *offset
= new(mem_ctx
) ir_constant(0u);
382 *row_major
= is_dereferenced_thing_row_major(deref
);
385 /* Calculate the offset to the start of the region of the UBO
386 * dereferenced by *rvalue. This may be a variable offset if an
387 * array dereference has a variable index.
390 switch (deref
->ir_type
) {
391 case ir_type_dereference_variable
: {
392 *const_offset
+= ubo_var
->Offset
;
397 case ir_type_dereference_array
: {
398 ir_dereference_array
*deref_array
= (ir_dereference_array
*) deref
;
399 unsigned array_stride
;
400 if (deref_array
->array
->type
->is_vector()) {
401 /* We get this when storing or loading a component out of a vector
402 * with a non-constant index. This happens for v[i] = f where v is
403 * a vector (or m[i][j] = f where m is a matrix). If we don't
404 * lower that here, it gets turned into v = vector_insert(v, i,
405 * f), which loads the entire vector, modifies one component and
406 * then write the entire thing back. That breaks if another
407 * thread or SIMD channel is modifying the same vector.
410 if (deref_array
->array
->type
->is_double())
412 } else if (deref_array
->array
->type
->is_matrix() && *row_major
) {
413 /* When loading a vector out of a row major matrix, the
414 * step between the columns (vectors) is the size of a
415 * float, while the step between the rows (elements of a
416 * vector) is handled below in emit_ubo_loads.
419 if (deref_array
->array
->type
->is_double())
421 *matrix_columns
= deref_array
->array
->type
->matrix_columns
;
422 } else if (deref_array
->type
->without_array()->is_interface()) {
423 /* We're processing an array dereference of an interface instance
424 * array. The thing being dereferenced *must* be a variable
425 * dereference because interfaces cannot be embedded in other
426 * types. In terms of calculating the offsets for the lowering
427 * pass, we don't care about the array index. All elements of an
428 * interface instance array will have the same offsets relative to
429 * the base of the block that backs them.
431 deref
= deref_array
->array
->as_dereference();
434 /* Whether or not the field is row-major (because it might be a
435 * bvec2 or something) does not affect the array itself. We need
436 * to know whether an array element in its entirety is row-major.
438 const bool array_row_major
=
439 is_dereferenced_thing_row_major(deref_array
);
441 /* The array type will give the correct interface packing
444 if (packing
== GLSL_INTERFACE_PACKING_STD430
) {
445 array_stride
= deref_array
->type
->std430_array_stride(array_row_major
);
447 array_stride
= deref_array
->type
->std140_size(array_row_major
);
448 array_stride
= glsl_align(array_stride
, 16);
452 ir_rvalue
*array_index
= deref_array
->array_index
;
453 if (array_index
->type
->base_type
== GLSL_TYPE_INT
)
454 array_index
= i2u(array_index
);
456 ir_constant
*const_index
=
457 array_index
->constant_expression_value(NULL
);
459 *const_offset
+= array_stride
* const_index
->value
.u
[0];
461 *offset
= add(*offset
,
463 new(mem_ctx
) ir_constant(array_stride
)));
465 deref
= deref_array
->array
->as_dereference();
469 case ir_type_dereference_record
: {
470 ir_dereference_record
*deref_record
= (ir_dereference_record
*) deref
;
471 const glsl_type
*struct_type
= deref_record
->record
->type
;
472 unsigned intra_struct_offset
= 0;
474 for (unsigned int i
= 0; i
< struct_type
->length
; i
++) {
475 const glsl_type
*type
= struct_type
->fields
.structure
[i
].type
;
477 ir_dereference_record
*field_deref
= new(mem_ctx
)
478 ir_dereference_record(deref_record
->record
,
479 struct_type
->fields
.structure
[i
].name
);
480 const bool field_row_major
=
481 is_dereferenced_thing_row_major(field_deref
);
483 ralloc_free(field_deref
);
485 unsigned field_align
= 0;
487 if (packing
== GLSL_INTERFACE_PACKING_STD430
)
488 field_align
= type
->std430_base_alignment(field_row_major
);
490 field_align
= type
->std140_base_alignment(field_row_major
);
492 intra_struct_offset
= glsl_align(intra_struct_offset
, field_align
);
494 if (strcmp(struct_type
->fields
.structure
[i
].name
,
495 deref_record
->field
) == 0)
498 if (packing
== GLSL_INTERFACE_PACKING_STD430
)
499 intra_struct_offset
+= type
->std430_size(field_row_major
);
501 intra_struct_offset
+= type
->std140_size(field_row_major
);
503 /* If the field just examined was itself a structure, apply rule
506 * "The structure may have padding at the end; the base offset
507 * of the member following the sub-structure is rounded up to
508 * the next multiple of the base alignment of the structure."
510 if (type
->without_array()->is_record()) {
511 intra_struct_offset
= glsl_align(intra_struct_offset
,
517 *const_offset
+= intra_struct_offset
;
518 deref
= deref_record
->record
->as_dereference();
522 case ir_type_swizzle
: {
523 ir_swizzle
*deref_swizzle
= (ir_swizzle
*) deref
;
525 assert(deref_swizzle
->mask
.num_components
== 1);
527 *const_offset
+= deref_swizzle
->mask
.x
* sizeof(int);
528 deref
= deref_swizzle
->val
->as_dereference();
533 assert(!"not reached");
541 lower_ubo_reference_visitor::handle_rvalue(ir_rvalue
**rvalue
)
546 ir_dereference
*deref
= (*rvalue
)->as_dereference();
550 ir_variable
*var
= deref
->variable_referenced();
551 if (!var
|| !var
->is_in_buffer_block())
554 mem_ctx
= ralloc_parent(shader
->ir
);
556 ir_rvalue
*offset
= NULL
;
557 unsigned const_offset
;
560 unsigned packing
= var
->get_interface_type()->interface_packing
;
562 this->buffer_access_type
=
563 var
->is_in_shader_storage_block() ?
564 ssbo_load_access
: ubo_load_access
;
566 /* Compute the offset to the start if the dereference as well as other
567 * information we need to configure the write
569 setup_for_load_or_store(var
, deref
,
570 &offset
, &const_offset
,
571 &row_major
, &matrix_columns
,
575 /* Now that we've calculated the offset to the start of the
576 * dereference, walk over the type and emit loads into a temporary.
578 const glsl_type
*type
= (*rvalue
)->type
;
579 ir_variable
*load_var
= new(mem_ctx
) ir_variable(type
,
582 base_ir
->insert_before(load_var
);
584 ir_variable
*load_offset
= new(mem_ctx
) ir_variable(glsl_type::uint_type
,
585 "ubo_load_temp_offset",
587 base_ir
->insert_before(load_offset
);
588 base_ir
->insert_before(assign(load_offset
, offset
));
590 deref
= new(mem_ctx
) ir_dereference_variable(load_var
);
591 emit_access(mem_ctx
, false, deref
, load_offset
, const_offset
,
592 row_major
, matrix_columns
, packing
, 0);
599 lower_ubo_reference_visitor::ubo_load(const glsl_type
*type
,
602 ir_rvalue
*block_ref
= this->uniform_block
->clone(mem_ctx
, NULL
);
604 ir_expression(ir_binop_ubo_load
,
612 shader_storage_buffer_object(const _mesa_glsl_parse_state
*state
)
614 return state
->ARB_shader_storage_buffer_object_enable
;
618 lower_ubo_reference_visitor::ssbo_store(ir_rvalue
*deref
,
622 exec_list sig_params
;
624 ir_variable
*block_ref
= new(mem_ctx
)
625 ir_variable(glsl_type::uint_type
, "block_ref" , ir_var_function_in
);
626 sig_params
.push_tail(block_ref
);
628 ir_variable
*offset_ref
= new(mem_ctx
)
629 ir_variable(glsl_type::uint_type
, "offset" , ir_var_function_in
);
630 sig_params
.push_tail(offset_ref
);
632 ir_variable
*val_ref
= new(mem_ctx
)
633 ir_variable(deref
->type
, "value" , ir_var_function_in
);
634 sig_params
.push_tail(val_ref
);
636 ir_variable
*writemask_ref
= new(mem_ctx
)
637 ir_variable(glsl_type::uint_type
, "write_mask" , ir_var_function_in
);
638 sig_params
.push_tail(writemask_ref
);
640 ir_function_signature
*sig
= new(mem_ctx
)
641 ir_function_signature(glsl_type::void_type
, shader_storage_buffer_object
);
643 sig
->replace_parameters(&sig_params
);
644 sig
->is_intrinsic
= true;
646 ir_function
*f
= new(mem_ctx
) ir_function("__intrinsic_store_ssbo");
647 f
->add_signature(sig
);
649 exec_list call_params
;
650 call_params
.push_tail(this->uniform_block
->clone(mem_ctx
, NULL
));
651 call_params
.push_tail(offset
->clone(mem_ctx
, NULL
));
652 call_params
.push_tail(deref
->clone(mem_ctx
, NULL
));
653 call_params
.push_tail(new(mem_ctx
) ir_constant(write_mask
));
654 return new(mem_ctx
) ir_call(sig
, NULL
, &call_params
);
658 lower_ubo_reference_visitor::ssbo_load(const struct glsl_type
*type
,
661 exec_list sig_params
;
663 ir_variable
*block_ref
= new(mem_ctx
)
664 ir_variable(glsl_type::uint_type
, "block_ref" , ir_var_function_in
);
665 sig_params
.push_tail(block_ref
);
667 ir_variable
*offset_ref
= new(mem_ctx
)
668 ir_variable(glsl_type::uint_type
, "offset_ref" , ir_var_function_in
);
669 sig_params
.push_tail(offset_ref
);
671 ir_function_signature
*sig
=
672 new(mem_ctx
) ir_function_signature(type
, shader_storage_buffer_object
);
674 sig
->replace_parameters(&sig_params
);
675 sig
->is_intrinsic
= true;
677 ir_function
*f
= new(mem_ctx
) ir_function("__intrinsic_load_ssbo");
678 f
->add_signature(sig
);
680 ir_variable
*result
= new(mem_ctx
)
681 ir_variable(type
, "ssbo_load_result", ir_var_temporary
);
682 base_ir
->insert_before(result
);
683 ir_dereference_variable
*deref_result
= new(mem_ctx
)
684 ir_dereference_variable(result
);
686 exec_list call_params
;
687 call_params
.push_tail(this->uniform_block
->clone(mem_ctx
, NULL
));
688 call_params
.push_tail(offset
->clone(mem_ctx
, NULL
));
690 return new(mem_ctx
) ir_call(sig
, deref_result
, &call_params
);
694 lower_ubo_reference_visitor::insert_buffer_access(void *mem_ctx
,
695 ir_dereference
*deref
,
696 const glsl_type
*type
,
701 switch (this->buffer_access_type
) {
702 case ubo_load_access
:
703 base_ir
->insert_before(assign(deref
->clone(mem_ctx
, NULL
),
704 ubo_load(type
, offset
),
707 case ssbo_load_access
: {
708 ir_call
*load_ssbo
= ssbo_load(type
, offset
);
709 base_ir
->insert_before(load_ssbo
);
710 ir_rvalue
*value
= load_ssbo
->return_deref
->as_rvalue()->clone(mem_ctx
, NULL
);
711 ir_assignment
*assignment
=
712 assign(deref
->clone(mem_ctx
, NULL
), value
, mask
);
713 base_ir
->insert_before(assignment
);
716 case ssbo_store_access
:
718 base_ir
->insert_after(ssbo_store(swizzle(deref
, channel
, 1),
721 base_ir
->insert_after(ssbo_store(deref
, offset
, mask
));
725 unreachable("invalid buffer_access_type in insert_buffer_access");
730 lower_ubo_reference_visitor::write_to_memory(ir_dereference
*deref
,
732 ir_variable
*write_var
,
735 ir_rvalue
*offset
= NULL
;
736 unsigned const_offset
;
739 unsigned packing
= var
->get_interface_type()->interface_packing
;
741 this->buffer_access_type
= ssbo_store_access
;
743 /* Compute the offset to the start if the dereference as well as other
744 * information we need to configure the write
746 setup_for_load_or_store(var
, deref
,
747 &offset
, &const_offset
,
748 &row_major
, &matrix_columns
,
752 /* Now emit writes from the temporary to memory */
753 ir_variable
*write_offset
=
754 new(mem_ctx
) ir_variable(glsl_type::uint_type
,
755 "ssbo_store_temp_offset",
758 base_ir
->insert_before(write_offset
);
759 base_ir
->insert_before(assign(write_offset
, offset
));
761 deref
= new(mem_ctx
) ir_dereference_variable(write_var
);
762 emit_access(mem_ctx
, true, deref
, write_offset
, const_offset
,
763 row_major
, matrix_columns
, packing
, write_mask
);
767 lower_ubo_reference_visitor::visit_enter(ir_expression
*ir
)
769 check_ssbo_unsized_array_length_expression(ir
);
770 return rvalue_visit(ir
);
774 lower_ubo_reference_visitor::calculate_ssbo_unsized_array_length(ir_expression
*expr
)
776 if (expr
->operation
!=
777 ir_expression_operation(ir_unop_ssbo_unsized_array_length
))
780 ir_rvalue
*rvalue
= expr
->operands
[0]->as_rvalue();
782 !rvalue
->type
->is_array() || !rvalue
->type
->is_unsized_array())
785 ir_dereference
*deref
= expr
->operands
[0]->as_dereference();
789 ir_variable
*var
= expr
->operands
[0]->variable_referenced();
790 if (!var
|| !var
->is_in_shader_storage_block())
792 return process_ssbo_unsized_array_length(&rvalue
, deref
, var
);
796 lower_ubo_reference_visitor::check_ssbo_unsized_array_length_expression(ir_expression
*ir
)
799 ir_expression_operation(ir_unop_ssbo_unsized_array_length
)) {
800 /* Don't replace this unop if it is found alone. It is going to be
801 * removed by the optimization passes or replaced if it is part of
802 * an ir_assignment or another ir_expression.
807 for (unsigned i
= 0; i
< ir
->get_num_operands(); i
++) {
808 if (ir
->operands
[i
]->ir_type
!= ir_type_expression
)
810 ir_expression
*expr
= (ir_expression
*) ir
->operands
[i
];
811 ir_expression
*temp
= calculate_ssbo_unsized_array_length(expr
);
816 ir
->operands
[i
] = temp
;
821 lower_ubo_reference_visitor::check_ssbo_unsized_array_length_assignment(ir_assignment
*ir
)
823 if (!ir
->rhs
|| ir
->rhs
->ir_type
!= ir_type_expression
)
826 ir_expression
*expr
= (ir_expression
*) ir
->rhs
;
827 ir_expression
*temp
= calculate_ssbo_unsized_array_length(expr
);
837 lower_ubo_reference_visitor::emit_ssbo_get_buffer_size()
839 ir_rvalue
*block_ref
= this->uniform_block
->clone(mem_ctx
, NULL
);
840 return new(mem_ctx
) ir_expression(ir_unop_get_buffer_size
,
846 lower_ubo_reference_visitor::calculate_unsized_array_stride(ir_dereference
*deref
,
849 unsigned array_stride
= 0;
851 switch (deref
->ir_type
) {
852 case ir_type_dereference_variable
:
854 ir_dereference_variable
*deref_var
= (ir_dereference_variable
*)deref
;
855 const struct glsl_type
*unsized_array_type
= NULL
;
856 /* An unsized array can be sized by other lowering passes, so pick
857 * the first field of the array which has the data type of the unsized
860 unsized_array_type
= deref_var
->var
->type
->fields
.array
;
862 /* Whether or not the field is row-major (because it might be a
863 * bvec2 or something) does not affect the array itself. We need
864 * to know whether an array element in its entirety is row-major.
866 const bool array_row_major
=
867 is_dereferenced_thing_row_major(deref_var
);
869 if (packing
== GLSL_INTERFACE_PACKING_STD430
) {
870 array_stride
= unsized_array_type
->std430_array_stride(array_row_major
);
872 array_stride
= unsized_array_type
->std140_size(array_row_major
);
873 array_stride
= glsl_align(array_stride
, 16);
877 case ir_type_dereference_record
:
879 ir_dereference_record
*deref_record
= (ir_dereference_record
*) deref
;
880 ir_dereference
*interface_deref
=
881 deref_record
->record
->as_dereference();
882 assert(interface_deref
!= NULL
);
883 const struct glsl_type
*interface_type
= interface_deref
->type
;
884 unsigned record_length
= interface_type
->length
;
885 /* Unsized array is always the last element of the interface */
886 const struct glsl_type
*unsized_array_type
=
887 interface_type
->fields
.structure
[record_length
- 1].type
->fields
.array
;
889 const bool array_row_major
=
890 is_dereferenced_thing_row_major(deref_record
);
892 if (packing
== GLSL_INTERFACE_PACKING_STD430
) {
893 array_stride
= unsized_array_type
->std430_array_stride(array_row_major
);
895 array_stride
= unsized_array_type
->std140_size(array_row_major
);
896 array_stride
= glsl_align(array_stride
, 16);
901 unreachable("Unsupported dereference type");
907 lower_ubo_reference_visitor::process_ssbo_unsized_array_length(ir_rvalue
**rvalue
,
908 ir_dereference
*deref
,
911 mem_ctx
= ralloc_parent(*rvalue
);
913 ir_rvalue
*base_offset
= NULL
;
914 unsigned const_offset
;
917 unsigned packing
= var
->get_interface_type()->interface_packing
;
918 int unsized_array_stride
= calculate_unsized_array_stride(deref
, packing
);
920 this->buffer_access_type
= ssbo_unsized_array_length_access
;
922 /* Compute the offset to the start if the dereference as well as other
923 * information we need to calculate the length.
925 setup_for_load_or_store(var
, deref
,
926 &base_offset
, &const_offset
,
927 &row_major
, &matrix_columns
,
930 * max((buffer_object_size - offset_of_array) / stride_of_array, 0)
932 ir_expression
*buffer_size
= emit_ssbo_get_buffer_size();
934 ir_expression
*offset_of_array
= new(mem_ctx
)
935 ir_expression(ir_binop_add
, base_offset
,
936 new(mem_ctx
) ir_constant(const_offset
));
937 ir_expression
*offset_of_array_int
= new(mem_ctx
)
938 ir_expression(ir_unop_u2i
, offset_of_array
);
940 ir_expression
*sub
= new(mem_ctx
)
941 ir_expression(ir_binop_sub
, buffer_size
, offset_of_array_int
);
942 ir_expression
*div
= new(mem_ctx
)
943 ir_expression(ir_binop_div
, sub
,
944 new(mem_ctx
) ir_constant(unsized_array_stride
));
945 ir_expression
*max
= new(mem_ctx
)
946 ir_expression(ir_binop_max
, div
, new(mem_ctx
) ir_constant(0));
952 lower_ubo_reference_visitor::check_for_ssbo_store(ir_assignment
*ir
)
957 ir_rvalue
*rvalue
= ir
->lhs
->as_rvalue();
961 ir_dereference
*deref
= ir
->lhs
->as_dereference();
965 ir_variable
*var
= ir
->lhs
->variable_referenced();
966 if (!var
|| !var
->is_in_buffer_block())
969 /* We have a write to a buffer variable, so declare a temporary and rewrite
970 * the assignment so that the temporary is the LHS.
972 mem_ctx
= ralloc_parent(shader
->ir
);
974 const glsl_type
*type
= rvalue
->type
;
975 ir_variable
*write_var
= new(mem_ctx
) ir_variable(type
,
978 base_ir
->insert_before(write_var
);
979 ir
->lhs
= new(mem_ctx
) ir_dereference_variable(write_var
);
981 /* Now we have to write the value assigned to the temporary back to memory */
982 write_to_memory(deref
, var
, write_var
, ir
->write_mask
);
987 is_buffer_backed_variable(ir_variable
*var
)
989 return var
->is_in_buffer_block() ||
990 var
->data
.mode
== ir_var_shader_shared
;
994 lower_ubo_reference_visitor::check_for_buffer_array_copy(ir_assignment
*ir
)
996 if (!ir
|| !ir
->lhs
|| !ir
->rhs
)
999 /* LHS and RHS must be arrays
1000 * FIXME: arrays of arrays?
1002 if (!ir
->lhs
->type
->is_array() || !ir
->rhs
->type
->is_array())
1005 /* RHS must be a buffer-backed variable. This is what can cause the problem
1006 * since it would lead to a series of loads that need to live until we
1007 * see the writes to the LHS.
1009 ir_variable
*rhs_var
= ir
->rhs
->variable_referenced();
1010 if (!rhs_var
|| !is_buffer_backed_variable(rhs_var
))
1013 /* Split the array copy into individual element copies to reduce
1016 ir_dereference
*rhs_deref
= ir
->rhs
->as_dereference();
1020 ir_dereference
*lhs_deref
= ir
->lhs
->as_dereference();
1024 assert(lhs_deref
->type
->length
== rhs_deref
->type
->length
);
1025 mem_ctx
= ralloc_parent(shader
->ir
);
1027 for (unsigned i
= 0; i
< lhs_deref
->type
->length
; i
++) {
1028 ir_dereference
*lhs_i
=
1029 new(mem_ctx
) ir_dereference_array(lhs_deref
->clone(mem_ctx
, NULL
),
1030 new(mem_ctx
) ir_constant(i
));
1032 ir_dereference
*rhs_i
=
1033 new(mem_ctx
) ir_dereference_array(rhs_deref
->clone(mem_ctx
, NULL
),
1034 new(mem_ctx
) ir_constant(i
));
1035 ir
->insert_after(assign(lhs_i
, rhs_i
));
1044 lower_ubo_reference_visitor::check_for_buffer_struct_copy(ir_assignment
*ir
)
1046 if (!ir
|| !ir
->lhs
|| !ir
->rhs
)
1049 /* LHS and RHS must be records */
1050 if (!ir
->lhs
->type
->is_record() || !ir
->rhs
->type
->is_record())
1053 /* RHS must be a buffer-backed variable. This is what can cause the problem
1054 * since it would lead to a series of loads that need to live until we
1055 * see the writes to the LHS.
1057 ir_variable
*rhs_var
= ir
->rhs
->variable_referenced();
1058 if (!rhs_var
|| !is_buffer_backed_variable(rhs_var
))
1061 /* Split the struct copy into individual element copies to reduce
1064 ir_dereference
*rhs_deref
= ir
->rhs
->as_dereference();
1068 ir_dereference
*lhs_deref
= ir
->lhs
->as_dereference();
1072 assert(lhs_deref
->type
->record_compare(rhs_deref
->type
));
1073 mem_ctx
= ralloc_parent(shader
->ir
);
1075 for (unsigned i
= 0; i
< lhs_deref
->type
->length
; i
++) {
1076 const char *field_name
= lhs_deref
->type
->fields
.structure
[i
].name
;
1077 ir_dereference
*lhs_field
=
1078 new(mem_ctx
) ir_dereference_record(lhs_deref
->clone(mem_ctx
, NULL
),
1080 ir_dereference
*rhs_field
=
1081 new(mem_ctx
) ir_dereference_record(rhs_deref
->clone(mem_ctx
, NULL
),
1083 ir
->insert_after(assign(lhs_field
, rhs_field
));
1092 lower_ubo_reference_visitor::visit_enter(ir_assignment
*ir
)
1094 /* Array and struct copies could involve large amounts of load/store
1095 * operations. To improve register pressure we want to special-case
1096 * these and split them into individual element copies.
1097 * This way we avoid emitting all the loads for the RHS first and
1098 * all the writes for the LHS second and register usage is more
1101 if (check_for_buffer_array_copy(ir
))
1102 return visit_continue_with_parent
;
1104 if (check_for_buffer_struct_copy(ir
))
1105 return visit_continue_with_parent
;
1107 check_ssbo_unsized_array_length_assignment(ir
);
1108 check_for_ssbo_store(ir
);
1109 return rvalue_visit(ir
);
1112 /* Lowers the intrinsic call to a new internal intrinsic that swaps the
1113 * access to the buffer variable in the first parameter by an offset
1114 * and block index. This involves creating the new internal intrinsic
1115 * (i.e. the new function signature).
1118 lower_ubo_reference_visitor::lower_ssbo_atomic_intrinsic(ir_call
*ir
)
1120 /* SSBO atomics usually have 2 parameters, the buffer variable and an
1121 * integer argument. The exception is CompSwap, that has an additional
1122 * integer parameter.
1124 int param_count
= ir
->actual_parameters
.length();
1125 assert(param_count
== 2 || param_count
== 3);
1127 /* First argument must be a scalar integer buffer variable */
1128 exec_node
*param
= ir
->actual_parameters
.get_head();
1129 ir_instruction
*inst
= (ir_instruction
*) param
;
1130 assert(inst
->ir_type
== ir_type_dereference_variable
||
1131 inst
->ir_type
== ir_type_dereference_array
||
1132 inst
->ir_type
== ir_type_dereference_record
||
1133 inst
->ir_type
== ir_type_swizzle
);
1135 ir_rvalue
*deref
= (ir_rvalue
*) inst
;
1136 assert(deref
->type
->is_scalar() && deref
->type
->is_integer());
1138 ir_variable
*var
= deref
->variable_referenced();
1141 /* Compute the offset to the start if the dereference and the
1144 mem_ctx
= ralloc_parent(shader
->ir
);
1146 ir_rvalue
*offset
= NULL
;
1147 unsigned const_offset
;
1150 unsigned packing
= var
->get_interface_type()->interface_packing
;
1152 this->buffer_access_type
= ssbo_atomic_access
;
1154 setup_for_load_or_store(var
, deref
,
1155 &offset
, &const_offset
,
1156 &row_major
, &matrix_columns
,
1160 assert(matrix_columns
== 1);
1162 ir_rvalue
*deref_offset
=
1163 add(offset
, new(mem_ctx
) ir_constant(const_offset
));
1164 ir_rvalue
*block_index
= this->uniform_block
->clone(mem_ctx
, NULL
);
1166 /* Create the new internal function signature that will take a block
1167 * index and offset instead of a buffer variable
1169 exec_list sig_params
;
1170 ir_variable
*sig_param
= new(mem_ctx
)
1171 ir_variable(glsl_type::uint_type
, "block_ref" , ir_var_function_in
);
1172 sig_params
.push_tail(sig_param
);
1174 sig_param
= new(mem_ctx
)
1175 ir_variable(glsl_type::uint_type
, "offset" , ir_var_function_in
);
1176 sig_params
.push_tail(sig_param
);
1178 const glsl_type
*type
= deref
->type
->base_type
== GLSL_TYPE_INT
?
1179 glsl_type::int_type
: glsl_type::uint_type
;
1180 sig_param
= new(mem_ctx
)
1181 ir_variable(type
, "data1", ir_var_function_in
);
1182 sig_params
.push_tail(sig_param
);
1184 if (param_count
== 3) {
1185 sig_param
= new(mem_ctx
)
1186 ir_variable(type
, "data2", ir_var_function_in
);
1187 sig_params
.push_tail(sig_param
);
1190 ir_function_signature
*sig
=
1191 new(mem_ctx
) ir_function_signature(deref
->type
,
1192 shader_storage_buffer_object
);
1194 sig
->replace_parameters(&sig_params
);
1195 sig
->is_intrinsic
= true;
1198 sprintf(func_name
, "%s_internal", ir
->callee_name());
1199 ir_function
*f
= new(mem_ctx
) ir_function(func_name
);
1200 f
->add_signature(sig
);
1202 /* Now, create the call to the internal intrinsic */
1203 exec_list call_params
;
1204 call_params
.push_tail(block_index
);
1205 call_params
.push_tail(deref_offset
);
1206 param
= ir
->actual_parameters
.get_head()->get_next();
1207 ir_rvalue
*param_as_rvalue
= ((ir_instruction
*) param
)->as_rvalue();
1208 call_params
.push_tail(param_as_rvalue
->clone(mem_ctx
, NULL
));
1209 if (param_count
== 3) {
1210 param
= param
->get_next();
1211 param_as_rvalue
= ((ir_instruction
*) param
)->as_rvalue();
1212 call_params
.push_tail(param_as_rvalue
->clone(mem_ctx
, NULL
));
1214 ir_dereference_variable
*return_deref
=
1215 ir
->return_deref
->clone(mem_ctx
, NULL
);
1216 return new(mem_ctx
) ir_call(sig
, return_deref
, &call_params
);
1220 lower_ubo_reference_visitor::check_for_ssbo_atomic_intrinsic(ir_call
*ir
)
1222 const char *callee
= ir
->callee_name();
1223 if (!strcmp("__intrinsic_ssbo_atomic_add", callee
) ||
1224 !strcmp("__intrinsic_ssbo_atomic_min", callee
) ||
1225 !strcmp("__intrinsic_ssbo_atomic_max", callee
) ||
1226 !strcmp("__intrinsic_ssbo_atomic_and", callee
) ||
1227 !strcmp("__intrinsic_ssbo_atomic_or", callee
) ||
1228 !strcmp("__intrinsic_ssbo_atomic_xor", callee
) ||
1229 !strcmp("__intrinsic_ssbo_atomic_exchange", callee
) ||
1230 !strcmp("__intrinsic_ssbo_atomic_comp_swap", callee
)) {
1231 return lower_ssbo_atomic_intrinsic(ir
);
1239 lower_ubo_reference_visitor::visit_enter(ir_call
*ir
)
1241 ir_call
*new_ir
= check_for_ssbo_atomic_intrinsic(ir
);
1244 base_ir
->replace_with(new_ir
);
1245 return visit_continue_with_parent
;
1248 return rvalue_visit(ir
);
1252 } /* unnamed namespace */
1255 lower_ubo_reference(struct gl_shader
*shader
)
1257 lower_ubo_reference_visitor
v(shader
);
1259 /* Loop over the instructions lowering references, because we take
1260 * a deref of a UBO array using a UBO dereference as the index will
1261 * produce a collection of instructions all of which have cloned
1262 * UBO dereferences for that array index.
1266 visit_list_elements(&v
, shader
->ir
);
1267 } while (v
.progress
);