2 * Copyright © 2011 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 * @file brw_vec4_copy_propagation.cpp
27 * Implements tracking of values copied between registers, and
28 * optimizations based on that: copy propagation and constant
35 #include "main/macros.h"
46 is_direct_copy(vec4_instruction
*inst
)
48 return (inst
->opcode
== BRW_OPCODE_MOV
&&
50 inst
->dst
.file
== GRF
&&
52 !inst
->src
[0].reladdr
&&
53 (inst
->dst
.type
== inst
->src
[0].type
||
54 (inst
->dst
.type
== BRW_REGISTER_TYPE_F
&&
55 inst
->src
[0].type
== BRW_REGISTER_TYPE_VF
)));
59 is_dominated_by_previous_instruction(vec4_instruction
*inst
)
61 return (inst
->opcode
!= BRW_OPCODE_DO
&&
62 inst
->opcode
!= BRW_OPCODE_WHILE
&&
63 inst
->opcode
!= BRW_OPCODE_ELSE
&&
64 inst
->opcode
!= BRW_OPCODE_ENDIF
);
68 is_channel_updated(vec4_instruction
*inst
, src_reg
*values
[4], int ch
)
70 const src_reg
*src
= values
[ch
];
72 /* consider GRF only */
73 assert(inst
->dst
.file
== GRF
);
74 if (!src
|| src
->file
!= GRF
)
77 return (src
->reg
== inst
->dst
.reg
&&
78 src
->reg_offset
== inst
->dst
.reg_offset
&&
79 inst
->dst
.writemask
& (1 << BRW_GET_SWZ(src
->swizzle
, ch
)));
83 swizzle_vf_imm(unsigned vf4
, unsigned swizzle
)
90 ret
.vf
[0] = v
.vf
[BRW_GET_SWZ(swizzle
, 0)];
91 ret
.vf
[1] = v
.vf
[BRW_GET_SWZ(swizzle
, 1)];
92 ret
.vf
[2] = v
.vf
[BRW_GET_SWZ(swizzle
, 2)];
93 ret
.vf
[3] = v
.vf
[BRW_GET_SWZ(swizzle
, 3)];
99 try_constant_propagate(struct brw_context
*brw
, vec4_instruction
*inst
,
100 int arg
, struct copy_entry
*entry
)
102 /* For constant propagation, we only handle the same constant
103 * across all 4 channels. Some day, we should handle the 8-bit
104 * float vector format, which would let us constant propagate
107 src_reg value
= *entry
->value
[0];
108 for (int i
= 1; i
< 4; i
++) {
109 if (!value
.equals(*entry
->value
[i
]))
113 if (value
.file
!= IMM
)
116 if (inst
->src
[arg
].abs
) {
117 if (value
.type
== BRW_REGISTER_TYPE_F
) {
118 value
.fixed_hw_reg
.dw1
.f
= fabs(value
.fixed_hw_reg
.dw1
.f
);
119 } else if (value
.type
== BRW_REGISTER_TYPE_VF
) {
120 value
.fixed_hw_reg
.dw1
.ud
&= ~0x80808080;
121 } else if (value
.type
== BRW_REGISTER_TYPE_D
) {
122 if (value
.fixed_hw_reg
.dw1
.d
< 0)
123 value
.fixed_hw_reg
.dw1
.d
= -value
.fixed_hw_reg
.dw1
.d
;
127 if (inst
->src
[arg
].negate
) {
128 if (value
.type
== BRW_REGISTER_TYPE_F
)
129 value
.fixed_hw_reg
.dw1
.f
= -value
.fixed_hw_reg
.dw1
.f
;
130 else if (value
.type
== BRW_REGISTER_TYPE_VF
)
131 value
.fixed_hw_reg
.dw1
.ud
^= 0x80808080;
133 value
.fixed_hw_reg
.dw1
.ud
= -value
.fixed_hw_reg
.dw1
.ud
;
136 if (value
.type
== BRW_REGISTER_TYPE_VF
)
137 value
.fixed_hw_reg
.dw1
.ud
= swizzle_vf_imm(value
.fixed_hw_reg
.dw1
.ud
,
138 inst
->src
[arg
].swizzle
);
140 switch (inst
->opcode
) {
142 inst
->src
[arg
] = value
;
145 case SHADER_OPCODE_POW
:
146 case SHADER_OPCODE_INT_QUOTIENT
:
147 case SHADER_OPCODE_INT_REMAINDER
:
155 case BRW_OPCODE_BFI1
:
159 case BRW_OPCODE_SUBB
:
161 inst
->src
[arg
] = value
;
166 case BRW_OPCODE_MACH
:
172 case BRW_OPCODE_ADDC
:
174 inst
->src
[arg
] = value
;
176 } else if (arg
== 0 && inst
->src
[1].file
!= IMM
) {
177 /* Fit this constant in by commuting the operands. Exception: we
178 * can't do this for 32-bit integer MUL/MACH because it's asymmetric.
180 if ((inst
->opcode
== BRW_OPCODE_MUL
||
181 inst
->opcode
== BRW_OPCODE_MACH
) &&
182 (inst
->src
[1].type
== BRW_REGISTER_TYPE_D
||
183 inst
->src
[1].type
== BRW_REGISTER_TYPE_UD
))
185 inst
->src
[0] = inst
->src
[1];
186 inst
->src
[1] = value
;
193 inst
->src
[arg
] = value
;
195 } else if (arg
== 0 && inst
->src
[1].file
!= IMM
) {
196 enum brw_conditional_mod new_cmod
;
198 new_cmod
= brw_swap_cmod(inst
->conditional_mod
);
199 if (new_cmod
!= BRW_CONDITIONAL_NONE
) {
200 /* Fit this constant in by swapping the operands and
203 inst
->src
[0] = inst
->src
[1];
204 inst
->src
[1] = value
;
205 inst
->conditional_mod
= new_cmod
;
213 inst
->src
[arg
] = value
;
215 } else if (arg
== 0 && inst
->src
[1].file
!= IMM
) {
216 inst
->src
[0] = inst
->src
[1];
217 inst
->src
[1] = value
;
219 /* If this was predicated, flipping operands means
220 * we also need to flip the predicate.
222 if (inst
->conditional_mod
== BRW_CONDITIONAL_NONE
) {
223 inst
->predicate_inverse
= !inst
->predicate_inverse
;
237 is_logic_op(enum opcode opcode
)
239 return (opcode
== BRW_OPCODE_AND
||
240 opcode
== BRW_OPCODE_OR
||
241 opcode
== BRW_OPCODE_XOR
||
242 opcode
== BRW_OPCODE_NOT
);
246 try_copy_propagate(struct brw_context
*brw
, vec4_instruction
*inst
,
247 int arg
, struct copy_entry
*entry
, int reg
)
249 /* For constant propagation, we only handle the same constant
250 * across all 4 channels. Some day, we should handle the 8-bit
251 * float vector format, which would let us constant propagate
254 src_reg value
= *entry
->value
[0];
255 for (int i
= 1; i
< 4; i
++) {
256 /* This is equals() except we don't care about the swizzle. */
257 if (value
.file
!= entry
->value
[i
]->file
||
258 value
.reg
!= entry
->value
[i
]->reg
||
259 value
.reg_offset
!= entry
->value
[i
]->reg_offset
||
260 value
.type
!= entry
->value
[i
]->type
||
261 value
.negate
!= entry
->value
[i
]->negate
||
262 value
.abs
!= entry
->value
[i
]->abs
) {
267 /* Compute the swizzle of the original register by swizzling the
268 * component loaded from each value according to the swizzle of
269 * operand we're going to change.
272 for (int i
= 0; i
< 4; i
++) {
273 s
[i
] = BRW_GET_SWZ(entry
->value
[i
]->swizzle
,
274 BRW_GET_SWZ(inst
->src
[arg
].swizzle
, i
));
276 value
.swizzle
= BRW_SWIZZLE4(s
[0], s
[1], s
[2], s
[3]);
278 if (value
.file
!= UNIFORM
&&
283 if (brw
->gen
>= 8 && (value
.negate
|| value
.abs
) &&
284 is_logic_op(inst
->opcode
)) {
288 if (inst
->src
[arg
].abs
) {
289 value
.negate
= false;
292 if (inst
->src
[arg
].negate
)
293 value
.negate
= !value
.negate
;
295 bool has_source_modifiers
= value
.negate
|| value
.abs
;
297 /* gen6 math and gen7+ SENDs from GRFs ignore source modifiers on
300 if ((has_source_modifiers
|| value
.file
== UNIFORM
||
301 value
.swizzle
!= BRW_SWIZZLE_XYZW
) && !inst
->can_do_source_mods(brw
))
304 if (has_source_modifiers
&& value
.type
!= inst
->src
[arg
].type
)
307 if (has_source_modifiers
&&
308 inst
->opcode
== SHADER_OPCODE_GEN4_SCRATCH_WRITE
)
311 bool is_3src_inst
= (inst
->opcode
== BRW_OPCODE_LRP
||
312 inst
->opcode
== BRW_OPCODE_MAD
||
313 inst
->opcode
== BRW_OPCODE_BFE
||
314 inst
->opcode
== BRW_OPCODE_BFI2
);
315 if (is_3src_inst
&& value
.file
== UNIFORM
)
318 if (inst
->is_send_from_grf())
321 /* We can't copy-propagate a UD negation into a condmod
322 * instruction, because the condmod ends up looking at the 33-bit
323 * signed accumulator value instead of the 32-bit value we wanted
325 if (inst
->conditional_mod
&&
327 value
.type
== BRW_REGISTER_TYPE_UD
)
330 /* Don't report progress if this is a noop. */
331 if (value
.equals(inst
->src
[arg
]))
334 /* Limit saturate propagation only to SEL with src1 bounded within 1.0 and 1.0
335 * otherwise, skip copy propagate altogether
337 if (entry
->saturatemask
& (1 << arg
)) {
338 switch(inst
->opcode
) {
340 if (inst
->src
[1].file
!= IMM
||
341 inst
->src
[1].fixed_hw_reg
.dw1
.f
< 0.0 ||
342 inst
->src
[1].fixed_hw_reg
.dw1
.f
> 1.0) {
346 inst
->saturate
= true;
353 value
.type
= inst
->src
[arg
].type
;
354 inst
->src
[arg
] = value
;
359 vec4_visitor::opt_copy_propagation(bool do_constant_prop
)
361 bool progress
= false;
362 struct copy_entry entries
[virtual_grf_reg_count
];
364 memset(&entries
, 0, sizeof(entries
));
366 foreach_block_and_inst(block
, vec4_instruction
, inst
, cfg
) {
367 /* This pass only works on basic blocks. If there's flow
368 * control, throw out all our information and start from
371 * This should really be fixed by using a structure like in
372 * src/glsl/opt_copy_propagation.cpp to track available copies.
374 if (!is_dominated_by_previous_instruction(inst
)) {
375 memset(&entries
, 0, sizeof(entries
));
379 /* For each source arg, see if each component comes from a copy
380 * from the same type file (IMM, GRF, UNIFORM), and try
381 * optimizing out access to the copy result
383 for (int i
= 2; i
>= 0; i
--) {
384 /* Copied values end up in GRFs, and we don't track reladdr
387 if (inst
->src
[i
].file
!= GRF
||
388 inst
->src
[i
].reladdr
)
391 int reg
= (virtual_grf_reg_map
[inst
->src
[i
].reg
] +
392 inst
->src
[i
].reg_offset
);
394 /* Find the regs that each swizzle component came from.
396 struct copy_entry entry
;
397 memset(&entry
, 0, sizeof(copy_entry
));
399 for (c
= 0; c
< 4; c
++) {
400 int channel
= BRW_GET_SWZ(inst
->src
[i
].swizzle
, c
);
401 entry
.value
[c
] = entries
[reg
].value
[channel
];
403 /* If there's no available copy for this channel, bail.
404 * We could be more aggressive here -- some channels might
405 * not get used based on the destination writemask.
410 entry
.saturatemask
|=
411 (entries
[reg
].saturatemask
& (1 << channel
) ? 1 : 0) << c
;
413 /* We'll only be able to copy propagate if the sources are
414 * all from the same file -- there's no ability to swizzle
415 * 0 or 1 constants in with source registers like in i915.
417 if (c
> 0 && entry
.value
[c
- 1]->file
!= entry
.value
[c
]->file
)
424 if (do_constant_prop
&& try_constant_propagate(brw
, inst
, i
, &entry
))
427 if (try_copy_propagate(brw
, inst
, i
, &entry
, reg
))
431 /* Track available source registers. */
432 if (inst
->dst
.file
== GRF
) {
434 virtual_grf_reg_map
[inst
->dst
.reg
] + inst
->dst
.reg_offset
;
436 /* Update our destination's current channel values. For a direct copy,
437 * the value is the newly propagated source. Otherwise, we don't know
438 * the new value, so clear it.
440 bool direct_copy
= is_direct_copy(inst
);
441 entries
[reg
].saturatemask
= 0x0;
442 for (int i
= 0; i
< 4; i
++) {
443 if (inst
->dst
.writemask
& (1 << i
)) {
444 entries
[reg
].value
[i
] = direct_copy
? &inst
->src
[0] : NULL
;
445 entries
[reg
].saturatemask
|= (((inst
->saturate
&& direct_copy
) ? 1 : 0) << i
);
449 /* Clear the records for any registers whose current value came from
450 * our destination's updated channels, as the two are no longer equal.
452 if (inst
->dst
.reladdr
)
453 memset(&entries
, 0, sizeof(entries
));
455 for (int i
= 0; i
< virtual_grf_reg_count
; i
++) {
456 for (int j
= 0; j
< 4; j
++) {
457 if (is_channel_updated(inst
, entries
[i
].value
, j
)){
458 entries
[i
].value
[j
] = NULL
;
459 entries
[i
].saturatemask
&= ~(1 << j
);
468 invalidate_live_intervals();
473 } /* namespace brw */