2 * Copyright © 2015-2019 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 /** @file brw_eu_validate.c
26 * This file implements a pass that validates shader assembly.
28 * The restrictions implemented herein are intended to verify that instructions
29 * in shader assembly do not violate restrictions documented in the graphics
30 * programming reference manuals.
32 * The restrictions are difficult for humans to quickly verify due to their
33 * complexity and abundance.
35 * It is critical that this code is thoroughly unit tested because false
36 * results will lead developers astray, which is worse than having no validator
37 * at all. Functional changes to this file without corresponding unit tests (in
38 * test_eu_validate.cpp) will be rejected.
43 /* We're going to do lots of string concatenation, so this should help. */
50 cat(struct string
*dest
, const struct string src
)
52 dest
->str
= realloc(dest
->str
, dest
->len
+ src
.len
+ 1);
53 memcpy(dest
->str
+ dest
->len
, src
.str
, src
.len
);
54 dest
->str
[dest
->len
+ src
.len
] = '\0';
55 dest
->len
= dest
->len
+ src
.len
;
57 #define CAT(dest, src) cat(&dest, (struct string){src, strlen(src)})
60 contains(const struct string haystack
, const struct string needle
)
62 return haystack
.str
&& memmem(haystack
.str
, haystack
.len
,
63 needle
.str
, needle
.len
) != NULL
;
65 #define CONTAINS(haystack, needle) \
66 contains(haystack, (struct string){needle, strlen(needle)})
68 #define error(str) "\tERROR: " str "\n"
69 #define ERROR_INDENT "\t "
71 #define ERROR(msg) ERROR_IF(true, msg)
72 #define ERROR_IF(cond, msg) \
74 if ((cond) && !CONTAINS(error_msg, error(msg))) { \
75 CAT(error_msg, error(msg)); \
79 #define CHECK(func, args...) \
81 struct string __msg = func(devinfo, inst, ##args); \
83 cat(&error_msg, __msg); \
88 #define STRIDE(stride) (stride != 0 ? 1 << ((stride) - 1) : 0)
89 #define WIDTH(width) (1 << (width))
92 inst_is_send(const struct gen_device_info
*devinfo
, const brw_inst
*inst
)
94 switch (brw_inst_opcode(devinfo
, inst
)) {
96 case BRW_OPCODE_SENDC
:
97 case BRW_OPCODE_SENDS
:
98 case BRW_OPCODE_SENDSC
:
106 inst_is_split_send(const struct gen_device_info
*devinfo
, const brw_inst
*inst
)
108 switch (brw_inst_opcode(devinfo
, inst
)) {
109 case BRW_OPCODE_SENDS
:
110 case BRW_OPCODE_SENDSC
:
118 signed_type(unsigned type
)
121 case BRW_REGISTER_TYPE_UD
: return BRW_REGISTER_TYPE_D
;
122 case BRW_REGISTER_TYPE_UW
: return BRW_REGISTER_TYPE_W
;
123 case BRW_REGISTER_TYPE_UB
: return BRW_REGISTER_TYPE_B
;
124 case BRW_REGISTER_TYPE_UQ
: return BRW_REGISTER_TYPE_Q
;
125 default: return type
;
130 inst_is_raw_move(const struct gen_device_info
*devinfo
, const brw_inst
*inst
)
132 unsigned dst_type
= signed_type(brw_inst_dst_type(devinfo
, inst
));
133 unsigned src_type
= signed_type(brw_inst_src0_type(devinfo
, inst
));
135 if (brw_inst_src0_reg_file(devinfo
, inst
) == BRW_IMMEDIATE_VALUE
) {
136 /* FIXME: not strictly true */
137 if (brw_inst_src0_type(devinfo
, inst
) == BRW_REGISTER_TYPE_VF
||
138 brw_inst_src0_type(devinfo
, inst
) == BRW_REGISTER_TYPE_UV
||
139 brw_inst_src0_type(devinfo
, inst
) == BRW_REGISTER_TYPE_V
) {
142 } else if (brw_inst_src0_negate(devinfo
, inst
) ||
143 brw_inst_src0_abs(devinfo
, inst
)) {
147 return brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_MOV
&&
148 brw_inst_saturate(devinfo
, inst
) == 0 &&
149 dst_type
== src_type
;
153 dst_is_null(const struct gen_device_info
*devinfo
, const brw_inst
*inst
)
155 return brw_inst_dst_reg_file(devinfo
, inst
) == BRW_ARCHITECTURE_REGISTER_FILE
&&
156 brw_inst_dst_da_reg_nr(devinfo
, inst
) == BRW_ARF_NULL
;
160 src0_is_null(const struct gen_device_info
*devinfo
, const brw_inst
*inst
)
162 return brw_inst_src0_reg_file(devinfo
, inst
) == BRW_ARCHITECTURE_REGISTER_FILE
&&
163 brw_inst_src0_da_reg_nr(devinfo
, inst
) == BRW_ARF_NULL
;
167 src1_is_null(const struct gen_device_info
*devinfo
, const brw_inst
*inst
)
169 return brw_inst_src1_reg_file(devinfo
, inst
) == BRW_ARCHITECTURE_REGISTER_FILE
&&
170 brw_inst_src1_da_reg_nr(devinfo
, inst
) == BRW_ARF_NULL
;
174 src0_is_acc(const struct gen_device_info
*devinfo
, const brw_inst
*inst
)
176 return brw_inst_src0_reg_file(devinfo
, inst
) == BRW_ARCHITECTURE_REGISTER_FILE
&&
177 (brw_inst_src0_da_reg_nr(devinfo
, inst
) & 0xF0) == BRW_ARF_ACCUMULATOR
;
181 src1_is_acc(const struct gen_device_info
*devinfo
, const brw_inst
*inst
)
183 return brw_inst_src1_reg_file(devinfo
, inst
) == BRW_ARCHITECTURE_REGISTER_FILE
&&
184 (brw_inst_src1_da_reg_nr(devinfo
, inst
) & 0xF0) == BRW_ARF_ACCUMULATOR
;
188 src0_is_grf(const struct gen_device_info
*devinfo
, const brw_inst
*inst
)
190 return brw_inst_src0_reg_file(devinfo
, inst
) == BRW_GENERAL_REGISTER_FILE
;
194 src0_has_scalar_region(const struct gen_device_info
*devinfo
, const brw_inst
*inst
)
196 return brw_inst_src0_vstride(devinfo
, inst
) == BRW_VERTICAL_STRIDE_0
&&
197 brw_inst_src0_width(devinfo
, inst
) == BRW_WIDTH_1
&&
198 brw_inst_src0_hstride(devinfo
, inst
) == BRW_HORIZONTAL_STRIDE_0
;
202 src1_has_scalar_region(const struct gen_device_info
*devinfo
, const brw_inst
*inst
)
204 return brw_inst_src1_vstride(devinfo
, inst
) == BRW_VERTICAL_STRIDE_0
&&
205 brw_inst_src1_width(devinfo
, inst
) == BRW_WIDTH_1
&&
206 brw_inst_src1_hstride(devinfo
, inst
) == BRW_HORIZONTAL_STRIDE_0
;
210 num_sources_from_inst(const struct gen_device_info
*devinfo
,
211 const brw_inst
*inst
)
213 const struct opcode_desc
*desc
=
214 brw_opcode_desc(devinfo
, brw_inst_opcode(devinfo
, inst
));
215 unsigned math_function
;
217 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_MATH
) {
218 math_function
= brw_inst_math_function(devinfo
, inst
);
219 } else if (devinfo
->gen
< 6 &&
220 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
) {
221 if (brw_inst_sfid(devinfo
, inst
) == BRW_SFID_MATH
) {
222 /* src1 must be a descriptor (including the information to determine
223 * that the SEND is doing an extended math operation), but src0 can
224 * actually be null since it serves as the source of the implicit GRF
227 * If we stop using that functionality, we'll have to revisit this.
231 /* Send instructions are allowed to have null sources since they use
232 * the base_mrf field to specify which message register source.
237 assert(desc
->nsrc
< 4);
241 switch (math_function
) {
242 case BRW_MATH_FUNCTION_INV
:
243 case BRW_MATH_FUNCTION_LOG
:
244 case BRW_MATH_FUNCTION_EXP
:
245 case BRW_MATH_FUNCTION_SQRT
:
246 case BRW_MATH_FUNCTION_RSQ
:
247 case BRW_MATH_FUNCTION_SIN
:
248 case BRW_MATH_FUNCTION_COS
:
249 case BRW_MATH_FUNCTION_SINCOS
:
250 case GEN8_MATH_FUNCTION_INVM
:
251 case GEN8_MATH_FUNCTION_RSQRTM
:
253 case BRW_MATH_FUNCTION_FDIV
:
254 case BRW_MATH_FUNCTION_POW
:
255 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
256 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
:
257 case BRW_MATH_FUNCTION_INT_DIV_REMAINDER
:
260 unreachable("not reached");
265 sources_not_null(const struct gen_device_info
*devinfo
,
266 const brw_inst
*inst
)
268 unsigned num_sources
= num_sources_from_inst(devinfo
, inst
);
269 struct string error_msg
= { .str
= NULL
, .len
= 0 };
271 /* Nothing to test. 3-src instructions can only have GRF sources, and
272 * there's no bit to control the file.
274 if (num_sources
== 3)
275 return (struct string
){};
277 /* Nothing to test. Split sends can only encode a file in sources that are
278 * allowed to be NULL.
280 if (inst_is_split_send(devinfo
, inst
))
281 return (struct string
){};
283 if (num_sources
>= 1)
284 ERROR_IF(src0_is_null(devinfo
, inst
), "src0 is null");
286 if (num_sources
== 2)
287 ERROR_IF(src1_is_null(devinfo
, inst
), "src1 is null");
293 inst_uses_src_acc(const struct gen_device_info
*devinfo
, const brw_inst
*inst
)
295 /* Check instructions that use implicit accumulator sources */
296 switch (brw_inst_opcode(devinfo
, inst
)) {
298 case BRW_OPCODE_MACH
:
299 case BRW_OPCODE_SADA2
:
303 /* FIXME: support 3-src instructions */
304 unsigned num_sources
= num_sources_from_inst(devinfo
, inst
);
305 assert(num_sources
< 3);
307 return src0_is_acc(devinfo
, inst
) || (num_sources
> 1 && src1_is_acc(devinfo
, inst
));
311 send_restrictions(const struct gen_device_info
*devinfo
,
312 const brw_inst
*inst
)
314 struct string error_msg
= { .str
= NULL
, .len
= 0 };
316 if (inst_is_split_send(devinfo
, inst
)) {
317 ERROR_IF(brw_inst_send_src1_reg_file(devinfo
, inst
) == BRW_ARCHITECTURE_REGISTER_FILE
&&
318 brw_inst_send_src1_reg_nr(devinfo
, inst
) != BRW_ARF_NULL
,
319 "src1 of split send must be a GRF or NULL");
321 ERROR_IF(brw_inst_eot(devinfo
, inst
) &&
322 brw_inst_src0_da_reg_nr(devinfo
, inst
) < 112,
323 "send with EOT must use g112-g127");
324 ERROR_IF(brw_inst_eot(devinfo
, inst
) &&
325 brw_inst_send_src1_reg_file(devinfo
, inst
) == BRW_GENERAL_REGISTER_FILE
&&
326 brw_inst_send_src1_reg_nr(devinfo
, inst
) < 112,
327 "send with EOT must use g112-g127");
329 if (brw_inst_send_src1_reg_file(devinfo
, inst
) == BRW_GENERAL_REGISTER_FILE
) {
330 /* Assume minimums if we don't know */
332 if (!brw_inst_send_sel_reg32_desc(devinfo
, inst
)) {
333 const uint32_t desc
= brw_inst_send_desc(devinfo
, inst
);
334 mlen
= brw_message_desc_mlen(devinfo
, desc
);
337 unsigned ex_mlen
= 1;
338 if (!brw_inst_send_sel_reg32_ex_desc(devinfo
, inst
)) {
339 const uint32_t ex_desc
= brw_inst_send_ex_desc(devinfo
, inst
);
340 ex_mlen
= brw_message_ex_desc_ex_mlen(devinfo
, ex_desc
);
342 const unsigned src0_reg_nr
= brw_inst_src0_da_reg_nr(devinfo
, inst
);
343 const unsigned src1_reg_nr
= brw_inst_send_src1_reg_nr(devinfo
, inst
);
344 ERROR_IF((src0_reg_nr
<= src1_reg_nr
&&
345 src1_reg_nr
< src0_reg_nr
+ mlen
) ||
346 (src1_reg_nr
<= src0_reg_nr
&&
347 src0_reg_nr
< src1_reg_nr
+ ex_mlen
),
348 "split send payloads must not overlap");
350 } else if (inst_is_send(devinfo
, inst
)) {
351 ERROR_IF(brw_inst_src0_address_mode(devinfo
, inst
) != BRW_ADDRESS_DIRECT
,
352 "send must use direct addressing");
354 if (devinfo
->gen
>= 7) {
355 ERROR_IF(!src0_is_grf(devinfo
, inst
), "send from non-GRF");
356 ERROR_IF(brw_inst_eot(devinfo
, inst
) &&
357 brw_inst_src0_da_reg_nr(devinfo
, inst
) < 112,
358 "send with EOT must use g112-g127");
361 if (devinfo
->gen
>= 8) {
362 ERROR_IF(!dst_is_null(devinfo
, inst
) &&
363 (brw_inst_dst_da_reg_nr(devinfo
, inst
) +
364 brw_inst_rlen(devinfo
, inst
) > 127) &&
365 (brw_inst_src0_da_reg_nr(devinfo
, inst
) +
366 brw_inst_mlen(devinfo
, inst
) >
367 brw_inst_dst_da_reg_nr(devinfo
, inst
)),
368 "r127 must not be used for return address when there is "
369 "a src and dest overlap");
377 is_unsupported_inst(const struct gen_device_info
*devinfo
,
378 const brw_inst
*inst
)
380 return brw_opcode_desc(devinfo
, brw_inst_opcode(devinfo
, inst
)) == NULL
;
384 * Returns whether a combination of two types would qualify as mixed float
388 types_are_mixed_float(enum brw_reg_type t0
, enum brw_reg_type t1
)
390 return (t0
== BRW_REGISTER_TYPE_F
&& t1
== BRW_REGISTER_TYPE_HF
) ||
391 (t1
== BRW_REGISTER_TYPE_F
&& t0
== BRW_REGISTER_TYPE_HF
);
394 static enum brw_reg_type
395 execution_type_for_type(enum brw_reg_type type
)
398 case BRW_REGISTER_TYPE_NF
:
399 case BRW_REGISTER_TYPE_DF
:
400 case BRW_REGISTER_TYPE_F
:
401 case BRW_REGISTER_TYPE_HF
:
404 case BRW_REGISTER_TYPE_VF
:
405 return BRW_REGISTER_TYPE_F
;
407 case BRW_REGISTER_TYPE_Q
:
408 case BRW_REGISTER_TYPE_UQ
:
409 return BRW_REGISTER_TYPE_Q
;
411 case BRW_REGISTER_TYPE_D
:
412 case BRW_REGISTER_TYPE_UD
:
413 return BRW_REGISTER_TYPE_D
;
415 case BRW_REGISTER_TYPE_W
:
416 case BRW_REGISTER_TYPE_UW
:
417 case BRW_REGISTER_TYPE_B
:
418 case BRW_REGISTER_TYPE_UB
:
419 case BRW_REGISTER_TYPE_V
:
420 case BRW_REGISTER_TYPE_UV
:
421 return BRW_REGISTER_TYPE_W
;
423 unreachable("not reached");
427 * Returns the execution type of an instruction \p inst
429 static enum brw_reg_type
430 execution_type(const struct gen_device_info
*devinfo
, const brw_inst
*inst
)
432 unsigned num_sources
= num_sources_from_inst(devinfo
, inst
);
433 enum brw_reg_type src0_exec_type
, src1_exec_type
;
435 /* Execution data type is independent of destination data type, except in
436 * mixed F/HF instructions.
438 enum brw_reg_type dst_exec_type
= brw_inst_dst_type(devinfo
, inst
);
440 src0_exec_type
= execution_type_for_type(brw_inst_src0_type(devinfo
, inst
));
441 if (num_sources
== 1) {
442 if (src0_exec_type
== BRW_REGISTER_TYPE_HF
)
443 return dst_exec_type
;
444 return src0_exec_type
;
447 src1_exec_type
= execution_type_for_type(brw_inst_src1_type(devinfo
, inst
));
448 if (types_are_mixed_float(src0_exec_type
, src1_exec_type
) ||
449 types_are_mixed_float(src0_exec_type
, dst_exec_type
) ||
450 types_are_mixed_float(src1_exec_type
, dst_exec_type
)) {
451 return BRW_REGISTER_TYPE_F
;
454 if (src0_exec_type
== src1_exec_type
)
455 return src0_exec_type
;
457 /* Mixed operand types where one is float is float on Gen < 6
458 * (and not allowed on later platforms)
460 if (devinfo
->gen
< 6 &&
461 (src0_exec_type
== BRW_REGISTER_TYPE_F
||
462 src1_exec_type
== BRW_REGISTER_TYPE_F
))
463 return BRW_REGISTER_TYPE_F
;
465 if (src0_exec_type
== BRW_REGISTER_TYPE_Q
||
466 src1_exec_type
== BRW_REGISTER_TYPE_Q
)
467 return BRW_REGISTER_TYPE_Q
;
469 if (src0_exec_type
== BRW_REGISTER_TYPE_D
||
470 src1_exec_type
== BRW_REGISTER_TYPE_D
)
471 return BRW_REGISTER_TYPE_D
;
473 if (src0_exec_type
== BRW_REGISTER_TYPE_W
||
474 src1_exec_type
== BRW_REGISTER_TYPE_W
)
475 return BRW_REGISTER_TYPE_W
;
477 if (src0_exec_type
== BRW_REGISTER_TYPE_DF
||
478 src1_exec_type
== BRW_REGISTER_TYPE_DF
)
479 return BRW_REGISTER_TYPE_DF
;
481 unreachable("not reached");
485 * Returns whether a region is packed
487 * A region is packed if its elements are adjacent in memory, with no
488 * intervening space, no overlap, and no replicated values.
491 is_packed(unsigned vstride
, unsigned width
, unsigned hstride
)
493 if (vstride
== width
) {
505 * Returns whether an instruction is an explicit or implicit conversion
506 * to/from half-float.
509 is_half_float_conversion(const struct gen_device_info
*devinfo
,
510 const brw_inst
*inst
)
512 enum brw_reg_type dst_type
= brw_inst_dst_type(devinfo
, inst
);
514 unsigned num_sources
= num_sources_from_inst(devinfo
, inst
);
515 enum brw_reg_type src0_type
= brw_inst_src0_type(devinfo
, inst
);
517 if (dst_type
!= src0_type
&&
518 (dst_type
== BRW_REGISTER_TYPE_HF
|| src0_type
== BRW_REGISTER_TYPE_HF
)) {
520 } else if (num_sources
> 1) {
521 enum brw_reg_type src1_type
= brw_inst_src1_type(devinfo
, inst
);
522 return dst_type
!= src1_type
&&
523 (dst_type
== BRW_REGISTER_TYPE_HF
||
524 src1_type
== BRW_REGISTER_TYPE_HF
);
531 * Returns whether an instruction is using mixed float operation mode
534 is_mixed_float(const struct gen_device_info
*devinfo
, const brw_inst
*inst
)
536 if (devinfo
->gen
< 8)
539 if (inst_is_send(devinfo
, inst
))
542 unsigned opcode
= brw_inst_opcode(devinfo
, inst
);
543 const struct opcode_desc
*desc
= brw_opcode_desc(devinfo
, opcode
);
547 /* FIXME: support 3-src instructions */
548 unsigned num_sources
= num_sources_from_inst(devinfo
, inst
);
549 assert(num_sources
< 3);
551 enum brw_reg_type dst_type
= brw_inst_dst_type(devinfo
, inst
);
552 enum brw_reg_type src0_type
= brw_inst_src0_type(devinfo
, inst
);
554 if (num_sources
== 1)
555 return types_are_mixed_float(src0_type
, dst_type
);
557 enum brw_reg_type src1_type
= brw_inst_src1_type(devinfo
, inst
);
559 return types_are_mixed_float(src0_type
, src1_type
) ||
560 types_are_mixed_float(src0_type
, dst_type
) ||
561 types_are_mixed_float(src1_type
, dst_type
);
565 * Returns whether an instruction is an explicit or implicit conversion
569 is_byte_conversion(const struct gen_device_info
*devinfo
,
570 const brw_inst
*inst
)
572 enum brw_reg_type dst_type
= brw_inst_dst_type(devinfo
, inst
);
574 unsigned num_sources
= num_sources_from_inst(devinfo
, inst
);
575 enum brw_reg_type src0_type
= brw_inst_src0_type(devinfo
, inst
);
577 if (dst_type
!= src0_type
&&
578 (type_sz(dst_type
) == 1 || type_sz(src0_type
) == 1)) {
580 } else if (num_sources
> 1) {
581 enum brw_reg_type src1_type
= brw_inst_src1_type(devinfo
, inst
);
582 return dst_type
!= src1_type
&&
583 (type_sz(dst_type
) == 1 || type_sz(src1_type
) == 1);
590 * Checks restrictions listed in "General Restrictions Based on Operand Types"
591 * in the "Register Region Restrictions" section.
594 general_restrictions_based_on_operand_types(const struct gen_device_info
*devinfo
,
595 const brw_inst
*inst
)
597 const struct opcode_desc
*desc
=
598 brw_opcode_desc(devinfo
, brw_inst_opcode(devinfo
, inst
));
599 unsigned num_sources
= num_sources_from_inst(devinfo
, inst
);
600 unsigned exec_size
= 1 << brw_inst_exec_size(devinfo
, inst
);
601 struct string error_msg
= { .str
= NULL
, .len
= 0 };
603 if (num_sources
== 3)
604 return (struct string
){};
606 if (inst_is_send(devinfo
, inst
))
607 return (struct string
){};
610 return (struct string
){};
613 return (struct string
){};
617 * Where n is the largest element size in bytes for any source or
618 * destination operand type, ExecSize * n must be <= 64.
620 * But we do not attempt to enforce it, because it is implied by other
623 * - that the destination stride must match the execution data type
624 * - sources may not span more than two adjacent GRF registers
625 * - destination may not span more than two adjacent GRF registers
627 * In fact, checking it would weaken testing of the other rules.
630 unsigned dst_stride
= STRIDE(brw_inst_dst_hstride(devinfo
, inst
));
631 enum brw_reg_type dst_type
= brw_inst_dst_type(devinfo
, inst
);
632 bool dst_type_is_byte
=
633 brw_inst_dst_type(devinfo
, inst
) == BRW_REGISTER_TYPE_B
||
634 brw_inst_dst_type(devinfo
, inst
) == BRW_REGISTER_TYPE_UB
;
636 if (dst_type_is_byte
) {
637 if (is_packed(exec_size
* dst_stride
, exec_size
, dst_stride
)) {
638 if (!inst_is_raw_move(devinfo
, inst
)) {
639 ERROR("Only raw MOV supports a packed-byte destination");
642 return (struct string
){};
647 unsigned exec_type
= execution_type(devinfo
, inst
);
648 unsigned exec_type_size
= brw_reg_type_to_size(exec_type
);
649 unsigned dst_type_size
= brw_reg_type_to_size(dst_type
);
651 /* On IVB/BYT, region parameters and execution size for DF are in terms of
652 * 32-bit elements, so they are doubled. For evaluating the validity of an
653 * instruction, we halve them.
655 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
656 exec_type_size
== 8 && dst_type_size
== 4)
659 if (is_byte_conversion(devinfo
, inst
)) {
660 /* From the BDW+ PRM, Volume 2a, Command Reference, Instructions - MOV:
662 * "There is no direct conversion from B/UB to DF or DF to B/UB.
663 * There is no direct conversion from B/UB to Q/UQ or Q/UQ to B/UB."
665 * Even if these restrictions are listed for the MOV instruction, we
666 * validate this more generally, since there is the possibility
667 * of implicit conversions from other instructions.
669 enum brw_reg_type src0_type
= brw_inst_src0_type(devinfo
, inst
);
670 enum brw_reg_type src1_type
= num_sources
> 1 ?
671 brw_inst_src1_type(devinfo
, inst
) : 0;
673 ERROR_IF(type_sz(dst_type
) == 1 &&
674 (type_sz(src0_type
) == 8 ||
675 (num_sources
> 1 && type_sz(src1_type
) == 8)),
676 "There are no direct conversions between 64-bit types and B/UB");
678 ERROR_IF(type_sz(dst_type
) == 8 &&
679 (type_sz(src0_type
) == 1 ||
680 (num_sources
> 1 && type_sz(src1_type
) == 1)),
681 "There are no direct conversions between 64-bit types and B/UB");
684 if (is_half_float_conversion(devinfo
, inst
)) {
686 * A helper to validate used in the validation of the following restriction
687 * from the BDW+ PRM, Volume 2a, Command Reference, Instructions - MOV:
689 * "There is no direct conversion from HF to DF or DF to HF.
690 * There is no direct conversion from HF to Q/UQ or Q/UQ to HF."
692 * Even if these restrictions are listed for the MOV instruction, we
693 * validate this more generally, since there is the possibility
694 * of implicit conversions from other instructions, such us implicit
695 * conversion from integer to HF with the ADD instruction in SKL+.
697 enum brw_reg_type src0_type
= brw_inst_src0_type(devinfo
, inst
);
698 enum brw_reg_type src1_type
= num_sources
> 1 ?
699 brw_inst_src1_type(devinfo
, inst
) : 0;
700 ERROR_IF(dst_type
== BRW_REGISTER_TYPE_HF
&&
701 (type_sz(src0_type
) == 8 ||
702 (num_sources
> 1 && type_sz(src1_type
) == 8)),
703 "There are no direct conversions between 64-bit types and HF");
705 ERROR_IF(type_sz(dst_type
) == 8 &&
706 (src0_type
== BRW_REGISTER_TYPE_HF
||
707 (num_sources
> 1 && src1_type
== BRW_REGISTER_TYPE_HF
)),
708 "There are no direct conversions between 64-bit types and HF");
710 /* From the BDW+ PRM:
712 * "Conversion between Integer and HF (Half Float) must be
713 * DWord-aligned and strided by a DWord on the destination."
715 * Also, the above restrictions seems to be expanded on CHV and SKL+ by:
717 * "There is a relaxed alignment rule for word destinations. When
718 * the destination type is word (UW, W, HF), destination data types
719 * can be aligned to either the lowest word or the second lowest
720 * word of the execution channel. This means the destination data
721 * words can be either all in the even word locations or all in the
722 * odd word locations."
724 * We do not implement the second rule as is though, since empirical
725 * testing shows inconsistencies:
726 * - It suggests that packed 16-bit is not allowed, which is not true.
727 * - It suggests that conversions from Q/DF to W (which need to be
728 * 64-bit aligned on the destination) are not possible, which is
731 * So from this rule we only validate the implication that conversions
732 * from F to HF need to be DWord strided (except in Align1 mixed
733 * float mode where packed fp16 destination is allowed so long as the
734 * destination is oword-aligned).
736 * Finally, we only validate this for Align1 because Align16 always
737 * requires packed destinations, so these restrictions can't possibly
738 * apply to Align16 mode.
740 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
741 if ((dst_type
== BRW_REGISTER_TYPE_HF
&&
742 (brw_reg_type_is_integer(src0_type
) ||
743 (num_sources
> 1 && brw_reg_type_is_integer(src1_type
)))) ||
744 (brw_reg_type_is_integer(dst_type
) &&
745 (src0_type
== BRW_REGISTER_TYPE_HF
||
746 (num_sources
> 1 && src1_type
== BRW_REGISTER_TYPE_HF
)))) {
747 ERROR_IF(dst_stride
* dst_type_size
!= 4,
748 "Conversions between integer and half-float must be "
749 "strided by a DWord on the destination");
751 unsigned subreg
= brw_inst_dst_da1_subreg_nr(devinfo
, inst
);
752 ERROR_IF(subreg
% 4 != 0,
753 "Conversions between integer and half-float must be "
754 "aligned to a DWord on the destination");
755 } else if ((devinfo
->is_cherryview
|| devinfo
->gen
>= 9) &&
756 dst_type
== BRW_REGISTER_TYPE_HF
) {
757 unsigned subreg
= brw_inst_dst_da1_subreg_nr(devinfo
, inst
);
758 ERROR_IF(dst_stride
!= 2 &&
759 !(is_mixed_float(devinfo
, inst
) &&
760 dst_stride
== 1 && subreg
% 16 == 0),
761 "Conversions to HF must have either all words in even "
762 "word locations or all words in odd word locations or "
763 "be mixed-float with Oword-aligned packed destination");
768 /* There are special regioning rules for mixed-float mode in CHV and SKL that
769 * override the general rule for the ratio of sizes of the destination type
770 * and the execution type. We will add validation for those in a later patch.
772 bool validate_dst_size_and_exec_size_ratio
=
773 !is_mixed_float(devinfo
, inst
) ||
774 !(devinfo
->is_cherryview
|| devinfo
->gen
>= 9);
776 if (validate_dst_size_and_exec_size_ratio
&&
777 exec_type_size
> dst_type_size
) {
778 if (!(dst_type_is_byte
&& inst_is_raw_move(devinfo
, inst
))) {
779 ERROR_IF(dst_stride
* dst_type_size
!= exec_type_size
,
780 "Destination stride must be equal to the ratio of the sizes "
781 "of the execution data type to the destination type");
784 unsigned subreg
= brw_inst_dst_da1_subreg_nr(devinfo
, inst
);
786 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
&&
787 brw_inst_dst_address_mode(devinfo
, inst
) == BRW_ADDRESS_DIRECT
) {
788 /* The i965 PRM says:
790 * Implementation Restriction: The relaxed alignment rule for byte
791 * destination (#10.5) is not supported.
793 if ((devinfo
->gen
> 4 || devinfo
->is_g4x
) && dst_type_is_byte
) {
794 ERROR_IF(subreg
% exec_type_size
!= 0 &&
795 subreg
% exec_type_size
!= 1,
796 "Destination subreg must be aligned to the size of the "
797 "execution data type (or to the next lowest byte for byte "
800 ERROR_IF(subreg
% exec_type_size
!= 0,
801 "Destination subreg must be aligned to the size of the "
802 "execution data type");
811 * Checks restrictions listed in "General Restrictions on Regioning Parameters"
812 * in the "Register Region Restrictions" section.
815 general_restrictions_on_region_parameters(const struct gen_device_info
*devinfo
,
816 const brw_inst
*inst
)
818 const struct opcode_desc
*desc
=
819 brw_opcode_desc(devinfo
, brw_inst_opcode(devinfo
, inst
));
820 unsigned num_sources
= num_sources_from_inst(devinfo
, inst
);
821 unsigned exec_size
= 1 << brw_inst_exec_size(devinfo
, inst
);
822 struct string error_msg
= { .str
= NULL
, .len
= 0 };
824 if (num_sources
== 3)
825 return (struct string
){};
827 /* Split sends don't have the bits in the instruction to encode regions so
828 * there's nothing to check.
830 if (inst_is_split_send(devinfo
, inst
))
831 return (struct string
){};
833 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_16
) {
834 if (desc
->ndst
!= 0 && !dst_is_null(devinfo
, inst
))
835 ERROR_IF(brw_inst_dst_hstride(devinfo
, inst
) != BRW_HORIZONTAL_STRIDE_1
,
836 "Destination Horizontal Stride must be 1");
838 if (num_sources
>= 1) {
839 if (devinfo
->is_haswell
|| devinfo
->gen
>= 8) {
840 ERROR_IF(brw_inst_src0_reg_file(devinfo
, inst
) != BRW_IMMEDIATE_VALUE
&&
841 brw_inst_src0_vstride(devinfo
, inst
) != BRW_VERTICAL_STRIDE_0
&&
842 brw_inst_src0_vstride(devinfo
, inst
) != BRW_VERTICAL_STRIDE_2
&&
843 brw_inst_src0_vstride(devinfo
, inst
) != BRW_VERTICAL_STRIDE_4
,
844 "In Align16 mode, only VertStride of 0, 2, or 4 is allowed");
846 ERROR_IF(brw_inst_src0_reg_file(devinfo
, inst
) != BRW_IMMEDIATE_VALUE
&&
847 brw_inst_src0_vstride(devinfo
, inst
) != BRW_VERTICAL_STRIDE_0
&&
848 brw_inst_src0_vstride(devinfo
, inst
) != BRW_VERTICAL_STRIDE_4
,
849 "In Align16 mode, only VertStride of 0 or 4 is allowed");
853 if (num_sources
== 2) {
854 if (devinfo
->is_haswell
|| devinfo
->gen
>= 8) {
855 ERROR_IF(brw_inst_src1_reg_file(devinfo
, inst
) != BRW_IMMEDIATE_VALUE
&&
856 brw_inst_src1_vstride(devinfo
, inst
) != BRW_VERTICAL_STRIDE_0
&&
857 brw_inst_src1_vstride(devinfo
, inst
) != BRW_VERTICAL_STRIDE_2
&&
858 brw_inst_src1_vstride(devinfo
, inst
) != BRW_VERTICAL_STRIDE_4
,
859 "In Align16 mode, only VertStride of 0, 2, or 4 is allowed");
861 ERROR_IF(brw_inst_src1_reg_file(devinfo
, inst
) != BRW_IMMEDIATE_VALUE
&&
862 brw_inst_src1_vstride(devinfo
, inst
) != BRW_VERTICAL_STRIDE_0
&&
863 brw_inst_src1_vstride(devinfo
, inst
) != BRW_VERTICAL_STRIDE_4
,
864 "In Align16 mode, only VertStride of 0 or 4 is allowed");
871 for (unsigned i
= 0; i
< num_sources
; i
++) {
872 unsigned vstride
, width
, hstride
, element_size
, subreg
;
873 enum brw_reg_type type
;
876 if (brw_inst_src ## n ## _reg_file(devinfo, inst) == \
877 BRW_IMMEDIATE_VALUE) \
880 vstride = STRIDE(brw_inst_src ## n ## _vstride(devinfo, inst)); \
881 width = WIDTH(brw_inst_src ## n ## _width(devinfo, inst)); \
882 hstride = STRIDE(brw_inst_src ## n ## _hstride(devinfo, inst)); \
883 type = brw_inst_src ## n ## _type(devinfo, inst); \
884 element_size = brw_reg_type_to_size(type); \
885 subreg = brw_inst_src ## n ## _da1_subreg_nr(devinfo, inst)
894 /* On IVB/BYT, region parameters and execution size for DF are in terms of
895 * 32-bit elements, so they are doubled. For evaluating the validity of an
896 * instruction, we halve them.
898 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
902 /* ExecSize must be greater than or equal to Width. */
903 ERROR_IF(exec_size
< width
, "ExecSize must be greater than or equal "
906 /* If ExecSize = Width and HorzStride ≠ 0,
907 * VertStride must be set to Width * HorzStride.
909 if (exec_size
== width
&& hstride
!= 0) {
910 ERROR_IF(vstride
!= width
* hstride
,
911 "If ExecSize = Width and HorzStride ≠ 0, "
912 "VertStride must be set to Width * HorzStride");
915 /* If Width = 1, HorzStride must be 0 regardless of the values of
916 * ExecSize and VertStride.
919 ERROR_IF(hstride
!= 0,
920 "If Width = 1, HorzStride must be 0 regardless "
921 "of the values of ExecSize and VertStride");
924 /* If ExecSize = Width = 1, both VertStride and HorzStride must be 0. */
925 if (exec_size
== 1 && width
== 1) {
926 ERROR_IF(vstride
!= 0 || hstride
!= 0,
927 "If ExecSize = Width = 1, both VertStride "
928 "and HorzStride must be 0");
931 /* If VertStride = HorzStride = 0, Width must be 1 regardless of the
934 if (vstride
== 0 && hstride
== 0) {
936 "If VertStride = HorzStride = 0, Width must be "
937 "1 regardless of the value of ExecSize");
940 /* VertStride must be used to cross GRF register boundaries. This rule
941 * implies that elements within a 'Width' cannot cross GRF boundaries.
943 const uint64_t mask
= (1ULL << element_size
) - 1;
944 unsigned rowbase
= subreg
;
946 for (int y
= 0; y
< exec_size
/ width
; y
++) {
947 uint64_t access_mask
= 0;
948 unsigned offset
= rowbase
;
950 for (int x
= 0; x
< width
; x
++) {
951 access_mask
|= mask
<< offset
;
952 offset
+= hstride
* element_size
;
955 rowbase
+= vstride
* element_size
;
957 if ((uint32_t)access_mask
!= 0 && (access_mask
>> 32) != 0) {
958 ERROR("VertStride must be used to cross GRF register boundaries");
964 /* Dst.HorzStride must not be 0. */
965 if (desc
->ndst
!= 0 && !dst_is_null(devinfo
, inst
)) {
966 ERROR_IF(brw_inst_dst_hstride(devinfo
, inst
) == BRW_HORIZONTAL_STRIDE_0
,
967 "Destination Horizontal Stride must not be 0");
974 special_restrictions_for_mixed_float_mode(const struct gen_device_info
*devinfo
,
975 const brw_inst
*inst
)
977 struct string error_msg
= { .str
= NULL
, .len
= 0 };
979 const unsigned opcode
= brw_inst_opcode(devinfo
, inst
);
980 const unsigned num_sources
= num_sources_from_inst(devinfo
, inst
);
981 if (num_sources
>= 3)
984 if (!is_mixed_float(devinfo
, inst
))
987 unsigned exec_size
= 1 << brw_inst_exec_size(devinfo
, inst
);
988 bool is_align16
= brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_16
;
990 enum brw_reg_type src0_type
= brw_inst_src0_type(devinfo
, inst
);
991 enum brw_reg_type src1_type
= num_sources
> 1 ?
992 brw_inst_src1_type(devinfo
, inst
) : 0;
993 enum brw_reg_type dst_type
= brw_inst_dst_type(devinfo
, inst
);
995 unsigned dst_stride
= STRIDE(brw_inst_dst_hstride(devinfo
, inst
));
996 bool dst_is_packed
= is_packed(exec_size
* dst_stride
, exec_size
, dst_stride
);
998 /* From the SKL PRM, Special Restrictions for Handling Mixed Mode
1001 * "Indirect addressing on source is not supported when source and
1002 * destination data types are mixed float."
1004 ERROR_IF(brw_inst_src0_address_mode(devinfo
, inst
) != BRW_ADDRESS_DIRECT
||
1006 brw_inst_src1_address_mode(devinfo
, inst
) != BRW_ADDRESS_DIRECT
),
1007 "Indirect addressing on source is not supported when source and "
1008 "destination data types are mixed float");
1010 /* From the SKL PRM, Special Restrictions for Handling Mixed Mode
1013 * "No SIMD16 in mixed mode when destination is f32. Instruction
1014 * execution size must be no more than 8."
1016 ERROR_IF(exec_size
> 8 && dst_type
== BRW_REGISTER_TYPE_F
,
1017 "Mixed float mode with 32-bit float destination is limited "
1021 /* From the SKL PRM, Special Restrictions for Handling Mixed Mode
1024 * "In Align16 mode, when half float and float data types are mixed
1025 * between source operands OR between source and destination operands,
1026 * the register content are assumed to be packed."
1028 * Since Align16 doesn't have a concept of horizontal stride (or width),
1029 * it means that vertical stride must always be 4, since 0 and 2 would
1030 * lead to replicated data, and any other value is disallowed in Align16.
1032 ERROR_IF(brw_inst_src0_vstride(devinfo
, inst
) != BRW_VERTICAL_STRIDE_4
,
1033 "Align16 mixed float mode assumes packed data (vstride must be 4");
1035 ERROR_IF(num_sources
>= 2 &&
1036 brw_inst_src1_vstride(devinfo
, inst
) != BRW_VERTICAL_STRIDE_4
,
1037 "Align16 mixed float mode assumes packed data (vstride must be 4");
1039 /* From the SKL PRM, Special Restrictions for Handling Mixed Mode
1042 * "For Align16 mixed mode, both input and output packed f16 data
1043 * must be oword aligned, no oword crossing in packed f16."
1045 * The previous rule requires that Align16 operands are always packed,
1046 * and since there is only one bit for Align16 subnr, which represents
1047 * offsets 0B and 16B, this rule is always enforced and we don't need to
1051 /* From the SKL PRM, Special Restrictions for Handling Mixed Mode
1054 * "No SIMD16 in mixed mode when destination is packed f16 for both
1055 * Align1 and Align16."
1059 * "In Align16 mode, when half float and float data types are mixed
1060 * between source operands OR between source and destination operands,
1061 * the register content are assumed to be packed."
1063 * Which implies that SIMD16 is not available in Align16. This is further
1066 * "For Align16 mixed mode, both input and output packed f16 data
1067 * must be oword aligned, no oword crossing in packed f16"
1069 * Since oword-aligned packed f16 data would cross oword boundaries when
1070 * the execution size is larger than 8.
1072 ERROR_IF(exec_size
> 8, "Align16 mixed float mode is limited to SIMD8");
1074 /* From the SKL PRM, Special Restrictions for Handling Mixed Mode
1077 * "No accumulator read access for Align16 mixed float."
1079 ERROR_IF(inst_uses_src_acc(devinfo
, inst
),
1080 "No accumulator read access for Align16 mixed float");
1082 assert(!is_align16
);
1084 /* From the SKL PRM, Special Restrictions for Handling Mixed Mode
1087 * "No SIMD16 in mixed mode when destination is packed f16 for both
1088 * Align1 and Align16."
1090 ERROR_IF(exec_size
> 8 && dst_is_packed
&&
1091 dst_type
== BRW_REGISTER_TYPE_HF
,
1092 "Align1 mixed float mode is limited to SIMD8 when destination "
1093 "is packed half-float");
1095 /* From the SKL PRM, Special Restrictions for Handling Mixed Mode
1098 * "Math operations for mixed mode:
1099 * - In Align1, f16 inputs need to be strided"
1101 if (opcode
== BRW_OPCODE_MATH
) {
1102 if (src0_type
== BRW_REGISTER_TYPE_HF
) {
1103 ERROR_IF(STRIDE(brw_inst_src0_hstride(devinfo
, inst
)) <= 1,
1104 "Align1 mixed mode math needs strided half-float inputs");
1107 if (num_sources
>= 2 && src1_type
== BRW_REGISTER_TYPE_HF
) {
1108 ERROR_IF(STRIDE(brw_inst_src1_hstride(devinfo
, inst
)) <= 1,
1109 "Align1 mixed mode math needs strided half-float inputs");
1113 if (dst_type
== BRW_REGISTER_TYPE_HF
&& dst_stride
== 1) {
1114 /* From the SKL PRM, Special Restrictions for Handling Mixed Mode
1117 * "In Align1, destination stride can be smaller than execution
1118 * type. When destination is stride of 1, 16 bit packed data is
1119 * updated on the destination. However, output packed f16 data
1120 * must be oword aligned, no oword crossing in packed f16."
1122 * The requirement of not crossing oword boundaries for 16-bit oword
1123 * aligned data means that execution size is limited to 8.
1126 if (brw_inst_dst_address_mode(devinfo
, inst
) == BRW_ADDRESS_DIRECT
)
1127 subreg
= brw_inst_dst_da1_subreg_nr(devinfo
, inst
);
1129 subreg
= brw_inst_dst_ia_subreg_nr(devinfo
, inst
);
1130 ERROR_IF(subreg
% 16 != 0,
1131 "Align1 mixed mode packed half-float output must be "
1133 ERROR_IF(exec_size
> 8,
1134 "Align1 mixed mode packed half-float output must not "
1135 "cross oword boundaries (max exec size is 8)");
1137 /* From the SKL PRM, Special Restrictions for Handling Mixed Mode
1140 * "When source is float or half float from accumulator register and
1141 * destination is half float with a stride of 1, the source must
1142 * register aligned. i.e., source must have offset zero."
1144 * Align16 mixed float mode doesn't allow accumulator access on sources,
1145 * so we only need to check this for Align1.
1147 if (src0_is_acc(devinfo
, inst
) &&
1148 (src0_type
== BRW_REGISTER_TYPE_F
||
1149 src0_type
== BRW_REGISTER_TYPE_HF
)) {
1150 ERROR_IF(brw_inst_src0_da1_subreg_nr(devinfo
, inst
) != 0,
1151 "Mixed float mode requires register-aligned accumulator "
1152 "source reads when destination is packed half-float");
1156 if (num_sources
> 1 &&
1157 src1_is_acc(devinfo
, inst
) &&
1158 (src1_type
== BRW_REGISTER_TYPE_F
||
1159 src1_type
== BRW_REGISTER_TYPE_HF
)) {
1160 ERROR_IF(brw_inst_src1_da1_subreg_nr(devinfo
, inst
) != 0,
1161 "Mixed float mode requires register-aligned accumulator "
1162 "source reads when destination is packed half-float");
1166 /* From the SKL PRM, Special Restrictions for Handling Mixed Mode
1169 * "No swizzle is allowed when an accumulator is used as an implicit
1170 * source or an explicit source in an instruction. i.e. when
1171 * destination is half float with an implicit accumulator source,
1172 * destination stride needs to be 2."
1174 * FIXME: it is not quite clear what the first sentence actually means
1175 * or its link to the implication described after it, so we only
1176 * validate the explicit implication, which is clearly described.
1178 if (dst_type
== BRW_REGISTER_TYPE_HF
&&
1179 inst_uses_src_acc(devinfo
, inst
)) {
1180 ERROR_IF(dst_stride
!= 2,
1181 "Mixed float mode with implicit/explicit accumulator "
1182 "source and half-float destination requires a stride "
1183 "of 2 on the destination");
1191 * Creates an \p access_mask for an \p exec_size, \p element_size, and a region
1193 * An \p access_mask is a 32-element array of uint64_t, where each uint64_t is
1194 * a bitmask of bytes accessed by the region.
1196 * For instance the access mask of the source gX.1<4,2,2>F in an exec_size = 4
1197 * instruction would be
1199 * access_mask[0] = 0x00000000000000F0
1200 * access_mask[1] = 0x000000000000F000
1201 * access_mask[2] = 0x0000000000F00000
1202 * access_mask[3] = 0x00000000F0000000
1203 * access_mask[4-31] = 0
1205 * because the first execution channel accesses bytes 7-4 and the second
1206 * execution channel accesses bytes 15-12, etc.
1209 align1_access_mask(uint64_t access_mask
[static 32],
1210 unsigned exec_size
, unsigned element_size
, unsigned subreg
,
1211 unsigned vstride
, unsigned width
, unsigned hstride
)
1213 const uint64_t mask
= (1ULL << element_size
) - 1;
1214 unsigned rowbase
= subreg
;
1215 unsigned element
= 0;
1217 for (int y
= 0; y
< exec_size
/ width
; y
++) {
1218 unsigned offset
= rowbase
;
1220 for (int x
= 0; x
< width
; x
++) {
1221 access_mask
[element
++] = mask
<< offset
;
1222 offset
+= hstride
* element_size
;
1225 rowbase
+= vstride
* element_size
;
1228 assert(element
== 0 || element
== exec_size
);
1232 * Returns the number of registers accessed according to the \p access_mask
1235 registers_read(const uint64_t access_mask
[static 32])
1239 for (unsigned i
= 0; i
< 32; i
++) {
1240 if (access_mask
[i
] > 0xFFFFFFFF) {
1242 } else if (access_mask
[i
]) {
1251 * Checks restrictions listed in "Region Alignment Rules" in the "Register
1252 * Region Restrictions" section.
1254 static struct string
1255 region_alignment_rules(const struct gen_device_info
*devinfo
,
1256 const brw_inst
*inst
)
1258 const struct opcode_desc
*desc
=
1259 brw_opcode_desc(devinfo
, brw_inst_opcode(devinfo
, inst
));
1260 unsigned num_sources
= num_sources_from_inst(devinfo
, inst
);
1261 unsigned exec_size
= 1 << brw_inst_exec_size(devinfo
, inst
);
1262 uint64_t dst_access_mask
[32], src0_access_mask
[32], src1_access_mask
[32];
1263 struct string error_msg
= { .str
= NULL
, .len
= 0 };
1265 if (num_sources
== 3)
1266 return (struct string
){};
1268 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_16
)
1269 return (struct string
){};
1271 if (inst_is_send(devinfo
, inst
))
1272 return (struct string
){};
1274 memset(dst_access_mask
, 0, sizeof(dst_access_mask
));
1275 memset(src0_access_mask
, 0, sizeof(src0_access_mask
));
1276 memset(src1_access_mask
, 0, sizeof(src1_access_mask
));
1278 for (unsigned i
= 0; i
< num_sources
; i
++) {
1279 unsigned vstride
, width
, hstride
, element_size
, subreg
;
1280 enum brw_reg_type type
;
1282 /* In Direct Addressing mode, a source cannot span more than 2 adjacent
1287 if (brw_inst_src ## n ## _address_mode(devinfo, inst) != \
1288 BRW_ADDRESS_DIRECT) \
1291 if (brw_inst_src ## n ## _reg_file(devinfo, inst) == \
1292 BRW_IMMEDIATE_VALUE) \
1295 vstride = STRIDE(brw_inst_src ## n ## _vstride(devinfo, inst)); \
1296 width = WIDTH(brw_inst_src ## n ## _width(devinfo, inst)); \
1297 hstride = STRIDE(brw_inst_src ## n ## _hstride(devinfo, inst)); \
1298 type = brw_inst_src ## n ## _type(devinfo, inst); \
1299 element_size = brw_reg_type_to_size(type); \
1300 subreg = brw_inst_src ## n ## _da1_subreg_nr(devinfo, inst); \
1301 align1_access_mask(src ## n ## _access_mask, \
1302 exec_size, element_size, subreg, \
1303 vstride, width, hstride)
1312 unsigned num_vstride
= exec_size
/ width
;
1313 unsigned num_hstride
= width
;
1314 unsigned vstride_elements
= (num_vstride
- 1) * vstride
;
1315 unsigned hstride_elements
= (num_hstride
- 1) * hstride
;
1316 unsigned offset
= (vstride_elements
+ hstride_elements
) * element_size
+
1318 ERROR_IF(offset
>= 64,
1319 "A source cannot span more than 2 adjacent GRF registers");
1322 if (desc
->ndst
== 0 || dst_is_null(devinfo
, inst
))
1325 unsigned stride
= STRIDE(brw_inst_dst_hstride(devinfo
, inst
));
1326 enum brw_reg_type dst_type
= brw_inst_dst_type(devinfo
, inst
);
1327 unsigned element_size
= brw_reg_type_to_size(dst_type
);
1328 unsigned subreg
= brw_inst_dst_da1_subreg_nr(devinfo
, inst
);
1329 unsigned offset
= ((exec_size
- 1) * stride
* element_size
) + subreg
;
1330 ERROR_IF(offset
>= 64,
1331 "A destination cannot span more than 2 adjacent GRF registers");
1336 /* On IVB/BYT, region parameters and execution size for DF are in terms of
1337 * 32-bit elements, so they are doubled. For evaluating the validity of an
1338 * instruction, we halve them.
1340 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
1344 align1_access_mask(dst_access_mask
, exec_size
, element_size
, subreg
,
1345 exec_size
== 1 ? 0 : exec_size
* stride
,
1346 exec_size
== 1 ? 1 : exec_size
,
1347 exec_size
== 1 ? 0 : stride
);
1349 unsigned dst_regs
= registers_read(dst_access_mask
);
1350 unsigned src0_regs
= registers_read(src0_access_mask
);
1351 unsigned src1_regs
= registers_read(src1_access_mask
);
1353 /* The SNB, IVB, HSW, BDW, and CHV PRMs say:
1355 * When an instruction has a source region spanning two registers and a
1356 * destination region contained in one register, the number of elements
1357 * must be the same between two sources and one of the following must be
1360 * 1. The destination region is entirely contained in the lower OWord
1362 * 2. The destination region is entirely contained in the upper OWord
1364 * 3. The destination elements are evenly split between the two OWords
1367 if (devinfo
->gen
<= 8) {
1368 if (dst_regs
== 1 && (src0_regs
== 2 || src1_regs
== 2)) {
1369 unsigned upper_oword_writes
= 0, lower_oword_writes
= 0;
1371 for (unsigned i
= 0; i
< exec_size
; i
++) {
1372 if (dst_access_mask
[i
] > 0x0000FFFF) {
1373 upper_oword_writes
++;
1375 assert(dst_access_mask
[i
] != 0);
1376 lower_oword_writes
++;
1380 ERROR_IF(lower_oword_writes
!= 0 &&
1381 upper_oword_writes
!= 0 &&
1382 upper_oword_writes
!= lower_oword_writes
,
1383 "Writes must be to only one OWord or "
1384 "evenly split between OWords");
1388 /* The IVB and HSW PRMs say:
1390 * When an instruction has a source region that spans two registers and
1391 * the destination spans two registers, the destination elements must be
1392 * evenly split between the two registers [...]
1394 * The SNB PRM contains similar wording (but written in a much more
1395 * confusing manner).
1399 * When destination spans two registers, the source may be one or two
1400 * registers. The destination elements must be evenly split between the
1405 * When destination of MATH instruction spans two registers, the
1406 * destination elements must be evenly split between the two registers.
1408 * It is not known whether this restriction applies to KBL other Gens after
1411 if (devinfo
->gen
<= 8 ||
1412 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_MATH
) {
1414 /* Nothing explicitly states that on Gen < 8 elements must be evenly
1415 * split between two destination registers in the two exceptional
1416 * source-region-spans-one-register cases, but since Broadwell requires
1417 * evenly split writes regardless of source region, we assume that it was
1418 * an oversight and require it.
1420 if (dst_regs
== 2) {
1421 unsigned upper_reg_writes
= 0, lower_reg_writes
= 0;
1423 for (unsigned i
= 0; i
< exec_size
; i
++) {
1424 if (dst_access_mask
[i
] > 0xFFFFFFFF) {
1427 assert(dst_access_mask
[i
] != 0);
1432 ERROR_IF(upper_reg_writes
!= lower_reg_writes
,
1433 "Writes must be evenly split between the two "
1434 "destination registers");
1438 /* The IVB and HSW PRMs say:
1440 * When an instruction has a source region that spans two registers and
1441 * the destination spans two registers, the destination elements must be
1442 * evenly split between the two registers and each destination register
1443 * must be entirely derived from one source register.
1445 * Note: In such cases, the regioning parameters must ensure that the
1446 * offset from the two source registers is the same.
1448 * The SNB PRM contains similar wording (but written in a much more
1449 * confusing manner).
1451 * There are effectively three rules stated here:
1453 * For an instruction with a source and a destination spanning two
1456 * (1) destination elements must be evenly split between the two
1458 * (2) all destination elements in a register must be derived
1459 * from one source register
1460 * (3) the offset (i.e. the starting location in each of the two
1461 * registers spanned by a region) must be the same in the two
1462 * registers spanned by a region
1464 * It is impossible to violate rule (1) without violating (2) or (3), so we
1465 * do not attempt to validate it.
1467 if (devinfo
->gen
<= 7 && dst_regs
== 2) {
1468 for (unsigned i
= 0; i
< num_sources
; i
++) {
1470 if (src ## n ## _regs <= 1) \
1473 for (unsigned i = 0; i < exec_size; i++) { \
1474 if ((dst_access_mask[i] > 0xFFFFFFFF) != \
1475 (src ## n ## _access_mask[i] > 0xFFFFFFFF)) { \
1476 ERROR("Each destination register must be entirely derived " \
1477 "from one source register"); \
1482 unsigned offset_0 = \
1483 brw_inst_src ## n ## _da1_subreg_nr(devinfo, inst); \
1484 unsigned offset_1 = offset_0; \
1486 for (unsigned i = 0; i < exec_size; i++) { \
1487 if (src ## n ## _access_mask[i] > 0xFFFFFFFF) { \
1488 offset_1 = __builtin_ctzll(src ## n ## _access_mask[i]) - 32; \
1493 ERROR_IF(num_sources == 2 && offset_0 != offset_1, \
1494 "The offset from the two source registers " \
1506 /* The IVB and HSW PRMs say:
1508 * When destination spans two registers, the source MUST span two
1509 * registers. The exception to the above rule:
1510 * 1. When source is scalar, the source registers are not
1512 * 2. When source is packed integer Word and destination is packed
1513 * integer DWord, the source register is not incremented by the
1514 * source sub register is incremented.
1516 * The SNB PRM does not contain this rule, but the internal documentation
1517 * indicates that it applies to SNB as well. We assume that the rule applies
1518 * to Gen <= 5 although their PRMs do not state it.
1520 * While the documentation explicitly says in exception (2) that the
1521 * destination must be an integer DWord, the hardware allows at least a
1522 * float destination type as well. We emit such instructions from
1524 * fs_visitor::emit_interpolation_setup_gen6
1525 * fs_visitor::emit_fragcoord_interpolation
1527 * and have for years with no ill effects.
1529 * Additionally the simulator source code indicates that the real condition
1530 * is that the size of the destination type is 4 bytes.
1532 if (devinfo
->gen
<= 7 && dst_regs
== 2) {
1533 enum brw_reg_type dst_type
= brw_inst_dst_type(devinfo
, inst
);
1534 bool dst_is_packed_dword
=
1535 is_packed(exec_size
* stride
, exec_size
, stride
) &&
1536 brw_reg_type_to_size(dst_type
) == 4;
1538 for (unsigned i
= 0; i
< num_sources
; i
++) {
1540 unsigned vstride, width, hstride; \
1541 vstride = STRIDE(brw_inst_src ## n ## _vstride(devinfo, inst)); \
1542 width = WIDTH(brw_inst_src ## n ## _width(devinfo, inst)); \
1543 hstride = STRIDE(brw_inst_src ## n ## _hstride(devinfo, inst)); \
1544 bool src ## n ## _is_packed_word = \
1545 is_packed(vstride, width, hstride) && \
1546 (brw_inst_src ## n ## _type(devinfo, inst) == BRW_REGISTER_TYPE_W || \
1547 brw_inst_src ## n ## _type(devinfo, inst) == BRW_REGISTER_TYPE_UW); \
1549 ERROR_IF(src ## n ## _regs == 1 && \
1550 !src ## n ## _has_scalar_region(devinfo, inst) && \
1551 !(dst_is_packed_dword && src ## n ## _is_packed_word), \
1552 "When the destination spans two registers, the source must " \
1553 "span two registers\n" ERROR_INDENT "(exceptions for scalar " \
1554 "source and packed-word to packed-dword expansion)")
1568 static struct string
1569 vector_immediate_restrictions(const struct gen_device_info
*devinfo
,
1570 const brw_inst
*inst
)
1572 unsigned num_sources
= num_sources_from_inst(devinfo
, inst
);
1573 struct string error_msg
= { .str
= NULL
, .len
= 0 };
1575 if (num_sources
== 3 || num_sources
== 0)
1576 return (struct string
){};
1578 unsigned file
= num_sources
== 1 ?
1579 brw_inst_src0_reg_file(devinfo
, inst
) :
1580 brw_inst_src1_reg_file(devinfo
, inst
);
1581 if (file
!= BRW_IMMEDIATE_VALUE
)
1582 return (struct string
){};
1584 enum brw_reg_type dst_type
= brw_inst_dst_type(devinfo
, inst
);
1585 unsigned dst_type_size
= brw_reg_type_to_size(dst_type
);
1586 unsigned dst_subreg
= brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
?
1587 brw_inst_dst_da1_subreg_nr(devinfo
, inst
) : 0;
1588 unsigned dst_stride
= STRIDE(brw_inst_dst_hstride(devinfo
, inst
));
1589 enum brw_reg_type type
= num_sources
== 1 ?
1590 brw_inst_src0_type(devinfo
, inst
) :
1591 brw_inst_src1_type(devinfo
, inst
);
1595 * When an immediate vector is used in an instruction, the destination
1596 * must be 128-bit aligned with destination horizontal stride equivalent
1597 * to a word for an immediate integer vector (v) and equivalent to a
1598 * DWord for an immediate float vector (vf).
1600 * The text has not been updated for the addition of the immediate unsigned
1601 * integer vector type (uv) on SNB, but presumably the same restriction
1605 case BRW_REGISTER_TYPE_V
:
1606 case BRW_REGISTER_TYPE_UV
:
1607 case BRW_REGISTER_TYPE_VF
:
1608 ERROR_IF(dst_subreg
% (128 / 8) != 0,
1609 "Destination must be 128-bit aligned in order to use immediate "
1612 if (type
== BRW_REGISTER_TYPE_VF
) {
1613 ERROR_IF(dst_type_size
* dst_stride
!= 4,
1614 "Destination must have stride equivalent to dword in order "
1615 "to use the VF type");
1617 ERROR_IF(dst_type_size
* dst_stride
!= 2,
1618 "Destination must have stride equivalent to word in order "
1619 "to use the V or UV type");
1629 static struct string
1630 special_requirements_for_handling_double_precision_data_types(
1631 const struct gen_device_info
*devinfo
,
1632 const brw_inst
*inst
)
1634 unsigned num_sources
= num_sources_from_inst(devinfo
, inst
);
1635 struct string error_msg
= { .str
= NULL
, .len
= 0 };
1637 if (num_sources
== 3 || num_sources
== 0)
1638 return (struct string
){};
1640 /* Split sends don't have types so there's no doubles there. */
1641 if (inst_is_split_send(devinfo
, inst
))
1642 return (struct string
){};
1644 enum brw_reg_type exec_type
= execution_type(devinfo
, inst
);
1645 unsigned exec_type_size
= brw_reg_type_to_size(exec_type
);
1647 enum brw_reg_file dst_file
= brw_inst_dst_reg_file(devinfo
, inst
);
1648 enum brw_reg_type dst_type
= brw_inst_dst_type(devinfo
, inst
);
1649 unsigned dst_type_size
= brw_reg_type_to_size(dst_type
);
1650 unsigned dst_hstride
= STRIDE(brw_inst_dst_hstride(devinfo
, inst
));
1651 unsigned dst_reg
= brw_inst_dst_da_reg_nr(devinfo
, inst
);
1652 unsigned dst_subreg
= brw_inst_dst_da1_subreg_nr(devinfo
, inst
);
1653 unsigned dst_address_mode
= brw_inst_dst_address_mode(devinfo
, inst
);
1655 bool is_integer_dword_multiply
=
1656 devinfo
->gen
>= 8 &&
1657 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_MUL
&&
1658 (brw_inst_src0_type(devinfo
, inst
) == BRW_REGISTER_TYPE_D
||
1659 brw_inst_src0_type(devinfo
, inst
) == BRW_REGISTER_TYPE_UD
) &&
1660 (brw_inst_src1_type(devinfo
, inst
) == BRW_REGISTER_TYPE_D
||
1661 brw_inst_src1_type(devinfo
, inst
) == BRW_REGISTER_TYPE_UD
);
1663 if (dst_type_size
!= 8 && exec_type_size
!= 8 && !is_integer_dword_multiply
)
1664 return (struct string
){};
1666 for (unsigned i
= 0; i
< num_sources
; i
++) {
1667 unsigned vstride
, width
, hstride
, type_size
, reg
, subreg
, address_mode
;
1668 bool is_scalar_region
;
1669 enum brw_reg_file file
;
1670 enum brw_reg_type type
;
1673 if (brw_inst_src ## n ## _reg_file(devinfo, inst) == \
1674 BRW_IMMEDIATE_VALUE) \
1677 is_scalar_region = src ## n ## _has_scalar_region(devinfo, inst); \
1678 vstride = STRIDE(brw_inst_src ## n ## _vstride(devinfo, inst)); \
1679 width = WIDTH(brw_inst_src ## n ## _width(devinfo, inst)); \
1680 hstride = STRIDE(brw_inst_src ## n ## _hstride(devinfo, inst)); \
1681 file = brw_inst_src ## n ## _reg_file(devinfo, inst); \
1682 type = brw_inst_src ## n ## _type(devinfo, inst); \
1683 type_size = brw_reg_type_to_size(type); \
1684 reg = brw_inst_src ## n ## _da_reg_nr(devinfo, inst); \
1685 subreg = brw_inst_src ## n ## _da1_subreg_nr(devinfo, inst); \
1686 address_mode = brw_inst_src ## n ## _address_mode(devinfo, inst)
1695 /* The PRMs say that for CHV, BXT:
1697 * When source or destination datatype is 64b or operation is integer
1698 * DWord multiply, regioning in Align1 must follow these rules:
1700 * 1. Source and Destination horizontal stride must be aligned to the
1702 * 2. Regioning must ensure Src.Vstride = Src.Width * Src.Hstride.
1703 * 3. Source and Destination offset must be the same, except the case
1706 * We assume that the restriction applies to GLK as well.
1708 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
&&
1709 (devinfo
->is_cherryview
|| gen_device_info_is_9lp(devinfo
))) {
1710 unsigned src_stride
= hstride
* type_size
;
1711 unsigned dst_stride
= dst_hstride
* dst_type_size
;
1713 ERROR_IF(!is_scalar_region
&&
1714 (src_stride
% 8 != 0 ||
1715 dst_stride
% 8 != 0 ||
1716 src_stride
!= dst_stride
),
1717 "Source and destination horizontal stride must equal and a "
1718 "multiple of a qword when the execution type is 64-bit");
1720 ERROR_IF(vstride
!= width
* hstride
,
1721 "Vstride must be Width * Hstride when the execution type is "
1724 ERROR_IF(!is_scalar_region
&& dst_subreg
!= subreg
,
1725 "Source and destination offset must be the same when the "
1726 "execution type is 64-bit");
1729 /* The PRMs say that for CHV, BXT:
1731 * When source or destination datatype is 64b or operation is integer
1732 * DWord multiply, indirect addressing must not be used.
1734 * We assume that the restriction applies to GLK as well.
1736 if (devinfo
->is_cherryview
|| gen_device_info_is_9lp(devinfo
)) {
1737 ERROR_IF(BRW_ADDRESS_REGISTER_INDIRECT_REGISTER
== address_mode
||
1738 BRW_ADDRESS_REGISTER_INDIRECT_REGISTER
== dst_address_mode
,
1739 "Indirect addressing is not allowed when the execution type "
1743 /* The PRMs say that for CHV, BXT:
1745 * ARF registers must never be used with 64b datatype or when
1746 * operation is integer DWord multiply.
1748 * We assume that the restriction applies to GLK as well.
1750 * We assume that the restriction does not apply to the null register.
1752 if (devinfo
->is_cherryview
|| gen_device_info_is_9lp(devinfo
)) {
1753 ERROR_IF(brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_MAC
||
1754 brw_inst_acc_wr_control(devinfo
, inst
) ||
1755 (BRW_ARCHITECTURE_REGISTER_FILE
== file
&&
1756 reg
!= BRW_ARF_NULL
) ||
1757 (BRW_ARCHITECTURE_REGISTER_FILE
== dst_file
&&
1758 dst_reg
!= BRW_ARF_NULL
),
1759 "Architecture registers cannot be used when the execution "
1764 /* The PRMs say that for BDW, SKL:
1766 * If Align16 is required for an operation with QW destination and non-QW
1767 * source datatypes, the execution size cannot exceed 2.
1769 * We assume that the restriction applies to all Gen8+ parts.
1771 if (devinfo
->gen
>= 8) {
1772 enum brw_reg_type src0_type
= brw_inst_src0_type(devinfo
, inst
);
1773 enum brw_reg_type src1_type
=
1774 num_sources
> 1 ? brw_inst_src1_type(devinfo
, inst
) : src0_type
;
1775 unsigned src0_type_size
= brw_reg_type_to_size(src0_type
);
1776 unsigned src1_type_size
= brw_reg_type_to_size(src1_type
);
1778 ERROR_IF(brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_16
&&
1779 dst_type_size
== 8 &&
1780 (src0_type_size
!= 8 || src1_type_size
!= 8) &&
1781 brw_inst_exec_size(devinfo
, inst
) > BRW_EXECUTE_2
,
1782 "In Align16 exec size cannot exceed 2 with a QWord destination "
1783 "and a non-QWord source");
1786 /* The PRMs say that for CHV, BXT:
1788 * When source or destination datatype is 64b or operation is integer
1789 * DWord multiply, DepCtrl must not be used.
1791 * We assume that the restriction applies to GLK as well.
1793 if (devinfo
->is_cherryview
|| gen_device_info_is_9lp(devinfo
)) {
1794 ERROR_IF(brw_inst_no_dd_check(devinfo
, inst
) ||
1795 brw_inst_no_dd_clear(devinfo
, inst
),
1796 "DepCtrl is not allowed when the execution type is 64-bit");
1803 brw_validate_instructions(const struct gen_device_info
*devinfo
,
1804 const void *assembly
, int start_offset
, int end_offset
,
1805 struct disasm_info
*disasm
)
1809 for (int src_offset
= start_offset
; src_offset
< end_offset
;) {
1810 struct string error_msg
= { .str
= NULL
, .len
= 0 };
1811 const brw_inst
*inst
= assembly
+ src_offset
;
1812 bool is_compact
= brw_inst_cmpt_control(devinfo
, inst
);
1813 brw_inst uncompacted
;
1816 brw_compact_inst
*compacted
= (void *)inst
;
1817 brw_uncompact_instruction(devinfo
, &uncompacted
, compacted
);
1818 inst
= &uncompacted
;
1821 if (is_unsupported_inst(devinfo
, inst
)) {
1822 ERROR("Instruction not supported on this Gen");
1824 CHECK(sources_not_null
);
1825 CHECK(send_restrictions
);
1826 CHECK(general_restrictions_based_on_operand_types
);
1827 CHECK(general_restrictions_on_region_parameters
);
1828 CHECK(special_restrictions_for_mixed_float_mode
);
1829 CHECK(region_alignment_rules
);
1830 CHECK(vector_immediate_restrictions
);
1831 CHECK(special_requirements_for_handling_double_precision_data_types
);
1834 if (error_msg
.str
&& disasm
) {
1835 disasm_insert_error(disasm
, src_offset
, error_msg
.str
);
1837 valid
= valid
&& error_msg
.len
== 0;
1838 free(error_msg
.str
);
1841 src_offset
+= sizeof(brw_compact_inst
);
1843 src_offset
+= sizeof(brw_inst
);