projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
i965: Normalize types for FBL, FBH, etc
[mesa.git] / src / intel / compiler / brw_fs_generator.cpp
1 /*
2 * Copyright © 2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 /** @file brw_fs_generator.cpp
25 *
26 * This file supports generating code from the FS LIR to the actual
27 * native instructions.
28 */
29
30 #include "brw_eu.h"
31 #include "brw_fs.h"
32 #include "brw_cfg.h"
33
34 static enum brw_reg_file
35 brw_file_from_reg(fs_reg *reg)
36 {
37 switch (reg->file) {
38 case ARF:
39 return BRW_ARCHITECTURE_REGISTER_FILE;
40 case FIXED_GRF:
41 case VGRF:
42 return BRW_GENERAL_REGISTER_FILE;
43 case MRF:
44 return BRW_MESSAGE_REGISTER_FILE;
45 case IMM:
46 return BRW_IMMEDIATE_VALUE;
47 case BAD_FILE:
48 case ATTR:
49 case UNIFORM:
50 unreachable("not reached");
51 }
52 return BRW_ARCHITECTURE_REGISTER_FILE;
53 }
54
55 static struct brw_reg
56 brw_reg_from_fs_reg(const struct gen_device_info *devinfo, fs_inst *inst,
57 fs_reg *reg, bool compressed)
58 {
59 struct brw_reg brw_reg;
60
61 switch (reg->file) {
62 case MRF:
63 assert((reg->nr & ~BRW_MRF_COMPR4) < BRW_MAX_MRF(devinfo->gen));
64 /* Fallthrough */
65 case VGRF:
66 if (reg->stride == 0) {
67 brw_reg = brw_vec1_reg(brw_file_from_reg(reg), reg->nr, 0);
68 } else {
69 /* From the Haswell PRM:
70 *
71 * "VertStride must be used to cross GRF register boundaries. This
72 * rule implies that elements within a 'Width' cannot cross GRF
73 * boundaries."
74 *
75 * The maximum width value that could satisfy this restriction is:
76 */
77 const unsigned reg_width = REG_SIZE / (reg->stride * type_sz(reg->type));
78
79 /* Because the hardware can only split source regions at a whole
80 * multiple of width during decompression (i.e. vertically), clamp
81 * the value obtained above to the physical execution size of a
82 * single decompressed chunk of the instruction:
83 */
84 const unsigned phys_width = compressed ? inst->exec_size / 2 :
85 inst->exec_size;
86
87 /* XXX - The equation above is strictly speaking not correct on
88 * hardware that supports unbalanced GRF writes -- On Gen9+
89 * each decompressed chunk of the instruction may have a
90 * different execution size when the number of components
91 * written to each destination GRF is not the same.
92 */
93 const unsigned width = MIN2(reg_width, phys_width);
94 brw_reg = brw_vecn_reg(width, brw_file_from_reg(reg), reg->nr, 0);
95 brw_reg = stride(brw_reg, width * reg->stride, width, reg->stride);
96
97 if (devinfo->gen == 7 && !devinfo->is_haswell) {
98 /* From the IvyBridge PRM (EU Changes by Processor Generation, page 13):
99 * "Each DF (Double Float) operand uses an element size of 4 rather
100 * than 8 and all regioning parameters are twice what the values
101 * would be based on the true element size: ExecSize, Width,
102 * HorzStride, and VertStride. Each DF operand uses a pair of
103 * channels and all masking and swizzing should be adjusted
104 * appropriately."
105 *
106 * From the IvyBridge PRM (Special Requirements for Handling Double
107 * Precision Data Types, page 71):
108 * "In Align1 mode, all regioning parameters like stride, execution
109 * size, and width must use the syntax of a pair of packed
110 * floats. The offsets for these data types must be 64-bit
111 * aligned. The execution size and regioning parameters are in terms
112 * of floats."
113 *
114 * Summarized: when handling DF-typed arguments, ExecSize,
115 * VertStride, and Width must be doubled.
116 *
117 * It applies to BayTrail too.
118 */
119 if (type_sz(reg->type) == 8) {
120 brw_reg.width++;
121 if (brw_reg.vstride > 0)
122 brw_reg.vstride++;
123 assert(brw_reg.hstride == BRW_HORIZONTAL_STRIDE_1);
124 }
125
126 /* When converting from DF->F, we set the destination stride to 2
127 * because each d2f conversion implicitly writes 2 floats, being
128 * the first one the converted value. IVB/BYT actually writes two
129 * F components per SIMD channel, and every other component is
130 * filled with garbage.
131 */
132 if (reg == &inst->dst && get_exec_type_size(inst) == 8 &&
133 type_sz(inst->dst.type) < 8) {
134 assert(brw_reg.hstride > BRW_HORIZONTAL_STRIDE_1);
135 brw_reg.hstride--;
136 }
137 }
138 }
139
140 brw_reg = retype(brw_reg, reg->type);
141 brw_reg = byte_offset(brw_reg, reg->offset);
142 brw_reg.abs = reg->abs;
143 brw_reg.negate = reg->negate;
144 break;
145 case ARF:
146 case FIXED_GRF:
147 case IMM:
148 assert(reg->offset == 0);
149 brw_reg = reg->as_brw_reg();
150 break;
151 case BAD_FILE:
152 /* Probably unused. */
153 brw_reg = brw_null_reg();
154 break;
155 case ATTR:
156 case UNIFORM:
157 unreachable("not reached");
158 }
159
160 /* On HSW+, scalar DF sources can be accessed using the normal <0,1,0>
161 * region, but on IVB and BYT DF regions must be programmed in terms of
162 * floats. A <0,2,1> region accomplishes this.
163 */
164 if (devinfo->gen == 7 && !devinfo->is_haswell &&
165 type_sz(reg->type) == 8 &&
166 brw_reg.vstride == BRW_VERTICAL_STRIDE_0 &&
167 brw_reg.width == BRW_WIDTH_1 &&
168 brw_reg.hstride == BRW_HORIZONTAL_STRIDE_0) {
169 brw_reg.width = BRW_WIDTH_2;
170 brw_reg.hstride = BRW_HORIZONTAL_STRIDE_1;
171 }
172
173 return brw_reg;
174 }
175
176 fs_generator::fs_generator(const struct brw_compiler *compiler, void *log_data,
177 void *mem_ctx,
178 const void *key,
179 struct brw_stage_prog_data *prog_data,
180 unsigned promoted_constants,
181 bool runtime_check_aads_emit,
182 gl_shader_stage stage)
183
184 : compiler(compiler), log_data(log_data),
185 devinfo(compiler->devinfo), key(key),
186 prog_data(prog_data),
187 promoted_constants(promoted_constants),
188 runtime_check_aads_emit(runtime_check_aads_emit), debug_flag(false),
189 stage(stage), mem_ctx(mem_ctx)
190 {
191 p = rzalloc(mem_ctx, struct brw_codegen);
192 brw_init_codegen(devinfo, p, mem_ctx);
193 }
194
195 fs_generator::~fs_generator()
196 {
197 }
198
199 class ip_record : public exec_node {
200 public:
201 DECLARE_RALLOC_CXX_OPERATORS(ip_record)
202
203 ip_record(int ip)
204 {
205 this->ip = ip;
206 }
207
208 int ip;
209 };
210
211 bool
212 fs_generator::patch_discard_jumps_to_fb_writes()
213 {
214 if (devinfo->gen < 6 || this->discard_halt_patches.is_empty())
215 return false;
216
217 int scale = brw_jump_scale(p->devinfo);
218
219 /* There is a somewhat strange undocumented requirement of using
220 * HALT, according to the simulator. If some channel has HALTed to
221 * a particular UIP, then by the end of the program, every channel
222 * must have HALTed to that UIP. Furthermore, the tracking is a
223 * stack, so you can't do the final halt of a UIP after starting
224 * halting to a new UIP.
225 *
226 * Symptoms of not emitting this instruction on actual hardware
227 * included GPU hangs and sparkly rendering on the piglit discard
228 * tests.
229 */
230 brw_inst *last_halt = gen6_HALT(p);
231 brw_inst_set_uip(p->devinfo, last_halt, 1 * scale);
232 brw_inst_set_jip(p->devinfo, last_halt, 1 * scale);
233
234 int ip = p->nr_insn;
235
236 foreach_in_list(ip_record, patch_ip, &discard_halt_patches) {
237 brw_inst *patch = &p->store[patch_ip->ip];
238
239 assert(brw_inst_opcode(p->devinfo, patch) == BRW_OPCODE_HALT);
240 /* HALT takes a half-instruction distance from the pre-incremented IP. */
241 brw_inst_set_uip(p->devinfo, patch, (ip - patch_ip->ip) * scale);
242 }
243
244 this->discard_halt_patches.make_empty();
245 return true;
246 }
247
248 void
249 fs_generator::fire_fb_write(fs_inst *inst,
250 struct brw_reg payload,
251 struct brw_reg implied_header,
252 GLuint nr)
253 {
254 uint32_t msg_control;
255
256 struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
257
258 if (devinfo->gen < 6) {
259 brw_push_insn_state(p);
260 brw_set_default_exec_size(p, BRW_EXECUTE_8);
261 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
262 brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
263 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
264 brw_MOV(p, offset(payload, 1), brw_vec8_grf(1, 0));
265 brw_pop_insn_state(p);
266 }
267
268 if (inst->opcode == FS_OPCODE_REP_FB_WRITE)
269 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD16_SINGLE_SOURCE_REPLICATED;
270 else if (prog_data->dual_src_blend) {
271 if (!inst->group)
272 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_DUAL_SOURCE_SUBSPAN01;
273 else
274 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_DUAL_SOURCE_SUBSPAN23;
275 } else if (inst->exec_size == 16)
276 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD16_SINGLE_SOURCE;
277 else
278 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_SINGLE_SOURCE_SUBSPAN01;
279
280 /* We assume render targets start at 0, because headerless FB write
281 * messages set "Render Target Index" to 0. Using a different binding
282 * table index would make it impossible to use headerless messages.
283 */
284 assert(prog_data->binding_table.render_target_start == 0);
285
286 const uint32_t surf_index = inst->target;
287
288 bool last_render_target = inst->eot ||
289 (prog_data->dual_src_blend && dispatch_width == 16);
290
291
292 brw_fb_WRITE(p,
293 payload,
294 implied_header,
295 msg_control,
296 surf_index,
297 nr,
298 0,
299 inst->eot,
300 last_render_target,
301 inst->header_size != 0);
302
303 brw_mark_surface_used(&prog_data->base, surf_index);
304 }
305
306 void
307 fs_generator::generate_fb_write(fs_inst *inst, struct brw_reg payload)
308 {
309 struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
310 const brw_wm_prog_key * const key = (brw_wm_prog_key * const) this->key;
311 struct brw_reg implied_header;
312
313 if (devinfo->gen < 8 && !devinfo->is_haswell) {
314 brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
315 }
316
317 if (inst->base_mrf >= 0)
318 payload = brw_message_reg(inst->base_mrf);
319
320 /* Header is 2 regs, g0 and g1 are the contents. g0 will be implied
321 * move, here's g1.
322 */
323 if (inst->header_size != 0) {
324 brw_push_insn_state(p);
325 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
326 brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
327 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
328 brw_set_default_flag_reg(p, 0, 0);
329
330 /* On HSW, the GPU will use the predicate on SENDC, unless the header is
331 * present.
332 */
333 if (prog_data->uses_kill) {
334 struct brw_reg pixel_mask;
335
336 if (devinfo->gen >= 6)
337 pixel_mask = retype(brw_vec1_grf(1, 7), BRW_REGISTER_TYPE_UW);
338 else
339 pixel_mask = retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UW);
340
341 brw_MOV(p, pixel_mask, brw_flag_reg(0, 1));
342 }
343
344 if (devinfo->gen >= 6) {
345 brw_push_insn_state(p);
346 brw_set_default_exec_size(p, BRW_EXECUTE_16);
347 brw_set_default_compression_control(p, BRW_COMPRESSION_COMPRESSED);
348 brw_MOV(p,
349 retype(payload, BRW_REGISTER_TYPE_UD),
350 retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));
351 brw_pop_insn_state(p);
352
353 if (inst->target > 0 && key->replicate_alpha) {
354 /* Set "Source0 Alpha Present to RenderTarget" bit in message
355 * header.
356 */
357 brw_OR(p,
358 vec1(retype(payload, BRW_REGISTER_TYPE_UD)),
359 vec1(retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD)),
360 brw_imm_ud(0x1 << 11));
361 }
362
363 if (inst->target > 0) {
364 /* Set the render target index for choosing BLEND_STATE. */
365 brw_MOV(p, retype(vec1(suboffset(payload, 2)),
366 BRW_REGISTER_TYPE_UD),
367 brw_imm_ud(inst->target));
368 }
369
370 /* Set computes stencil to render target */
371 if (prog_data->computed_stencil) {
372 brw_OR(p,
373 vec1(retype(payload, BRW_REGISTER_TYPE_UD)),
374 vec1(retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD)),
375 brw_imm_ud(0x1 << 14));
376 }
377
378 implied_header = brw_null_reg();
379 } else {
380 implied_header = retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UW);
381 }
382
383 brw_pop_insn_state(p);
384 } else {
385 implied_header = brw_null_reg();
386 }
387
388 if (!runtime_check_aads_emit) {
389 fire_fb_write(inst, payload, implied_header, inst->mlen);
390 } else {
391 /* This can only happen in gen < 6 */
392 assert(devinfo->gen < 6);
393
394 struct brw_reg v1_null_ud = vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_UD));
395
396 /* Check runtime bit to detect if we have to send AA data or not */
397 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
398 brw_AND(p,
399 v1_null_ud,
400 retype(brw_vec1_grf(1, 6), BRW_REGISTER_TYPE_UD),
401 brw_imm_ud(1<<26));
402 brw_inst_set_cond_modifier(p->devinfo, brw_last_inst, BRW_CONDITIONAL_NZ);
403
404 int jmp = brw_JMPI(p, brw_imm_ud(0), BRW_PREDICATE_NORMAL) - p->store;
405 brw_inst_set_exec_size(p->devinfo, brw_last_inst, BRW_EXECUTE_1);
406 {
407 /* Don't send AA data */
408 fire_fb_write(inst, offset(payload, 1), implied_header, inst->mlen-1);
409 }
410 brw_land_fwd_jump(p, jmp);
411 fire_fb_write(inst, payload, implied_header, inst->mlen);
412 }
413 }
414
415 void
416 fs_generator::generate_fb_read(fs_inst *inst, struct brw_reg dst,
417 struct brw_reg payload)
418 {
419 assert(inst->size_written % REG_SIZE == 0);
420 struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
421 const unsigned surf_index =
422 prog_data->binding_table.render_target_start + inst->target;
423
424 gen9_fb_READ(p, dst, payload, surf_index,
425 inst->header_size, inst->size_written / REG_SIZE,
426 prog_data->persample_dispatch);
427
428 brw_mark_surface_used(&prog_data->base, surf_index);
429 }
430
431 void
432 fs_generator::generate_mov_indirect(fs_inst *inst,
433 struct brw_reg dst,
434 struct brw_reg reg,
435 struct brw_reg indirect_byte_offset)
436 {
437 assert(indirect_byte_offset.type == BRW_REGISTER_TYPE_UD);
438 assert(indirect_byte_offset.file == BRW_GENERAL_REGISTER_FILE);
439
440 unsigned imm_byte_offset = reg.nr * REG_SIZE + reg.subnr;
441
442 if (indirect_byte_offset.file == BRW_IMMEDIATE_VALUE) {
443 imm_byte_offset += indirect_byte_offset.ud;
444
445 reg.nr = imm_byte_offset / REG_SIZE;
446 reg.subnr = imm_byte_offset % REG_SIZE;
447 brw_MOV(p, dst, reg);
448 } else {
449 /* Prior to Broadwell, there are only 8 address registers. */
450 assert(inst->exec_size <= 8 || devinfo->gen >= 8);
451
452 /* We use VxH indirect addressing, clobbering a0.0 through a0.7. */
453 struct brw_reg addr = vec8(brw_address_reg(0));
454
455 /* The destination stride of an instruction (in bytes) must be greater
456 * than or equal to the size of the rest of the instruction. Since the
457 * address register is of type UW, we can't use a D-type instruction.
458 * In order to get around this, re retype to UW and use a stride.
459 */
460 indirect_byte_offset =
461 retype(spread(indirect_byte_offset, 2), BRW_REGISTER_TYPE_UW);
462
463 /* There are a number of reasons why we don't use the base offset here.
464 * One reason is that the field is only 9 bits which means we can only
465 * use it to access the first 16 GRFs. Also, from the Haswell PRM
466 * section "Register Region Restrictions":
467 *
468 * "The lower bits of the AddressImmediate must not overflow to
469 * change the register address. The lower 5 bits of Address
470 * Immediate when added to lower 5 bits of address register gives
471 * the sub-register offset. The upper bits of Address Immediate
472 * when added to upper bits of address register gives the register
473 * address. Any overflow from sub-register offset is dropped."
474 *
475 * Since the indirect may cause us to cross a register boundary, this
476 * makes the base offset almost useless. We could try and do something
477 * clever where we use a actual base offset if base_offset % 32 == 0 but
478 * that would mean we were generating different code depending on the
479 * base offset. Instead, for the sake of consistency, we'll just do the
480 * add ourselves. This restriction is only listed in the Haswell PRM
481 * but empirical testing indicates that it applies on all older
482 * generations and is lifted on Broadwell.
483 *
484 * In the end, while base_offset is nice to look at in the generated
485 * code, using it saves us 0 instructions and would require quite a bit
486 * of case-by-case work. It's just not worth it.
487 */
488 if (devinfo->gen >= 8 || devinfo->is_haswell || type_sz(reg.type) < 8) {
489 brw_ADD(p, addr, indirect_byte_offset, brw_imm_uw(imm_byte_offset));
490 } else {
491 /* IVB reads two address register components per channel for
492 * indirectly addressed 64-bit sources, so we need to initialize
493 * adjacent address components to consecutive dwords of the source
494 * region by emitting two separate ADD instructions. Found
495 * empirically.
496 */
497 assert(inst->exec_size <= 4);
498 brw_push_insn_state(p);
499 brw_set_default_exec_size(p, cvt(inst->exec_size) - 1);
500
501 brw_ADD(p, spread(addr, 2), indirect_byte_offset,
502 brw_imm_uw(imm_byte_offset));
503 brw_inst_set_no_dd_clear(devinfo, brw_last_inst, true);
504
505 brw_ADD(p, spread(suboffset(addr, 1), 2), indirect_byte_offset,
506 brw_imm_uw(imm_byte_offset + 4));
507 brw_inst_set_no_dd_check(devinfo, brw_last_inst, true);
508
509 brw_pop_insn_state(p);
510 }
511
512 struct brw_reg ind_src = brw_VxH_indirect(0, 0);
513
514 brw_inst *mov = brw_MOV(p, dst, retype(ind_src, reg.type));
515
516 if (devinfo->gen == 6 && dst.file == BRW_MESSAGE_REGISTER_FILE &&
517 !inst->get_next()->is_tail_sentinel() &&
518 ((fs_inst *)inst->get_next())->mlen > 0) {
519 /* From the Sandybridge PRM:
520 *
521 * "[Errata: DevSNB(SNB)] If MRF register is updated by any
522 * instruction that “indexed/indirect” source AND is followed by a
523 * send, the instruction requires a “Switch”. This is to avoid
524 * race condition where send may dispatch before MRF is updated."
525 */
526 brw_inst_set_thread_control(devinfo, mov, BRW_THREAD_SWITCH);
527 }
528 }
529 }
530
531 void
532 fs_generator::generate_urb_read(fs_inst *inst,
533 struct brw_reg dst,
534 struct brw_reg header)
535 {
536 assert(inst->size_written % REG_SIZE == 0);
537 assert(header.file == BRW_GENERAL_REGISTER_FILE);
538 assert(header.type == BRW_REGISTER_TYPE_UD);
539
540 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
541 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UD));
542 brw_set_src0(p, send, header);
543 brw_set_src1(p, send, brw_imm_ud(0u));
544
545 brw_inst_set_sfid(p->devinfo, send, BRW_SFID_URB);
546 brw_inst_set_urb_opcode(p->devinfo, send, GEN8_URB_OPCODE_SIMD8_READ);
547
548 if (inst->opcode == SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT)
549 brw_inst_set_urb_per_slot_offset(p->devinfo, send, true);
550
551 brw_inst_set_mlen(p->devinfo, send, inst->mlen);
552 brw_inst_set_rlen(p->devinfo, send, inst->size_written / REG_SIZE);
553 brw_inst_set_header_present(p->devinfo, send, true);
554 brw_inst_set_urb_global_offset(p->devinfo, send, inst->offset);
555 }
556
557 void
558 fs_generator::generate_urb_write(fs_inst *inst, struct brw_reg payload)
559 {
560 brw_inst *insn;
561
562 insn = brw_next_insn(p, BRW_OPCODE_SEND);
563
564 brw_set_dest(p, insn, brw_null_reg());
565 brw_set_src0(p, insn, payload);
566 brw_set_src1(p, insn, brw_imm_d(0));
567
568 brw_inst_set_sfid(p->devinfo, insn, BRW_SFID_URB);
569 brw_inst_set_urb_opcode(p->devinfo, insn, GEN8_URB_OPCODE_SIMD8_WRITE);
570
571 if (inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT ||
572 inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT)
573 brw_inst_set_urb_per_slot_offset(p->devinfo, insn, true);
574
575 if (inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_MASKED ||
576 inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT)
577 brw_inst_set_urb_channel_mask_present(p->devinfo, insn, true);
578
579 brw_inst_set_mlen(p->devinfo, insn, inst->mlen);
580 brw_inst_set_rlen(p->devinfo, insn, 0);
581 brw_inst_set_eot(p->devinfo, insn, inst->eot);
582 brw_inst_set_header_present(p->devinfo, insn, true);
583 brw_inst_set_urb_global_offset(p->devinfo, insn, inst->offset);
584 }
585
586 void
587 fs_generator::generate_cs_terminate(fs_inst *inst, struct brw_reg payload)
588 {
589 struct brw_inst *insn;
590
591 insn = brw_next_insn(p, BRW_OPCODE_SEND);
592
593 brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW));
594 brw_set_src0(p, insn, retype(payload, BRW_REGISTER_TYPE_UW));
595 brw_set_src1(p, insn, brw_imm_d(0));
596
597 /* Terminate a compute shader by sending a message to the thread spawner.
598 */
599 brw_inst_set_sfid(devinfo, insn, BRW_SFID_THREAD_SPAWNER);
600 brw_inst_set_mlen(devinfo, insn, 1);
601 brw_inst_set_rlen(devinfo, insn, 0);
602 brw_inst_set_eot(devinfo, insn, inst->eot);
603 brw_inst_set_header_present(devinfo, insn, false);
604
605 brw_inst_set_ts_opcode(devinfo, insn, 0); /* Dereference resource */
606 brw_inst_set_ts_request_type(devinfo, insn, 0); /* Root thread */
607
608 /* Note that even though the thread has a URB resource associated with it,
609 * we set the "do not dereference URB" bit, because the URB resource is
610 * managed by the fixed-function unit, so it will free it automatically.
611 */
612 brw_inst_set_ts_resource_select(devinfo, insn, 1); /* Do not dereference URB */
613
614 brw_inst_set_mask_control(devinfo, insn, BRW_MASK_DISABLE);
615 }
616
617 void
618 fs_generator::generate_barrier(fs_inst *inst, struct brw_reg src)
619 {
620 brw_barrier(p, src);
621 brw_WAIT(p);
622 }
623
624 void
625 fs_generator::generate_linterp(fs_inst *inst,
626 struct brw_reg dst, struct brw_reg *src)
627 {
628 /* PLN reads:
629 * / in SIMD16 \
630 * -----------------------------------
631 * | src1+0 | src1+1 | src1+2 | src1+3 |
632 * |-----------------------------------|
633 * |(x0, x1)|(y0, y1)|(x2, x3)|(y2, y3)|
634 * -----------------------------------
635 *
636 * but for the LINE/MAC pair, the LINE reads Xs and the MAC reads Ys:
637 *
638 * -----------------------------------
639 * | src1+0 | src1+1 | src1+2 | src1+3 |
640 * |-----------------------------------|
641 * |(x0, x1)|(y0, y1)| | | in SIMD8
642 * |-----------------------------------|
643 * |(x0, x1)|(x2, x3)|(y0, y1)|(y2, y3)| in SIMD16
644 * -----------------------------------
645 *
646 * See also: emit_interpolation_setup_gen4().
647 */
648 struct brw_reg delta_x = src[0];
649 struct brw_reg delta_y = offset(src[0], inst->exec_size / 8);
650 struct brw_reg interp = src[1];
651
652 if (devinfo->has_pln &&
653 (devinfo->gen >= 7 || (delta_x.nr & 1) == 0)) {
654 brw_PLN(p, dst, interp, delta_x);
655 } else {
656 brw_LINE(p, brw_null_reg(), interp, delta_x);
657 brw_MAC(p, dst, suboffset(interp, 1), delta_y);
658 }
659 }
660
661 void
662 fs_generator::generate_get_buffer_size(fs_inst *inst,
663 struct brw_reg dst,
664 struct brw_reg src,
665 struct brw_reg surf_index)
666 {
667 assert(devinfo->gen >= 7);
668 assert(surf_index.file == BRW_IMMEDIATE_VALUE);
669
670 uint32_t simd_mode;
671 int rlen = 4;
672
673 switch (inst->exec_size) {
674 case 8:
675 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
676 break;
677 case 16:
678 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
679 break;
680 default:
681 unreachable("Invalid width for texture instruction");
682 }
683
684 if (simd_mode == BRW_SAMPLER_SIMD_MODE_SIMD16) {
685 rlen = 8;
686 dst = vec16(dst);
687 }
688
689 brw_SAMPLE(p,
690 retype(dst, BRW_REGISTER_TYPE_UW),
691 inst->base_mrf,
692 src,
693 surf_index.ud,
694 0,
695 GEN5_SAMPLER_MESSAGE_SAMPLE_RESINFO,
696 rlen, /* response length */
697 inst->mlen,
698 inst->header_size > 0,
699 simd_mode,
700 BRW_SAMPLER_RETURN_FORMAT_SINT32);
701
702 brw_mark_surface_used(prog_data, surf_index.ud);
703 }
704
705 void
706 fs_generator::generate_tex(fs_inst *inst, struct brw_reg dst, struct brw_reg src,
707 struct brw_reg surface_index,
708 struct brw_reg sampler_index)
709 {
710 assert(inst->size_written % REG_SIZE == 0);
711 int msg_type = -1;
712 uint32_t simd_mode;
713 uint32_t return_format;
714 bool is_combined_send = inst->eot;
715
716 switch (dst.type) {
717 case BRW_REGISTER_TYPE_D:
718 return_format = BRW_SAMPLER_RETURN_FORMAT_SINT32;
719 break;
720 case BRW_REGISTER_TYPE_UD:
721 return_format = BRW_SAMPLER_RETURN_FORMAT_UINT32;
722 break;
723 default:
724 return_format = BRW_SAMPLER_RETURN_FORMAT_FLOAT32;
725 break;
726 }
727
728 /* Stomp the resinfo output type to UINT32. On gens 4-5, the output type
729 * is set as part of the message descriptor. On gen4, the PRM seems to
730 * allow UINT32 and FLOAT32 (i965 PRM, Vol. 4 Section 4.8.1.1), but on
731 * later gens UINT32 is required. Once you hit Sandy Bridge, the bit is
732 * gone from the message descriptor entirely and you just get UINT32 all
733 * the time regasrdless. Since we can really only do non-UINT32 on gen4,
734 * just stomp it to UINT32 all the time.
735 */
736 if (inst->opcode == SHADER_OPCODE_TXS)
737 return_format = BRW_SAMPLER_RETURN_FORMAT_UINT32;
738
739 switch (inst->exec_size) {
740 case 8:
741 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
742 break;
743 case 16:
744 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
745 break;
746 default:
747 unreachable("Invalid width for texture instruction");
748 }
749
750 if (devinfo->gen >= 5) {
751 switch (inst->opcode) {
752 case SHADER_OPCODE_TEX:
753 if (inst->shadow_compare) {
754 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_COMPARE;
755 } else {
756 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE;
757 }
758 break;
759 case FS_OPCODE_TXB:
760 if (inst->shadow_compare) {
761 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_BIAS_COMPARE;
762 } else {
763 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_BIAS;
764 }
765 break;
766 case SHADER_OPCODE_TXL:
767 if (inst->shadow_compare) {
768 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LOD_COMPARE;
769 } else {
770 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LOD;
771 }
772 break;
773 case SHADER_OPCODE_TXL_LZ:
774 assert(devinfo->gen >= 9);
775 if (inst->shadow_compare) {
776 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_C_LZ;
777 } else {
778 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_LZ;
779 }
780 break;
781 case SHADER_OPCODE_TXS:
782 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_RESINFO;
783 break;
784 case SHADER_OPCODE_TXD:
785 if (inst->shadow_compare) {
786 /* Gen7.5+. Otherwise, lowered in NIR */
787 assert(devinfo->gen >= 8 || devinfo->is_haswell);
788 msg_type = HSW_SAMPLER_MESSAGE_SAMPLE_DERIV_COMPARE;
789 } else {
790 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_DERIVS;
791 }
792 break;
793 case SHADER_OPCODE_TXF:
794 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
795 break;
796 case SHADER_OPCODE_TXF_LZ:
797 assert(devinfo->gen >= 9);
798 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_LD_LZ;
799 break;
800 case SHADER_OPCODE_TXF_CMS_W:
801 assert(devinfo->gen >= 9);
802 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_LD2DMS_W;
803 break;
804 case SHADER_OPCODE_TXF_CMS:
805 if (devinfo->gen >= 7)
806 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD2DMS;
807 else
808 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
809 break;
810 case SHADER_OPCODE_TXF_UMS:
811 assert(devinfo->gen >= 7);
812 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD2DSS;
813 break;
814 case SHADER_OPCODE_TXF_MCS:
815 assert(devinfo->gen >= 7);
816 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD_MCS;
817 break;
818 case SHADER_OPCODE_LOD:
819 msg_type = GEN5_SAMPLER_MESSAGE_LOD;
820 break;
821 case SHADER_OPCODE_TG4:
822 if (inst->shadow_compare) {
823 assert(devinfo->gen >= 7);
824 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_C;
825 } else {
826 assert(devinfo->gen >= 6);
827 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4;
828 }
829 break;
830 case SHADER_OPCODE_TG4_OFFSET:
831 assert(devinfo->gen >= 7);
832 if (inst->shadow_compare) {
833 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_PO_C;
834 } else {
835 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_PO;
836 }
837 break;
838 case SHADER_OPCODE_SAMPLEINFO:
839 msg_type = GEN6_SAMPLER_MESSAGE_SAMPLE_SAMPLEINFO;
840 break;
841 default:
842 unreachable("not reached");
843 }
844 } else {
845 switch (inst->opcode) {
846 case SHADER_OPCODE_TEX:
847 /* Note that G45 and older determines shadow compare and dispatch width
848 * from message length for most messages.
849 */
850 if (inst->exec_size == 8) {
851 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE;
852 if (inst->shadow_compare) {
853 assert(inst->mlen == 6);
854 } else {
855 assert(inst->mlen <= 4);
856 }
857 } else {
858 if (inst->shadow_compare) {
859 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE_COMPARE;
860 assert(inst->mlen == 9);
861 } else {
862 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE;
863 assert(inst->mlen <= 7 && inst->mlen % 2 == 1);
864 }
865 }
866 break;
867 case FS_OPCODE_TXB:
868 if (inst->shadow_compare) {
869 assert(inst->exec_size == 8);
870 assert(inst->mlen == 6);
871 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE_BIAS_COMPARE;
872 } else {
873 assert(inst->mlen == 9);
874 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE_BIAS;
875 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
876 }
877 break;
878 case SHADER_OPCODE_TXL:
879 if (inst->shadow_compare) {
880 assert(inst->exec_size == 8);
881 assert(inst->mlen == 6);
882 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE_LOD_COMPARE;
883 } else {
884 assert(inst->mlen == 9);
885 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE_LOD;
886 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
887 }
888 break;
889 case SHADER_OPCODE_TXD:
890 /* There is no sample_d_c message; comparisons are done manually */
891 assert(inst->exec_size == 8);
892 assert(inst->mlen == 7 || inst->mlen == 10);
893 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE_GRADIENTS;
894 break;
895 case SHADER_OPCODE_TXF:
896 assert(inst->mlen <= 9 && inst->mlen % 2 == 1);
897 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_LD;
898 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
899 break;
900 case SHADER_OPCODE_TXS:
901 assert(inst->mlen == 3);
902 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_RESINFO;
903 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
904 break;
905 default:
906 unreachable("not reached");
907 }
908 }
909 assert(msg_type != -1);
910
911 if (simd_mode == BRW_SAMPLER_SIMD_MODE_SIMD16) {
912 dst = vec16(dst);
913 }
914
915 assert(devinfo->gen < 7 || inst->header_size == 0 ||
916 src.file == BRW_GENERAL_REGISTER_FILE);
917
918 assert(sampler_index.type == BRW_REGISTER_TYPE_UD);
919
920 /* Load the message header if present. If there's a texture offset,
921 * we need to set it up explicitly and load the offset bitfield.
922 * Otherwise, we can use an implied move from g0 to the first message reg.
923 */
924 if (inst->header_size != 0) {
925 if (devinfo->gen < 6 && !inst->offset) {
926 /* Set up an implied move from g0 to the MRF. */
927 src = retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UW);
928 } else {
929 struct brw_reg header_reg;
930
931 if (devinfo->gen >= 7) {
932 header_reg = src;
933 } else {
934 assert(inst->base_mrf != -1);
935 header_reg = brw_message_reg(inst->base_mrf);
936 }
937
938 brw_push_insn_state(p);
939 brw_set_default_exec_size(p, BRW_EXECUTE_8);
940 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
941 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
942 /* Explicitly set up the message header by copying g0 to the MRF. */
943 brw_MOV(p, header_reg, brw_vec8_grf(0, 0));
944
945 if (inst->offset) {
946 /* Set the offset bits in DWord 2. */
947 brw_MOV(p, get_element_ud(header_reg, 2),
948 brw_imm_ud(inst->offset));
949 } else if (stage != MESA_SHADER_VERTEX &&
950 stage != MESA_SHADER_FRAGMENT) {
951 /* The vertex and fragment stages have g0.2 set to 0, so
952 * header0.2 is 0 when g0 is copied. Other stages may not, so we
953 * must set it to 0 to avoid setting undesirable bits in the
954 * message.
955 */
956 brw_MOV(p, get_element_ud(header_reg, 2), brw_imm_ud(0));
957 }
958
959 brw_adjust_sampler_state_pointer(p, header_reg, sampler_index);
960 brw_pop_insn_state(p);
961 }
962 }
963
964 uint32_t base_binding_table_index = (inst->opcode == SHADER_OPCODE_TG4 ||
965 inst->opcode == SHADER_OPCODE_TG4_OFFSET)
966 ? prog_data->binding_table.gather_texture_start
967 : prog_data->binding_table.texture_start;
968
969 if (surface_index.file == BRW_IMMEDIATE_VALUE &&
970 sampler_index.file == BRW_IMMEDIATE_VALUE) {
971 uint32_t surface = surface_index.ud;
972 uint32_t sampler = sampler_index.ud;
973
974 brw_SAMPLE(p,
975 retype(dst, BRW_REGISTER_TYPE_UW),
976 inst->base_mrf,
977 src,
978 surface + base_binding_table_index,
979 sampler % 16,
980 msg_type,
981 inst->size_written / REG_SIZE,
982 inst->mlen,
983 inst->header_size != 0,
984 simd_mode,
985 return_format);
986
987 brw_mark_surface_used(prog_data, surface + base_binding_table_index);
988 } else {
989 /* Non-const sampler index */
990
991 struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
992 struct brw_reg surface_reg = vec1(retype(surface_index, BRW_REGISTER_TYPE_UD));
993 struct brw_reg sampler_reg = vec1(retype(sampler_index, BRW_REGISTER_TYPE_UD));
994
995 brw_push_insn_state(p);
996 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
997 brw_set_default_access_mode(p, BRW_ALIGN_1);
998
999 if (brw_regs_equal(&surface_reg, &sampler_reg)) {
1000 brw_MUL(p, addr, sampler_reg, brw_imm_uw(0x101));
1001 } else {
1002 if (sampler_reg.file == BRW_IMMEDIATE_VALUE) {
1003 brw_OR(p, addr, surface_reg, brw_imm_ud(sampler_reg.ud << 8));
1004 } else {
1005 brw_SHL(p, addr, sampler_reg, brw_imm_ud(8));
1006 brw_OR(p, addr, addr, surface_reg);
1007 }
1008 }
1009 if (base_binding_table_index)
1010 brw_ADD(p, addr, addr, brw_imm_ud(base_binding_table_index));
1011 brw_AND(p, addr, addr, brw_imm_ud(0xfff));
1012
1013 brw_pop_insn_state(p);
1014
1015 /* dst = send(offset, a0.0 | <descriptor>) */
1016 brw_inst *insn = brw_send_indirect_message(
1017 p, BRW_SFID_SAMPLER, dst, src, addr);
1018 brw_set_sampler_message(p, insn,
1019 0 /* surface */,
1020 0 /* sampler */,
1021 msg_type,
1022 inst->size_written / REG_SIZE,
1023 inst->mlen /* mlen */,
1024 inst->header_size != 0 /* header */,
1025 simd_mode,
1026 return_format);
1027
1028 /* visitor knows more than we do about the surface limit required,
1029 * so has already done marking.
1030 */
1031 }
1032
1033 if (is_combined_send) {
1034 brw_inst_set_eot(p->devinfo, brw_last_inst, true);
1035 brw_inst_set_opcode(p->devinfo, brw_last_inst, BRW_OPCODE_SENDC);
1036 }
1037 }
1038
1039
1040 /* For OPCODE_DDX and OPCODE_DDY, per channel of output we've got input
1041 * looking like:
1042 *
1043 * arg0: ss0.tl ss0.tr ss0.bl ss0.br ss1.tl ss1.tr ss1.bl ss1.br
1044 *
1045 * Ideally, we want to produce:
1046 *
1047 * DDX DDY
1048 * dst: (ss0.tr - ss0.tl) (ss0.tl - ss0.bl)
1049 * (ss0.tr - ss0.tl) (ss0.tr - ss0.br)
1050 * (ss0.br - ss0.bl) (ss0.tl - ss0.bl)
1051 * (ss0.br - ss0.bl) (ss0.tr - ss0.br)
1052 * (ss1.tr - ss1.tl) (ss1.tl - ss1.bl)
1053 * (ss1.tr - ss1.tl) (ss1.tr - ss1.br)
1054 * (ss1.br - ss1.bl) (ss1.tl - ss1.bl)
1055 * (ss1.br - ss1.bl) (ss1.tr - ss1.br)
1056 *
1057 * and add another set of two more subspans if in 16-pixel dispatch mode.
1058 *
1059 * For DDX, it ends up being easy: width = 2, horiz=0 gets us the same result
1060 * for each pair, and vertstride = 2 jumps us 2 elements after processing a
1061 * pair. But the ideal approximation may impose a huge performance cost on
1062 * sample_d. On at least Haswell, sample_d instruction does some
1063 * optimizations if the same LOD is used for all pixels in the subspan.
1064 *
1065 * For DDY, we need to use ALIGN16 mode since it's capable of doing the
1066 * appropriate swizzling.
1067 */
1068 void
1069 fs_generator::generate_ddx(enum opcode opcode,
1070 struct brw_reg dst, struct brw_reg src)
1071 {
1072 unsigned vstride, width;
1073
1074 if (opcode == FS_OPCODE_DDX_FINE) {
1075 /* produce accurate derivatives */
1076 vstride = BRW_VERTICAL_STRIDE_2;
1077 width = BRW_WIDTH_2;
1078 } else {
1079 /* replicate the derivative at the top-left pixel to other pixels */
1080 vstride = BRW_VERTICAL_STRIDE_4;
1081 width = BRW_WIDTH_4;
1082 }
1083
1084 struct brw_reg src0 = brw_reg(src.file, src.nr, 1,
1085 src.negate, src.abs,
1086 BRW_REGISTER_TYPE_F,
1087 vstride,
1088 width,
1089 BRW_HORIZONTAL_STRIDE_0,
1090 BRW_SWIZZLE_XYZW, WRITEMASK_XYZW);
1091 struct brw_reg src1 = brw_reg(src.file, src.nr, 0,
1092 src.negate, src.abs,
1093 BRW_REGISTER_TYPE_F,
1094 vstride,
1095 width,
1096 BRW_HORIZONTAL_STRIDE_0,
1097 BRW_SWIZZLE_XYZW, WRITEMASK_XYZW);
1098 brw_ADD(p, dst, src0, negate(src1));
1099 }
1100
1101 /* The negate_value boolean is used to negate the derivative computation for
1102 * FBOs, since they place the origin at the upper left instead of the lower
1103 * left.
1104 */
1105 void
1106 fs_generator::generate_ddy(enum opcode opcode,
1107 struct brw_reg dst, struct brw_reg src)
1108 {
1109 if (opcode == FS_OPCODE_DDY_FINE) {
1110 /* produce accurate derivatives */
1111 struct brw_reg src0 = brw_reg(src.file, src.nr, 0,
1112 src.negate, src.abs,
1113 BRW_REGISTER_TYPE_F,
1114 BRW_VERTICAL_STRIDE_4,
1115 BRW_WIDTH_4,
1116 BRW_HORIZONTAL_STRIDE_1,
1117 BRW_SWIZZLE_XYXY, WRITEMASK_XYZW);
1118 struct brw_reg src1 = brw_reg(src.file, src.nr, 0,
1119 src.negate, src.abs,
1120 BRW_REGISTER_TYPE_F,
1121 BRW_VERTICAL_STRIDE_4,
1122 BRW_WIDTH_4,
1123 BRW_HORIZONTAL_STRIDE_1,
1124 BRW_SWIZZLE_ZWZW, WRITEMASK_XYZW);
1125 brw_push_insn_state(p);
1126 brw_set_default_access_mode(p, BRW_ALIGN_16);
1127 brw_ADD(p, dst, negate(src0), src1);
1128 brw_pop_insn_state(p);
1129 } else {
1130 /* replicate the derivative at the top-left pixel to other pixels */
1131 struct brw_reg src0 = brw_reg(src.file, src.nr, 0,
1132 src.negate, src.abs,
1133 BRW_REGISTER_TYPE_F,
1134 BRW_VERTICAL_STRIDE_4,
1135 BRW_WIDTH_4,
1136 BRW_HORIZONTAL_STRIDE_0,
1137 BRW_SWIZZLE_XYZW, WRITEMASK_XYZW);
1138 struct brw_reg src1 = brw_reg(src.file, src.nr, 2,
1139 src.negate, src.abs,
1140 BRW_REGISTER_TYPE_F,
1141 BRW_VERTICAL_STRIDE_4,
1142 BRW_WIDTH_4,
1143 BRW_HORIZONTAL_STRIDE_0,
1144 BRW_SWIZZLE_XYZW, WRITEMASK_XYZW);
1145 brw_ADD(p, dst, negate(src0), src1);
1146 }
1147 }
1148
1149 void
1150 fs_generator::generate_discard_jump(fs_inst *inst)
1151 {
1152 assert(devinfo->gen >= 6);
1153
1154 /* This HALT will be patched up at FB write time to point UIP at the end of
1155 * the program, and at brw_uip_jip() JIP will be set to the end of the
1156 * current block (or the program).
1157 */
1158 this->discard_halt_patches.push_tail(new(mem_ctx) ip_record(p->nr_insn));
1159 gen6_HALT(p);
1160 }
1161
1162 void
1163 fs_generator::generate_scratch_write(fs_inst *inst, struct brw_reg src)
1164 {
1165 /* The 32-wide messages only respect the first 16-wide half of the channel
1166 * enable signals which are replicated identically for the second group of
1167 * 16 channels, so we cannot use them unless the write is marked
1168 * force_writemask_all.
1169 */
1170 const unsigned lower_size = inst->force_writemask_all ? inst->exec_size :
1171 MIN2(16, inst->exec_size);
1172 const unsigned block_size = 4 * lower_size / REG_SIZE;
1173 assert(inst->mlen != 0);
1174
1175 brw_push_insn_state(p);
1176 brw_set_default_exec_size(p, cvt(lower_size) - 1);
1177 brw_set_default_compression(p, lower_size > 8);
1178
1179 for (unsigned i = 0; i < inst->exec_size / lower_size; i++) {
1180 brw_set_default_group(p, inst->group + lower_size * i);
1181
1182 brw_MOV(p, brw_uvec_mrf(lower_size, inst->base_mrf + 1, 0),
1183 retype(offset(src, block_size * i), BRW_REGISTER_TYPE_UD));
1184
1185 brw_oword_block_write_scratch(p, brw_message_reg(inst->base_mrf),
1186 block_size,
1187 inst->offset + block_size * REG_SIZE * i);
1188 }
1189
1190 brw_pop_insn_state(p);
1191 }
1192
1193 void
1194 fs_generator::generate_scratch_read(fs_inst *inst, struct brw_reg dst)
1195 {
1196 assert(inst->exec_size <= 16 || inst->force_writemask_all);
1197 assert(inst->mlen != 0);
1198
1199 brw_oword_block_read_scratch(p, dst, brw_message_reg(inst->base_mrf),
1200 inst->exec_size / 8, inst->offset);
1201 }
1202
1203 void
1204 fs_generator::generate_scratch_read_gen7(fs_inst *inst, struct brw_reg dst)
1205 {
1206 assert(inst->exec_size <= 16 || inst->force_writemask_all);
1207
1208 gen7_block_read_scratch(p, dst, inst->exec_size / 8, inst->offset);
1209 }
1210
1211 void
1212 fs_generator::generate_uniform_pull_constant_load(fs_inst *inst,
1213 struct brw_reg dst,
1214 struct brw_reg index,
1215 struct brw_reg offset)
1216 {
1217 assert(type_sz(dst.type) == 4);
1218 assert(inst->mlen != 0);
1219
1220 assert(index.file == BRW_IMMEDIATE_VALUE &&
1221 index.type == BRW_REGISTER_TYPE_UD);
1222 uint32_t surf_index = index.ud;
1223
1224 assert(offset.file == BRW_IMMEDIATE_VALUE &&
1225 offset.type == BRW_REGISTER_TYPE_UD);
1226 uint32_t read_offset = offset.ud;
1227
1228 brw_oword_block_read(p, dst, brw_message_reg(inst->base_mrf),
1229 read_offset, surf_index);
1230 }
1231
1232 void
1233 fs_generator::generate_uniform_pull_constant_load_gen7(fs_inst *inst,
1234 struct brw_reg dst,
1235 struct brw_reg index,
1236 struct brw_reg payload)
1237 {
1238 assert(index.type == BRW_REGISTER_TYPE_UD);
1239 assert(payload.file == BRW_GENERAL_REGISTER_FILE);
1240 assert(type_sz(dst.type) == 4);
1241
1242 if (index.file == BRW_IMMEDIATE_VALUE) {
1243 const uint32_t surf_index = index.ud;
1244
1245 brw_push_insn_state(p);
1246 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1247 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
1248 brw_pop_insn_state(p);
1249
1250 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UD));
1251 brw_set_src0(p, send, retype(payload, BRW_REGISTER_TYPE_UD));
1252 brw_set_dp_read_message(p, send, surf_index,
1253 BRW_DATAPORT_OWORD_BLOCK_DWORDS(inst->exec_size),
1254 GEN7_DATAPORT_DC_OWORD_BLOCK_READ,
1255 GEN6_SFID_DATAPORT_CONSTANT_CACHE,
1256 1, /* mlen */
1257 true, /* header */
1258 DIV_ROUND_UP(inst->size_written, REG_SIZE));
1259
1260 } else {
1261 struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
1262
1263 brw_push_insn_state(p);
1264 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1265
1266 /* a0.0 = surf_index & 0xff */
1267 brw_inst *insn_and = brw_next_insn(p, BRW_OPCODE_AND);
1268 brw_inst_set_exec_size(p->devinfo, insn_and, BRW_EXECUTE_1);
1269 brw_set_dest(p, insn_and, addr);
1270 brw_set_src0(p, insn_and, vec1(retype(index, BRW_REGISTER_TYPE_UD)));
1271 brw_set_src1(p, insn_and, brw_imm_ud(0x0ff));
1272
1273 /* dst = send(payload, a0.0 | <descriptor>) */
1274 brw_inst *insn = brw_send_indirect_message(
1275 p, GEN6_SFID_DATAPORT_CONSTANT_CACHE,
1276 retype(dst, BRW_REGISTER_TYPE_UD),
1277 retype(payload, BRW_REGISTER_TYPE_UD), addr);
1278 brw_set_dp_read_message(p, insn, 0 /* surface */,
1279 BRW_DATAPORT_OWORD_BLOCK_DWORDS(inst->exec_size),
1280 GEN7_DATAPORT_DC_OWORD_BLOCK_READ,
1281 GEN6_SFID_DATAPORT_CONSTANT_CACHE,
1282 1, /* mlen */
1283 true, /* header */
1284 DIV_ROUND_UP(inst->size_written, REG_SIZE));
1285
1286 brw_pop_insn_state(p);
1287 }
1288 }
1289
1290 void
1291 fs_generator::generate_varying_pull_constant_load_gen4(fs_inst *inst,
1292 struct brw_reg dst,
1293 struct brw_reg index)
1294 {
1295 assert(devinfo->gen < 7); /* Should use the gen7 variant. */
1296 assert(inst->header_size != 0);
1297 assert(inst->mlen);
1298
1299 assert(index.file == BRW_IMMEDIATE_VALUE &&
1300 index.type == BRW_REGISTER_TYPE_UD);
1301 uint32_t surf_index = index.ud;
1302
1303 uint32_t simd_mode, rlen, msg_type;
1304 if (inst->exec_size == 16) {
1305 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1306 rlen = 8;
1307 } else {
1308 assert(inst->exec_size == 8);
1309 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
1310 rlen = 4;
1311 }
1312
1313 if (devinfo->gen >= 5)
1314 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
1315 else {
1316 /* We always use the SIMD16 message so that we only have to load U, and
1317 * not V or R.
1318 */
1319 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_LD;
1320 assert(inst->mlen == 3);
1321 assert(inst->size_written == 8 * REG_SIZE);
1322 rlen = 8;
1323 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1324 }
1325
1326 struct brw_reg header = brw_vec8_grf(0, 0);
1327 gen6_resolve_implied_move(p, &header, inst->base_mrf);
1328
1329 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
1330 brw_inst_set_compression(devinfo, send, false);
1331 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UW));
1332 brw_set_src0(p, send, header);
1333 if (devinfo->gen < 6)
1334 brw_inst_set_base_mrf(p->devinfo, send, inst->base_mrf);
1335
1336 /* Our surface is set up as floats, regardless of what actual data is
1337 * stored in it.
1338 */
1339 uint32_t return_format = BRW_SAMPLER_RETURN_FORMAT_FLOAT32;
1340 brw_set_sampler_message(p, send,
1341 surf_index,
1342 0, /* sampler (unused) */
1343 msg_type,
1344 rlen,
1345 inst->mlen,
1346 inst->header_size != 0,
1347 simd_mode,
1348 return_format);
1349 }
1350
1351 void
1352 fs_generator::generate_varying_pull_constant_load_gen7(fs_inst *inst,
1353 struct brw_reg dst,
1354 struct brw_reg index,
1355 struct brw_reg offset)
1356 {
1357 assert(devinfo->gen >= 7);
1358 /* Varying-offset pull constant loads are treated as a normal expression on
1359 * gen7, so the fact that it's a send message is hidden at the IR level.
1360 */
1361 assert(inst->header_size == 0);
1362 assert(!inst->mlen);
1363 assert(index.type == BRW_REGISTER_TYPE_UD);
1364
1365 uint32_t simd_mode, rlen, mlen;
1366 if (inst->exec_size == 16) {
1367 mlen = 2;
1368 rlen = 8;
1369 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1370 } else {
1371 assert(inst->exec_size == 8);
1372 mlen = 1;
1373 rlen = 4;
1374 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
1375 }
1376
1377 if (index.file == BRW_IMMEDIATE_VALUE) {
1378
1379 uint32_t surf_index = index.ud;
1380
1381 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
1382 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UW));
1383 brw_set_src0(p, send, offset);
1384 brw_set_sampler_message(p, send,
1385 surf_index,
1386 0, /* LD message ignores sampler unit */
1387 GEN5_SAMPLER_MESSAGE_SAMPLE_LD,
1388 rlen,
1389 mlen,
1390 false, /* no header */
1391 simd_mode,
1392 0);
1393
1394 } else {
1395
1396 struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
1397
1398 brw_push_insn_state(p);
1399 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1400
1401 /* a0.0 = surf_index & 0xff */
1402 brw_inst *insn_and = brw_next_insn(p, BRW_OPCODE_AND);
1403 brw_inst_set_exec_size(p->devinfo, insn_and, BRW_EXECUTE_1);
1404 brw_set_dest(p, insn_and, addr);
1405 brw_set_src0(p, insn_and, vec1(retype(index, BRW_REGISTER_TYPE_UD)));
1406 brw_set_src1(p, insn_and, brw_imm_ud(0x0ff));
1407
1408 brw_pop_insn_state(p);
1409
1410 /* dst = send(offset, a0.0 | <descriptor>) */
1411 brw_inst *insn = brw_send_indirect_message(
1412 p, BRW_SFID_SAMPLER, retype(dst, BRW_REGISTER_TYPE_UW),
1413 offset, addr);
1414 brw_set_sampler_message(p, insn,
1415 0 /* surface */,
1416 0 /* sampler */,
1417 GEN5_SAMPLER_MESSAGE_SAMPLE_LD,
1418 rlen /* rlen */,
1419 mlen /* mlen */,
1420 false /* header */,
1421 simd_mode,
1422 0);
1423 }
1424 }
1425
1426 /**
1427 * Cause the current pixel/sample mask (from R1.7 bits 15:0) to be transferred
1428 * into the flags register (f0.0).
1429 *
1430 * Used only on Gen6 and above.
1431 */
1432 void
1433 fs_generator::generate_mov_dispatch_to_flags(fs_inst *inst)
1434 {
1435 struct brw_reg flags = brw_flag_reg(0, inst->flag_subreg);
1436 struct brw_reg dispatch_mask;
1437
1438 if (devinfo->gen >= 6)
1439 dispatch_mask = retype(brw_vec1_grf(1, 7), BRW_REGISTER_TYPE_UW);
1440 else
1441 dispatch_mask = retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UW);
1442
1443 brw_push_insn_state(p);
1444 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1445 brw_MOV(p, flags, dispatch_mask);
1446 brw_pop_insn_state(p);
1447 }
1448
1449 void
1450 fs_generator::generate_pixel_interpolator_query(fs_inst *inst,
1451 struct brw_reg dst,
1452 struct brw_reg src,
1453 struct brw_reg msg_data,
1454 unsigned msg_type)
1455 {
1456 assert(inst->size_written % REG_SIZE == 0);
1457 assert(msg_data.type == BRW_REGISTER_TYPE_UD);
1458
1459 brw_pixel_interpolator_query(p,
1460 retype(dst, BRW_REGISTER_TYPE_UW),
1461 src,
1462 inst->pi_noperspective,
1463 msg_type,
1464 msg_data,
1465 inst->mlen,
1466 inst->size_written / REG_SIZE);
1467 }
1468
1469 /* Sets vstride=1, width=4, hstride=0 of register src1 during
1470 * the ADD instruction.
1471 */
1472 void
1473 fs_generator::generate_set_sample_id(fs_inst *inst,
1474 struct brw_reg dst,
1475 struct brw_reg src0,
1476 struct brw_reg src1)
1477 {
1478 assert(dst.type == BRW_REGISTER_TYPE_D ||
1479 dst.type == BRW_REGISTER_TYPE_UD);
1480 assert(src0.type == BRW_REGISTER_TYPE_D ||
1481 src0.type == BRW_REGISTER_TYPE_UD);
1482
1483 struct brw_reg reg = stride(src1, 1, 4, 0);
1484 if (devinfo->gen >= 8 || inst->exec_size == 8) {
1485 brw_ADD(p, dst, src0, reg);
1486 } else if (inst->exec_size == 16) {
1487 brw_push_insn_state(p);
1488 brw_set_default_exec_size(p, BRW_EXECUTE_8);
1489 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
1490 brw_ADD(p, firsthalf(dst), firsthalf(src0), reg);
1491 brw_set_default_compression_control(p, BRW_COMPRESSION_2NDHALF);
1492 brw_ADD(p, sechalf(dst), sechalf(src0), suboffset(reg, 2));
1493 brw_pop_insn_state(p);
1494 }
1495 }
1496
1497 void
1498 fs_generator::generate_pack_half_2x16_split(fs_inst *inst,
1499 struct brw_reg dst,
1500 struct brw_reg x,
1501 struct brw_reg y)
1502 {
1503 assert(devinfo->gen >= 7);
1504 assert(dst.type == BRW_REGISTER_TYPE_UD);
1505 assert(x.type == BRW_REGISTER_TYPE_F);
1506 assert(y.type == BRW_REGISTER_TYPE_F);
1507
1508 /* From the Ivybridge PRM, Vol4, Part3, Section 6.27 f32to16:
1509 *
1510 * Because this instruction does not have a 16-bit floating-point type,
1511 * the destination data type must be Word (W).
1512 *
1513 * The destination must be DWord-aligned and specify a horizontal stride
1514 * (HorzStride) of 2. The 16-bit result is stored in the lower word of
1515 * each destination channel and the upper word is not modified.
1516 */
1517 struct brw_reg dst_w = spread(retype(dst, BRW_REGISTER_TYPE_W), 2);
1518
1519 /* Give each 32-bit channel of dst the form below, where "." means
1520 * unchanged.
1521 * 0x....hhhh
1522 */
1523 brw_F32TO16(p, dst_w, y);
1524
1525 /* Now the form:
1526 * 0xhhhh0000
1527 */
1528 brw_SHL(p, dst, dst, brw_imm_ud(16u));
1529
1530 /* And, finally the form of packHalf2x16's output:
1531 * 0xhhhhllll
1532 */
1533 brw_F32TO16(p, dst_w, x);
1534 }
1535
1536 void
1537 fs_generator::generate_unpack_half_2x16_split(fs_inst *inst,
1538 struct brw_reg dst,
1539 struct brw_reg src)
1540 {
1541 assert(devinfo->gen >= 7);
1542 assert(dst.type == BRW_REGISTER_TYPE_F);
1543 assert(src.type == BRW_REGISTER_TYPE_UD);
1544
1545 /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
1546 *
1547 * Because this instruction does not have a 16-bit floating-point type,
1548 * the source data type must be Word (W). The destination type must be
1549 * F (Float).
1550 */
1551 struct brw_reg src_w = spread(retype(src, BRW_REGISTER_TYPE_W), 2);
1552
1553 /* Each channel of src has the form of unpackHalf2x16's input: 0xhhhhllll.
1554 * For the Y case, we wish to access only the upper word; therefore
1555 * a 16-bit subregister offset is needed.
1556 */
1557 assert(inst->opcode == FS_OPCODE_UNPACK_HALF_2x16_SPLIT_X ||
1558 inst->opcode == FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y);
1559 if (inst->opcode == FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y)
1560 src_w.subnr += 2;
1561
1562 brw_F16TO32(p, dst, src_w);
1563 }
1564
1565 void
1566 fs_generator::generate_shader_time_add(fs_inst *inst,
1567 struct brw_reg payload,
1568 struct brw_reg offset,
1569 struct brw_reg value)
1570 {
1571 assert(devinfo->gen >= 7);
1572 brw_push_insn_state(p);
1573 brw_set_default_mask_control(p, true);
1574
1575 assert(payload.file == BRW_GENERAL_REGISTER_FILE);
1576 struct brw_reg payload_offset = retype(brw_vec1_grf(payload.nr, 0),
1577 offset.type);
1578 struct brw_reg payload_value = retype(brw_vec1_grf(payload.nr + 1, 0),
1579 value.type);
1580
1581 assert(offset.file == BRW_IMMEDIATE_VALUE);
1582 if (value.file == BRW_GENERAL_REGISTER_FILE) {
1583 value.width = BRW_WIDTH_1;
1584 value.hstride = BRW_HORIZONTAL_STRIDE_0;
1585 value.vstride = BRW_VERTICAL_STRIDE_0;
1586 } else {
1587 assert(value.file == BRW_IMMEDIATE_VALUE);
1588 }
1589
1590 /* Trying to deal with setup of the params from the IR is crazy in the FS8
1591 * case, and we don't really care about squeezing every bit of performance
1592 * out of this path, so we just emit the MOVs from here.
1593 */
1594 brw_MOV(p, payload_offset, offset);
1595 brw_MOV(p, payload_value, value);
1596 brw_shader_time_add(p, payload,
1597 prog_data->binding_table.shader_time_start);
1598 brw_pop_insn_state(p);
1599
1600 brw_mark_surface_used(prog_data,
1601 prog_data->binding_table.shader_time_start);
1602 }
1603
1604 void
1605 fs_generator::enable_debug(const char *shader_name)
1606 {
1607 debug_flag = true;
1608 this->shader_name = shader_name;
1609 }
1610
1611 int
1612 fs_generator::generate_code(const cfg_t *cfg, int dispatch_width)
1613 {
1614 /* align to 64 byte boundary. */
1615 while (p->next_insn_offset % 64)
1616 brw_NOP(p);
1617
1618 this->dispatch_width = dispatch_width;
1619
1620 int start_offset = p->next_insn_offset;
1621 int spill_count = 0, fill_count = 0;
1622 int loop_count = 0;
1623
1624 struct annotation_info annotation;
1625 memset(&annotation, 0, sizeof(annotation));
1626
1627 foreach_block_and_inst (block, fs_inst, inst, cfg) {
1628 struct brw_reg src[3], dst;
1629 unsigned int last_insn_offset = p->next_insn_offset;
1630 bool multiple_instructions_emitted = false;
1631
1632 /* From the Broadwell PRM, Volume 7, "3D-Media-GPGPU", in the
1633 * "Register Region Restrictions" section: for BDW, SKL:
1634 *
1635 * "A POW/FDIV operation must not be followed by an instruction
1636 * that requires two destination registers."
1637 *
1638 * The documentation is often lacking annotations for Atom parts,
1639 * and empirically this affects CHV as well.
1640 */
1641 if (devinfo->gen >= 8 &&
1642 p->nr_insn > 1 &&
1643 brw_inst_opcode(devinfo, brw_last_inst) == BRW_OPCODE_MATH &&
1644 brw_inst_math_function(devinfo, brw_last_inst) == BRW_MATH_FUNCTION_POW &&
1645 inst->dst.component_size(inst->exec_size) > REG_SIZE) {
1646 brw_NOP(p);
1647 last_insn_offset = p->next_insn_offset;
1648 }
1649
1650 if (unlikely(debug_flag))
1651 annotate(p->devinfo, &annotation, cfg, inst, p->next_insn_offset);
1652
1653 /* If the instruction writes to more than one register, it needs to be
1654 * explicitly marked as compressed on Gen <= 5. On Gen >= 6 the
1655 * hardware figures out by itself what the right compression mode is,
1656 * but we still need to know whether the instruction is compressed to
1657 * set up the source register regions appropriately.
1658 *
1659 * XXX - This is wrong for instructions that write a single register but
1660 * read more than one which should strictly speaking be treated as
1661 * compressed. For instructions that don't write any registers it
1662 * relies on the destination being a null register of the correct
1663 * type and regioning so the instruction is considered compressed
1664 * or not accordingly.
1665 */
1666 const bool compressed =
1667 inst->dst.component_size(inst->exec_size) > REG_SIZE;
1668 brw_set_default_compression(p, compressed);
1669 brw_set_default_group(p, inst->group);
1670
1671 for (unsigned int i = 0; i < inst->sources; i++) {
1672 src[i] = brw_reg_from_fs_reg(devinfo, inst,
1673 &inst->src[i], compressed);
1674 /* The accumulator result appears to get used for the
1675 * conditional modifier generation. When negating a UD
1676 * value, there is a 33rd bit generated for the sign in the
1677 * accumulator value, so now you can't check, for example,
1678 * equality with a 32-bit value. See piglit fs-op-neg-uvec4.
1679 */
1680 assert(!inst->conditional_mod ||
1681 inst->src[i].type != BRW_REGISTER_TYPE_UD ||
1682 !inst->src[i].negate);
1683 }
1684 dst = brw_reg_from_fs_reg(devinfo, inst,
1685 &inst->dst, compressed);
1686
1687 brw_set_default_access_mode(p, BRW_ALIGN_1);
1688 brw_set_default_predicate_control(p, inst->predicate);
1689 brw_set_default_predicate_inverse(p, inst->predicate_inverse);
1690 brw_set_default_flag_reg(p, 0, inst->flag_subreg);
1691 brw_set_default_saturate(p, inst->saturate);
1692 brw_set_default_mask_control(p, inst->force_writemask_all);
1693 brw_set_default_acc_write_control(p, inst->writes_accumulator);
1694
1695 unsigned exec_size = inst->exec_size;
1696 if (devinfo->gen == 7 && !devinfo->is_haswell &&
1697 (get_exec_type_size(inst) == 8 || type_sz(inst->dst.type) == 8)) {
1698 exec_size *= 2;
1699 }
1700
1701 brw_set_default_exec_size(p, cvt(exec_size) - 1);
1702
1703 assert(inst->force_writemask_all || inst->exec_size >= 4);
1704 assert(inst->force_writemask_all || inst->group % inst->exec_size == 0);
1705 assert(inst->base_mrf + inst->mlen <= BRW_MAX_MRF(devinfo->gen));
1706 assert(inst->mlen <= BRW_MAX_MSG_LENGTH);
1707
1708 switch (inst->opcode) {
1709 case BRW_OPCODE_MOV:
1710 brw_MOV(p, dst, src[0]);
1711 break;
1712 case BRW_OPCODE_ADD:
1713 brw_ADD(p, dst, src[0], src[1]);
1714 break;
1715 case BRW_OPCODE_MUL:
1716 brw_MUL(p, dst, src[0], src[1]);
1717 break;
1718 case BRW_OPCODE_AVG:
1719 brw_AVG(p, dst, src[0], src[1]);
1720 break;
1721 case BRW_OPCODE_MACH:
1722 brw_MACH(p, dst, src[0], src[1]);
1723 break;
1724
1725 case BRW_OPCODE_LINE:
1726 brw_LINE(p, dst, src[0], src[1]);
1727 break;
1728
1729 case BRW_OPCODE_MAD:
1730 assert(devinfo->gen >= 6);
1731 brw_set_default_access_mode(p, BRW_ALIGN_16);
1732 brw_MAD(p, dst, src[0], src[1], src[2]);
1733 break;
1734
1735 case BRW_OPCODE_LRP:
1736 assert(devinfo->gen >= 6);
1737 brw_set_default_access_mode(p, BRW_ALIGN_16);
1738 brw_LRP(p, dst, src[0], src[1], src[2]);
1739 break;
1740
1741 case BRW_OPCODE_FRC:
1742 brw_FRC(p, dst, src[0]);
1743 break;
1744 case BRW_OPCODE_RNDD:
1745 brw_RNDD(p, dst, src[0]);
1746 break;
1747 case BRW_OPCODE_RNDE:
1748 brw_RNDE(p, dst, src[0]);
1749 break;
1750 case BRW_OPCODE_RNDZ:
1751 brw_RNDZ(p, dst, src[0]);
1752 break;
1753
1754 case BRW_OPCODE_AND:
1755 brw_AND(p, dst, src[0], src[1]);
1756 break;
1757 case BRW_OPCODE_OR:
1758 brw_OR(p, dst, src[0], src[1]);
1759 break;
1760 case BRW_OPCODE_XOR:
1761 brw_XOR(p, dst, src[0], src[1]);
1762 break;
1763 case BRW_OPCODE_NOT:
1764 brw_NOT(p, dst, src[0]);
1765 break;
1766 case BRW_OPCODE_ASR:
1767 brw_ASR(p, dst, src[0], src[1]);
1768 break;
1769 case BRW_OPCODE_SHR:
1770 brw_SHR(p, dst, src[0], src[1]);
1771 break;
1772 case BRW_OPCODE_SHL:
1773 brw_SHL(p, dst, src[0], src[1]);
1774 break;
1775 case BRW_OPCODE_F32TO16:
1776 assert(devinfo->gen >= 7);
1777 brw_F32TO16(p, dst, src[0]);
1778 break;
1779 case BRW_OPCODE_F16TO32:
1780 assert(devinfo->gen >= 7);
1781 brw_F16TO32(p, dst, src[0]);
1782 break;
1783 case BRW_OPCODE_CMP:
1784 if (inst->exec_size >= 16 && devinfo->gen == 7 && !devinfo->is_haswell &&
1785 dst.file == BRW_ARCHITECTURE_REGISTER_FILE) {
1786 /* For unknown reasons the WaCMPInstFlagDepClearedEarly workaround
1787 * implemented in the compiler is not sufficient. Overriding the
1788 * type when the destination is the null register is necessary but
1789 * not sufficient by itself.
1790 */
1791 assert(dst.nr == BRW_ARF_NULL);
1792 dst.type = BRW_REGISTER_TYPE_D;
1793 }
1794 brw_CMP(p, dst, inst->conditional_mod, src[0], src[1]);
1795 break;
1796 case BRW_OPCODE_SEL:
1797 brw_SEL(p, dst, src[0], src[1]);
1798 break;
1799 case BRW_OPCODE_BFREV:
1800 assert(devinfo->gen >= 7);
1801 brw_BFREV(p, retype(dst, BRW_REGISTER_TYPE_UD),
1802 retype(src[0], BRW_REGISTER_TYPE_UD));
1803 break;
1804 case BRW_OPCODE_FBH:
1805 assert(devinfo->gen >= 7);
1806 brw_FBH(p, retype(dst, src[0].type), src[0]);
1807 break;
1808 case BRW_OPCODE_FBL:
1809 assert(devinfo->gen >= 7);
1810 brw_FBL(p, retype(dst, BRW_REGISTER_TYPE_UD),
1811 retype(src[0], BRW_REGISTER_TYPE_UD));
1812 break;
1813 case BRW_OPCODE_LZD:
1814 brw_LZD(p, dst, src[0]);
1815 break;
1816 case BRW_OPCODE_CBIT:
1817 assert(devinfo->gen >= 7);
1818 brw_CBIT(p, retype(dst, BRW_REGISTER_TYPE_UD),
1819 retype(src[0], BRW_REGISTER_TYPE_UD));
1820 break;
1821 case BRW_OPCODE_ADDC:
1822 assert(devinfo->gen >= 7);
1823 brw_ADDC(p, dst, src[0], src[1]);
1824 break;
1825 case BRW_OPCODE_SUBB:
1826 assert(devinfo->gen >= 7);
1827 brw_SUBB(p, dst, src[0], src[1]);
1828 break;
1829 case BRW_OPCODE_MAC:
1830 brw_MAC(p, dst, src[0], src[1]);
1831 break;
1832
1833 case BRW_OPCODE_BFE:
1834 assert(devinfo->gen >= 7);
1835 brw_set_default_access_mode(p, BRW_ALIGN_16);
1836 brw_BFE(p, dst, src[0], src[1], src[2]);
1837 break;
1838
1839 case BRW_OPCODE_BFI1:
1840 assert(devinfo->gen >= 7);
1841 brw_BFI1(p, dst, src[0], src[1]);
1842 break;
1843 case BRW_OPCODE_BFI2:
1844 assert(devinfo->gen >= 7);
1845 brw_set_default_access_mode(p, BRW_ALIGN_16);
1846 brw_BFI2(p, dst, src[0], src[1], src[2]);
1847 break;
1848
1849 case BRW_OPCODE_IF:
1850 if (inst->src[0].file != BAD_FILE) {
1851 /* The instruction has an embedded compare (only allowed on gen6) */
1852 assert(devinfo->gen == 6);
1853 gen6_IF(p, inst->conditional_mod, src[0], src[1]);
1854 } else {
1855 brw_IF(p, brw_inst_exec_size(devinfo, p->current));
1856 }
1857 break;
1858
1859 case BRW_OPCODE_ELSE:
1860 brw_ELSE(p);
1861 break;
1862 case BRW_OPCODE_ENDIF:
1863 brw_ENDIF(p);
1864 break;
1865
1866 case BRW_OPCODE_DO:
1867 brw_DO(p, brw_inst_exec_size(devinfo, p->current));
1868 break;
1869
1870 case BRW_OPCODE_BREAK:
1871 brw_BREAK(p);
1872 break;
1873 case BRW_OPCODE_CONTINUE:
1874 brw_CONT(p);
1875 break;
1876
1877 case BRW_OPCODE_WHILE:
1878 brw_WHILE(p);
1879 loop_count++;
1880 break;
1881
1882 case SHADER_OPCODE_RCP:
1883 case SHADER_OPCODE_RSQ:
1884 case SHADER_OPCODE_SQRT:
1885 case SHADER_OPCODE_EXP2:
1886 case SHADER_OPCODE_LOG2:
1887 case SHADER_OPCODE_SIN:
1888 case SHADER_OPCODE_COS:
1889 assert(inst->conditional_mod == BRW_CONDITIONAL_NONE);
1890 if (devinfo->gen >= 6) {
1891 assert(inst->mlen == 0);
1892 assert(devinfo->gen >= 7 || inst->exec_size == 8);
1893 gen6_math(p, dst, brw_math_function(inst->opcode),
1894 src[0], brw_null_reg());
1895 } else {
1896 assert(inst->mlen >= 1);
1897 assert(devinfo->gen == 5 || devinfo->is_g4x || inst->exec_size == 8);
1898 gen4_math(p, dst,
1899 brw_math_function(inst->opcode),
1900 inst->base_mrf, src[0],
1901 BRW_MATH_PRECISION_FULL);
1902 }
1903 break;
1904 case SHADER_OPCODE_INT_QUOTIENT:
1905 case SHADER_OPCODE_INT_REMAINDER:
1906 case SHADER_OPCODE_POW:
1907 assert(inst->conditional_mod == BRW_CONDITIONAL_NONE);
1908 if (devinfo->gen >= 6) {
1909 assert(inst->mlen == 0);
1910 assert((devinfo->gen >= 7 && inst->opcode == SHADER_OPCODE_POW) ||
1911 inst->exec_size == 8);
1912 gen6_math(p, dst, brw_math_function(inst->opcode), src[0], src[1]);
1913 } else {
1914 assert(inst->mlen >= 1);
1915 assert(inst->exec_size == 8);
1916 gen4_math(p, dst, brw_math_function(inst->opcode),
1917 inst->base_mrf, src[0],
1918 BRW_MATH_PRECISION_FULL);
1919 }
1920 break;
1921 case FS_OPCODE_CINTERP:
1922 brw_MOV(p, dst, src[0]);
1923 break;
1924 case FS_OPCODE_LINTERP:
1925 generate_linterp(inst, dst, src);
1926 break;
1927 case FS_OPCODE_PIXEL_X:
1928 assert(src[0].type == BRW_REGISTER_TYPE_UW);
1929 src[0].subnr = 0 * type_sz(src[0].type);
1930 brw_MOV(p, dst, stride(src[0], 8, 4, 1));
1931 break;
1932 case FS_OPCODE_PIXEL_Y:
1933 assert(src[0].type == BRW_REGISTER_TYPE_UW);
1934 src[0].subnr = 4 * type_sz(src[0].type);
1935 brw_MOV(p, dst, stride(src[0], 8, 4, 1));
1936 break;
1937 case FS_OPCODE_GET_BUFFER_SIZE:
1938 generate_get_buffer_size(inst, dst, src[0], src[1]);
1939 break;
1940 case SHADER_OPCODE_TEX:
1941 case FS_OPCODE_TXB:
1942 case SHADER_OPCODE_TXD:
1943 case SHADER_OPCODE_TXF:
1944 case SHADER_OPCODE_TXF_LZ:
1945 case SHADER_OPCODE_TXF_CMS:
1946 case SHADER_OPCODE_TXF_CMS_W:
1947 case SHADER_OPCODE_TXF_UMS:
1948 case SHADER_OPCODE_TXF_MCS:
1949 case SHADER_OPCODE_TXL:
1950 case SHADER_OPCODE_TXL_LZ:
1951 case SHADER_OPCODE_TXS:
1952 case SHADER_OPCODE_LOD:
1953 case SHADER_OPCODE_TG4:
1954 case SHADER_OPCODE_TG4_OFFSET:
1955 case SHADER_OPCODE_SAMPLEINFO:
1956 generate_tex(inst, dst, src[0], src[1], src[2]);
1957 break;
1958 case FS_OPCODE_DDX_COARSE:
1959 case FS_OPCODE_DDX_FINE:
1960 generate_ddx(inst->opcode, dst, src[0]);
1961 break;
1962 case FS_OPCODE_DDY_COARSE:
1963 case FS_OPCODE_DDY_FINE:
1964 generate_ddy(inst->opcode, dst, src[0]);
1965 break;
1966
1967 case SHADER_OPCODE_GEN4_SCRATCH_WRITE:
1968 generate_scratch_write(inst, src[0]);
1969 spill_count++;
1970 break;
1971
1972 case SHADER_OPCODE_GEN4_SCRATCH_READ:
1973 generate_scratch_read(inst, dst);
1974 fill_count++;
1975 break;
1976
1977 case SHADER_OPCODE_GEN7_SCRATCH_READ:
1978 generate_scratch_read_gen7(inst, dst);
1979 fill_count++;
1980 break;
1981
1982 case SHADER_OPCODE_MOV_INDIRECT:
1983 generate_mov_indirect(inst, dst, src[0], src[1]);
1984 break;
1985
1986 case SHADER_OPCODE_URB_READ_SIMD8:
1987 case SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT:
1988 generate_urb_read(inst, dst, src[0]);
1989 break;
1990
1991 case SHADER_OPCODE_URB_WRITE_SIMD8:
1992 case SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT:
1993 case SHADER_OPCODE_URB_WRITE_SIMD8_MASKED:
1994 case SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT:
1995 generate_urb_write(inst, src[0]);
1996 break;
1997
1998 case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD:
1999 assert(inst->force_writemask_all);
2000 generate_uniform_pull_constant_load(inst, dst, src[0], src[1]);
2001 break;
2002
2003 case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD_GEN7:
2004 assert(inst->force_writemask_all);
2005 generate_uniform_pull_constant_load_gen7(inst, dst, src[0], src[1]);
2006 break;
2007
2008 case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_GEN4:
2009 generate_varying_pull_constant_load_gen4(inst, dst, src[0]);
2010 break;
2011
2012 case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_GEN7:
2013 generate_varying_pull_constant_load_gen7(inst, dst, src[0], src[1]);
2014 break;
2015
2016 case FS_OPCODE_REP_FB_WRITE:
2017 case FS_OPCODE_FB_WRITE:
2018 generate_fb_write(inst, src[0]);
2019 break;
2020
2021 case FS_OPCODE_FB_READ:
2022 generate_fb_read(inst, dst, src[0]);
2023 break;
2024
2025 case FS_OPCODE_MOV_DISPATCH_TO_FLAGS:
2026 generate_mov_dispatch_to_flags(inst);
2027 break;
2028
2029 case FS_OPCODE_DISCARD_JUMP:
2030 generate_discard_jump(inst);
2031 break;
2032
2033 case SHADER_OPCODE_SHADER_TIME_ADD:
2034 generate_shader_time_add(inst, src[0], src[1], src[2]);
2035 break;
2036
2037 case SHADER_OPCODE_UNTYPED_ATOMIC:
2038 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2039 brw_untyped_atomic(p, dst, src[0], src[1], src[2].ud,
2040 inst->mlen, !inst->dst.is_null());
2041 break;
2042
2043 case SHADER_OPCODE_UNTYPED_SURFACE_READ:
2044 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2045 brw_untyped_surface_read(p, dst, src[0], src[1],
2046 inst->mlen, src[2].ud);
2047 break;
2048
2049 case SHADER_OPCODE_UNTYPED_SURFACE_WRITE:
2050 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2051 brw_untyped_surface_write(p, src[0], src[1],
2052 inst->mlen, src[2].ud);
2053 break;
2054
2055 case SHADER_OPCODE_TYPED_ATOMIC:
2056 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2057 brw_typed_atomic(p, dst, src[0], src[1],
2058 src[2].ud, inst->mlen, !inst->dst.is_null());
2059 break;
2060
2061 case SHADER_OPCODE_TYPED_SURFACE_READ:
2062 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2063 brw_typed_surface_read(p, dst, src[0], src[1],
2064 inst->mlen, src[2].ud);
2065 break;
2066
2067 case SHADER_OPCODE_TYPED_SURFACE_WRITE:
2068 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2069 brw_typed_surface_write(p, src[0], src[1], inst->mlen, src[2].ud);
2070 break;
2071
2072 case SHADER_OPCODE_MEMORY_FENCE:
2073 brw_memory_fence(p, dst);
2074 break;
2075
2076 case SHADER_OPCODE_FIND_LIVE_CHANNEL: {
2077 const struct brw_reg mask =
2078 brw_stage_has_packed_dispatch(devinfo, stage,
2079 prog_data) ? brw_imm_ud(~0u) :
2080 stage == MESA_SHADER_FRAGMENT ? brw_vmask_reg() :
2081 brw_dmask_reg();
2082 brw_find_live_channel(p, dst, mask);
2083 break;
2084 }
2085
2086 case SHADER_OPCODE_BROADCAST:
2087 assert(inst->force_writemask_all);
2088 brw_broadcast(p, dst, src[0], src[1]);
2089 break;
2090
2091 case FS_OPCODE_SET_SAMPLE_ID:
2092 generate_set_sample_id(inst, dst, src[0], src[1]);
2093 break;
2094
2095 case FS_OPCODE_PACK_HALF_2x16_SPLIT:
2096 generate_pack_half_2x16_split(inst, dst, src[0], src[1]);
2097 break;
2098
2099 case FS_OPCODE_UNPACK_HALF_2x16_SPLIT_X:
2100 case FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y:
2101 generate_unpack_half_2x16_split(inst, dst, src[0]);
2102 break;
2103
2104 case FS_OPCODE_PLACEHOLDER_HALT:
2105 /* This is the place where the final HALT needs to be inserted if
2106 * we've emitted any discards. If not, this will emit no code.
2107 */
2108 if (!patch_discard_jumps_to_fb_writes()) {
2109 if (unlikely(debug_flag)) {
2110 annotation.ann_count--;
2111 }
2112 }
2113 break;
2114
2115 case FS_OPCODE_INTERPOLATE_AT_SAMPLE:
2116 generate_pixel_interpolator_query(inst, dst, src[0], src[1],
2117 GEN7_PIXEL_INTERPOLATOR_LOC_SAMPLE);
2118 break;
2119
2120 case FS_OPCODE_INTERPOLATE_AT_SHARED_OFFSET:
2121 generate_pixel_interpolator_query(inst, dst, src[0], src[1],
2122 GEN7_PIXEL_INTERPOLATOR_LOC_SHARED_OFFSET);
2123 break;
2124
2125 case FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET:
2126 generate_pixel_interpolator_query(inst, dst, src[0], src[1],
2127 GEN7_PIXEL_INTERPOLATOR_LOC_PER_SLOT_OFFSET);
2128 break;
2129
2130 case CS_OPCODE_CS_TERMINATE:
2131 generate_cs_terminate(inst, src[0]);
2132 break;
2133
2134 case SHADER_OPCODE_BARRIER:
2135 generate_barrier(inst, src[0]);
2136 break;
2137
2138 case BRW_OPCODE_DIM:
2139 assert(devinfo->is_haswell);
2140 assert(src[0].type == BRW_REGISTER_TYPE_DF);
2141 assert(dst.type == BRW_REGISTER_TYPE_DF);
2142 brw_DIM(p, dst, retype(src[0], BRW_REGISTER_TYPE_F));
2143 break;
2144
2145 default:
2146 unreachable("Unsupported opcode");
2147
2148 case SHADER_OPCODE_LOAD_PAYLOAD:
2149 unreachable("Should be lowered by lower_load_payload()");
2150 }
2151
2152 if (multiple_instructions_emitted)
2153 continue;
2154
2155 if (inst->no_dd_clear || inst->no_dd_check || inst->conditional_mod) {
2156 assert(p->next_insn_offset == last_insn_offset + 16 ||
2157 !"conditional_mod, no_dd_check, or no_dd_clear set for IR "
2158 "emitting more than 1 instruction");
2159
2160 brw_inst *last = &p->store[last_insn_offset / 16];
2161
2162 if (inst->conditional_mod)
2163 brw_inst_set_cond_modifier(p->devinfo, last, inst->conditional_mod);
2164 brw_inst_set_no_dd_clear(p->devinfo, last, inst->no_dd_clear);
2165 brw_inst_set_no_dd_check(p->devinfo, last, inst->no_dd_check);
2166 }
2167 }
2168
2169 brw_set_uip_jip(p, start_offset);
2170 annotation_finalize(&annotation, p->next_insn_offset);
2171
2172 #ifndef NDEBUG
2173 bool validated = brw_validate_instructions(devinfo, p->store,
2174 start_offset,
2175 p->next_insn_offset,
2176 &annotation);
2177 #else
2178 if (unlikely(debug_flag))
2179 brw_validate_instructions(devinfo, p->store,
2180 start_offset,
2181 p->next_insn_offset,
2182 &annotation);
2183 #endif
2184
2185 int before_size = p->next_insn_offset - start_offset;
2186 brw_compact_instructions(p, start_offset, annotation.ann_count,
2187 annotation.ann);
2188 int after_size = p->next_insn_offset - start_offset;
2189
2190 if (unlikely(debug_flag)) {
2191 fprintf(stderr, "Native code for %s\n"
2192 "SIMD%d shader: %d instructions. %d loops. %u cycles. %d:%d spills:fills. Promoted %u constants. Compacted %d to %d"
2193 " bytes (%.0f%%)\n",
2194 shader_name, dispatch_width, before_size / 16, loop_count, cfg->cycle_count,
2195 spill_count, fill_count, promoted_constants, before_size, after_size,
2196 100.0f * (before_size - after_size) / before_size);
2197
2198 dump_assembly(p->store, annotation.ann_count, annotation.ann,
2199 p->devinfo);
2200 ralloc_free(annotation.mem_ctx);
2201 }
2202 assert(validated);
2203
2204 compiler->shader_debug_log(log_data,
2205 "%s SIMD%d shader: %d inst, %d loops, %u cycles, "
2206 "%d:%d spills:fills, Promoted %u constants, "
2207 "compacted %d to %d bytes.",
2208 _mesa_shader_stage_to_abbrev(stage),
2209 dispatch_width, before_size / 16,
2210 loop_count, cfg->cycle_count, spill_count,
2211 fill_count, promoted_constants, before_size,
2212 after_size);
2213
2214 return start_offset;
2215 }
2216
2217 const unsigned *
2218 fs_generator::get_assembly(unsigned int *assembly_size)
2219 {
2220 return brw_get_program(p, assembly_size);
2221 }