projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
anv/image: Fix VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT
[mesa.git] / src / intel / compiler / brw_fs_generator.cpp
1 /*
2 * Copyright © 2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 /** @file brw_fs_generator.cpp
25 *
26 * This file supports generating code from the FS LIR to the actual
27 * native instructions.
28 */
29
30 #include "brw_eu.h"
31 #include "brw_fs.h"
32 #include "brw_cfg.h"
33
34 static enum brw_reg_file
35 brw_file_from_reg(fs_reg *reg)
36 {
37 switch (reg->file) {
38 case ARF:
39 return BRW_ARCHITECTURE_REGISTER_FILE;
40 case FIXED_GRF:
41 case VGRF:
42 return BRW_GENERAL_REGISTER_FILE;
43 case MRF:
44 return BRW_MESSAGE_REGISTER_FILE;
45 case IMM:
46 return BRW_IMMEDIATE_VALUE;
47 case BAD_FILE:
48 case ATTR:
49 case UNIFORM:
50 unreachable("not reached");
51 }
52 return BRW_ARCHITECTURE_REGISTER_FILE;
53 }
54
55 static struct brw_reg
56 brw_reg_from_fs_reg(const struct gen_device_info *devinfo, fs_inst *inst,
57 fs_reg *reg, bool compressed)
58 {
59 struct brw_reg brw_reg;
60
61 switch (reg->file) {
62 case MRF:
63 assert((reg->nr & ~BRW_MRF_COMPR4) < BRW_MAX_MRF(devinfo->gen));
64 /* Fallthrough */
65 case VGRF:
66 if (reg->stride == 0) {
67 brw_reg = brw_vec1_reg(brw_file_from_reg(reg), reg->nr, 0);
68 } else {
69 /* From the Haswell PRM:
70 *
71 * "VertStride must be used to cross GRF register boundaries. This
72 * rule implies that elements within a 'Width' cannot cross GRF
73 * boundaries."
74 *
75 * The maximum width value that could satisfy this restriction is:
76 */
77 const unsigned reg_width = REG_SIZE / (reg->stride * type_sz(reg->type));
78
79 /* Because the hardware can only split source regions at a whole
80 * multiple of width during decompression (i.e. vertically), clamp
81 * the value obtained above to the physical execution size of a
82 * single decompressed chunk of the instruction:
83 */
84 const unsigned phys_width = compressed ? inst->exec_size / 2 :
85 inst->exec_size;
86
87 /* XXX - The equation above is strictly speaking not correct on
88 * hardware that supports unbalanced GRF writes -- On Gen9+
89 * each decompressed chunk of the instruction may have a
90 * different execution size when the number of components
91 * written to each destination GRF is not the same.
92 */
93 const unsigned width = MIN2(reg_width, phys_width);
94 brw_reg = brw_vecn_reg(width, brw_file_from_reg(reg), reg->nr, 0);
95 brw_reg = stride(brw_reg, width * reg->stride, width, reg->stride);
96
97 if (devinfo->gen == 7 && !devinfo->is_haswell) {
98 /* From the IvyBridge PRM (EU Changes by Processor Generation, page 13):
99 * "Each DF (Double Float) operand uses an element size of 4 rather
100 * than 8 and all regioning parameters are twice what the values
101 * would be based on the true element size: ExecSize, Width,
102 * HorzStride, and VertStride. Each DF operand uses a pair of
103 * channels and all masking and swizzing should be adjusted
104 * appropriately."
105 *
106 * From the IvyBridge PRM (Special Requirements for Handling Double
107 * Precision Data Types, page 71):
108 * "In Align1 mode, all regioning parameters like stride, execution
109 * size, and width must use the syntax of a pair of packed
110 * floats. The offsets for these data types must be 64-bit
111 * aligned. The execution size and regioning parameters are in terms
112 * of floats."
113 *
114 * Summarized: when handling DF-typed arguments, ExecSize,
115 * VertStride, and Width must be doubled.
116 *
117 * It applies to BayTrail too.
118 */
119 if (type_sz(reg->type) == 8) {
120 brw_reg.width++;
121 if (brw_reg.vstride > 0)
122 brw_reg.vstride++;
123 assert(brw_reg.hstride == BRW_HORIZONTAL_STRIDE_1);
124 }
125
126 /* When converting from DF->F, we set the destination stride to 2
127 * because each d2f conversion implicitly writes 2 floats, being
128 * the first one the converted value. IVB/BYT actually writes two
129 * F components per SIMD channel, and every other component is
130 * filled with garbage.
131 */
132 if (reg == &inst->dst && get_exec_type_size(inst) == 8 &&
133 type_sz(inst->dst.type) < 8) {
134 assert(brw_reg.hstride > BRW_HORIZONTAL_STRIDE_1);
135 brw_reg.hstride--;
136 }
137 }
138 }
139
140 brw_reg = retype(brw_reg, reg->type);
141 brw_reg = byte_offset(brw_reg, reg->offset);
142 brw_reg.abs = reg->abs;
143 brw_reg.negate = reg->negate;
144 break;
145 case ARF:
146 case FIXED_GRF:
147 case IMM:
148 assert(reg->offset == 0);
149 brw_reg = reg->as_brw_reg();
150 break;
151 case BAD_FILE:
152 /* Probably unused. */
153 brw_reg = brw_null_reg();
154 break;
155 case ATTR:
156 case UNIFORM:
157 unreachable("not reached");
158 }
159
160 /* On HSW+, scalar DF sources can be accessed using the normal <0,1,0>
161 * region, but on IVB and BYT DF regions must be programmed in terms of
162 * floats. A <0,2,1> region accomplishes this.
163 */
164 if (devinfo->gen == 7 && !devinfo->is_haswell &&
165 type_sz(reg->type) == 8 &&
166 brw_reg.vstride == BRW_VERTICAL_STRIDE_0 &&
167 brw_reg.width == BRW_WIDTH_1 &&
168 brw_reg.hstride == BRW_HORIZONTAL_STRIDE_0) {
169 brw_reg.width = BRW_WIDTH_2;
170 brw_reg.hstride = BRW_HORIZONTAL_STRIDE_1;
171 }
172
173 return brw_reg;
174 }
175
176 fs_generator::fs_generator(const struct brw_compiler *compiler, void *log_data,
177 void *mem_ctx,
178 const void *key,
179 struct brw_stage_prog_data *prog_data,
180 unsigned promoted_constants,
181 bool runtime_check_aads_emit,
182 gl_shader_stage stage)
183
184 : compiler(compiler), log_data(log_data),
185 devinfo(compiler->devinfo), key(key),
186 prog_data(prog_data),
187 promoted_constants(promoted_constants),
188 runtime_check_aads_emit(runtime_check_aads_emit), debug_flag(false),
189 stage(stage), mem_ctx(mem_ctx)
190 {
191 p = rzalloc(mem_ctx, struct brw_codegen);
192 brw_init_codegen(devinfo, p, mem_ctx);
193 }
194
195 fs_generator::~fs_generator()
196 {
197 }
198
199 class ip_record : public exec_node {
200 public:
201 DECLARE_RALLOC_CXX_OPERATORS(ip_record)
202
203 ip_record(int ip)
204 {
205 this->ip = ip;
206 }
207
208 int ip;
209 };
210
211 bool
212 fs_generator::patch_discard_jumps_to_fb_writes()
213 {
214 if (devinfo->gen < 6 || this->discard_halt_patches.is_empty())
215 return false;
216
217 int scale = brw_jump_scale(p->devinfo);
218
219 /* There is a somewhat strange undocumented requirement of using
220 * HALT, according to the simulator. If some channel has HALTed to
221 * a particular UIP, then by the end of the program, every channel
222 * must have HALTed to that UIP. Furthermore, the tracking is a
223 * stack, so you can't do the final halt of a UIP after starting
224 * halting to a new UIP.
225 *
226 * Symptoms of not emitting this instruction on actual hardware
227 * included GPU hangs and sparkly rendering on the piglit discard
228 * tests.
229 */
230 brw_inst *last_halt = gen6_HALT(p);
231 brw_inst_set_uip(p->devinfo, last_halt, 1 * scale);
232 brw_inst_set_jip(p->devinfo, last_halt, 1 * scale);
233
234 int ip = p->nr_insn;
235
236 foreach_in_list(ip_record, patch_ip, &discard_halt_patches) {
237 brw_inst *patch = &p->store[patch_ip->ip];
238
239 assert(brw_inst_opcode(p->devinfo, patch) == BRW_OPCODE_HALT);
240 /* HALT takes a half-instruction distance from the pre-incremented IP. */
241 brw_inst_set_uip(p->devinfo, patch, (ip - patch_ip->ip) * scale);
242 }
243
244 this->discard_halt_patches.make_empty();
245 return true;
246 }
247
248 void
249 fs_generator::fire_fb_write(fs_inst *inst,
250 struct brw_reg payload,
251 struct brw_reg implied_header,
252 GLuint nr)
253 {
254 uint32_t msg_control;
255
256 struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
257
258 if (devinfo->gen < 6) {
259 brw_push_insn_state(p);
260 brw_set_default_exec_size(p, BRW_EXECUTE_8);
261 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
262 brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
263 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
264 brw_MOV(p, offset(payload, 1), brw_vec8_grf(1, 0));
265 brw_pop_insn_state(p);
266 }
267
268 if (inst->opcode == FS_OPCODE_REP_FB_WRITE)
269 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD16_SINGLE_SOURCE_REPLICATED;
270 else if (prog_data->dual_src_blend) {
271 if (!inst->group)
272 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_DUAL_SOURCE_SUBSPAN01;
273 else
274 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_DUAL_SOURCE_SUBSPAN23;
275 } else if (inst->exec_size == 16)
276 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD16_SINGLE_SOURCE;
277 else
278 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_SINGLE_SOURCE_SUBSPAN01;
279
280 uint32_t surf_index =
281 prog_data->binding_table.render_target_start + inst->target;
282
283 bool last_render_target = inst->eot ||
284 (prog_data->dual_src_blend && dispatch_width == 16);
285
286
287 brw_fb_WRITE(p,
288 payload,
289 implied_header,
290 msg_control,
291 surf_index,
292 nr,
293 0,
294 inst->eot,
295 last_render_target,
296 inst->header_size != 0);
297
298 brw_mark_surface_used(&prog_data->base, surf_index);
299 }
300
301 void
302 fs_generator::generate_fb_write(fs_inst *inst, struct brw_reg payload)
303 {
304 struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
305 const brw_wm_prog_key * const key = (brw_wm_prog_key * const) this->key;
306 struct brw_reg implied_header;
307
308 if (devinfo->gen < 8 && !devinfo->is_haswell) {
309 brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
310 }
311
312 if (inst->base_mrf >= 0)
313 payload = brw_message_reg(inst->base_mrf);
314
315 /* Header is 2 regs, g0 and g1 are the contents. g0 will be implied
316 * move, here's g1.
317 */
318 if (inst->header_size != 0) {
319 brw_push_insn_state(p);
320 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
321 brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
322 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
323 brw_set_default_flag_reg(p, 0, 0);
324
325 /* On HSW, the GPU will use the predicate on SENDC, unless the header is
326 * present.
327 */
328 if (prog_data->uses_kill) {
329 struct brw_reg pixel_mask;
330
331 if (devinfo->gen >= 6)
332 pixel_mask = retype(brw_vec1_grf(1, 7), BRW_REGISTER_TYPE_UW);
333 else
334 pixel_mask = retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UW);
335
336 brw_MOV(p, pixel_mask, brw_flag_reg(0, 1));
337 }
338
339 if (devinfo->gen >= 6) {
340 brw_push_insn_state(p);
341 brw_set_default_exec_size(p, BRW_EXECUTE_16);
342 brw_set_default_compression_control(p, BRW_COMPRESSION_COMPRESSED);
343 brw_MOV(p,
344 retype(payload, BRW_REGISTER_TYPE_UD),
345 retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));
346 brw_pop_insn_state(p);
347
348 if (inst->target > 0 && key->replicate_alpha) {
349 /* Set "Source0 Alpha Present to RenderTarget" bit in message
350 * header.
351 */
352 brw_OR(p,
353 vec1(retype(payload, BRW_REGISTER_TYPE_UD)),
354 vec1(retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD)),
355 brw_imm_ud(0x1 << 11));
356 }
357
358 if (inst->target > 0) {
359 /* Set the render target index for choosing BLEND_STATE. */
360 brw_MOV(p, retype(vec1(suboffset(payload, 2)),
361 BRW_REGISTER_TYPE_UD),
362 brw_imm_ud(inst->target));
363 }
364
365 /* Set computes stencil to render target */
366 if (prog_data->computed_stencil) {
367 brw_OR(p,
368 vec1(retype(payload, BRW_REGISTER_TYPE_UD)),
369 vec1(retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD)),
370 brw_imm_ud(0x1 << 14));
371 }
372
373 implied_header = brw_null_reg();
374 } else {
375 implied_header = retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UW);
376 }
377
378 brw_pop_insn_state(p);
379 } else {
380 implied_header = brw_null_reg();
381 }
382
383 if (!runtime_check_aads_emit) {
384 fire_fb_write(inst, payload, implied_header, inst->mlen);
385 } else {
386 /* This can only happen in gen < 6 */
387 assert(devinfo->gen < 6);
388
389 struct brw_reg v1_null_ud = vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_UD));
390
391 /* Check runtime bit to detect if we have to send AA data or not */
392 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
393 brw_AND(p,
394 v1_null_ud,
395 retype(brw_vec1_grf(1, 6), BRW_REGISTER_TYPE_UD),
396 brw_imm_ud(1<<26));
397 brw_inst_set_cond_modifier(p->devinfo, brw_last_inst, BRW_CONDITIONAL_NZ);
398
399 int jmp = brw_JMPI(p, brw_imm_ud(0), BRW_PREDICATE_NORMAL) - p->store;
400 brw_inst_set_exec_size(p->devinfo, brw_last_inst, BRW_EXECUTE_1);
401 {
402 /* Don't send AA data */
403 fire_fb_write(inst, offset(payload, 1), implied_header, inst->mlen-1);
404 }
405 brw_land_fwd_jump(p, jmp);
406 fire_fb_write(inst, payload, implied_header, inst->mlen);
407 }
408 }
409
410 void
411 fs_generator::generate_fb_read(fs_inst *inst, struct brw_reg dst,
412 struct brw_reg payload)
413 {
414 assert(inst->size_written % REG_SIZE == 0);
415 struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
416 const unsigned surf_index =
417 prog_data->binding_table.render_target_start + inst->target;
418
419 gen9_fb_READ(p, dst, payload, surf_index,
420 inst->header_size, inst->size_written / REG_SIZE,
421 prog_data->persample_dispatch);
422
423 brw_mark_surface_used(&prog_data->base, surf_index);
424 }
425
426 void
427 fs_generator::generate_mov_indirect(fs_inst *inst,
428 struct brw_reg dst,
429 struct brw_reg reg,
430 struct brw_reg indirect_byte_offset)
431 {
432 assert(indirect_byte_offset.type == BRW_REGISTER_TYPE_UD);
433 assert(indirect_byte_offset.file == BRW_GENERAL_REGISTER_FILE);
434
435 unsigned imm_byte_offset = reg.nr * REG_SIZE + reg.subnr;
436
437 if (indirect_byte_offset.file == BRW_IMMEDIATE_VALUE) {
438 imm_byte_offset += indirect_byte_offset.ud;
439
440 reg.nr = imm_byte_offset / REG_SIZE;
441 reg.subnr = imm_byte_offset % REG_SIZE;
442 brw_MOV(p, dst, reg);
443 } else {
444 /* Prior to Broadwell, there are only 8 address registers. */
445 assert(inst->exec_size <= 8 || devinfo->gen >= 8);
446
447 /* We use VxH indirect addressing, clobbering a0.0 through a0.7. */
448 struct brw_reg addr = vec8(brw_address_reg(0));
449
450 /* The destination stride of an instruction (in bytes) must be greater
451 * than or equal to the size of the rest of the instruction. Since the
452 * address register is of type UW, we can't use a D-type instruction.
453 * In order to get around this, re retype to UW and use a stride.
454 */
455 indirect_byte_offset =
456 retype(spread(indirect_byte_offset, 2), BRW_REGISTER_TYPE_UW);
457
458 /* There are a number of reasons why we don't use the base offset here.
459 * One reason is that the field is only 9 bits which means we can only
460 * use it to access the first 16 GRFs. Also, from the Haswell PRM
461 * section "Register Region Restrictions":
462 *
463 * "The lower bits of the AddressImmediate must not overflow to
464 * change the register address. The lower 5 bits of Address
465 * Immediate when added to lower 5 bits of address register gives
466 * the sub-register offset. The upper bits of Address Immediate
467 * when added to upper bits of address register gives the register
468 * address. Any overflow from sub-register offset is dropped."
469 *
470 * Since the indirect may cause us to cross a register boundary, this
471 * makes the base offset almost useless. We could try and do something
472 * clever where we use a actual base offset if base_offset % 32 == 0 but
473 * that would mean we were generating different code depending on the
474 * base offset. Instead, for the sake of consistency, we'll just do the
475 * add ourselves. This restriction is only listed in the Haswell PRM
476 * but empirical testing indicates that it applies on all older
477 * generations and is lifted on Broadwell.
478 *
479 * In the end, while base_offset is nice to look at in the generated
480 * code, using it saves us 0 instructions and would require quite a bit
481 * of case-by-case work. It's just not worth it.
482 */
483 if (devinfo->gen >= 8 || devinfo->is_haswell || type_sz(reg.type) < 8) {
484 brw_ADD(p, addr, indirect_byte_offset, brw_imm_uw(imm_byte_offset));
485 } else {
486 /* IVB reads two address register components per channel for
487 * indirectly addressed 64-bit sources, so we need to initialize
488 * adjacent address components to consecutive dwords of the source
489 * region by emitting two separate ADD instructions. Found
490 * empirically.
491 */
492 assert(inst->exec_size <= 4);
493 brw_push_insn_state(p);
494 brw_set_default_exec_size(p, cvt(inst->exec_size) - 1);
495
496 brw_ADD(p, spread(addr, 2), indirect_byte_offset,
497 brw_imm_uw(imm_byte_offset));
498 brw_inst_set_no_dd_clear(devinfo, brw_last_inst, true);
499
500 brw_ADD(p, spread(suboffset(addr, 1), 2), indirect_byte_offset,
501 brw_imm_uw(imm_byte_offset + 4));
502 brw_inst_set_no_dd_check(devinfo, brw_last_inst, true);
503
504 brw_pop_insn_state(p);
505 }
506
507 struct brw_reg ind_src = brw_VxH_indirect(0, 0);
508
509 brw_inst *mov = brw_MOV(p, dst, retype(ind_src, reg.type));
510
511 if (devinfo->gen == 6 && dst.file == BRW_MESSAGE_REGISTER_FILE &&
512 !inst->get_next()->is_tail_sentinel() &&
513 ((fs_inst *)inst->get_next())->mlen > 0) {
514 /* From the Sandybridge PRM:
515 *
516 * "[Errata: DevSNB(SNB)] If MRF register is updated by any
517 * instruction that “indexed/indirect” source AND is followed by a
518 * send, the instruction requires a “Switch”. This is to avoid
519 * race condition where send may dispatch before MRF is updated."
520 */
521 brw_inst_set_thread_control(devinfo, mov, BRW_THREAD_SWITCH);
522 }
523 }
524 }
525
526 void
527 fs_generator::generate_urb_read(fs_inst *inst,
528 struct brw_reg dst,
529 struct brw_reg header)
530 {
531 assert(inst->size_written % REG_SIZE == 0);
532 assert(header.file == BRW_GENERAL_REGISTER_FILE);
533 assert(header.type == BRW_REGISTER_TYPE_UD);
534
535 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
536 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UD));
537 brw_set_src0(p, send, header);
538 brw_set_src1(p, send, brw_imm_ud(0u));
539
540 brw_inst_set_sfid(p->devinfo, send, BRW_SFID_URB);
541 brw_inst_set_urb_opcode(p->devinfo, send, GEN8_URB_OPCODE_SIMD8_READ);
542
543 if (inst->opcode == SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT)
544 brw_inst_set_urb_per_slot_offset(p->devinfo, send, true);
545
546 brw_inst_set_mlen(p->devinfo, send, inst->mlen);
547 brw_inst_set_rlen(p->devinfo, send, inst->size_written / REG_SIZE);
548 brw_inst_set_header_present(p->devinfo, send, true);
549 brw_inst_set_urb_global_offset(p->devinfo, send, inst->offset);
550 }
551
552 void
553 fs_generator::generate_urb_write(fs_inst *inst, struct brw_reg payload)
554 {
555 brw_inst *insn;
556
557 insn = brw_next_insn(p, BRW_OPCODE_SEND);
558
559 brw_set_dest(p, insn, brw_null_reg());
560 brw_set_src0(p, insn, payload);
561 brw_set_src1(p, insn, brw_imm_d(0));
562
563 brw_inst_set_sfid(p->devinfo, insn, BRW_SFID_URB);
564 brw_inst_set_urb_opcode(p->devinfo, insn, GEN8_URB_OPCODE_SIMD8_WRITE);
565
566 if (inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT ||
567 inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT)
568 brw_inst_set_urb_per_slot_offset(p->devinfo, insn, true);
569
570 if (inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_MASKED ||
571 inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT)
572 brw_inst_set_urb_channel_mask_present(p->devinfo, insn, true);
573
574 brw_inst_set_mlen(p->devinfo, insn, inst->mlen);
575 brw_inst_set_rlen(p->devinfo, insn, 0);
576 brw_inst_set_eot(p->devinfo, insn, inst->eot);
577 brw_inst_set_header_present(p->devinfo, insn, true);
578 brw_inst_set_urb_global_offset(p->devinfo, insn, inst->offset);
579 }
580
581 void
582 fs_generator::generate_cs_terminate(fs_inst *inst, struct brw_reg payload)
583 {
584 struct brw_inst *insn;
585
586 insn = brw_next_insn(p, BRW_OPCODE_SEND);
587
588 brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW));
589 brw_set_src0(p, insn, retype(payload, BRW_REGISTER_TYPE_UW));
590 brw_set_src1(p, insn, brw_imm_d(0));
591
592 /* Terminate a compute shader by sending a message to the thread spawner.
593 */
594 brw_inst_set_sfid(devinfo, insn, BRW_SFID_THREAD_SPAWNER);
595 brw_inst_set_mlen(devinfo, insn, 1);
596 brw_inst_set_rlen(devinfo, insn, 0);
597 brw_inst_set_eot(devinfo, insn, inst->eot);
598 brw_inst_set_header_present(devinfo, insn, false);
599
600 brw_inst_set_ts_opcode(devinfo, insn, 0); /* Dereference resource */
601 brw_inst_set_ts_request_type(devinfo, insn, 0); /* Root thread */
602
603 /* Note that even though the thread has a URB resource associated with it,
604 * we set the "do not dereference URB" bit, because the URB resource is
605 * managed by the fixed-function unit, so it will free it automatically.
606 */
607 brw_inst_set_ts_resource_select(devinfo, insn, 1); /* Do not dereference URB */
608
609 brw_inst_set_mask_control(devinfo, insn, BRW_MASK_DISABLE);
610 }
611
612 void
613 fs_generator::generate_barrier(fs_inst *inst, struct brw_reg src)
614 {
615 brw_barrier(p, src);
616 brw_WAIT(p);
617 }
618
619 void
620 fs_generator::generate_linterp(fs_inst *inst,
621 struct brw_reg dst, struct brw_reg *src)
622 {
623 /* PLN reads:
624 * / in SIMD16 \
625 * -----------------------------------
626 * | src1+0 | src1+1 | src1+2 | src1+3 |
627 * |-----------------------------------|
628 * |(x0, x1)|(y0, y1)|(x2, x3)|(y2, y3)|
629 * -----------------------------------
630 *
631 * but for the LINE/MAC pair, the LINE reads Xs and the MAC reads Ys:
632 *
633 * -----------------------------------
634 * | src1+0 | src1+1 | src1+2 | src1+3 |
635 * |-----------------------------------|
636 * |(x0, x1)|(y0, y1)| | | in SIMD8
637 * |-----------------------------------|
638 * |(x0, x1)|(x2, x3)|(y0, y1)|(y2, y3)| in SIMD16
639 * -----------------------------------
640 *
641 * See also: emit_interpolation_setup_gen4().
642 */
643 struct brw_reg delta_x = src[0];
644 struct brw_reg delta_y = offset(src[0], inst->exec_size / 8);
645 struct brw_reg interp = src[1];
646
647 if (devinfo->has_pln &&
648 (devinfo->gen >= 7 || (delta_x.nr & 1) == 0)) {
649 brw_PLN(p, dst, interp, delta_x);
650 } else {
651 brw_LINE(p, brw_null_reg(), interp, delta_x);
652 brw_MAC(p, dst, suboffset(interp, 1), delta_y);
653 }
654 }
655
656 void
657 fs_generator::generate_get_buffer_size(fs_inst *inst,
658 struct brw_reg dst,
659 struct brw_reg src,
660 struct brw_reg surf_index)
661 {
662 assert(devinfo->gen >= 7);
663 assert(surf_index.file == BRW_IMMEDIATE_VALUE);
664
665 uint32_t simd_mode;
666 int rlen = 4;
667
668 switch (inst->exec_size) {
669 case 8:
670 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
671 break;
672 case 16:
673 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
674 break;
675 default:
676 unreachable("Invalid width for texture instruction");
677 }
678
679 if (simd_mode == BRW_SAMPLER_SIMD_MODE_SIMD16) {
680 rlen = 8;
681 dst = vec16(dst);
682 }
683
684 brw_SAMPLE(p,
685 retype(dst, BRW_REGISTER_TYPE_UW),
686 inst->base_mrf,
687 src,
688 surf_index.ud,
689 0,
690 GEN5_SAMPLER_MESSAGE_SAMPLE_RESINFO,
691 rlen, /* response length */
692 inst->mlen,
693 inst->header_size > 0,
694 simd_mode,
695 BRW_SAMPLER_RETURN_FORMAT_SINT32);
696
697 brw_mark_surface_used(prog_data, surf_index.ud);
698 }
699
700 void
701 fs_generator::generate_tex(fs_inst *inst, struct brw_reg dst, struct brw_reg src,
702 struct brw_reg surface_index,
703 struct brw_reg sampler_index)
704 {
705 assert(inst->size_written % REG_SIZE == 0);
706 int msg_type = -1;
707 uint32_t simd_mode;
708 uint32_t return_format;
709 bool is_combined_send = inst->eot;
710
711 switch (dst.type) {
712 case BRW_REGISTER_TYPE_D:
713 return_format = BRW_SAMPLER_RETURN_FORMAT_SINT32;
714 break;
715 case BRW_REGISTER_TYPE_UD:
716 return_format = BRW_SAMPLER_RETURN_FORMAT_UINT32;
717 break;
718 default:
719 return_format = BRW_SAMPLER_RETURN_FORMAT_FLOAT32;
720 break;
721 }
722
723 /* Stomp the resinfo output type to UINT32. On gens 4-5, the output type
724 * is set as part of the message descriptor. On gen4, the PRM seems to
725 * allow UINT32 and FLOAT32 (i965 PRM, Vol. 4 Section 4.8.1.1), but on
726 * later gens UINT32 is required. Once you hit Sandy Bridge, the bit is
727 * gone from the message descriptor entirely and you just get UINT32 all
728 * the time regasrdless. Since we can really only do non-UINT32 on gen4,
729 * just stomp it to UINT32 all the time.
730 */
731 if (inst->opcode == SHADER_OPCODE_TXS)
732 return_format = BRW_SAMPLER_RETURN_FORMAT_UINT32;
733
734 switch (inst->exec_size) {
735 case 8:
736 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
737 break;
738 case 16:
739 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
740 break;
741 default:
742 unreachable("Invalid width for texture instruction");
743 }
744
745 if (devinfo->gen >= 5) {
746 switch (inst->opcode) {
747 case SHADER_OPCODE_TEX:
748 if (inst->shadow_compare) {
749 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_COMPARE;
750 } else {
751 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE;
752 }
753 break;
754 case FS_OPCODE_TXB:
755 if (inst->shadow_compare) {
756 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_BIAS_COMPARE;
757 } else {
758 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_BIAS;
759 }
760 break;
761 case SHADER_OPCODE_TXL:
762 if (inst->shadow_compare) {
763 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LOD_COMPARE;
764 } else {
765 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LOD;
766 }
767 break;
768 case SHADER_OPCODE_TXL_LZ:
769 assert(devinfo->gen >= 9);
770 if (inst->shadow_compare) {
771 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_C_LZ;
772 } else {
773 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_LZ;
774 }
775 break;
776 case SHADER_OPCODE_TXS:
777 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_RESINFO;
778 break;
779 case SHADER_OPCODE_TXD:
780 if (inst->shadow_compare) {
781 /* Gen7.5+. Otherwise, lowered in NIR */
782 assert(devinfo->gen >= 8 || devinfo->is_haswell);
783 msg_type = HSW_SAMPLER_MESSAGE_SAMPLE_DERIV_COMPARE;
784 } else {
785 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_DERIVS;
786 }
787 break;
788 case SHADER_OPCODE_TXF:
789 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
790 break;
791 case SHADER_OPCODE_TXF_LZ:
792 assert(devinfo->gen >= 9);
793 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_LD_LZ;
794 break;
795 case SHADER_OPCODE_TXF_CMS_W:
796 assert(devinfo->gen >= 9);
797 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_LD2DMS_W;
798 break;
799 case SHADER_OPCODE_TXF_CMS:
800 if (devinfo->gen >= 7)
801 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD2DMS;
802 else
803 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
804 break;
805 case SHADER_OPCODE_TXF_UMS:
806 assert(devinfo->gen >= 7);
807 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD2DSS;
808 break;
809 case SHADER_OPCODE_TXF_MCS:
810 assert(devinfo->gen >= 7);
811 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD_MCS;
812 break;
813 case SHADER_OPCODE_LOD:
814 msg_type = GEN5_SAMPLER_MESSAGE_LOD;
815 break;
816 case SHADER_OPCODE_TG4:
817 if (inst->shadow_compare) {
818 assert(devinfo->gen >= 7);
819 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_C;
820 } else {
821 assert(devinfo->gen >= 6);
822 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4;
823 }
824 break;
825 case SHADER_OPCODE_TG4_OFFSET:
826 assert(devinfo->gen >= 7);
827 if (inst->shadow_compare) {
828 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_PO_C;
829 } else {
830 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_PO;
831 }
832 break;
833 case SHADER_OPCODE_SAMPLEINFO:
834 msg_type = GEN6_SAMPLER_MESSAGE_SAMPLE_SAMPLEINFO;
835 break;
836 default:
837 unreachable("not reached");
838 }
839 } else {
840 switch (inst->opcode) {
841 case SHADER_OPCODE_TEX:
842 /* Note that G45 and older determines shadow compare and dispatch width
843 * from message length for most messages.
844 */
845 if (inst->exec_size == 8) {
846 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE;
847 if (inst->shadow_compare) {
848 assert(inst->mlen == 6);
849 } else {
850 assert(inst->mlen <= 4);
851 }
852 } else {
853 if (inst->shadow_compare) {
854 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE_COMPARE;
855 assert(inst->mlen == 9);
856 } else {
857 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE;
858 assert(inst->mlen <= 7 && inst->mlen % 2 == 1);
859 }
860 }
861 break;
862 case FS_OPCODE_TXB:
863 if (inst->shadow_compare) {
864 assert(inst->exec_size == 8);
865 assert(inst->mlen == 6);
866 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE_BIAS_COMPARE;
867 } else {
868 assert(inst->mlen == 9);
869 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE_BIAS;
870 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
871 }
872 break;
873 case SHADER_OPCODE_TXL:
874 if (inst->shadow_compare) {
875 assert(inst->exec_size == 8);
876 assert(inst->mlen == 6);
877 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE_LOD_COMPARE;
878 } else {
879 assert(inst->mlen == 9);
880 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE_LOD;
881 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
882 }
883 break;
884 case SHADER_OPCODE_TXD:
885 /* There is no sample_d_c message; comparisons are done manually */
886 assert(inst->exec_size == 8);
887 assert(inst->mlen == 7 || inst->mlen == 10);
888 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE_GRADIENTS;
889 break;
890 case SHADER_OPCODE_TXF:
891 assert(inst->mlen <= 9 && inst->mlen % 2 == 1);
892 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_LD;
893 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
894 break;
895 case SHADER_OPCODE_TXS:
896 assert(inst->mlen == 3);
897 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_RESINFO;
898 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
899 break;
900 default:
901 unreachable("not reached");
902 }
903 }
904 assert(msg_type != -1);
905
906 if (simd_mode == BRW_SAMPLER_SIMD_MODE_SIMD16) {
907 dst = vec16(dst);
908 }
909
910 assert(devinfo->gen < 7 || inst->header_size == 0 ||
911 src.file == BRW_GENERAL_REGISTER_FILE);
912
913 assert(sampler_index.type == BRW_REGISTER_TYPE_UD);
914
915 /* Load the message header if present. If there's a texture offset,
916 * we need to set it up explicitly and load the offset bitfield.
917 * Otherwise, we can use an implied move from g0 to the first message reg.
918 */
919 if (inst->header_size != 0) {
920 if (devinfo->gen < 6 && !inst->offset) {
921 /* Set up an implied move from g0 to the MRF. */
922 src = retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UW);
923 } else {
924 struct brw_reg header_reg;
925
926 if (devinfo->gen >= 7) {
927 header_reg = src;
928 } else {
929 assert(inst->base_mrf != -1);
930 header_reg = brw_message_reg(inst->base_mrf);
931 }
932
933 brw_push_insn_state(p);
934 brw_set_default_exec_size(p, BRW_EXECUTE_8);
935 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
936 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
937 /* Explicitly set up the message header by copying g0 to the MRF. */
938 brw_MOV(p, header_reg, brw_vec8_grf(0, 0));
939
940 if (inst->offset) {
941 /* Set the offset bits in DWord 2. */
942 brw_MOV(p, get_element_ud(header_reg, 2),
943 brw_imm_ud(inst->offset));
944 } else if (stage != MESA_SHADER_VERTEX &&
945 stage != MESA_SHADER_FRAGMENT) {
946 /* The vertex and fragment stages have g0.2 set to 0, so
947 * header0.2 is 0 when g0 is copied. Other stages may not, so we
948 * must set it to 0 to avoid setting undesirable bits in the
949 * message.
950 */
951 brw_MOV(p, get_element_ud(header_reg, 2), brw_imm_ud(0));
952 }
953
954 brw_adjust_sampler_state_pointer(p, header_reg, sampler_index);
955 brw_pop_insn_state(p);
956 }
957 }
958
959 uint32_t base_binding_table_index = (inst->opcode == SHADER_OPCODE_TG4 ||
960 inst->opcode == SHADER_OPCODE_TG4_OFFSET)
961 ? prog_data->binding_table.gather_texture_start
962 : prog_data->binding_table.texture_start;
963
964 if (surface_index.file == BRW_IMMEDIATE_VALUE &&
965 sampler_index.file == BRW_IMMEDIATE_VALUE) {
966 uint32_t surface = surface_index.ud;
967 uint32_t sampler = sampler_index.ud;
968
969 brw_SAMPLE(p,
970 retype(dst, BRW_REGISTER_TYPE_UW),
971 inst->base_mrf,
972 src,
973 surface + base_binding_table_index,
974 sampler % 16,
975 msg_type,
976 inst->size_written / REG_SIZE,
977 inst->mlen,
978 inst->header_size != 0,
979 simd_mode,
980 return_format);
981
982 brw_mark_surface_used(prog_data, surface + base_binding_table_index);
983 } else {
984 /* Non-const sampler index */
985
986 struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
987 struct brw_reg surface_reg = vec1(retype(surface_index, BRW_REGISTER_TYPE_UD));
988 struct brw_reg sampler_reg = vec1(retype(sampler_index, BRW_REGISTER_TYPE_UD));
989
990 brw_push_insn_state(p);
991 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
992 brw_set_default_access_mode(p, BRW_ALIGN_1);
993
994 if (brw_regs_equal(&surface_reg, &sampler_reg)) {
995 brw_MUL(p, addr, sampler_reg, brw_imm_uw(0x101));
996 } else {
997 if (sampler_reg.file == BRW_IMMEDIATE_VALUE) {
998 brw_OR(p, addr, surface_reg, brw_imm_ud(sampler_reg.ud << 8));
999 } else {
1000 brw_SHL(p, addr, sampler_reg, brw_imm_ud(8));
1001 brw_OR(p, addr, addr, surface_reg);
1002 }
1003 }
1004 if (base_binding_table_index)
1005 brw_ADD(p, addr, addr, brw_imm_ud(base_binding_table_index));
1006 brw_AND(p, addr, addr, brw_imm_ud(0xfff));
1007
1008 brw_pop_insn_state(p);
1009
1010 /* dst = send(offset, a0.0 | <descriptor>) */
1011 brw_inst *insn = brw_send_indirect_message(
1012 p, BRW_SFID_SAMPLER, dst, src, addr);
1013 brw_set_sampler_message(p, insn,
1014 0 /* surface */,
1015 0 /* sampler */,
1016 msg_type,
1017 inst->size_written / REG_SIZE,
1018 inst->mlen /* mlen */,
1019 inst->header_size != 0 /* header */,
1020 simd_mode,
1021 return_format);
1022
1023 /* visitor knows more than we do about the surface limit required,
1024 * so has already done marking.
1025 */
1026 }
1027
1028 if (is_combined_send) {
1029 brw_inst_set_eot(p->devinfo, brw_last_inst, true);
1030 brw_inst_set_opcode(p->devinfo, brw_last_inst, BRW_OPCODE_SENDC);
1031 }
1032 }
1033
1034
1035 /* For OPCODE_DDX and OPCODE_DDY, per channel of output we've got input
1036 * looking like:
1037 *
1038 * arg0: ss0.tl ss0.tr ss0.bl ss0.br ss1.tl ss1.tr ss1.bl ss1.br
1039 *
1040 * Ideally, we want to produce:
1041 *
1042 * DDX DDY
1043 * dst: (ss0.tr - ss0.tl) (ss0.tl - ss0.bl)
1044 * (ss0.tr - ss0.tl) (ss0.tr - ss0.br)
1045 * (ss0.br - ss0.bl) (ss0.tl - ss0.bl)
1046 * (ss0.br - ss0.bl) (ss0.tr - ss0.br)
1047 * (ss1.tr - ss1.tl) (ss1.tl - ss1.bl)
1048 * (ss1.tr - ss1.tl) (ss1.tr - ss1.br)
1049 * (ss1.br - ss1.bl) (ss1.tl - ss1.bl)
1050 * (ss1.br - ss1.bl) (ss1.tr - ss1.br)
1051 *
1052 * and add another set of two more subspans if in 16-pixel dispatch mode.
1053 *
1054 * For DDX, it ends up being easy: width = 2, horiz=0 gets us the same result
1055 * for each pair, and vertstride = 2 jumps us 2 elements after processing a
1056 * pair. But the ideal approximation may impose a huge performance cost on
1057 * sample_d. On at least Haswell, sample_d instruction does some
1058 * optimizations if the same LOD is used for all pixels in the subspan.
1059 *
1060 * For DDY, we need to use ALIGN16 mode since it's capable of doing the
1061 * appropriate swizzling.
1062 */
1063 void
1064 fs_generator::generate_ddx(enum opcode opcode,
1065 struct brw_reg dst, struct brw_reg src)
1066 {
1067 unsigned vstride, width;
1068
1069 if (opcode == FS_OPCODE_DDX_FINE) {
1070 /* produce accurate derivatives */
1071 vstride = BRW_VERTICAL_STRIDE_2;
1072 width = BRW_WIDTH_2;
1073 } else {
1074 /* replicate the derivative at the top-left pixel to other pixels */
1075 vstride = BRW_VERTICAL_STRIDE_4;
1076 width = BRW_WIDTH_4;
1077 }
1078
1079 struct brw_reg src0 = brw_reg(src.file, src.nr, 1,
1080 src.negate, src.abs,
1081 BRW_REGISTER_TYPE_F,
1082 vstride,
1083 width,
1084 BRW_HORIZONTAL_STRIDE_0,
1085 BRW_SWIZZLE_XYZW, WRITEMASK_XYZW);
1086 struct brw_reg src1 = brw_reg(src.file, src.nr, 0,
1087 src.negate, src.abs,
1088 BRW_REGISTER_TYPE_F,
1089 vstride,
1090 width,
1091 BRW_HORIZONTAL_STRIDE_0,
1092 BRW_SWIZZLE_XYZW, WRITEMASK_XYZW);
1093 brw_ADD(p, dst, src0, negate(src1));
1094 }
1095
1096 /* The negate_value boolean is used to negate the derivative computation for
1097 * FBOs, since they place the origin at the upper left instead of the lower
1098 * left.
1099 */
1100 void
1101 fs_generator::generate_ddy(enum opcode opcode,
1102 struct brw_reg dst, struct brw_reg src)
1103 {
1104 if (opcode == FS_OPCODE_DDY_FINE) {
1105 /* produce accurate derivatives */
1106 struct brw_reg src0 = brw_reg(src.file, src.nr, 0,
1107 src.negate, src.abs,
1108 BRW_REGISTER_TYPE_F,
1109 BRW_VERTICAL_STRIDE_4,
1110 BRW_WIDTH_4,
1111 BRW_HORIZONTAL_STRIDE_1,
1112 BRW_SWIZZLE_XYXY, WRITEMASK_XYZW);
1113 struct brw_reg src1 = brw_reg(src.file, src.nr, 0,
1114 src.negate, src.abs,
1115 BRW_REGISTER_TYPE_F,
1116 BRW_VERTICAL_STRIDE_4,
1117 BRW_WIDTH_4,
1118 BRW_HORIZONTAL_STRIDE_1,
1119 BRW_SWIZZLE_ZWZW, WRITEMASK_XYZW);
1120 brw_push_insn_state(p);
1121 brw_set_default_access_mode(p, BRW_ALIGN_16);
1122 brw_ADD(p, dst, negate(src0), src1);
1123 brw_pop_insn_state(p);
1124 } else {
1125 /* replicate the derivative at the top-left pixel to other pixels */
1126 struct brw_reg src0 = brw_reg(src.file, src.nr, 0,
1127 src.negate, src.abs,
1128 BRW_REGISTER_TYPE_F,
1129 BRW_VERTICAL_STRIDE_4,
1130 BRW_WIDTH_4,
1131 BRW_HORIZONTAL_STRIDE_0,
1132 BRW_SWIZZLE_XYZW, WRITEMASK_XYZW);
1133 struct brw_reg src1 = brw_reg(src.file, src.nr, 2,
1134 src.negate, src.abs,
1135 BRW_REGISTER_TYPE_F,
1136 BRW_VERTICAL_STRIDE_4,
1137 BRW_WIDTH_4,
1138 BRW_HORIZONTAL_STRIDE_0,
1139 BRW_SWIZZLE_XYZW, WRITEMASK_XYZW);
1140 brw_ADD(p, dst, negate(src0), src1);
1141 }
1142 }
1143
1144 void
1145 fs_generator::generate_discard_jump(fs_inst *inst)
1146 {
1147 assert(devinfo->gen >= 6);
1148
1149 /* This HALT will be patched up at FB write time to point UIP at the end of
1150 * the program, and at brw_uip_jip() JIP will be set to the end of the
1151 * current block (or the program).
1152 */
1153 this->discard_halt_patches.push_tail(new(mem_ctx) ip_record(p->nr_insn));
1154 gen6_HALT(p);
1155 }
1156
1157 void
1158 fs_generator::generate_scratch_write(fs_inst *inst, struct brw_reg src)
1159 {
1160 /* The 32-wide messages only respect the first 16-wide half of the channel
1161 * enable signals which are replicated identically for the second group of
1162 * 16 channels, so we cannot use them unless the write is marked
1163 * force_writemask_all.
1164 */
1165 const unsigned lower_size = inst->force_writemask_all ? inst->exec_size :
1166 MIN2(16, inst->exec_size);
1167 const unsigned block_size = 4 * lower_size / REG_SIZE;
1168 assert(inst->mlen != 0);
1169
1170 brw_push_insn_state(p);
1171 brw_set_default_exec_size(p, cvt(lower_size) - 1);
1172 brw_set_default_compression(p, lower_size > 8);
1173
1174 for (unsigned i = 0; i < inst->exec_size / lower_size; i++) {
1175 brw_set_default_group(p, inst->group + lower_size * i);
1176
1177 brw_MOV(p, brw_uvec_mrf(lower_size, inst->base_mrf + 1, 0),
1178 retype(offset(src, block_size * i), BRW_REGISTER_TYPE_UD));
1179
1180 brw_oword_block_write_scratch(p, brw_message_reg(inst->base_mrf),
1181 block_size,
1182 inst->offset + block_size * REG_SIZE * i);
1183 }
1184
1185 brw_pop_insn_state(p);
1186 }
1187
1188 void
1189 fs_generator::generate_scratch_read(fs_inst *inst, struct brw_reg dst)
1190 {
1191 assert(inst->exec_size <= 16 || inst->force_writemask_all);
1192 assert(inst->mlen != 0);
1193
1194 brw_oword_block_read_scratch(p, dst, brw_message_reg(inst->base_mrf),
1195 inst->exec_size / 8, inst->offset);
1196 }
1197
1198 void
1199 fs_generator::generate_scratch_read_gen7(fs_inst *inst, struct brw_reg dst)
1200 {
1201 assert(inst->exec_size <= 16 || inst->force_writemask_all);
1202
1203 gen7_block_read_scratch(p, dst, inst->exec_size / 8, inst->offset);
1204 }
1205
1206 void
1207 fs_generator::generate_uniform_pull_constant_load(fs_inst *inst,
1208 struct brw_reg dst,
1209 struct brw_reg index,
1210 struct brw_reg offset)
1211 {
1212 assert(type_sz(dst.type) == 4);
1213 assert(inst->mlen != 0);
1214
1215 assert(index.file == BRW_IMMEDIATE_VALUE &&
1216 index.type == BRW_REGISTER_TYPE_UD);
1217 uint32_t surf_index = index.ud;
1218
1219 assert(offset.file == BRW_IMMEDIATE_VALUE &&
1220 offset.type == BRW_REGISTER_TYPE_UD);
1221 uint32_t read_offset = offset.ud;
1222
1223 brw_oword_block_read(p, dst, brw_message_reg(inst->base_mrf),
1224 read_offset, surf_index);
1225 }
1226
1227 void
1228 fs_generator::generate_uniform_pull_constant_load_gen7(fs_inst *inst,
1229 struct brw_reg dst,
1230 struct brw_reg index,
1231 struct brw_reg payload)
1232 {
1233 assert(index.type == BRW_REGISTER_TYPE_UD);
1234 assert(payload.file == BRW_GENERAL_REGISTER_FILE);
1235 assert(type_sz(dst.type) == 4);
1236
1237 if (index.file == BRW_IMMEDIATE_VALUE) {
1238 const uint32_t surf_index = index.ud;
1239
1240 brw_push_insn_state(p);
1241 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1242 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
1243 brw_pop_insn_state(p);
1244
1245 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UD));
1246 brw_set_src0(p, send, retype(payload, BRW_REGISTER_TYPE_UD));
1247 brw_set_dp_read_message(p, send, surf_index,
1248 BRW_DATAPORT_OWORD_BLOCK_DWORDS(inst->exec_size),
1249 GEN7_DATAPORT_DC_OWORD_BLOCK_READ,
1250 GEN6_SFID_DATAPORT_CONSTANT_CACHE,
1251 1, /* mlen */
1252 true, /* header */
1253 DIV_ROUND_UP(inst->size_written, REG_SIZE));
1254
1255 } else {
1256 struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
1257
1258 brw_push_insn_state(p);
1259 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1260
1261 /* a0.0 = surf_index & 0xff */
1262 brw_inst *insn_and = brw_next_insn(p, BRW_OPCODE_AND);
1263 brw_inst_set_exec_size(p->devinfo, insn_and, BRW_EXECUTE_1);
1264 brw_set_dest(p, insn_and, addr);
1265 brw_set_src0(p, insn_and, vec1(retype(index, BRW_REGISTER_TYPE_UD)));
1266 brw_set_src1(p, insn_and, brw_imm_ud(0x0ff));
1267
1268 /* dst = send(payload, a0.0 | <descriptor>) */
1269 brw_inst *insn = brw_send_indirect_message(
1270 p, GEN6_SFID_DATAPORT_CONSTANT_CACHE,
1271 retype(dst, BRW_REGISTER_TYPE_UD),
1272 retype(payload, BRW_REGISTER_TYPE_UD), addr);
1273 brw_set_dp_read_message(p, insn, 0 /* surface */,
1274 BRW_DATAPORT_OWORD_BLOCK_DWORDS(inst->exec_size),
1275 GEN7_DATAPORT_DC_OWORD_BLOCK_READ,
1276 GEN6_SFID_DATAPORT_CONSTANT_CACHE,
1277 1, /* mlen */
1278 true, /* header */
1279 DIV_ROUND_UP(inst->size_written, REG_SIZE));
1280
1281 brw_pop_insn_state(p);
1282 }
1283 }
1284
1285 void
1286 fs_generator::generate_varying_pull_constant_load_gen4(fs_inst *inst,
1287 struct brw_reg dst,
1288 struct brw_reg index)
1289 {
1290 assert(devinfo->gen < 7); /* Should use the gen7 variant. */
1291 assert(inst->header_size != 0);
1292 assert(inst->mlen);
1293
1294 assert(index.file == BRW_IMMEDIATE_VALUE &&
1295 index.type == BRW_REGISTER_TYPE_UD);
1296 uint32_t surf_index = index.ud;
1297
1298 uint32_t simd_mode, rlen, msg_type;
1299 if (inst->exec_size == 16) {
1300 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1301 rlen = 8;
1302 } else {
1303 assert(inst->exec_size == 8);
1304 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
1305 rlen = 4;
1306 }
1307
1308 if (devinfo->gen >= 5)
1309 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
1310 else {
1311 /* We always use the SIMD16 message so that we only have to load U, and
1312 * not V or R.
1313 */
1314 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_LD;
1315 assert(inst->mlen == 3);
1316 assert(inst->size_written == 8 * REG_SIZE);
1317 rlen = 8;
1318 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1319 }
1320
1321 struct brw_reg header = brw_vec8_grf(0, 0);
1322 gen6_resolve_implied_move(p, &header, inst->base_mrf);
1323
1324 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
1325 brw_inst_set_compression(devinfo, send, false);
1326 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UW));
1327 brw_set_src0(p, send, header);
1328 if (devinfo->gen < 6)
1329 brw_inst_set_base_mrf(p->devinfo, send, inst->base_mrf);
1330
1331 /* Our surface is set up as floats, regardless of what actual data is
1332 * stored in it.
1333 */
1334 uint32_t return_format = BRW_SAMPLER_RETURN_FORMAT_FLOAT32;
1335 brw_set_sampler_message(p, send,
1336 surf_index,
1337 0, /* sampler (unused) */
1338 msg_type,
1339 rlen,
1340 inst->mlen,
1341 inst->header_size != 0,
1342 simd_mode,
1343 return_format);
1344 }
1345
1346 void
1347 fs_generator::generate_varying_pull_constant_load_gen7(fs_inst *inst,
1348 struct brw_reg dst,
1349 struct brw_reg index,
1350 struct brw_reg offset)
1351 {
1352 assert(devinfo->gen >= 7);
1353 /* Varying-offset pull constant loads are treated as a normal expression on
1354 * gen7, so the fact that it's a send message is hidden at the IR level.
1355 */
1356 assert(inst->header_size == 0);
1357 assert(!inst->mlen);
1358 assert(index.type == BRW_REGISTER_TYPE_UD);
1359
1360 uint32_t simd_mode, rlen, mlen;
1361 if (inst->exec_size == 16) {
1362 mlen = 2;
1363 rlen = 8;
1364 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1365 } else {
1366 assert(inst->exec_size == 8);
1367 mlen = 1;
1368 rlen = 4;
1369 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
1370 }
1371
1372 if (index.file == BRW_IMMEDIATE_VALUE) {
1373
1374 uint32_t surf_index = index.ud;
1375
1376 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
1377 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UW));
1378 brw_set_src0(p, send, offset);
1379 brw_set_sampler_message(p, send,
1380 surf_index,
1381 0, /* LD message ignores sampler unit */
1382 GEN5_SAMPLER_MESSAGE_SAMPLE_LD,
1383 rlen,
1384 mlen,
1385 false, /* no header */
1386 simd_mode,
1387 0);
1388
1389 } else {
1390
1391 struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
1392
1393 brw_push_insn_state(p);
1394 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1395
1396 /* a0.0 = surf_index & 0xff */
1397 brw_inst *insn_and = brw_next_insn(p, BRW_OPCODE_AND);
1398 brw_inst_set_exec_size(p->devinfo, insn_and, BRW_EXECUTE_1);
1399 brw_set_dest(p, insn_and, addr);
1400 brw_set_src0(p, insn_and, vec1(retype(index, BRW_REGISTER_TYPE_UD)));
1401 brw_set_src1(p, insn_and, brw_imm_ud(0x0ff));
1402
1403 brw_pop_insn_state(p);
1404
1405 /* dst = send(offset, a0.0 | <descriptor>) */
1406 brw_inst *insn = brw_send_indirect_message(
1407 p, BRW_SFID_SAMPLER, retype(dst, BRW_REGISTER_TYPE_UW),
1408 offset, addr);
1409 brw_set_sampler_message(p, insn,
1410 0 /* surface */,
1411 0 /* sampler */,
1412 GEN5_SAMPLER_MESSAGE_SAMPLE_LD,
1413 rlen /* rlen */,
1414 mlen /* mlen */,
1415 false /* header */,
1416 simd_mode,
1417 0);
1418 }
1419 }
1420
1421 /**
1422 * Cause the current pixel/sample mask (from R1.7 bits 15:0) to be transferred
1423 * into the flags register (f0.0).
1424 *
1425 * Used only on Gen6 and above.
1426 */
1427 void
1428 fs_generator::generate_mov_dispatch_to_flags(fs_inst *inst)
1429 {
1430 struct brw_reg flags = brw_flag_reg(0, inst->flag_subreg);
1431 struct brw_reg dispatch_mask;
1432
1433 if (devinfo->gen >= 6)
1434 dispatch_mask = retype(brw_vec1_grf(1, 7), BRW_REGISTER_TYPE_UW);
1435 else
1436 dispatch_mask = retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UW);
1437
1438 brw_push_insn_state(p);
1439 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1440 brw_MOV(p, flags, dispatch_mask);
1441 brw_pop_insn_state(p);
1442 }
1443
1444 void
1445 fs_generator::generate_pixel_interpolator_query(fs_inst *inst,
1446 struct brw_reg dst,
1447 struct brw_reg src,
1448 struct brw_reg msg_data,
1449 unsigned msg_type)
1450 {
1451 assert(inst->size_written % REG_SIZE == 0);
1452 assert(msg_data.type == BRW_REGISTER_TYPE_UD);
1453
1454 brw_pixel_interpolator_query(p,
1455 retype(dst, BRW_REGISTER_TYPE_UW),
1456 src,
1457 inst->pi_noperspective,
1458 msg_type,
1459 msg_data,
1460 inst->mlen,
1461 inst->size_written / REG_SIZE);
1462 }
1463
1464 /* Sets vstride=1, width=4, hstride=0 of register src1 during
1465 * the ADD instruction.
1466 */
1467 void
1468 fs_generator::generate_set_sample_id(fs_inst *inst,
1469 struct brw_reg dst,
1470 struct brw_reg src0,
1471 struct brw_reg src1)
1472 {
1473 assert(dst.type == BRW_REGISTER_TYPE_D ||
1474 dst.type == BRW_REGISTER_TYPE_UD);
1475 assert(src0.type == BRW_REGISTER_TYPE_D ||
1476 src0.type == BRW_REGISTER_TYPE_UD);
1477
1478 struct brw_reg reg = stride(src1, 1, 4, 0);
1479 if (devinfo->gen >= 8 || inst->exec_size == 8) {
1480 brw_ADD(p, dst, src0, reg);
1481 } else if (inst->exec_size == 16) {
1482 brw_push_insn_state(p);
1483 brw_set_default_exec_size(p, BRW_EXECUTE_8);
1484 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
1485 brw_ADD(p, firsthalf(dst), firsthalf(src0), reg);
1486 brw_set_default_compression_control(p, BRW_COMPRESSION_2NDHALF);
1487 brw_ADD(p, sechalf(dst), sechalf(src0), suboffset(reg, 2));
1488 brw_pop_insn_state(p);
1489 }
1490 }
1491
1492 void
1493 fs_generator::generate_pack_half_2x16_split(fs_inst *inst,
1494 struct brw_reg dst,
1495 struct brw_reg x,
1496 struct brw_reg y)
1497 {
1498 assert(devinfo->gen >= 7);
1499 assert(dst.type == BRW_REGISTER_TYPE_UD);
1500 assert(x.type == BRW_REGISTER_TYPE_F);
1501 assert(y.type == BRW_REGISTER_TYPE_F);
1502
1503 /* From the Ivybridge PRM, Vol4, Part3, Section 6.27 f32to16:
1504 *
1505 * Because this instruction does not have a 16-bit floating-point type,
1506 * the destination data type must be Word (W).
1507 *
1508 * The destination must be DWord-aligned and specify a horizontal stride
1509 * (HorzStride) of 2. The 16-bit result is stored in the lower word of
1510 * each destination channel and the upper word is not modified.
1511 */
1512 struct brw_reg dst_w = spread(retype(dst, BRW_REGISTER_TYPE_W), 2);
1513
1514 /* Give each 32-bit channel of dst the form below, where "." means
1515 * unchanged.
1516 * 0x....hhhh
1517 */
1518 brw_F32TO16(p, dst_w, y);
1519
1520 /* Now the form:
1521 * 0xhhhh0000
1522 */
1523 brw_SHL(p, dst, dst, brw_imm_ud(16u));
1524
1525 /* And, finally the form of packHalf2x16's output:
1526 * 0xhhhhllll
1527 */
1528 brw_F32TO16(p, dst_w, x);
1529 }
1530
1531 void
1532 fs_generator::generate_unpack_half_2x16_split(fs_inst *inst,
1533 struct brw_reg dst,
1534 struct brw_reg src)
1535 {
1536 assert(devinfo->gen >= 7);
1537 assert(dst.type == BRW_REGISTER_TYPE_F);
1538 assert(src.type == BRW_REGISTER_TYPE_UD);
1539
1540 /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
1541 *
1542 * Because this instruction does not have a 16-bit floating-point type,
1543 * the source data type must be Word (W). The destination type must be
1544 * F (Float).
1545 */
1546 struct brw_reg src_w = spread(retype(src, BRW_REGISTER_TYPE_W), 2);
1547
1548 /* Each channel of src has the form of unpackHalf2x16's input: 0xhhhhllll.
1549 * For the Y case, we wish to access only the upper word; therefore
1550 * a 16-bit subregister offset is needed.
1551 */
1552 assert(inst->opcode == FS_OPCODE_UNPACK_HALF_2x16_SPLIT_X ||
1553 inst->opcode == FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y);
1554 if (inst->opcode == FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y)
1555 src_w.subnr += 2;
1556
1557 brw_F16TO32(p, dst, src_w);
1558 }
1559
1560 void
1561 fs_generator::generate_shader_time_add(fs_inst *inst,
1562 struct brw_reg payload,
1563 struct brw_reg offset,
1564 struct brw_reg value)
1565 {
1566 assert(devinfo->gen >= 7);
1567 brw_push_insn_state(p);
1568 brw_set_default_mask_control(p, true);
1569
1570 assert(payload.file == BRW_GENERAL_REGISTER_FILE);
1571 struct brw_reg payload_offset = retype(brw_vec1_grf(payload.nr, 0),
1572 offset.type);
1573 struct brw_reg payload_value = retype(brw_vec1_grf(payload.nr + 1, 0),
1574 value.type);
1575
1576 assert(offset.file == BRW_IMMEDIATE_VALUE);
1577 if (value.file == BRW_GENERAL_REGISTER_FILE) {
1578 value.width = BRW_WIDTH_1;
1579 value.hstride = BRW_HORIZONTAL_STRIDE_0;
1580 value.vstride = BRW_VERTICAL_STRIDE_0;
1581 } else {
1582 assert(value.file == BRW_IMMEDIATE_VALUE);
1583 }
1584
1585 /* Trying to deal with setup of the params from the IR is crazy in the FS8
1586 * case, and we don't really care about squeezing every bit of performance
1587 * out of this path, so we just emit the MOVs from here.
1588 */
1589 brw_MOV(p, payload_offset, offset);
1590 brw_MOV(p, payload_value, value);
1591 brw_shader_time_add(p, payload,
1592 prog_data->binding_table.shader_time_start);
1593 brw_pop_insn_state(p);
1594
1595 brw_mark_surface_used(prog_data,
1596 prog_data->binding_table.shader_time_start);
1597 }
1598
1599 void
1600 fs_generator::enable_debug(const char *shader_name)
1601 {
1602 debug_flag = true;
1603 this->shader_name = shader_name;
1604 }
1605
1606 int
1607 fs_generator::generate_code(const cfg_t *cfg, int dispatch_width)
1608 {
1609 /* align to 64 byte boundary. */
1610 while (p->next_insn_offset % 64)
1611 brw_NOP(p);
1612
1613 this->dispatch_width = dispatch_width;
1614
1615 int start_offset = p->next_insn_offset;
1616 int spill_count = 0, fill_count = 0;
1617 int loop_count = 0;
1618
1619 struct annotation_info annotation;
1620 memset(&annotation, 0, sizeof(annotation));
1621
1622 foreach_block_and_inst (block, fs_inst, inst, cfg) {
1623 struct brw_reg src[3], dst;
1624 unsigned int last_insn_offset = p->next_insn_offset;
1625 bool multiple_instructions_emitted = false;
1626
1627 /* From the Broadwell PRM, Volume 7, "3D-Media-GPGPU", in the
1628 * "Register Region Restrictions" section: for BDW, SKL:
1629 *
1630 * "A POW/FDIV operation must not be followed by an instruction
1631 * that requires two destination registers."
1632 *
1633 * The documentation is often lacking annotations for Atom parts,
1634 * and empirically this affects CHV as well.
1635 */
1636 if (devinfo->gen >= 8 &&
1637 p->nr_insn > 1 &&
1638 brw_inst_opcode(devinfo, brw_last_inst) == BRW_OPCODE_MATH &&
1639 brw_inst_math_function(devinfo, brw_last_inst) == BRW_MATH_FUNCTION_POW &&
1640 inst->dst.component_size(inst->exec_size) > REG_SIZE) {
1641 brw_NOP(p);
1642 last_insn_offset = p->next_insn_offset;
1643 }
1644
1645 if (unlikely(debug_flag))
1646 annotate(p->devinfo, &annotation, cfg, inst, p->next_insn_offset);
1647
1648 /* If the instruction writes to more than one register, it needs to be
1649 * explicitly marked as compressed on Gen <= 5. On Gen >= 6 the
1650 * hardware figures out by itself what the right compression mode is,
1651 * but we still need to know whether the instruction is compressed to
1652 * set up the source register regions appropriately.
1653 *
1654 * XXX - This is wrong for instructions that write a single register but
1655 * read more than one which should strictly speaking be treated as
1656 * compressed. For instructions that don't write any registers it
1657 * relies on the destination being a null register of the correct
1658 * type and regioning so the instruction is considered compressed
1659 * or not accordingly.
1660 */
1661 const bool compressed =
1662 inst->dst.component_size(inst->exec_size) > REG_SIZE;
1663 brw_set_default_compression(p, compressed);
1664 brw_set_default_group(p, inst->group);
1665
1666 for (unsigned int i = 0; i < inst->sources; i++) {
1667 src[i] = brw_reg_from_fs_reg(devinfo, inst,
1668 &inst->src[i], compressed);
1669 /* The accumulator result appears to get used for the
1670 * conditional modifier generation. When negating a UD
1671 * value, there is a 33rd bit generated for the sign in the
1672 * accumulator value, so now you can't check, for example,
1673 * equality with a 32-bit value. See piglit fs-op-neg-uvec4.
1674 */
1675 assert(!inst->conditional_mod ||
1676 inst->src[i].type != BRW_REGISTER_TYPE_UD ||
1677 !inst->src[i].negate);
1678 }
1679 dst = brw_reg_from_fs_reg(devinfo, inst,
1680 &inst->dst, compressed);
1681
1682 brw_set_default_access_mode(p, BRW_ALIGN_1);
1683 brw_set_default_predicate_control(p, inst->predicate);
1684 brw_set_default_predicate_inverse(p, inst->predicate_inverse);
1685 brw_set_default_flag_reg(p, 0, inst->flag_subreg);
1686 brw_set_default_saturate(p, inst->saturate);
1687 brw_set_default_mask_control(p, inst->force_writemask_all);
1688 brw_set_default_acc_write_control(p, inst->writes_accumulator);
1689
1690 unsigned exec_size = inst->exec_size;
1691 if (devinfo->gen == 7 && !devinfo->is_haswell &&
1692 (get_exec_type_size(inst) == 8 || type_sz(inst->dst.type) == 8)) {
1693 exec_size *= 2;
1694 }
1695
1696 brw_set_default_exec_size(p, cvt(exec_size) - 1);
1697
1698 assert(inst->force_writemask_all || inst->exec_size >= 4);
1699 assert(inst->force_writemask_all || inst->group % inst->exec_size == 0);
1700 assert(inst->base_mrf + inst->mlen <= BRW_MAX_MRF(devinfo->gen));
1701 assert(inst->mlen <= BRW_MAX_MSG_LENGTH);
1702
1703 switch (inst->opcode) {
1704 case BRW_OPCODE_MOV:
1705 brw_MOV(p, dst, src[0]);
1706 break;
1707 case BRW_OPCODE_ADD:
1708 brw_ADD(p, dst, src[0], src[1]);
1709 break;
1710 case BRW_OPCODE_MUL:
1711 brw_MUL(p, dst, src[0], src[1]);
1712 break;
1713 case BRW_OPCODE_AVG:
1714 brw_AVG(p, dst, src[0], src[1]);
1715 break;
1716 case BRW_OPCODE_MACH:
1717 brw_MACH(p, dst, src[0], src[1]);
1718 break;
1719
1720 case BRW_OPCODE_LINE:
1721 brw_LINE(p, dst, src[0], src[1]);
1722 break;
1723
1724 case BRW_OPCODE_MAD:
1725 assert(devinfo->gen >= 6);
1726 brw_set_default_access_mode(p, BRW_ALIGN_16);
1727 brw_MAD(p, dst, src[0], src[1], src[2]);
1728 break;
1729
1730 case BRW_OPCODE_LRP:
1731 assert(devinfo->gen >= 6);
1732 brw_set_default_access_mode(p, BRW_ALIGN_16);
1733 brw_LRP(p, dst, src[0], src[1], src[2]);
1734 break;
1735
1736 case BRW_OPCODE_FRC:
1737 brw_FRC(p, dst, src[0]);
1738 break;
1739 case BRW_OPCODE_RNDD:
1740 brw_RNDD(p, dst, src[0]);
1741 break;
1742 case BRW_OPCODE_RNDE:
1743 brw_RNDE(p, dst, src[0]);
1744 break;
1745 case BRW_OPCODE_RNDZ:
1746 brw_RNDZ(p, dst, src[0]);
1747 break;
1748
1749 case BRW_OPCODE_AND:
1750 brw_AND(p, dst, src[0], src[1]);
1751 break;
1752 case BRW_OPCODE_OR:
1753 brw_OR(p, dst, src[0], src[1]);
1754 break;
1755 case BRW_OPCODE_XOR:
1756 brw_XOR(p, dst, src[0], src[1]);
1757 break;
1758 case BRW_OPCODE_NOT:
1759 brw_NOT(p, dst, src[0]);
1760 break;
1761 case BRW_OPCODE_ASR:
1762 brw_ASR(p, dst, src[0], src[1]);
1763 break;
1764 case BRW_OPCODE_SHR:
1765 brw_SHR(p, dst, src[0], src[1]);
1766 break;
1767 case BRW_OPCODE_SHL:
1768 brw_SHL(p, dst, src[0], src[1]);
1769 break;
1770 case BRW_OPCODE_F32TO16:
1771 assert(devinfo->gen >= 7);
1772 brw_F32TO16(p, dst, src[0]);
1773 break;
1774 case BRW_OPCODE_F16TO32:
1775 assert(devinfo->gen >= 7);
1776 brw_F16TO32(p, dst, src[0]);
1777 break;
1778 case BRW_OPCODE_CMP:
1779 if (inst->exec_size >= 16 && devinfo->gen == 7 && !devinfo->is_haswell &&
1780 dst.file == BRW_ARCHITECTURE_REGISTER_FILE) {
1781 /* For unknown reasons the WaCMPInstFlagDepClearedEarly workaround
1782 * implemented in the compiler is not sufficient. Overriding the
1783 * type when the destination is the null register is necessary but
1784 * not sufficient by itself.
1785 */
1786 assert(dst.nr == BRW_ARF_NULL);
1787 dst.type = BRW_REGISTER_TYPE_D;
1788 }
1789 brw_CMP(p, dst, inst->conditional_mod, src[0], src[1]);
1790 break;
1791 case BRW_OPCODE_SEL:
1792 brw_SEL(p, dst, src[0], src[1]);
1793 break;
1794 case BRW_OPCODE_BFREV:
1795 assert(devinfo->gen >= 7);
1796 /* BFREV only supports UD type for src and dst. */
1797 brw_BFREV(p, retype(dst, BRW_REGISTER_TYPE_UD),
1798 retype(src[0], BRW_REGISTER_TYPE_UD));
1799 break;
1800 case BRW_OPCODE_FBH:
1801 assert(devinfo->gen >= 7);
1802 /* FBH only supports UD type for dst. */
1803 brw_FBH(p, retype(dst, BRW_REGISTER_TYPE_UD), src[0]);
1804 break;
1805 case BRW_OPCODE_FBL:
1806 assert(devinfo->gen >= 7);
1807 /* FBL only supports UD type for dst. */
1808 brw_FBL(p, retype(dst, BRW_REGISTER_TYPE_UD), src[0]);
1809 break;
1810 case BRW_OPCODE_LZD:
1811 brw_LZD(p, dst, src[0]);
1812 break;
1813 case BRW_OPCODE_CBIT:
1814 assert(devinfo->gen >= 7);
1815 /* CBIT only supports UD type for dst. */
1816 brw_CBIT(p, retype(dst, BRW_REGISTER_TYPE_UD), src[0]);
1817 break;
1818 case BRW_OPCODE_ADDC:
1819 assert(devinfo->gen >= 7);
1820 brw_ADDC(p, dst, src[0], src[1]);
1821 break;
1822 case BRW_OPCODE_SUBB:
1823 assert(devinfo->gen >= 7);
1824 brw_SUBB(p, dst, src[0], src[1]);
1825 break;
1826 case BRW_OPCODE_MAC:
1827 brw_MAC(p, dst, src[0], src[1]);
1828 break;
1829
1830 case BRW_OPCODE_BFE:
1831 assert(devinfo->gen >= 7);
1832 brw_set_default_access_mode(p, BRW_ALIGN_16);
1833 brw_BFE(p, dst, src[0], src[1], src[2]);
1834 break;
1835
1836 case BRW_OPCODE_BFI1:
1837 assert(devinfo->gen >= 7);
1838 brw_BFI1(p, dst, src[0], src[1]);
1839 break;
1840 case BRW_OPCODE_BFI2:
1841 assert(devinfo->gen >= 7);
1842 brw_set_default_access_mode(p, BRW_ALIGN_16);
1843 brw_BFI2(p, dst, src[0], src[1], src[2]);
1844 break;
1845
1846 case BRW_OPCODE_IF:
1847 if (inst->src[0].file != BAD_FILE) {
1848 /* The instruction has an embedded compare (only allowed on gen6) */
1849 assert(devinfo->gen == 6);
1850 gen6_IF(p, inst->conditional_mod, src[0], src[1]);
1851 } else {
1852 brw_IF(p, brw_inst_exec_size(devinfo, p->current));
1853 }
1854 break;
1855
1856 case BRW_OPCODE_ELSE:
1857 brw_ELSE(p);
1858 break;
1859 case BRW_OPCODE_ENDIF:
1860 brw_ENDIF(p);
1861 break;
1862
1863 case BRW_OPCODE_DO:
1864 brw_DO(p, brw_inst_exec_size(devinfo, p->current));
1865 break;
1866
1867 case BRW_OPCODE_BREAK:
1868 brw_BREAK(p);
1869 break;
1870 case BRW_OPCODE_CONTINUE:
1871 brw_CONT(p);
1872 break;
1873
1874 case BRW_OPCODE_WHILE:
1875 brw_WHILE(p);
1876 loop_count++;
1877 break;
1878
1879 case SHADER_OPCODE_RCP:
1880 case SHADER_OPCODE_RSQ:
1881 case SHADER_OPCODE_SQRT:
1882 case SHADER_OPCODE_EXP2:
1883 case SHADER_OPCODE_LOG2:
1884 case SHADER_OPCODE_SIN:
1885 case SHADER_OPCODE_COS:
1886 assert(inst->conditional_mod == BRW_CONDITIONAL_NONE);
1887 if (devinfo->gen >= 6) {
1888 assert(inst->mlen == 0);
1889 assert(devinfo->gen >= 7 || inst->exec_size == 8);
1890 gen6_math(p, dst, brw_math_function(inst->opcode),
1891 src[0], brw_null_reg());
1892 } else {
1893 assert(inst->mlen >= 1);
1894 assert(devinfo->gen == 5 || devinfo->is_g4x || inst->exec_size == 8);
1895 gen4_math(p, dst,
1896 brw_math_function(inst->opcode),
1897 inst->base_mrf, src[0],
1898 BRW_MATH_PRECISION_FULL);
1899 }
1900 break;
1901 case SHADER_OPCODE_INT_QUOTIENT:
1902 case SHADER_OPCODE_INT_REMAINDER:
1903 case SHADER_OPCODE_POW:
1904 assert(inst->conditional_mod == BRW_CONDITIONAL_NONE);
1905 if (devinfo->gen >= 6) {
1906 assert(inst->mlen == 0);
1907 assert((devinfo->gen >= 7 && inst->opcode == SHADER_OPCODE_POW) ||
1908 inst->exec_size == 8);
1909 gen6_math(p, dst, brw_math_function(inst->opcode), src[0], src[1]);
1910 } else {
1911 assert(inst->mlen >= 1);
1912 assert(inst->exec_size == 8);
1913 gen4_math(p, dst, brw_math_function(inst->opcode),
1914 inst->base_mrf, src[0],
1915 BRW_MATH_PRECISION_FULL);
1916 }
1917 break;
1918 case FS_OPCODE_CINTERP:
1919 brw_MOV(p, dst, src[0]);
1920 break;
1921 case FS_OPCODE_LINTERP:
1922 generate_linterp(inst, dst, src);
1923 break;
1924 case FS_OPCODE_PIXEL_X:
1925 assert(src[0].type == BRW_REGISTER_TYPE_UW);
1926 src[0].subnr = 0 * type_sz(src[0].type);
1927 brw_MOV(p, dst, stride(src[0], 8, 4, 1));
1928 break;
1929 case FS_OPCODE_PIXEL_Y:
1930 assert(src[0].type == BRW_REGISTER_TYPE_UW);
1931 src[0].subnr = 4 * type_sz(src[0].type);
1932 brw_MOV(p, dst, stride(src[0], 8, 4, 1));
1933 break;
1934 case FS_OPCODE_GET_BUFFER_SIZE:
1935 generate_get_buffer_size(inst, dst, src[0], src[1]);
1936 break;
1937 case SHADER_OPCODE_TEX:
1938 case FS_OPCODE_TXB:
1939 case SHADER_OPCODE_TXD:
1940 case SHADER_OPCODE_TXF:
1941 case SHADER_OPCODE_TXF_LZ:
1942 case SHADER_OPCODE_TXF_CMS:
1943 case SHADER_OPCODE_TXF_CMS_W:
1944 case SHADER_OPCODE_TXF_UMS:
1945 case SHADER_OPCODE_TXF_MCS:
1946 case SHADER_OPCODE_TXL:
1947 case SHADER_OPCODE_TXL_LZ:
1948 case SHADER_OPCODE_TXS:
1949 case SHADER_OPCODE_LOD:
1950 case SHADER_OPCODE_TG4:
1951 case SHADER_OPCODE_TG4_OFFSET:
1952 case SHADER_OPCODE_SAMPLEINFO:
1953 generate_tex(inst, dst, src[0], src[1], src[2]);
1954 break;
1955 case FS_OPCODE_DDX_COARSE:
1956 case FS_OPCODE_DDX_FINE:
1957 generate_ddx(inst->opcode, dst, src[0]);
1958 break;
1959 case FS_OPCODE_DDY_COARSE:
1960 case FS_OPCODE_DDY_FINE:
1961 generate_ddy(inst->opcode, dst, src[0]);
1962 break;
1963
1964 case SHADER_OPCODE_GEN4_SCRATCH_WRITE:
1965 generate_scratch_write(inst, src[0]);
1966 spill_count++;
1967 break;
1968
1969 case SHADER_OPCODE_GEN4_SCRATCH_READ:
1970 generate_scratch_read(inst, dst);
1971 fill_count++;
1972 break;
1973
1974 case SHADER_OPCODE_GEN7_SCRATCH_READ:
1975 generate_scratch_read_gen7(inst, dst);
1976 fill_count++;
1977 break;
1978
1979 case SHADER_OPCODE_MOV_INDIRECT:
1980 generate_mov_indirect(inst, dst, src[0], src[1]);
1981 break;
1982
1983 case SHADER_OPCODE_URB_READ_SIMD8:
1984 case SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT:
1985 generate_urb_read(inst, dst, src[0]);
1986 break;
1987
1988 case SHADER_OPCODE_URB_WRITE_SIMD8:
1989 case SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT:
1990 case SHADER_OPCODE_URB_WRITE_SIMD8_MASKED:
1991 case SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT:
1992 generate_urb_write(inst, src[0]);
1993 break;
1994
1995 case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD:
1996 assert(inst->force_writemask_all);
1997 generate_uniform_pull_constant_load(inst, dst, src[0], src[1]);
1998 break;
1999
2000 case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD_GEN7:
2001 assert(inst->force_writemask_all);
2002 generate_uniform_pull_constant_load_gen7(inst, dst, src[0], src[1]);
2003 break;
2004
2005 case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_GEN4:
2006 generate_varying_pull_constant_load_gen4(inst, dst, src[0]);
2007 break;
2008
2009 case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_GEN7:
2010 generate_varying_pull_constant_load_gen7(inst, dst, src[0], src[1]);
2011 break;
2012
2013 case FS_OPCODE_REP_FB_WRITE:
2014 case FS_OPCODE_FB_WRITE:
2015 generate_fb_write(inst, src[0]);
2016 break;
2017
2018 case FS_OPCODE_FB_READ:
2019 generate_fb_read(inst, dst, src[0]);
2020 break;
2021
2022 case FS_OPCODE_MOV_DISPATCH_TO_FLAGS:
2023 generate_mov_dispatch_to_flags(inst);
2024 break;
2025
2026 case FS_OPCODE_DISCARD_JUMP:
2027 generate_discard_jump(inst);
2028 break;
2029
2030 case SHADER_OPCODE_SHADER_TIME_ADD:
2031 generate_shader_time_add(inst, src[0], src[1], src[2]);
2032 break;
2033
2034 case SHADER_OPCODE_UNTYPED_ATOMIC:
2035 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2036 brw_untyped_atomic(p, dst, src[0], src[1], src[2].ud,
2037 inst->mlen, !inst->dst.is_null());
2038 break;
2039
2040 case SHADER_OPCODE_UNTYPED_SURFACE_READ:
2041 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2042 brw_untyped_surface_read(p, dst, src[0], src[1],
2043 inst->mlen, src[2].ud);
2044 break;
2045
2046 case SHADER_OPCODE_UNTYPED_SURFACE_WRITE:
2047 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2048 brw_untyped_surface_write(p, src[0], src[1],
2049 inst->mlen, src[2].ud);
2050 break;
2051
2052 case SHADER_OPCODE_TYPED_ATOMIC:
2053 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2054 brw_typed_atomic(p, dst, src[0], src[1],
2055 src[2].ud, inst->mlen, !inst->dst.is_null());
2056 break;
2057
2058 case SHADER_OPCODE_TYPED_SURFACE_READ:
2059 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2060 brw_typed_surface_read(p, dst, src[0], src[1],
2061 inst->mlen, src[2].ud);
2062 break;
2063
2064 case SHADER_OPCODE_TYPED_SURFACE_WRITE:
2065 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2066 brw_typed_surface_write(p, src[0], src[1], inst->mlen, src[2].ud);
2067 break;
2068
2069 case SHADER_OPCODE_MEMORY_FENCE:
2070 brw_memory_fence(p, dst);
2071 break;
2072
2073 case SHADER_OPCODE_FIND_LIVE_CHANNEL: {
2074 const struct brw_reg mask =
2075 brw_stage_has_packed_dispatch(devinfo, stage,
2076 prog_data) ? brw_imm_ud(~0u) :
2077 stage == MESA_SHADER_FRAGMENT ? brw_vmask_reg() :
2078 brw_dmask_reg();
2079 brw_find_live_channel(p, dst, mask);
2080 break;
2081 }
2082
2083 case SHADER_OPCODE_BROADCAST:
2084 assert(inst->force_writemask_all);
2085 brw_broadcast(p, dst, src[0], src[1]);
2086 break;
2087
2088 case FS_OPCODE_SET_SAMPLE_ID:
2089 generate_set_sample_id(inst, dst, src[0], src[1]);
2090 break;
2091
2092 case FS_OPCODE_PACK_HALF_2x16_SPLIT:
2093 generate_pack_half_2x16_split(inst, dst, src[0], src[1]);
2094 break;
2095
2096 case FS_OPCODE_UNPACK_HALF_2x16_SPLIT_X:
2097 case FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y:
2098 generate_unpack_half_2x16_split(inst, dst, src[0]);
2099 break;
2100
2101 case FS_OPCODE_PLACEHOLDER_HALT:
2102 /* This is the place where the final HALT needs to be inserted if
2103 * we've emitted any discards. If not, this will emit no code.
2104 */
2105 if (!patch_discard_jumps_to_fb_writes()) {
2106 if (unlikely(debug_flag)) {
2107 annotation.ann_count--;
2108 }
2109 }
2110 break;
2111
2112 case FS_OPCODE_INTERPOLATE_AT_SAMPLE:
2113 generate_pixel_interpolator_query(inst, dst, src[0], src[1],
2114 GEN7_PIXEL_INTERPOLATOR_LOC_SAMPLE);
2115 break;
2116
2117 case FS_OPCODE_INTERPOLATE_AT_SHARED_OFFSET:
2118 generate_pixel_interpolator_query(inst, dst, src[0], src[1],
2119 GEN7_PIXEL_INTERPOLATOR_LOC_SHARED_OFFSET);
2120 break;
2121
2122 case FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET:
2123 generate_pixel_interpolator_query(inst, dst, src[0], src[1],
2124 GEN7_PIXEL_INTERPOLATOR_LOC_PER_SLOT_OFFSET);
2125 break;
2126
2127 case CS_OPCODE_CS_TERMINATE:
2128 generate_cs_terminate(inst, src[0]);
2129 break;
2130
2131 case SHADER_OPCODE_BARRIER:
2132 generate_barrier(inst, src[0]);
2133 break;
2134
2135 case BRW_OPCODE_DIM:
2136 assert(devinfo->is_haswell);
2137 assert(src[0].type == BRW_REGISTER_TYPE_DF);
2138 assert(dst.type == BRW_REGISTER_TYPE_DF);
2139 brw_DIM(p, dst, retype(src[0], BRW_REGISTER_TYPE_F));
2140 break;
2141
2142 default:
2143 unreachable("Unsupported opcode");
2144
2145 case SHADER_OPCODE_LOAD_PAYLOAD:
2146 unreachable("Should be lowered by lower_load_payload()");
2147 }
2148
2149 if (multiple_instructions_emitted)
2150 continue;
2151
2152 if (inst->no_dd_clear || inst->no_dd_check || inst->conditional_mod) {
2153 assert(p->next_insn_offset == last_insn_offset + 16 ||
2154 !"conditional_mod, no_dd_check, or no_dd_clear set for IR "
2155 "emitting more than 1 instruction");
2156
2157 brw_inst *last = &p->store[last_insn_offset / 16];
2158
2159 if (inst->conditional_mod)
2160 brw_inst_set_cond_modifier(p->devinfo, last, inst->conditional_mod);
2161 brw_inst_set_no_dd_clear(p->devinfo, last, inst->no_dd_clear);
2162 brw_inst_set_no_dd_check(p->devinfo, last, inst->no_dd_check);
2163 }
2164 }
2165
2166 brw_set_uip_jip(p, start_offset);
2167 annotation_finalize(&annotation, p->next_insn_offset);
2168
2169 #ifndef NDEBUG
2170 bool validated = brw_validate_instructions(devinfo, p->store,
2171 start_offset,
2172 p->next_insn_offset,
2173 &annotation);
2174 #else
2175 if (unlikely(debug_flag))
2176 brw_validate_instructions(devinfo, p->store,
2177 start_offset,
2178 p->next_insn_offset,
2179 &annotation);
2180 #endif
2181
2182 int before_size = p->next_insn_offset - start_offset;
2183 brw_compact_instructions(p, start_offset, annotation.ann_count,
2184 annotation.ann);
2185 int after_size = p->next_insn_offset - start_offset;
2186
2187 if (unlikely(debug_flag)) {
2188 fprintf(stderr, "Native code for %s\n"
2189 "SIMD%d shader: %d instructions. %d loops. %u cycles. %d:%d spills:fills. Promoted %u constants. Compacted %d to %d"
2190 " bytes (%.0f%%)\n",
2191 shader_name, dispatch_width, before_size / 16, loop_count, cfg->cycle_count,
2192 spill_count, fill_count, promoted_constants, before_size, after_size,
2193 100.0f * (before_size - after_size) / before_size);
2194
2195 dump_assembly(p->store, annotation.ann_count, annotation.ann,
2196 p->devinfo);
2197 ralloc_free(annotation.mem_ctx);
2198 }
2199 assert(validated);
2200
2201 compiler->shader_debug_log(log_data,
2202 "%s SIMD%d shader: %d inst, %d loops, %u cycles, "
2203 "%d:%d spills:fills, Promoted %u constants, "
2204 "compacted %d to %d bytes.",
2205 _mesa_shader_stage_to_abbrev(stage),
2206 dispatch_width, before_size / 16,
2207 loop_count, cfg->cycle_count, spill_count,
2208 fill_count, promoted_constants, before_size,
2209 after_size);
2210
2211 return start_offset;
2212 }
2213
2214 const unsigned *
2215 fs_generator::get_assembly(unsigned int *assembly_size)
2216 {
2217 return brw_get_program(p, assembly_size);
2218 }