projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
intel/compiler: Lower flrp32 on Gen11+
[mesa.git] / src / intel / compiler / brw_fs_generator.cpp
1 /*
2 * Copyright © 2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 /** @file brw_fs_generator.cpp
25 *
26 * This file supports generating code from the FS LIR to the actual
27 * native instructions.
28 */
29
30 #include "brw_eu.h"
31 #include "brw_fs.h"
32 #include "brw_cfg.h"
33
34 static enum brw_reg_file
35 brw_file_from_reg(fs_reg *reg)
36 {
37 switch (reg->file) {
38 case ARF:
39 return BRW_ARCHITECTURE_REGISTER_FILE;
40 case FIXED_GRF:
41 case VGRF:
42 return BRW_GENERAL_REGISTER_FILE;
43 case MRF:
44 return BRW_MESSAGE_REGISTER_FILE;
45 case IMM:
46 return BRW_IMMEDIATE_VALUE;
47 case BAD_FILE:
48 case ATTR:
49 case UNIFORM:
50 unreachable("not reached");
51 }
52 return BRW_ARCHITECTURE_REGISTER_FILE;
53 }
54
55 static struct brw_reg
56 brw_reg_from_fs_reg(const struct gen_device_info *devinfo, fs_inst *inst,
57 fs_reg *reg, bool compressed)
58 {
59 struct brw_reg brw_reg;
60
61 switch (reg->file) {
62 case MRF:
63 assert((reg->nr & ~BRW_MRF_COMPR4) < BRW_MAX_MRF(devinfo->gen));
64 /* Fallthrough */
65 case VGRF:
66 if (reg->stride == 0) {
67 brw_reg = brw_vec1_reg(brw_file_from_reg(reg), reg->nr, 0);
68 } else {
69 /* From the Haswell PRM:
70 *
71 * "VertStride must be used to cross GRF register boundaries. This
72 * rule implies that elements within a 'Width' cannot cross GRF
73 * boundaries."
74 *
75 * The maximum width value that could satisfy this restriction is:
76 */
77 const unsigned reg_width = REG_SIZE / (reg->stride * type_sz(reg->type));
78
79 /* Because the hardware can only split source regions at a whole
80 * multiple of width during decompression (i.e. vertically), clamp
81 * the value obtained above to the physical execution size of a
82 * single decompressed chunk of the instruction:
83 */
84 const unsigned phys_width = compressed ? inst->exec_size / 2 :
85 inst->exec_size;
86
87 /* XXX - The equation above is strictly speaking not correct on
88 * hardware that supports unbalanced GRF writes -- On Gen9+
89 * each decompressed chunk of the instruction may have a
90 * different execution size when the number of components
91 * written to each destination GRF is not the same.
92 */
93 const unsigned width = MIN2(reg_width, phys_width);
94 brw_reg = brw_vecn_reg(width, brw_file_from_reg(reg), reg->nr, 0);
95 brw_reg = stride(brw_reg, width * reg->stride, width, reg->stride);
96
97 if (devinfo->gen == 7 && !devinfo->is_haswell) {
98 /* From the IvyBridge PRM (EU Changes by Processor Generation, page 13):
99 * "Each DF (Double Float) operand uses an element size of 4 rather
100 * than 8 and all regioning parameters are twice what the values
101 * would be based on the true element size: ExecSize, Width,
102 * HorzStride, and VertStride. Each DF operand uses a pair of
103 * channels and all masking and swizzing should be adjusted
104 * appropriately."
105 *
106 * From the IvyBridge PRM (Special Requirements for Handling Double
107 * Precision Data Types, page 71):
108 * "In Align1 mode, all regioning parameters like stride, execution
109 * size, and width must use the syntax of a pair of packed
110 * floats. The offsets for these data types must be 64-bit
111 * aligned. The execution size and regioning parameters are in terms
112 * of floats."
113 *
114 * Summarized: when handling DF-typed arguments, ExecSize,
115 * VertStride, and Width must be doubled.
116 *
117 * It applies to BayTrail too.
118 */
119 if (type_sz(reg->type) == 8) {
120 brw_reg.width++;
121 if (brw_reg.vstride > 0)
122 brw_reg.vstride++;
123 assert(brw_reg.hstride == BRW_HORIZONTAL_STRIDE_1);
124 }
125
126 /* When converting from DF->F, we set the destination stride to 2
127 * because each d2f conversion implicitly writes 2 floats, being
128 * the first one the converted value. IVB/BYT actually writes two
129 * F components per SIMD channel, and every other component is
130 * filled with garbage.
131 */
132 if (reg == &inst->dst && get_exec_type_size(inst) == 8 &&
133 type_sz(inst->dst.type) < 8) {
134 assert(brw_reg.hstride > BRW_HORIZONTAL_STRIDE_1);
135 brw_reg.hstride--;
136 }
137 }
138 }
139
140 brw_reg = retype(brw_reg, reg->type);
141 brw_reg = byte_offset(brw_reg, reg->offset);
142 brw_reg.abs = reg->abs;
143 brw_reg.negate = reg->negate;
144 break;
145 case ARF:
146 case FIXED_GRF:
147 case IMM:
148 assert(reg->offset == 0);
149 brw_reg = reg->as_brw_reg();
150 break;
151 case BAD_FILE:
152 /* Probably unused. */
153 brw_reg = brw_null_reg();
154 break;
155 case ATTR:
156 case UNIFORM:
157 unreachable("not reached");
158 }
159
160 /* On HSW+, scalar DF sources can be accessed using the normal <0,1,0>
161 * region, but on IVB and BYT DF regions must be programmed in terms of
162 * floats. A <0,2,1> region accomplishes this.
163 */
164 if (devinfo->gen == 7 && !devinfo->is_haswell &&
165 type_sz(reg->type) == 8 &&
166 brw_reg.vstride == BRW_VERTICAL_STRIDE_0 &&
167 brw_reg.width == BRW_WIDTH_1 &&
168 brw_reg.hstride == BRW_HORIZONTAL_STRIDE_0) {
169 brw_reg.width = BRW_WIDTH_2;
170 brw_reg.hstride = BRW_HORIZONTAL_STRIDE_1;
171 }
172
173 return brw_reg;
174 }
175
176 fs_generator::fs_generator(const struct brw_compiler *compiler, void *log_data,
177 void *mem_ctx,
178 const void *key,
179 struct brw_stage_prog_data *prog_data,
180 unsigned promoted_constants,
181 bool runtime_check_aads_emit,
182 gl_shader_stage stage)
183
184 : compiler(compiler), log_data(log_data),
185 devinfo(compiler->devinfo), key(key),
186 prog_data(prog_data),
187 promoted_constants(promoted_constants),
188 runtime_check_aads_emit(runtime_check_aads_emit), debug_flag(false),
189 stage(stage), mem_ctx(mem_ctx)
190 {
191 p = rzalloc(mem_ctx, struct brw_codegen);
192 brw_init_codegen(devinfo, p, mem_ctx);
193
194 /* In the FS code generator, we are very careful to ensure that we always
195 * set the right execution size so we don't need the EU code to "help" us
196 * by trying to infer it. Sometimes, it infers the wrong thing.
197 */
198 p->automatic_exec_sizes = false;
199 }
200
201 fs_generator::~fs_generator()
202 {
203 }
204
205 class ip_record : public exec_node {
206 public:
207 DECLARE_RALLOC_CXX_OPERATORS(ip_record)
208
209 ip_record(int ip)
210 {
211 this->ip = ip;
212 }
213
214 int ip;
215 };
216
217 bool
218 fs_generator::patch_discard_jumps_to_fb_writes()
219 {
220 if (devinfo->gen < 6 || this->discard_halt_patches.is_empty())
221 return false;
222
223 int scale = brw_jump_scale(p->devinfo);
224
225 /* There is a somewhat strange undocumented requirement of using
226 * HALT, according to the simulator. If some channel has HALTed to
227 * a particular UIP, then by the end of the program, every channel
228 * must have HALTed to that UIP. Furthermore, the tracking is a
229 * stack, so you can't do the final halt of a UIP after starting
230 * halting to a new UIP.
231 *
232 * Symptoms of not emitting this instruction on actual hardware
233 * included GPU hangs and sparkly rendering on the piglit discard
234 * tests.
235 */
236 brw_inst *last_halt = gen6_HALT(p);
237 brw_inst_set_uip(p->devinfo, last_halt, 1 * scale);
238 brw_inst_set_jip(p->devinfo, last_halt, 1 * scale);
239
240 int ip = p->nr_insn;
241
242 foreach_in_list(ip_record, patch_ip, &discard_halt_patches) {
243 brw_inst *patch = &p->store[patch_ip->ip];
244
245 assert(brw_inst_opcode(p->devinfo, patch) == BRW_OPCODE_HALT);
246 /* HALT takes a half-instruction distance from the pre-incremented IP. */
247 brw_inst_set_uip(p->devinfo, patch, (ip - patch_ip->ip) * scale);
248 }
249
250 this->discard_halt_patches.make_empty();
251 return true;
252 }
253
254 void
255 fs_generator::fire_fb_write(fs_inst *inst,
256 struct brw_reg payload,
257 struct brw_reg implied_header,
258 GLuint nr)
259 {
260 uint32_t msg_control;
261
262 struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
263
264 if (devinfo->gen < 6) {
265 brw_push_insn_state(p);
266 brw_set_default_exec_size(p, BRW_EXECUTE_8);
267 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
268 brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
269 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
270 brw_MOV(p, offset(payload, 1), brw_vec8_grf(1, 0));
271 brw_pop_insn_state(p);
272 }
273
274 if (inst->opcode == FS_OPCODE_REP_FB_WRITE)
275 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD16_SINGLE_SOURCE_REPLICATED;
276 else if (prog_data->dual_src_blend) {
277 if (!inst->group)
278 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_DUAL_SOURCE_SUBSPAN01;
279 else
280 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_DUAL_SOURCE_SUBSPAN23;
281 } else if (inst->exec_size == 16)
282 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD16_SINGLE_SOURCE;
283 else
284 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_SINGLE_SOURCE_SUBSPAN01;
285
286 /* We assume render targets start at 0, because headerless FB write
287 * messages set "Render Target Index" to 0. Using a different binding
288 * table index would make it impossible to use headerless messages.
289 */
290 const uint32_t surf_index = inst->target;
291
292 bool last_render_target = inst->eot ||
293 (prog_data->dual_src_blend && dispatch_width == 16);
294
295
296 brw_fb_WRITE(p,
297 payload,
298 implied_header,
299 msg_control,
300 surf_index,
301 nr,
302 0,
303 inst->eot,
304 last_render_target,
305 inst->header_size != 0);
306
307 brw_mark_surface_used(&prog_data->base, surf_index);
308 }
309
310 void
311 fs_generator::generate_fb_write(fs_inst *inst, struct brw_reg payload)
312 {
313 struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
314 const brw_wm_prog_key * const key = (brw_wm_prog_key * const) this->key;
315 struct brw_reg implied_header;
316
317 if (devinfo->gen < 8 && !devinfo->is_haswell) {
318 brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
319 }
320
321 if (inst->base_mrf >= 0)
322 payload = brw_message_reg(inst->base_mrf);
323
324 /* Header is 2 regs, g0 and g1 are the contents. g0 will be implied
325 * move, here's g1.
326 */
327 if (inst->header_size != 0) {
328 brw_push_insn_state(p);
329 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
330 brw_set_default_exec_size(p, BRW_EXECUTE_1);
331 brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
332 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
333 brw_set_default_flag_reg(p, 0, 0);
334
335 /* On HSW, the GPU will use the predicate on SENDC, unless the header is
336 * present.
337 */
338 if (prog_data->uses_kill) {
339 struct brw_reg pixel_mask;
340
341 if (devinfo->gen >= 6)
342 pixel_mask = retype(brw_vec1_grf(1, 7), BRW_REGISTER_TYPE_UW);
343 else
344 pixel_mask = retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UW);
345
346 brw_MOV(p, pixel_mask, brw_flag_reg(0, 1));
347 }
348
349 if (devinfo->gen >= 6) {
350 brw_push_insn_state(p);
351 brw_set_default_exec_size(p, BRW_EXECUTE_16);
352 brw_set_default_compression_control(p, BRW_COMPRESSION_COMPRESSED);
353 brw_MOV(p,
354 retype(payload, BRW_REGISTER_TYPE_UD),
355 retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));
356 brw_pop_insn_state(p);
357
358 if (inst->target > 0 && key->replicate_alpha) {
359 /* Set "Source0 Alpha Present to RenderTarget" bit in message
360 * header.
361 */
362 brw_OR(p,
363 vec1(retype(payload, BRW_REGISTER_TYPE_UD)),
364 vec1(retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD)),
365 brw_imm_ud(0x1 << 11));
366 }
367
368 if (inst->target > 0) {
369 /* Set the render target index for choosing BLEND_STATE. */
370 brw_MOV(p, retype(vec1(suboffset(payload, 2)),
371 BRW_REGISTER_TYPE_UD),
372 brw_imm_ud(inst->target));
373 }
374
375 /* Set computes stencil to render target */
376 if (prog_data->computed_stencil) {
377 brw_OR(p,
378 vec1(retype(payload, BRW_REGISTER_TYPE_UD)),
379 vec1(retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD)),
380 brw_imm_ud(0x1 << 14));
381 }
382
383 implied_header = brw_null_reg();
384 } else {
385 implied_header = retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UW);
386 }
387
388 brw_pop_insn_state(p);
389 } else {
390 implied_header = brw_null_reg();
391 }
392
393 if (!runtime_check_aads_emit) {
394 fire_fb_write(inst, payload, implied_header, inst->mlen);
395 } else {
396 /* This can only happen in gen < 6 */
397 assert(devinfo->gen < 6);
398
399 struct brw_reg v1_null_ud = vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_UD));
400
401 /* Check runtime bit to detect if we have to send AA data or not */
402 brw_push_insn_state(p);
403 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
404 brw_set_default_exec_size(p, BRW_EXECUTE_1);
405 brw_AND(p,
406 v1_null_ud,
407 retype(brw_vec1_grf(1, 6), BRW_REGISTER_TYPE_UD),
408 brw_imm_ud(1<<26));
409 brw_inst_set_cond_modifier(p->devinfo, brw_last_inst, BRW_CONDITIONAL_NZ);
410
411 int jmp = brw_JMPI(p, brw_imm_ud(0), BRW_PREDICATE_NORMAL) - p->store;
412 brw_pop_insn_state(p);
413 {
414 /* Don't send AA data */
415 fire_fb_write(inst, offset(payload, 1), implied_header, inst->mlen-1);
416 }
417 brw_land_fwd_jump(p, jmp);
418 fire_fb_write(inst, payload, implied_header, inst->mlen);
419 }
420 }
421
422 void
423 fs_generator::generate_fb_read(fs_inst *inst, struct brw_reg dst,
424 struct brw_reg payload)
425 {
426 assert(inst->size_written % REG_SIZE == 0);
427 struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
428 /* We assume that render targets start at binding table index 0. */
429 const unsigned surf_index = inst->target;
430
431 gen9_fb_READ(p, dst, payload, surf_index,
432 inst->header_size, inst->size_written / REG_SIZE,
433 prog_data->persample_dispatch);
434
435 brw_mark_surface_used(&prog_data->base, surf_index);
436 }
437
438 void
439 fs_generator::generate_mov_indirect(fs_inst *inst,
440 struct brw_reg dst,
441 struct brw_reg reg,
442 struct brw_reg indirect_byte_offset)
443 {
444 assert(indirect_byte_offset.type == BRW_REGISTER_TYPE_UD);
445 assert(indirect_byte_offset.file == BRW_GENERAL_REGISTER_FILE);
446 assert(!reg.abs && !reg.negate);
447 assert(reg.type == dst.type);
448
449 unsigned imm_byte_offset = reg.nr * REG_SIZE + reg.subnr;
450
451 if (indirect_byte_offset.file == BRW_IMMEDIATE_VALUE) {
452 imm_byte_offset += indirect_byte_offset.ud;
453
454 reg.nr = imm_byte_offset / REG_SIZE;
455 reg.subnr = imm_byte_offset % REG_SIZE;
456 brw_MOV(p, dst, reg);
457 } else {
458 /* Prior to Broadwell, there are only 8 address registers. */
459 assert(inst->exec_size <= 8 || devinfo->gen >= 8);
460
461 /* We use VxH indirect addressing, clobbering a0.0 through a0.7. */
462 struct brw_reg addr = vec8(brw_address_reg(0));
463
464 /* The destination stride of an instruction (in bytes) must be greater
465 * than or equal to the size of the rest of the instruction. Since the
466 * address register is of type UW, we can't use a D-type instruction.
467 * In order to get around this, re retype to UW and use a stride.
468 */
469 indirect_byte_offset =
470 retype(spread(indirect_byte_offset, 2), BRW_REGISTER_TYPE_UW);
471
472 /* There are a number of reasons why we don't use the base offset here.
473 * One reason is that the field is only 9 bits which means we can only
474 * use it to access the first 16 GRFs. Also, from the Haswell PRM
475 * section "Register Region Restrictions":
476 *
477 * "The lower bits of the AddressImmediate must not overflow to
478 * change the register address. The lower 5 bits of Address
479 * Immediate when added to lower 5 bits of address register gives
480 * the sub-register offset. The upper bits of Address Immediate
481 * when added to upper bits of address register gives the register
482 * address. Any overflow from sub-register offset is dropped."
483 *
484 * Since the indirect may cause us to cross a register boundary, this
485 * makes the base offset almost useless. We could try and do something
486 * clever where we use a actual base offset if base_offset % 32 == 0 but
487 * that would mean we were generating different code depending on the
488 * base offset. Instead, for the sake of consistency, we'll just do the
489 * add ourselves. This restriction is only listed in the Haswell PRM
490 * but empirical testing indicates that it applies on all older
491 * generations and is lifted on Broadwell.
492 *
493 * In the end, while base_offset is nice to look at in the generated
494 * code, using it saves us 0 instructions and would require quite a bit
495 * of case-by-case work. It's just not worth it.
496 */
497 brw_ADD(p, addr, indirect_byte_offset, brw_imm_uw(imm_byte_offset));
498
499 if (type_sz(reg.type) > 4 &&
500 ((devinfo->gen == 7 && !devinfo->is_haswell) ||
501 devinfo->is_cherryview || gen_device_info_is_9lp(devinfo))) {
502 /* IVB has an issue (which we found empirically) where it reads two
503 * address register components per channel for indirectly addressed
504 * 64-bit sources.
505 *
506 * From the Cherryview PRM Vol 7. "Register Region Restrictions":
507 *
508 * "When source or destination datatype is 64b or operation is
509 * integer DWord multiply, indirect addressing must not be used."
510 *
511 * To work around both of these, we do two integer MOVs insead of one
512 * 64-bit MOV. Because no double value should ever cross a register
513 * boundary, it's safe to use the immediate offset in the indirect
514 * here to handle adding 4 bytes to the offset and avoid the extra
515 * ADD to the register file.
516 */
517 brw_MOV(p, subscript(dst, BRW_REGISTER_TYPE_D, 0),
518 retype(brw_VxH_indirect(0, 0), BRW_REGISTER_TYPE_D));
519 brw_MOV(p, subscript(dst, BRW_REGISTER_TYPE_D, 1),
520 retype(brw_VxH_indirect(0, 4), BRW_REGISTER_TYPE_D));
521 } else {
522 struct brw_reg ind_src = brw_VxH_indirect(0, 0);
523
524 brw_inst *mov = brw_MOV(p, dst, retype(ind_src, reg.type));
525
526 if (devinfo->gen == 6 && dst.file == BRW_MESSAGE_REGISTER_FILE &&
527 !inst->get_next()->is_tail_sentinel() &&
528 ((fs_inst *)inst->get_next())->mlen > 0) {
529 /* From the Sandybridge PRM:
530 *
531 * "[Errata: DevSNB(SNB)] If MRF register is updated by any
532 * instruction that “indexed/indirect” source AND is followed
533 * by a send, the instruction requires a “Switch”. This is to
534 * avoid race condition where send may dispatch before MRF is
535 * updated."
536 */
537 brw_inst_set_thread_control(devinfo, mov, BRW_THREAD_SWITCH);
538 }
539 }
540 }
541 }
542
543 void
544 fs_generator::generate_urb_read(fs_inst *inst,
545 struct brw_reg dst,
546 struct brw_reg header)
547 {
548 assert(inst->size_written % REG_SIZE == 0);
549 assert(header.file == BRW_GENERAL_REGISTER_FILE);
550 assert(header.type == BRW_REGISTER_TYPE_UD);
551
552 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
553 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UD));
554 brw_set_src0(p, send, header);
555 brw_set_src1(p, send, brw_imm_ud(0u));
556
557 brw_inst_set_sfid(p->devinfo, send, BRW_SFID_URB);
558 brw_inst_set_urb_opcode(p->devinfo, send, GEN8_URB_OPCODE_SIMD8_READ);
559
560 if (inst->opcode == SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT)
561 brw_inst_set_urb_per_slot_offset(p->devinfo, send, true);
562
563 brw_inst_set_mlen(p->devinfo, send, inst->mlen);
564 brw_inst_set_rlen(p->devinfo, send, inst->size_written / REG_SIZE);
565 brw_inst_set_header_present(p->devinfo, send, true);
566 brw_inst_set_urb_global_offset(p->devinfo, send, inst->offset);
567 }
568
569 void
570 fs_generator::generate_urb_write(fs_inst *inst, struct brw_reg payload)
571 {
572 brw_inst *insn;
573
574 /* WaClearTDRRegBeforeEOTForNonPS.
575 *
576 * WA: Clear tdr register before send EOT in all non-PS shader kernels
577 *
578 * mov(8) tdr0:ud 0x0:ud {NoMask}"
579 */
580 if (inst->eot && p->devinfo->gen == 10) {
581 brw_push_insn_state(p);
582 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
583 brw_MOV(p, brw_tdr_reg(), brw_imm_uw(0));
584 brw_pop_insn_state(p);
585 }
586
587 insn = brw_next_insn(p, BRW_OPCODE_SEND);
588
589 brw_set_dest(p, insn, brw_null_reg());
590 brw_set_src0(p, insn, payload);
591 brw_set_src1(p, insn, brw_imm_d(0));
592
593 brw_inst_set_sfid(p->devinfo, insn, BRW_SFID_URB);
594 brw_inst_set_urb_opcode(p->devinfo, insn, GEN8_URB_OPCODE_SIMD8_WRITE);
595
596 if (inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT ||
597 inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT)
598 brw_inst_set_urb_per_slot_offset(p->devinfo, insn, true);
599
600 if (inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_MASKED ||
601 inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT)
602 brw_inst_set_urb_channel_mask_present(p->devinfo, insn, true);
603
604 brw_inst_set_mlen(p->devinfo, insn, inst->mlen);
605 brw_inst_set_rlen(p->devinfo, insn, 0);
606 brw_inst_set_eot(p->devinfo, insn, inst->eot);
607 brw_inst_set_header_present(p->devinfo, insn, true);
608 brw_inst_set_urb_global_offset(p->devinfo, insn, inst->offset);
609 }
610
611 void
612 fs_generator::generate_cs_terminate(fs_inst *inst, struct brw_reg payload)
613 {
614 struct brw_inst *insn;
615
616 insn = brw_next_insn(p, BRW_OPCODE_SEND);
617
618 brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW));
619 brw_set_src0(p, insn, retype(payload, BRW_REGISTER_TYPE_UW));
620 brw_set_src1(p, insn, brw_imm_d(0));
621
622 /* Terminate a compute shader by sending a message to the thread spawner.
623 */
624 brw_inst_set_sfid(devinfo, insn, BRW_SFID_THREAD_SPAWNER);
625 brw_inst_set_mlen(devinfo, insn, 1);
626 brw_inst_set_rlen(devinfo, insn, 0);
627 brw_inst_set_eot(devinfo, insn, inst->eot);
628 brw_inst_set_header_present(devinfo, insn, false);
629
630 brw_inst_set_ts_opcode(devinfo, insn, 0); /* Dereference resource */
631 brw_inst_set_ts_request_type(devinfo, insn, 0); /* Root thread */
632
633 /* Note that even though the thread has a URB resource associated with it,
634 * we set the "do not dereference URB" bit, because the URB resource is
635 * managed by the fixed-function unit, so it will free it automatically.
636 */
637 brw_inst_set_ts_resource_select(devinfo, insn, 1); /* Do not dereference URB */
638
639 brw_inst_set_mask_control(devinfo, insn, BRW_MASK_DISABLE);
640 }
641
642 void
643 fs_generator::generate_barrier(fs_inst *inst, struct brw_reg src)
644 {
645 brw_barrier(p, src);
646 brw_WAIT(p);
647 }
648
649 bool
650 fs_generator::generate_linterp(fs_inst *inst,
651 struct brw_reg dst, struct brw_reg *src)
652 {
653 /* PLN reads:
654 * / in SIMD16 \
655 * -----------------------------------
656 * | src1+0 | src1+1 | src1+2 | src1+3 |
657 * |-----------------------------------|
658 * |(x0, x1)|(y0, y1)|(x2, x3)|(y2, y3)|
659 * -----------------------------------
660 *
661 * but for the LINE/MAC pair, the LINE reads Xs and the MAC reads Ys:
662 *
663 * -----------------------------------
664 * | src1+0 | src1+1 | src1+2 | src1+3 |
665 * |-----------------------------------|
666 * |(x0, x1)|(y0, y1)| | | in SIMD8
667 * |-----------------------------------|
668 * |(x0, x1)|(x2, x3)|(y0, y1)|(y2, y3)| in SIMD16
669 * -----------------------------------
670 *
671 * See also: emit_interpolation_setup_gen4().
672 */
673 struct brw_reg delta_x = src[0];
674 struct brw_reg delta_y = offset(src[0], inst->exec_size / 8);
675 struct brw_reg interp = src[1];
676 brw_inst *i[4];
677
678 if (devinfo->gen >= 11) {
679 struct brw_reg acc = retype(brw_acc_reg(8), BRW_REGISTER_TYPE_NF);
680 struct brw_reg dwP = suboffset(interp, 0);
681 struct brw_reg dwQ = suboffset(interp, 1);
682 struct brw_reg dwR = suboffset(interp, 3);
683
684 brw_set_default_exec_size(p, BRW_EXECUTE_8);
685
686 if (inst->exec_size == 8) {
687 i[0] = brw_MAD(p, acc, dwR, offset(delta_x, 0), dwP);
688 i[1] = brw_MAD(p, offset(dst, 0), acc, offset(delta_y, 0), dwQ);
689
690 brw_inst_set_cond_modifier(p->devinfo, i[1], inst->conditional_mod);
691
692 /* brw_set_default_saturate() is called before emitting instructions,
693 * so the saturate bit is set in each instruction, so we need to unset
694 * it on the first instruction of each pair.
695 */
696 brw_inst_set_saturate(p->devinfo, i[0], false);
697 } else {
698 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
699 i[0] = brw_MAD(p, acc, dwR, offset(delta_x, 0), dwP);
700 i[1] = brw_MAD(p, offset(dst, 0), acc, offset(delta_x, 1), dwQ);
701
702 brw_set_default_compression_control(p, BRW_COMPRESSION_2NDHALF);
703 i[2] = brw_MAD(p, acc, dwR, offset(delta_y, 0), dwP);
704 i[3] = brw_MAD(p, offset(dst, 1), acc, offset(delta_y, 1), dwQ);
705
706 brw_set_default_compression_control(p, BRW_COMPRESSION_COMPRESSED);
707
708 brw_inst_set_cond_modifier(p->devinfo, i[1], inst->conditional_mod);
709 brw_inst_set_cond_modifier(p->devinfo, i[3], inst->conditional_mod);
710
711 /* brw_set_default_saturate() is called before emitting instructions,
712 * so the saturate bit is set in each instruction, so we need to unset
713 * it on the first instruction of each pair.
714 */
715 brw_inst_set_saturate(p->devinfo, i[0], false);
716 brw_inst_set_saturate(p->devinfo, i[2], false);
717 }
718
719 return true;
720 } else if (devinfo->has_pln &&
721 (devinfo->gen >= 7 || (delta_x.nr & 1) == 0)) {
722 brw_PLN(p, dst, interp, delta_x);
723
724 return false;
725 } else {
726 i[0] = brw_LINE(p, brw_null_reg(), interp, delta_x);
727 i[1] = brw_MAC(p, dst, suboffset(interp, 1), delta_y);
728
729 brw_inst_set_cond_modifier(p->devinfo, i[1], inst->conditional_mod);
730
731 /* brw_set_default_saturate() is called before emitting instructions, so
732 * the saturate bit is set in each instruction, so we need to unset it on
733 * the first instruction.
734 */
735 brw_inst_set_saturate(p->devinfo, i[0], false);
736
737 return true;
738 }
739 }
740
741 void
742 fs_generator::generate_get_buffer_size(fs_inst *inst,
743 struct brw_reg dst,
744 struct brw_reg src,
745 struct brw_reg surf_index)
746 {
747 assert(devinfo->gen >= 7);
748 assert(surf_index.file == BRW_IMMEDIATE_VALUE);
749
750 uint32_t simd_mode;
751 int rlen = 4;
752
753 switch (inst->exec_size) {
754 case 8:
755 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
756 break;
757 case 16:
758 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
759 break;
760 default:
761 unreachable("Invalid width for texture instruction");
762 }
763
764 if (simd_mode == BRW_SAMPLER_SIMD_MODE_SIMD16) {
765 rlen = 8;
766 dst = vec16(dst);
767 }
768
769 brw_SAMPLE(p,
770 retype(dst, BRW_REGISTER_TYPE_UW),
771 inst->base_mrf,
772 src,
773 surf_index.ud,
774 0,
775 GEN5_SAMPLER_MESSAGE_SAMPLE_RESINFO,
776 rlen, /* response length */
777 inst->mlen,
778 inst->header_size > 0,
779 simd_mode,
780 BRW_SAMPLER_RETURN_FORMAT_SINT32);
781
782 brw_mark_surface_used(prog_data, surf_index.ud);
783 }
784
785 void
786 fs_generator::generate_tex(fs_inst *inst, struct brw_reg dst, struct brw_reg src,
787 struct brw_reg surface_index,
788 struct brw_reg sampler_index)
789 {
790 assert(inst->size_written % REG_SIZE == 0);
791 int msg_type = -1;
792 uint32_t simd_mode;
793 uint32_t return_format;
794 bool is_combined_send = inst->eot;
795
796 switch (dst.type) {
797 case BRW_REGISTER_TYPE_D:
798 return_format = BRW_SAMPLER_RETURN_FORMAT_SINT32;
799 break;
800 case BRW_REGISTER_TYPE_UD:
801 return_format = BRW_SAMPLER_RETURN_FORMAT_UINT32;
802 break;
803 default:
804 return_format = BRW_SAMPLER_RETURN_FORMAT_FLOAT32;
805 break;
806 }
807
808 /* Stomp the resinfo output type to UINT32. On gens 4-5, the output type
809 * is set as part of the message descriptor. On gen4, the PRM seems to
810 * allow UINT32 and FLOAT32 (i965 PRM, Vol. 4 Section 4.8.1.1), but on
811 * later gens UINT32 is required. Once you hit Sandy Bridge, the bit is
812 * gone from the message descriptor entirely and you just get UINT32 all
813 * the time regasrdless. Since we can really only do non-UINT32 on gen4,
814 * just stomp it to UINT32 all the time.
815 */
816 if (inst->opcode == SHADER_OPCODE_TXS)
817 return_format = BRW_SAMPLER_RETURN_FORMAT_UINT32;
818
819 switch (inst->exec_size) {
820 case 8:
821 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
822 break;
823 case 16:
824 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
825 break;
826 default:
827 unreachable("Invalid width for texture instruction");
828 }
829
830 if (devinfo->gen >= 5) {
831 switch (inst->opcode) {
832 case SHADER_OPCODE_TEX:
833 if (inst->shadow_compare) {
834 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_COMPARE;
835 } else {
836 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE;
837 }
838 break;
839 case FS_OPCODE_TXB:
840 if (inst->shadow_compare) {
841 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_BIAS_COMPARE;
842 } else {
843 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_BIAS;
844 }
845 break;
846 case SHADER_OPCODE_TXL:
847 if (inst->shadow_compare) {
848 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LOD_COMPARE;
849 } else {
850 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LOD;
851 }
852 break;
853 case SHADER_OPCODE_TXL_LZ:
854 assert(devinfo->gen >= 9);
855 if (inst->shadow_compare) {
856 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_C_LZ;
857 } else {
858 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_LZ;
859 }
860 break;
861 case SHADER_OPCODE_TXS:
862 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_RESINFO;
863 break;
864 case SHADER_OPCODE_TXD:
865 if (inst->shadow_compare) {
866 /* Gen7.5+. Otherwise, lowered in NIR */
867 assert(devinfo->gen >= 8 || devinfo->is_haswell);
868 msg_type = HSW_SAMPLER_MESSAGE_SAMPLE_DERIV_COMPARE;
869 } else {
870 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_DERIVS;
871 }
872 break;
873 case SHADER_OPCODE_TXF:
874 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
875 break;
876 case SHADER_OPCODE_TXF_LZ:
877 assert(devinfo->gen >= 9);
878 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_LD_LZ;
879 break;
880 case SHADER_OPCODE_TXF_CMS_W:
881 assert(devinfo->gen >= 9);
882 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_LD2DMS_W;
883 break;
884 case SHADER_OPCODE_TXF_CMS:
885 if (devinfo->gen >= 7)
886 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD2DMS;
887 else
888 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
889 break;
890 case SHADER_OPCODE_TXF_UMS:
891 assert(devinfo->gen >= 7);
892 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD2DSS;
893 break;
894 case SHADER_OPCODE_TXF_MCS:
895 assert(devinfo->gen >= 7);
896 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD_MCS;
897 break;
898 case SHADER_OPCODE_LOD:
899 msg_type = GEN5_SAMPLER_MESSAGE_LOD;
900 break;
901 case SHADER_OPCODE_TG4:
902 if (inst->shadow_compare) {
903 assert(devinfo->gen >= 7);
904 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_C;
905 } else {
906 assert(devinfo->gen >= 6);
907 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4;
908 }
909 break;
910 case SHADER_OPCODE_TG4_OFFSET:
911 assert(devinfo->gen >= 7);
912 if (inst->shadow_compare) {
913 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_PO_C;
914 } else {
915 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_PO;
916 }
917 break;
918 case SHADER_OPCODE_SAMPLEINFO:
919 msg_type = GEN6_SAMPLER_MESSAGE_SAMPLE_SAMPLEINFO;
920 break;
921 default:
922 unreachable("not reached");
923 }
924 } else {
925 switch (inst->opcode) {
926 case SHADER_OPCODE_TEX:
927 /* Note that G45 and older determines shadow compare and dispatch width
928 * from message length for most messages.
929 */
930 if (inst->exec_size == 8) {
931 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE;
932 if (inst->shadow_compare) {
933 assert(inst->mlen == 6);
934 } else {
935 assert(inst->mlen <= 4);
936 }
937 } else {
938 if (inst->shadow_compare) {
939 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE_COMPARE;
940 assert(inst->mlen == 9);
941 } else {
942 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE;
943 assert(inst->mlen <= 7 && inst->mlen % 2 == 1);
944 }
945 }
946 break;
947 case FS_OPCODE_TXB:
948 if (inst->shadow_compare) {
949 assert(inst->exec_size == 8);
950 assert(inst->mlen == 6);
951 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE_BIAS_COMPARE;
952 } else {
953 assert(inst->mlen == 9);
954 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE_BIAS;
955 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
956 }
957 break;
958 case SHADER_OPCODE_TXL:
959 if (inst->shadow_compare) {
960 assert(inst->exec_size == 8);
961 assert(inst->mlen == 6);
962 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE_LOD_COMPARE;
963 } else {
964 assert(inst->mlen == 9);
965 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE_LOD;
966 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
967 }
968 break;
969 case SHADER_OPCODE_TXD:
970 /* There is no sample_d_c message; comparisons are done manually */
971 assert(inst->exec_size == 8);
972 assert(inst->mlen == 7 || inst->mlen == 10);
973 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE_GRADIENTS;
974 break;
975 case SHADER_OPCODE_TXF:
976 assert(inst->mlen <= 9 && inst->mlen % 2 == 1);
977 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_LD;
978 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
979 break;
980 case SHADER_OPCODE_TXS:
981 assert(inst->mlen == 3);
982 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_RESINFO;
983 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
984 break;
985 default:
986 unreachable("not reached");
987 }
988 }
989 assert(msg_type != -1);
990
991 if (simd_mode == BRW_SAMPLER_SIMD_MODE_SIMD16) {
992 dst = vec16(dst);
993 }
994
995 assert(devinfo->gen < 7 || inst->header_size == 0 ||
996 src.file == BRW_GENERAL_REGISTER_FILE);
997
998 assert(sampler_index.type == BRW_REGISTER_TYPE_UD);
999
1000 /* Load the message header if present. If there's a texture offset,
1001 * we need to set it up explicitly and load the offset bitfield.
1002 * Otherwise, we can use an implied move from g0 to the first message reg.
1003 */
1004 if (inst->header_size != 0) {
1005 if (devinfo->gen < 6 && !inst->offset) {
1006 /* Set up an implied move from g0 to the MRF. */
1007 src = retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UW);
1008 } else {
1009 struct brw_reg header_reg;
1010
1011 if (devinfo->gen >= 7) {
1012 header_reg = src;
1013 } else {
1014 assert(inst->base_mrf != -1);
1015 header_reg = brw_message_reg(inst->base_mrf);
1016 }
1017
1018 brw_push_insn_state(p);
1019 brw_set_default_exec_size(p, BRW_EXECUTE_8);
1020 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1021 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
1022 /* Explicitly set up the message header by copying g0 to the MRF. */
1023 brw_MOV(p, header_reg, brw_vec8_grf(0, 0));
1024
1025 brw_set_default_exec_size(p, BRW_EXECUTE_1);
1026 if (inst->offset) {
1027 /* Set the offset bits in DWord 2. */
1028 brw_MOV(p, get_element_ud(header_reg, 2),
1029 brw_imm_ud(inst->offset));
1030 } else if (stage != MESA_SHADER_VERTEX &&
1031 stage != MESA_SHADER_FRAGMENT) {
1032 /* The vertex and fragment stages have g0.2 set to 0, so
1033 * header0.2 is 0 when g0 is copied. Other stages may not, so we
1034 * must set it to 0 to avoid setting undesirable bits in the
1035 * message.
1036 */
1037 brw_MOV(p, get_element_ud(header_reg, 2), brw_imm_ud(0));
1038 }
1039
1040 brw_adjust_sampler_state_pointer(p, header_reg, sampler_index);
1041 brw_pop_insn_state(p);
1042 }
1043 }
1044
1045 uint32_t base_binding_table_index = (inst->opcode == SHADER_OPCODE_TG4 ||
1046 inst->opcode == SHADER_OPCODE_TG4_OFFSET)
1047 ? prog_data->binding_table.gather_texture_start
1048 : prog_data->binding_table.texture_start;
1049
1050 if (surface_index.file == BRW_IMMEDIATE_VALUE &&
1051 sampler_index.file == BRW_IMMEDIATE_VALUE) {
1052 uint32_t surface = surface_index.ud;
1053 uint32_t sampler = sampler_index.ud;
1054
1055 brw_SAMPLE(p,
1056 retype(dst, BRW_REGISTER_TYPE_UW),
1057 inst->base_mrf,
1058 src,
1059 surface + base_binding_table_index,
1060 sampler % 16,
1061 msg_type,
1062 inst->size_written / REG_SIZE,
1063 inst->mlen,
1064 inst->header_size != 0,
1065 simd_mode,
1066 return_format);
1067
1068 brw_mark_surface_used(prog_data, surface + base_binding_table_index);
1069 } else {
1070 /* Non-const sampler index */
1071
1072 struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
1073 struct brw_reg surface_reg = vec1(retype(surface_index, BRW_REGISTER_TYPE_UD));
1074 struct brw_reg sampler_reg = vec1(retype(sampler_index, BRW_REGISTER_TYPE_UD));
1075
1076 brw_push_insn_state(p);
1077 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1078 brw_set_default_access_mode(p, BRW_ALIGN_1);
1079 brw_set_default_exec_size(p, BRW_EXECUTE_1);
1080
1081 if (brw_regs_equal(&surface_reg, &sampler_reg)) {
1082 brw_MUL(p, addr, sampler_reg, brw_imm_uw(0x101));
1083 } else {
1084 if (sampler_reg.file == BRW_IMMEDIATE_VALUE) {
1085 brw_OR(p, addr, surface_reg, brw_imm_ud(sampler_reg.ud << 8));
1086 } else {
1087 brw_SHL(p, addr, sampler_reg, brw_imm_ud(8));
1088 brw_OR(p, addr, addr, surface_reg);
1089 }
1090 }
1091 if (base_binding_table_index)
1092 brw_ADD(p, addr, addr, brw_imm_ud(base_binding_table_index));
1093 brw_AND(p, addr, addr, brw_imm_ud(0xfff));
1094
1095 brw_pop_insn_state(p);
1096
1097 /* dst = send(offset, a0.0 | <descriptor>) */
1098 brw_inst *insn = brw_send_indirect_message(
1099 p, BRW_SFID_SAMPLER, dst, src, addr);
1100 brw_set_sampler_message(p, insn,
1101 0 /* surface */,
1102 0 /* sampler */,
1103 msg_type,
1104 inst->size_written / REG_SIZE,
1105 inst->mlen /* mlen */,
1106 inst->header_size != 0 /* header */,
1107 simd_mode,
1108 return_format);
1109
1110 /* visitor knows more than we do about the surface limit required,
1111 * so has already done marking.
1112 */
1113 }
1114
1115 if (is_combined_send) {
1116 brw_inst_set_eot(p->devinfo, brw_last_inst, true);
1117 brw_inst_set_opcode(p->devinfo, brw_last_inst, BRW_OPCODE_SENDC);
1118 }
1119 }
1120
1121
1122 /* For OPCODE_DDX and OPCODE_DDY, per channel of output we've got input
1123 * looking like:
1124 *
1125 * arg0: ss0.tl ss0.tr ss0.bl ss0.br ss1.tl ss1.tr ss1.bl ss1.br
1126 *
1127 * Ideally, we want to produce:
1128 *
1129 * DDX DDY
1130 * dst: (ss0.tr - ss0.tl) (ss0.tl - ss0.bl)
1131 * (ss0.tr - ss0.tl) (ss0.tr - ss0.br)
1132 * (ss0.br - ss0.bl) (ss0.tl - ss0.bl)
1133 * (ss0.br - ss0.bl) (ss0.tr - ss0.br)
1134 * (ss1.tr - ss1.tl) (ss1.tl - ss1.bl)
1135 * (ss1.tr - ss1.tl) (ss1.tr - ss1.br)
1136 * (ss1.br - ss1.bl) (ss1.tl - ss1.bl)
1137 * (ss1.br - ss1.bl) (ss1.tr - ss1.br)
1138 *
1139 * and add another set of two more subspans if in 16-pixel dispatch mode.
1140 *
1141 * For DDX, it ends up being easy: width = 2, horiz=0 gets us the same result
1142 * for each pair, and vertstride = 2 jumps us 2 elements after processing a
1143 * pair. But the ideal approximation may impose a huge performance cost on
1144 * sample_d. On at least Haswell, sample_d instruction does some
1145 * optimizations if the same LOD is used for all pixels in the subspan.
1146 *
1147 * For DDY, we need to use ALIGN16 mode since it's capable of doing the
1148 * appropriate swizzling.
1149 */
1150 void
1151 fs_generator::generate_ddx(const fs_inst *inst,
1152 struct brw_reg dst, struct brw_reg src)
1153 {
1154 unsigned vstride, width;
1155
1156 if (inst->opcode == FS_OPCODE_DDX_FINE) {
1157 /* produce accurate derivatives */
1158 vstride = BRW_VERTICAL_STRIDE_2;
1159 width = BRW_WIDTH_2;
1160 } else {
1161 /* replicate the derivative at the top-left pixel to other pixels */
1162 vstride = BRW_VERTICAL_STRIDE_4;
1163 width = BRW_WIDTH_4;
1164 }
1165
1166 struct brw_reg src0 = src;
1167 struct brw_reg src1 = src;
1168
1169 src0.subnr = sizeof(float);
1170 src0.vstride = vstride;
1171 src0.width = width;
1172 src0.hstride = BRW_HORIZONTAL_STRIDE_0;
1173 src1.vstride = vstride;
1174 src1.width = width;
1175 src1.hstride = BRW_HORIZONTAL_STRIDE_0;
1176
1177 brw_ADD(p, dst, src0, negate(src1));
1178 }
1179
1180 /* The negate_value boolean is used to negate the derivative computation for
1181 * FBOs, since they place the origin at the upper left instead of the lower
1182 * left.
1183 */
1184 void
1185 fs_generator::generate_ddy(const fs_inst *inst,
1186 struct brw_reg dst, struct brw_reg src)
1187 {
1188 if (inst->opcode == FS_OPCODE_DDY_FINE) {
1189 /* produce accurate derivatives */
1190 if (devinfo->gen >= 11) {
1191 src = stride(src, 0, 2, 1);
1192 struct brw_reg src_0 = byte_offset(src, 0 * sizeof(float));
1193 struct brw_reg src_2 = byte_offset(src, 2 * sizeof(float));
1194 struct brw_reg src_4 = byte_offset(src, 4 * sizeof(float));
1195 struct brw_reg src_6 = byte_offset(src, 6 * sizeof(float));
1196 struct brw_reg src_8 = byte_offset(src, 8 * sizeof(float));
1197 struct brw_reg src_10 = byte_offset(src, 10 * sizeof(float));
1198 struct brw_reg src_12 = byte_offset(src, 12 * sizeof(float));
1199 struct brw_reg src_14 = byte_offset(src, 14 * sizeof(float));
1200
1201 struct brw_reg dst_0 = byte_offset(dst, 0 * sizeof(float));
1202 struct brw_reg dst_4 = byte_offset(dst, 4 * sizeof(float));
1203 struct brw_reg dst_8 = byte_offset(dst, 8 * sizeof(float));
1204 struct brw_reg dst_12 = byte_offset(dst, 12 * sizeof(float));
1205
1206 brw_push_insn_state(p);
1207 brw_set_default_exec_size(p, BRW_EXECUTE_4);
1208
1209 brw_ADD(p, dst_0, negate(src_0), src_2);
1210 brw_ADD(p, dst_4, negate(src_4), src_6);
1211
1212 if (inst->exec_size == 16) {
1213 brw_ADD(p, dst_8, negate(src_8), src_10);
1214 brw_ADD(p, dst_12, negate(src_12), src_14);
1215 }
1216
1217 brw_pop_insn_state(p);
1218 } else {
1219 struct brw_reg src0 = stride(src, 4, 4, 1);
1220 struct brw_reg src1 = stride(src, 4, 4, 1);
1221 src0.swizzle = BRW_SWIZZLE_XYXY;
1222 src1.swizzle = BRW_SWIZZLE_ZWZW;
1223
1224 brw_push_insn_state(p);
1225 brw_set_default_access_mode(p, BRW_ALIGN_16);
1226 brw_ADD(p, dst, negate(src0), src1);
1227 brw_pop_insn_state(p);
1228 }
1229 } else {
1230 /* replicate the derivative at the top-left pixel to other pixels */
1231 struct brw_reg src0 = stride(src, 4, 4, 0);
1232 struct brw_reg src1 = stride(src, 4, 4, 0);
1233 src0.subnr = 0 * sizeof(float);
1234 src1.subnr = 2 * sizeof(float);
1235
1236 brw_ADD(p, dst, negate(src0), src1);
1237 }
1238 }
1239
1240 void
1241 fs_generator::generate_discard_jump(fs_inst *inst)
1242 {
1243 assert(devinfo->gen >= 6);
1244
1245 /* This HALT will be patched up at FB write time to point UIP at the end of
1246 * the program, and at brw_uip_jip() JIP will be set to the end of the
1247 * current block (or the program).
1248 */
1249 this->discard_halt_patches.push_tail(new(mem_ctx) ip_record(p->nr_insn));
1250 gen6_HALT(p);
1251 }
1252
1253 void
1254 fs_generator::generate_scratch_write(fs_inst *inst, struct brw_reg src)
1255 {
1256 /* The 32-wide messages only respect the first 16-wide half of the channel
1257 * enable signals which are replicated identically for the second group of
1258 * 16 channels, so we cannot use them unless the write is marked
1259 * force_writemask_all.
1260 */
1261 const unsigned lower_size = inst->force_writemask_all ? inst->exec_size :
1262 MIN2(16, inst->exec_size);
1263 const unsigned block_size = 4 * lower_size / REG_SIZE;
1264 assert(inst->mlen != 0);
1265
1266 brw_push_insn_state(p);
1267 brw_set_default_exec_size(p, cvt(lower_size) - 1);
1268 brw_set_default_compression(p, lower_size > 8);
1269
1270 for (unsigned i = 0; i < inst->exec_size / lower_size; i++) {
1271 brw_set_default_group(p, inst->group + lower_size * i);
1272
1273 brw_MOV(p, brw_uvec_mrf(lower_size, inst->base_mrf + 1, 0),
1274 retype(offset(src, block_size * i), BRW_REGISTER_TYPE_UD));
1275
1276 brw_oword_block_write_scratch(p, brw_message_reg(inst->base_mrf),
1277 block_size,
1278 inst->offset + block_size * REG_SIZE * i);
1279 }
1280
1281 brw_pop_insn_state(p);
1282 }
1283
1284 void
1285 fs_generator::generate_scratch_read(fs_inst *inst, struct brw_reg dst)
1286 {
1287 assert(inst->exec_size <= 16 || inst->force_writemask_all);
1288 assert(inst->mlen != 0);
1289
1290 brw_oword_block_read_scratch(p, dst, brw_message_reg(inst->base_mrf),
1291 inst->exec_size / 8, inst->offset);
1292 }
1293
1294 void
1295 fs_generator::generate_scratch_read_gen7(fs_inst *inst, struct brw_reg dst)
1296 {
1297 assert(inst->exec_size <= 16 || inst->force_writemask_all);
1298
1299 gen7_block_read_scratch(p, dst, inst->exec_size / 8, inst->offset);
1300 }
1301
1302 void
1303 fs_generator::generate_uniform_pull_constant_load(fs_inst *inst,
1304 struct brw_reg dst,
1305 struct brw_reg index,
1306 struct brw_reg offset)
1307 {
1308 assert(type_sz(dst.type) == 4);
1309 assert(inst->mlen != 0);
1310
1311 assert(index.file == BRW_IMMEDIATE_VALUE &&
1312 index.type == BRW_REGISTER_TYPE_UD);
1313 uint32_t surf_index = index.ud;
1314
1315 assert(offset.file == BRW_IMMEDIATE_VALUE &&
1316 offset.type == BRW_REGISTER_TYPE_UD);
1317 uint32_t read_offset = offset.ud;
1318
1319 brw_oword_block_read(p, dst, brw_message_reg(inst->base_mrf),
1320 read_offset, surf_index);
1321 }
1322
1323 void
1324 fs_generator::generate_uniform_pull_constant_load_gen7(fs_inst *inst,
1325 struct brw_reg dst,
1326 struct brw_reg index,
1327 struct brw_reg payload)
1328 {
1329 assert(index.type == BRW_REGISTER_TYPE_UD);
1330 assert(payload.file == BRW_GENERAL_REGISTER_FILE);
1331 assert(type_sz(dst.type) == 4);
1332
1333 if (index.file == BRW_IMMEDIATE_VALUE) {
1334 const uint32_t surf_index = index.ud;
1335
1336 brw_push_insn_state(p);
1337 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1338 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
1339 brw_pop_insn_state(p);
1340
1341 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UD));
1342 brw_set_src0(p, send, retype(payload, BRW_REGISTER_TYPE_UD));
1343 brw_set_dp_read_message(p, send, surf_index,
1344 BRW_DATAPORT_OWORD_BLOCK_DWORDS(inst->exec_size),
1345 GEN7_DATAPORT_DC_OWORD_BLOCK_READ,
1346 GEN6_SFID_DATAPORT_CONSTANT_CACHE,
1347 1, /* mlen */
1348 true, /* header */
1349 DIV_ROUND_UP(inst->size_written, REG_SIZE));
1350
1351 } else {
1352 struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
1353
1354 brw_push_insn_state(p);
1355 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1356
1357 /* a0.0 = surf_index & 0xff */
1358 brw_inst *insn_and = brw_next_insn(p, BRW_OPCODE_AND);
1359 brw_inst_set_exec_size(p->devinfo, insn_and, BRW_EXECUTE_1);
1360 brw_set_dest(p, insn_and, addr);
1361 brw_set_src0(p, insn_and, vec1(retype(index, BRW_REGISTER_TYPE_UD)));
1362 brw_set_src1(p, insn_and, brw_imm_ud(0x0ff));
1363
1364 /* dst = send(payload, a0.0 | <descriptor>) */
1365 brw_inst *insn = brw_send_indirect_message(
1366 p, GEN6_SFID_DATAPORT_CONSTANT_CACHE,
1367 retype(dst, BRW_REGISTER_TYPE_UD),
1368 retype(payload, BRW_REGISTER_TYPE_UD), addr);
1369 brw_set_dp_read_message(p, insn, 0 /* surface */,
1370 BRW_DATAPORT_OWORD_BLOCK_DWORDS(inst->exec_size),
1371 GEN7_DATAPORT_DC_OWORD_BLOCK_READ,
1372 GEN6_SFID_DATAPORT_CONSTANT_CACHE,
1373 1, /* mlen */
1374 true, /* header */
1375 DIV_ROUND_UP(inst->size_written, REG_SIZE));
1376
1377 brw_pop_insn_state(p);
1378 }
1379 }
1380
1381 void
1382 fs_generator::generate_varying_pull_constant_load_gen4(fs_inst *inst,
1383 struct brw_reg dst,
1384 struct brw_reg index)
1385 {
1386 assert(devinfo->gen < 7); /* Should use the gen7 variant. */
1387 assert(inst->header_size != 0);
1388 assert(inst->mlen);
1389
1390 assert(index.file == BRW_IMMEDIATE_VALUE &&
1391 index.type == BRW_REGISTER_TYPE_UD);
1392 uint32_t surf_index = index.ud;
1393
1394 uint32_t simd_mode, rlen, msg_type;
1395 if (inst->exec_size == 16) {
1396 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1397 rlen = 8;
1398 } else {
1399 assert(inst->exec_size == 8);
1400 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
1401 rlen = 4;
1402 }
1403
1404 if (devinfo->gen >= 5)
1405 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
1406 else {
1407 /* We always use the SIMD16 message so that we only have to load U, and
1408 * not V or R.
1409 */
1410 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_LD;
1411 assert(inst->mlen == 3);
1412 assert(inst->size_written == 8 * REG_SIZE);
1413 rlen = 8;
1414 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1415 }
1416
1417 struct brw_reg header = brw_vec8_grf(0, 0);
1418 gen6_resolve_implied_move(p, &header, inst->base_mrf);
1419
1420 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
1421 brw_inst_set_compression(devinfo, send, false);
1422 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UW));
1423 brw_set_src0(p, send, header);
1424 if (devinfo->gen < 6)
1425 brw_inst_set_base_mrf(p->devinfo, send, inst->base_mrf);
1426
1427 /* Our surface is set up as floats, regardless of what actual data is
1428 * stored in it.
1429 */
1430 uint32_t return_format = BRW_SAMPLER_RETURN_FORMAT_FLOAT32;
1431 brw_set_sampler_message(p, send,
1432 surf_index,
1433 0, /* sampler (unused) */
1434 msg_type,
1435 rlen,
1436 inst->mlen,
1437 inst->header_size != 0,
1438 simd_mode,
1439 return_format);
1440 }
1441
1442 void
1443 fs_generator::generate_varying_pull_constant_load_gen7(fs_inst *inst,
1444 struct brw_reg dst,
1445 struct brw_reg index,
1446 struct brw_reg offset)
1447 {
1448 assert(devinfo->gen >= 7);
1449 /* Varying-offset pull constant loads are treated as a normal expression on
1450 * gen7, so the fact that it's a send message is hidden at the IR level.
1451 */
1452 assert(inst->header_size == 0);
1453 assert(!inst->mlen);
1454 assert(index.type == BRW_REGISTER_TYPE_UD);
1455
1456 uint32_t simd_mode, rlen, mlen;
1457 if (inst->exec_size == 16) {
1458 mlen = 2;
1459 rlen = 8;
1460 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1461 } else {
1462 assert(inst->exec_size == 8);
1463 mlen = 1;
1464 rlen = 4;
1465 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
1466 }
1467
1468 if (index.file == BRW_IMMEDIATE_VALUE) {
1469
1470 uint32_t surf_index = index.ud;
1471
1472 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
1473 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UW));
1474 brw_set_src0(p, send, offset);
1475 brw_set_sampler_message(p, send,
1476 surf_index,
1477 0, /* LD message ignores sampler unit */
1478 GEN5_SAMPLER_MESSAGE_SAMPLE_LD,
1479 rlen,
1480 mlen,
1481 false, /* no header */
1482 simd_mode,
1483 0);
1484
1485 } else {
1486
1487 struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
1488
1489 brw_push_insn_state(p);
1490 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1491
1492 /* a0.0 = surf_index & 0xff */
1493 brw_inst *insn_and = brw_next_insn(p, BRW_OPCODE_AND);
1494 brw_inst_set_exec_size(p->devinfo, insn_and, BRW_EXECUTE_1);
1495 brw_set_dest(p, insn_and, addr);
1496 brw_set_src0(p, insn_and, vec1(retype(index, BRW_REGISTER_TYPE_UD)));
1497 brw_set_src1(p, insn_and, brw_imm_ud(0x0ff));
1498
1499 brw_pop_insn_state(p);
1500
1501 /* dst = send(offset, a0.0 | <descriptor>) */
1502 brw_inst *insn = brw_send_indirect_message(
1503 p, BRW_SFID_SAMPLER, retype(dst, BRW_REGISTER_TYPE_UW),
1504 offset, addr);
1505 brw_set_sampler_message(p, insn,
1506 0 /* surface */,
1507 0 /* sampler */,
1508 GEN5_SAMPLER_MESSAGE_SAMPLE_LD,
1509 rlen /* rlen */,
1510 mlen /* mlen */,
1511 false /* header */,
1512 simd_mode,
1513 0);
1514 }
1515 }
1516
1517 /**
1518 * Cause the current pixel/sample mask (from R1.7 bits 15:0) to be transferred
1519 * into the flags register (f0.0).
1520 *
1521 * Used only on Gen6 and above.
1522 */
1523 void
1524 fs_generator::generate_mov_dispatch_to_flags(fs_inst *inst)
1525 {
1526 struct brw_reg flags = brw_flag_reg(0, inst->flag_subreg);
1527 struct brw_reg dispatch_mask;
1528
1529 if (devinfo->gen >= 6)
1530 dispatch_mask = retype(brw_vec1_grf(1, 7), BRW_REGISTER_TYPE_UW);
1531 else
1532 dispatch_mask = retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UW);
1533
1534 brw_push_insn_state(p);
1535 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1536 brw_set_default_exec_size(p, BRW_EXECUTE_1);
1537 brw_MOV(p, flags, dispatch_mask);
1538 brw_pop_insn_state(p);
1539 }
1540
1541 void
1542 fs_generator::generate_pixel_interpolator_query(fs_inst *inst,
1543 struct brw_reg dst,
1544 struct brw_reg src,
1545 struct brw_reg msg_data,
1546 unsigned msg_type)
1547 {
1548 assert(inst->size_written % REG_SIZE == 0);
1549 assert(msg_data.type == BRW_REGISTER_TYPE_UD);
1550
1551 brw_pixel_interpolator_query(p,
1552 retype(dst, BRW_REGISTER_TYPE_UW),
1553 src,
1554 inst->pi_noperspective,
1555 msg_type,
1556 msg_data,
1557 inst->mlen,
1558 inst->size_written / REG_SIZE);
1559 }
1560
1561 /* Sets vstride=1, width=4, hstride=0 of register src1 during
1562 * the ADD instruction.
1563 */
1564 void
1565 fs_generator::generate_set_sample_id(fs_inst *inst,
1566 struct brw_reg dst,
1567 struct brw_reg src0,
1568 struct brw_reg src1)
1569 {
1570 assert(dst.type == BRW_REGISTER_TYPE_D ||
1571 dst.type == BRW_REGISTER_TYPE_UD);
1572 assert(src0.type == BRW_REGISTER_TYPE_D ||
1573 src0.type == BRW_REGISTER_TYPE_UD);
1574
1575 struct brw_reg reg = stride(src1, 1, 4, 0);
1576 if (devinfo->gen >= 8 || inst->exec_size == 8) {
1577 brw_ADD(p, dst, src0, reg);
1578 } else if (inst->exec_size == 16) {
1579 brw_push_insn_state(p);
1580 brw_set_default_exec_size(p, BRW_EXECUTE_8);
1581 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
1582 brw_ADD(p, firsthalf(dst), firsthalf(src0), reg);
1583 brw_set_default_compression_control(p, BRW_COMPRESSION_2NDHALF);
1584 brw_ADD(p, sechalf(dst), sechalf(src0), suboffset(reg, 2));
1585 brw_pop_insn_state(p);
1586 }
1587 }
1588
1589 void
1590 fs_generator::generate_pack_half_2x16_split(fs_inst *inst,
1591 struct brw_reg dst,
1592 struct brw_reg x,
1593 struct brw_reg y)
1594 {
1595 assert(devinfo->gen >= 7);
1596 assert(dst.type == BRW_REGISTER_TYPE_UD);
1597 assert(x.type == BRW_REGISTER_TYPE_F);
1598 assert(y.type == BRW_REGISTER_TYPE_F);
1599
1600 /* From the Ivybridge PRM, Vol4, Part3, Section 6.27 f32to16:
1601 *
1602 * Because this instruction does not have a 16-bit floating-point type,
1603 * the destination data type must be Word (W).
1604 *
1605 * The destination must be DWord-aligned and specify a horizontal stride
1606 * (HorzStride) of 2. The 16-bit result is stored in the lower word of
1607 * each destination channel and the upper word is not modified.
1608 */
1609 struct brw_reg dst_w = spread(retype(dst, BRW_REGISTER_TYPE_W), 2);
1610
1611 /* Give each 32-bit channel of dst the form below, where "." means
1612 * unchanged.
1613 * 0x....hhhh
1614 */
1615 brw_F32TO16(p, dst_w, y);
1616
1617 /* Now the form:
1618 * 0xhhhh0000
1619 */
1620 brw_SHL(p, dst, dst, brw_imm_ud(16u));
1621
1622 /* And, finally the form of packHalf2x16's output:
1623 * 0xhhhhllll
1624 */
1625 brw_F32TO16(p, dst_w, x);
1626 }
1627
1628 void
1629 fs_generator::generate_unpack_half_2x16_split(fs_inst *inst,
1630 struct brw_reg dst,
1631 struct brw_reg src)
1632 {
1633 assert(devinfo->gen >= 7);
1634 assert(dst.type == BRW_REGISTER_TYPE_F);
1635 assert(src.type == BRW_REGISTER_TYPE_UD);
1636
1637 /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
1638 *
1639 * Because this instruction does not have a 16-bit floating-point type,
1640 * the source data type must be Word (W). The destination type must be
1641 * F (Float).
1642 */
1643 struct brw_reg src_w = spread(retype(src, BRW_REGISTER_TYPE_W), 2);
1644
1645 /* Each channel of src has the form of unpackHalf2x16's input: 0xhhhhllll.
1646 * For the Y case, we wish to access only the upper word; therefore
1647 * a 16-bit subregister offset is needed.
1648 */
1649 assert(inst->opcode == FS_OPCODE_UNPACK_HALF_2x16_SPLIT_X ||
1650 inst->opcode == FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y);
1651 if (inst->opcode == FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y)
1652 src_w.subnr += 2;
1653
1654 brw_F16TO32(p, dst, src_w);
1655 }
1656
1657 void
1658 fs_generator::generate_shader_time_add(fs_inst *inst,
1659 struct brw_reg payload,
1660 struct brw_reg offset,
1661 struct brw_reg value)
1662 {
1663 assert(devinfo->gen >= 7);
1664 brw_push_insn_state(p);
1665 brw_set_default_mask_control(p, true);
1666
1667 assert(payload.file == BRW_GENERAL_REGISTER_FILE);
1668 struct brw_reg payload_offset = retype(brw_vec1_grf(payload.nr, 0),
1669 offset.type);
1670 struct brw_reg payload_value = retype(brw_vec1_grf(payload.nr + 1, 0),
1671 value.type);
1672
1673 assert(offset.file == BRW_IMMEDIATE_VALUE);
1674 if (value.file == BRW_GENERAL_REGISTER_FILE) {
1675 value.width = BRW_WIDTH_1;
1676 value.hstride = BRW_HORIZONTAL_STRIDE_0;
1677 value.vstride = BRW_VERTICAL_STRIDE_0;
1678 } else {
1679 assert(value.file == BRW_IMMEDIATE_VALUE);
1680 }
1681
1682 /* Trying to deal with setup of the params from the IR is crazy in the FS8
1683 * case, and we don't really care about squeezing every bit of performance
1684 * out of this path, so we just emit the MOVs from here.
1685 */
1686 brw_MOV(p, payload_offset, offset);
1687 brw_MOV(p, payload_value, value);
1688 brw_shader_time_add(p, payload,
1689 prog_data->binding_table.shader_time_start);
1690 brw_pop_insn_state(p);
1691
1692 brw_mark_surface_used(prog_data,
1693 prog_data->binding_table.shader_time_start);
1694 }
1695
1696 void
1697 fs_generator::enable_debug(const char *shader_name)
1698 {
1699 debug_flag = true;
1700 this->shader_name = shader_name;
1701 }
1702
1703 int
1704 fs_generator::generate_code(const cfg_t *cfg, int dispatch_width)
1705 {
1706 /* align to 64 byte boundary. */
1707 while (p->next_insn_offset % 64)
1708 brw_NOP(p);
1709
1710 this->dispatch_width = dispatch_width;
1711
1712 int start_offset = p->next_insn_offset;
1713 int spill_count = 0, fill_count = 0;
1714 int loop_count = 0;
1715
1716 struct disasm_info *disasm_info = disasm_initialize(devinfo, cfg);
1717
1718 foreach_block_and_inst (block, fs_inst, inst, cfg) {
1719 struct brw_reg src[3], dst;
1720 unsigned int last_insn_offset = p->next_insn_offset;
1721 bool multiple_instructions_emitted = false;
1722
1723 /* From the Broadwell PRM, Volume 7, "3D-Media-GPGPU", in the
1724 * "Register Region Restrictions" section: for BDW, SKL:
1725 *
1726 * "A POW/FDIV operation must not be followed by an instruction
1727 * that requires two destination registers."
1728 *
1729 * The documentation is often lacking annotations for Atom parts,
1730 * and empirically this affects CHV as well.
1731 */
1732 if (devinfo->gen >= 8 &&
1733 devinfo->gen <= 9 &&
1734 p->nr_insn > 1 &&
1735 brw_inst_opcode(devinfo, brw_last_inst) == BRW_OPCODE_MATH &&
1736 brw_inst_math_function(devinfo, brw_last_inst) == BRW_MATH_FUNCTION_POW &&
1737 inst->dst.component_size(inst->exec_size) > REG_SIZE) {
1738 brw_NOP(p);
1739 last_insn_offset = p->next_insn_offset;
1740 }
1741
1742 if (unlikely(debug_flag))
1743 disasm_annotate(disasm_info, inst, p->next_insn_offset);
1744
1745 /* If the instruction writes to more than one register, it needs to be
1746 * explicitly marked as compressed on Gen <= 5. On Gen >= 6 the
1747 * hardware figures out by itself what the right compression mode is,
1748 * but we still need to know whether the instruction is compressed to
1749 * set up the source register regions appropriately.
1750 *
1751 * XXX - This is wrong for instructions that write a single register but
1752 * read more than one which should strictly speaking be treated as
1753 * compressed. For instructions that don't write any registers it
1754 * relies on the destination being a null register of the correct
1755 * type and regioning so the instruction is considered compressed
1756 * or not accordingly.
1757 */
1758 const bool compressed =
1759 inst->dst.component_size(inst->exec_size) > REG_SIZE;
1760 brw_set_default_compression(p, compressed);
1761 brw_set_default_group(p, inst->group);
1762
1763 for (unsigned int i = 0; i < inst->sources; i++) {
1764 src[i] = brw_reg_from_fs_reg(devinfo, inst,
1765 &inst->src[i], compressed);
1766 /* The accumulator result appears to get used for the
1767 * conditional modifier generation. When negating a UD
1768 * value, there is a 33rd bit generated for the sign in the
1769 * accumulator value, so now you can't check, for example,
1770 * equality with a 32-bit value. See piglit fs-op-neg-uvec4.
1771 */
1772 assert(!inst->conditional_mod ||
1773 inst->src[i].type != BRW_REGISTER_TYPE_UD ||
1774 !inst->src[i].negate);
1775 }
1776 dst = brw_reg_from_fs_reg(devinfo, inst,
1777 &inst->dst, compressed);
1778
1779 brw_set_default_access_mode(p, BRW_ALIGN_1);
1780 brw_set_default_predicate_control(p, inst->predicate);
1781 brw_set_default_predicate_inverse(p, inst->predicate_inverse);
1782 brw_set_default_flag_reg(p, 0, inst->flag_subreg);
1783 brw_set_default_saturate(p, inst->saturate);
1784 brw_set_default_mask_control(p, inst->force_writemask_all);
1785 brw_set_default_acc_write_control(p, inst->writes_accumulator);
1786
1787 unsigned exec_size = inst->exec_size;
1788 if (devinfo->gen == 7 && !devinfo->is_haswell &&
1789 (get_exec_type_size(inst) == 8 || type_sz(inst->dst.type) == 8)) {
1790 exec_size *= 2;
1791 }
1792
1793 brw_set_default_exec_size(p, cvt(exec_size) - 1);
1794
1795 assert(inst->force_writemask_all || inst->exec_size >= 4);
1796 assert(inst->force_writemask_all || inst->group % inst->exec_size == 0);
1797 assert(inst->base_mrf + inst->mlen <= BRW_MAX_MRF(devinfo->gen));
1798 assert(inst->mlen <= BRW_MAX_MSG_LENGTH);
1799
1800 switch (inst->opcode) {
1801 case BRW_OPCODE_MOV:
1802 brw_MOV(p, dst, src[0]);
1803 break;
1804 case BRW_OPCODE_ADD:
1805 brw_ADD(p, dst, src[0], src[1]);
1806 break;
1807 case BRW_OPCODE_MUL:
1808 brw_MUL(p, dst, src[0], src[1]);
1809 break;
1810 case BRW_OPCODE_AVG:
1811 brw_AVG(p, dst, src[0], src[1]);
1812 break;
1813 case BRW_OPCODE_MACH:
1814 brw_MACH(p, dst, src[0], src[1]);
1815 break;
1816
1817 case BRW_OPCODE_LINE:
1818 brw_LINE(p, dst, src[0], src[1]);
1819 break;
1820
1821 case BRW_OPCODE_MAD:
1822 assert(devinfo->gen >= 6);
1823 if (devinfo->gen < 10)
1824 brw_set_default_access_mode(p, BRW_ALIGN_16);
1825 brw_MAD(p, dst, src[0], src[1], src[2]);
1826 break;
1827
1828 case BRW_OPCODE_LRP:
1829 assert(devinfo->gen >= 6 && devinfo->gen <= 10);
1830 if (devinfo->gen < 10)
1831 brw_set_default_access_mode(p, BRW_ALIGN_16);
1832 brw_LRP(p, dst, src[0], src[1], src[2]);
1833 break;
1834
1835 case BRW_OPCODE_FRC:
1836 brw_FRC(p, dst, src[0]);
1837 break;
1838 case BRW_OPCODE_RNDD:
1839 brw_RNDD(p, dst, src[0]);
1840 break;
1841 case BRW_OPCODE_RNDE:
1842 brw_RNDE(p, dst, src[0]);
1843 break;
1844 case BRW_OPCODE_RNDZ:
1845 brw_RNDZ(p, dst, src[0]);
1846 break;
1847
1848 case BRW_OPCODE_AND:
1849 brw_AND(p, dst, src[0], src[1]);
1850 break;
1851 case BRW_OPCODE_OR:
1852 brw_OR(p, dst, src[0], src[1]);
1853 break;
1854 case BRW_OPCODE_XOR:
1855 brw_XOR(p, dst, src[0], src[1]);
1856 break;
1857 case BRW_OPCODE_NOT:
1858 brw_NOT(p, dst, src[0]);
1859 break;
1860 case BRW_OPCODE_ASR:
1861 brw_ASR(p, dst, src[0], src[1]);
1862 break;
1863 case BRW_OPCODE_SHR:
1864 brw_SHR(p, dst, src[0], src[1]);
1865 break;
1866 case BRW_OPCODE_SHL:
1867 brw_SHL(p, dst, src[0], src[1]);
1868 break;
1869 case BRW_OPCODE_F32TO16:
1870 assert(devinfo->gen >= 7);
1871 brw_F32TO16(p, dst, src[0]);
1872 break;
1873 case BRW_OPCODE_F16TO32:
1874 assert(devinfo->gen >= 7);
1875 brw_F16TO32(p, dst, src[0]);
1876 break;
1877 case BRW_OPCODE_CMP:
1878 if (inst->exec_size >= 16 && devinfo->gen == 7 && !devinfo->is_haswell &&
1879 dst.file == BRW_ARCHITECTURE_REGISTER_FILE) {
1880 /* For unknown reasons the WaCMPInstFlagDepClearedEarly workaround
1881 * implemented in the compiler is not sufficient. Overriding the
1882 * type when the destination is the null register is necessary but
1883 * not sufficient by itself.
1884 */
1885 assert(dst.nr == BRW_ARF_NULL);
1886 dst.type = BRW_REGISTER_TYPE_D;
1887 }
1888 brw_CMP(p, dst, inst->conditional_mod, src[0], src[1]);
1889 break;
1890 case BRW_OPCODE_SEL:
1891 brw_SEL(p, dst, src[0], src[1]);
1892 break;
1893 case BRW_OPCODE_BFREV:
1894 assert(devinfo->gen >= 7);
1895 brw_BFREV(p, retype(dst, BRW_REGISTER_TYPE_UD),
1896 retype(src[0], BRW_REGISTER_TYPE_UD));
1897 break;
1898 case BRW_OPCODE_FBH:
1899 assert(devinfo->gen >= 7);
1900 brw_FBH(p, retype(dst, src[0].type), src[0]);
1901 break;
1902 case BRW_OPCODE_FBL:
1903 assert(devinfo->gen >= 7);
1904 brw_FBL(p, retype(dst, BRW_REGISTER_TYPE_UD),
1905 retype(src[0], BRW_REGISTER_TYPE_UD));
1906 break;
1907 case BRW_OPCODE_LZD:
1908 brw_LZD(p, dst, src[0]);
1909 break;
1910 case BRW_OPCODE_CBIT:
1911 assert(devinfo->gen >= 7);
1912 brw_CBIT(p, retype(dst, BRW_REGISTER_TYPE_UD),
1913 retype(src[0], BRW_REGISTER_TYPE_UD));
1914 break;
1915 case BRW_OPCODE_ADDC:
1916 assert(devinfo->gen >= 7);
1917 brw_ADDC(p, dst, src[0], src[1]);
1918 break;
1919 case BRW_OPCODE_SUBB:
1920 assert(devinfo->gen >= 7);
1921 brw_SUBB(p, dst, src[0], src[1]);
1922 break;
1923 case BRW_OPCODE_MAC:
1924 brw_MAC(p, dst, src[0], src[1]);
1925 break;
1926
1927 case BRW_OPCODE_BFE:
1928 assert(devinfo->gen >= 7);
1929 if (devinfo->gen < 10)
1930 brw_set_default_access_mode(p, BRW_ALIGN_16);
1931 brw_BFE(p, dst, src[0], src[1], src[2]);
1932 break;
1933
1934 case BRW_OPCODE_BFI1:
1935 assert(devinfo->gen >= 7);
1936 brw_BFI1(p, dst, src[0], src[1]);
1937 break;
1938 case BRW_OPCODE_BFI2:
1939 assert(devinfo->gen >= 7);
1940 if (devinfo->gen < 10)
1941 brw_set_default_access_mode(p, BRW_ALIGN_16);
1942 brw_BFI2(p, dst, src[0], src[1], src[2]);
1943 break;
1944
1945 case BRW_OPCODE_IF:
1946 if (inst->src[0].file != BAD_FILE) {
1947 /* The instruction has an embedded compare (only allowed on gen6) */
1948 assert(devinfo->gen == 6);
1949 gen6_IF(p, inst->conditional_mod, src[0], src[1]);
1950 } else {
1951 brw_IF(p, brw_inst_exec_size(devinfo, p->current));
1952 }
1953 break;
1954
1955 case BRW_OPCODE_ELSE:
1956 brw_ELSE(p);
1957 break;
1958 case BRW_OPCODE_ENDIF:
1959 brw_ENDIF(p);
1960 break;
1961
1962 case BRW_OPCODE_DO:
1963 brw_DO(p, brw_inst_exec_size(devinfo, p->current));
1964 break;
1965
1966 case BRW_OPCODE_BREAK:
1967 brw_BREAK(p);
1968 break;
1969 case BRW_OPCODE_CONTINUE:
1970 brw_CONT(p);
1971 break;
1972
1973 case BRW_OPCODE_WHILE:
1974 brw_WHILE(p);
1975 loop_count++;
1976 break;
1977
1978 case SHADER_OPCODE_RCP:
1979 case SHADER_OPCODE_RSQ:
1980 case SHADER_OPCODE_SQRT:
1981 case SHADER_OPCODE_EXP2:
1982 case SHADER_OPCODE_LOG2:
1983 case SHADER_OPCODE_SIN:
1984 case SHADER_OPCODE_COS:
1985 assert(inst->conditional_mod == BRW_CONDITIONAL_NONE);
1986 if (devinfo->gen >= 6) {
1987 assert(inst->mlen == 0);
1988 assert(devinfo->gen >= 7 || inst->exec_size == 8);
1989 gen6_math(p, dst, brw_math_function(inst->opcode),
1990 src[0], brw_null_reg());
1991 } else {
1992 assert(inst->mlen >= 1);
1993 assert(devinfo->gen == 5 || devinfo->is_g4x || inst->exec_size == 8);
1994 gen4_math(p, dst,
1995 brw_math_function(inst->opcode),
1996 inst->base_mrf, src[0],
1997 BRW_MATH_PRECISION_FULL);
1998 }
1999 break;
2000 case SHADER_OPCODE_INT_QUOTIENT:
2001 case SHADER_OPCODE_INT_REMAINDER:
2002 case SHADER_OPCODE_POW:
2003 assert(inst->conditional_mod == BRW_CONDITIONAL_NONE);
2004 if (devinfo->gen >= 6) {
2005 assert(inst->mlen == 0);
2006 assert((devinfo->gen >= 7 && inst->opcode == SHADER_OPCODE_POW) ||
2007 inst->exec_size == 8);
2008 gen6_math(p, dst, brw_math_function(inst->opcode), src[0], src[1]);
2009 } else {
2010 assert(inst->mlen >= 1);
2011 assert(inst->exec_size == 8);
2012 gen4_math(p, dst, brw_math_function(inst->opcode),
2013 inst->base_mrf, src[0],
2014 BRW_MATH_PRECISION_FULL);
2015 }
2016 break;
2017 case FS_OPCODE_CINTERP:
2018 brw_MOV(p, dst, src[0]);
2019 break;
2020 case FS_OPCODE_LINTERP:
2021 multiple_instructions_emitted = generate_linterp(inst, dst, src);
2022 break;
2023 case FS_OPCODE_PIXEL_X:
2024 assert(src[0].type == BRW_REGISTER_TYPE_UW);
2025 src[0].subnr = 0 * type_sz(src[0].type);
2026 brw_MOV(p, dst, stride(src[0], 8, 4, 1));
2027 break;
2028 case FS_OPCODE_PIXEL_Y:
2029 assert(src[0].type == BRW_REGISTER_TYPE_UW);
2030 src[0].subnr = 4 * type_sz(src[0].type);
2031 brw_MOV(p, dst, stride(src[0], 8, 4, 1));
2032 break;
2033 case SHADER_OPCODE_GET_BUFFER_SIZE:
2034 generate_get_buffer_size(inst, dst, src[0], src[1]);
2035 break;
2036 case SHADER_OPCODE_TEX:
2037 case FS_OPCODE_TXB:
2038 case SHADER_OPCODE_TXD:
2039 case SHADER_OPCODE_TXF:
2040 case SHADER_OPCODE_TXF_LZ:
2041 case SHADER_OPCODE_TXF_CMS:
2042 case SHADER_OPCODE_TXF_CMS_W:
2043 case SHADER_OPCODE_TXF_UMS:
2044 case SHADER_OPCODE_TXF_MCS:
2045 case SHADER_OPCODE_TXL:
2046 case SHADER_OPCODE_TXL_LZ:
2047 case SHADER_OPCODE_TXS:
2048 case SHADER_OPCODE_LOD:
2049 case SHADER_OPCODE_TG4:
2050 case SHADER_OPCODE_TG4_OFFSET:
2051 case SHADER_OPCODE_SAMPLEINFO:
2052 generate_tex(inst, dst, src[0], src[1], src[2]);
2053 break;
2054 case FS_OPCODE_DDX_COARSE:
2055 case FS_OPCODE_DDX_FINE:
2056 generate_ddx(inst, dst, src[0]);
2057 break;
2058 case FS_OPCODE_DDY_COARSE:
2059 case FS_OPCODE_DDY_FINE:
2060 generate_ddy(inst, dst, src[0]);
2061 break;
2062
2063 case SHADER_OPCODE_GEN4_SCRATCH_WRITE:
2064 generate_scratch_write(inst, src[0]);
2065 spill_count++;
2066 break;
2067
2068 case SHADER_OPCODE_GEN4_SCRATCH_READ:
2069 generate_scratch_read(inst, dst);
2070 fill_count++;
2071 break;
2072
2073 case SHADER_OPCODE_GEN7_SCRATCH_READ:
2074 generate_scratch_read_gen7(inst, dst);
2075 fill_count++;
2076 break;
2077
2078 case SHADER_OPCODE_MOV_INDIRECT:
2079 generate_mov_indirect(inst, dst, src[0], src[1]);
2080 break;
2081
2082 case SHADER_OPCODE_URB_READ_SIMD8:
2083 case SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT:
2084 generate_urb_read(inst, dst, src[0]);
2085 break;
2086
2087 case SHADER_OPCODE_URB_WRITE_SIMD8:
2088 case SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT:
2089 case SHADER_OPCODE_URB_WRITE_SIMD8_MASKED:
2090 case SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT:
2091 generate_urb_write(inst, src[0]);
2092 break;
2093
2094 case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD:
2095 assert(inst->force_writemask_all);
2096 generate_uniform_pull_constant_load(inst, dst, src[0], src[1]);
2097 break;
2098
2099 case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD_GEN7:
2100 assert(inst->force_writemask_all);
2101 generate_uniform_pull_constant_load_gen7(inst, dst, src[0], src[1]);
2102 break;
2103
2104 case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_GEN4:
2105 generate_varying_pull_constant_load_gen4(inst, dst, src[0]);
2106 break;
2107
2108 case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_GEN7:
2109 generate_varying_pull_constant_load_gen7(inst, dst, src[0], src[1]);
2110 break;
2111
2112 case FS_OPCODE_REP_FB_WRITE:
2113 case FS_OPCODE_FB_WRITE:
2114 generate_fb_write(inst, src[0]);
2115 break;
2116
2117 case FS_OPCODE_FB_READ:
2118 generate_fb_read(inst, dst, src[0]);
2119 break;
2120
2121 case FS_OPCODE_MOV_DISPATCH_TO_FLAGS:
2122 generate_mov_dispatch_to_flags(inst);
2123 break;
2124
2125 case FS_OPCODE_DISCARD_JUMP:
2126 generate_discard_jump(inst);
2127 break;
2128
2129 case SHADER_OPCODE_SHADER_TIME_ADD:
2130 generate_shader_time_add(inst, src[0], src[1], src[2]);
2131 break;
2132
2133 case SHADER_OPCODE_UNTYPED_ATOMIC:
2134 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2135 brw_untyped_atomic(p, dst, src[0], src[1], src[2].ud,
2136 inst->mlen, !inst->dst.is_null());
2137 break;
2138
2139 case SHADER_OPCODE_UNTYPED_SURFACE_READ:
2140 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2141 brw_untyped_surface_read(p, dst, src[0], src[1],
2142 inst->mlen, src[2].ud);
2143 break;
2144
2145 case SHADER_OPCODE_UNTYPED_SURFACE_WRITE:
2146 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2147 brw_untyped_surface_write(p, src[0], src[1],
2148 inst->mlen, src[2].ud);
2149 break;
2150
2151 case SHADER_OPCODE_BYTE_SCATTERED_READ:
2152 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2153 brw_byte_scattered_read(p, dst, src[0], src[1],
2154 inst->mlen, src[2].ud);
2155 break;
2156
2157 case SHADER_OPCODE_BYTE_SCATTERED_WRITE:
2158 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2159 brw_byte_scattered_write(p, src[0], src[1],
2160 inst->mlen, src[2].ud);
2161 break;
2162
2163 case SHADER_OPCODE_TYPED_ATOMIC:
2164 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2165 brw_typed_atomic(p, dst, src[0], src[1],
2166 src[2].ud, inst->mlen, !inst->dst.is_null());
2167 break;
2168
2169 case SHADER_OPCODE_TYPED_SURFACE_READ:
2170 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2171 brw_typed_surface_read(p, dst, src[0], src[1],
2172 inst->mlen, src[2].ud);
2173 break;
2174
2175 case SHADER_OPCODE_TYPED_SURFACE_WRITE:
2176 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2177 brw_typed_surface_write(p, src[0], src[1], inst->mlen, src[2].ud);
2178 break;
2179
2180 case SHADER_OPCODE_MEMORY_FENCE:
2181 brw_memory_fence(p, dst);
2182 break;
2183
2184 case SHADER_OPCODE_FIND_LIVE_CHANNEL: {
2185 const struct brw_reg mask =
2186 brw_stage_has_packed_dispatch(devinfo, stage,
2187 prog_data) ? brw_imm_ud(~0u) :
2188 stage == MESA_SHADER_FRAGMENT ? brw_vmask_reg() :
2189 brw_dmask_reg();
2190 brw_find_live_channel(p, dst, mask);
2191 break;
2192 }
2193
2194 case SHADER_OPCODE_BROADCAST:
2195 assert(inst->force_writemask_all);
2196 brw_broadcast(p, dst, src[0], src[1]);
2197 break;
2198
2199 case FS_OPCODE_SET_SAMPLE_ID:
2200 generate_set_sample_id(inst, dst, src[0], src[1]);
2201 break;
2202
2203 case FS_OPCODE_PACK_HALF_2x16_SPLIT:
2204 generate_pack_half_2x16_split(inst, dst, src[0], src[1]);
2205 break;
2206
2207 case FS_OPCODE_UNPACK_HALF_2x16_SPLIT_X:
2208 case FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y:
2209 generate_unpack_half_2x16_split(inst, dst, src[0]);
2210 break;
2211
2212 case FS_OPCODE_PLACEHOLDER_HALT:
2213 /* This is the place where the final HALT needs to be inserted if
2214 * we've emitted any discards. If not, this will emit no code.
2215 */
2216 if (!patch_discard_jumps_to_fb_writes()) {
2217 if (unlikely(debug_flag)) {
2218 disasm_info->use_tail = true;
2219 }
2220 }
2221 break;
2222
2223 case FS_OPCODE_INTERPOLATE_AT_SAMPLE:
2224 generate_pixel_interpolator_query(inst, dst, src[0], src[1],
2225 GEN7_PIXEL_INTERPOLATOR_LOC_SAMPLE);
2226 break;
2227
2228 case FS_OPCODE_INTERPOLATE_AT_SHARED_OFFSET:
2229 generate_pixel_interpolator_query(inst, dst, src[0], src[1],
2230 GEN7_PIXEL_INTERPOLATOR_LOC_SHARED_OFFSET);
2231 break;
2232
2233 case FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET:
2234 generate_pixel_interpolator_query(inst, dst, src[0], src[1],
2235 GEN7_PIXEL_INTERPOLATOR_LOC_PER_SLOT_OFFSET);
2236 break;
2237
2238 case CS_OPCODE_CS_TERMINATE:
2239 generate_cs_terminate(inst, src[0]);
2240 break;
2241
2242 case SHADER_OPCODE_BARRIER:
2243 generate_barrier(inst, src[0]);
2244 break;
2245
2246 case BRW_OPCODE_DIM:
2247 assert(devinfo->is_haswell);
2248 assert(src[0].type == BRW_REGISTER_TYPE_DF);
2249 assert(dst.type == BRW_REGISTER_TYPE_DF);
2250 brw_DIM(p, dst, retype(src[0], BRW_REGISTER_TYPE_F));
2251 break;
2252
2253 case SHADER_OPCODE_RND_MODE:
2254 assert(src[0].file == BRW_IMMEDIATE_VALUE);
2255 brw_rounding_mode(p, (brw_rnd_mode) src[0].d);
2256 break;
2257
2258 default:
2259 unreachable("Unsupported opcode");
2260
2261 case SHADER_OPCODE_LOAD_PAYLOAD:
2262 unreachable("Should be lowered by lower_load_payload()");
2263 }
2264
2265 if (multiple_instructions_emitted)
2266 continue;
2267
2268 if (inst->no_dd_clear || inst->no_dd_check || inst->conditional_mod) {
2269 assert(p->next_insn_offset == last_insn_offset + 16 ||
2270 !"conditional_mod, no_dd_check, or no_dd_clear set for IR "
2271 "emitting more than 1 instruction");
2272
2273 brw_inst *last = &p->store[last_insn_offset / 16];
2274
2275 if (inst->conditional_mod)
2276 brw_inst_set_cond_modifier(p->devinfo, last, inst->conditional_mod);
2277 brw_inst_set_no_dd_clear(p->devinfo, last, inst->no_dd_clear);
2278 brw_inst_set_no_dd_check(p->devinfo, last, inst->no_dd_check);
2279 }
2280 }
2281
2282 brw_set_uip_jip(p, start_offset);
2283
2284 /* end of program sentinel */
2285 disasm_new_inst_group(disasm_info, p->next_insn_offset);
2286
2287 #ifndef NDEBUG
2288 bool validated =
2289 #else
2290 if (unlikely(debug_flag))
2291 #endif
2292 brw_validate_instructions(devinfo, p->store,
2293 start_offset,
2294 p->next_insn_offset,
2295 disasm_info);
2296
2297 int before_size = p->next_insn_offset - start_offset;
2298 brw_compact_instructions(p, start_offset, disasm_info);
2299 int after_size = p->next_insn_offset - start_offset;
2300
2301 if (unlikely(debug_flag)) {
2302 fprintf(stderr, "Native code for %s\n"
2303 "SIMD%d shader: %d instructions. %d loops. %u cycles. %d:%d spills:fills. Promoted %u constants. Compacted %d to %d"
2304 " bytes (%.0f%%)\n",
2305 shader_name, dispatch_width, before_size / 16, loop_count, cfg->cycle_count,
2306 spill_count, fill_count, promoted_constants, before_size, after_size,
2307 100.0f * (before_size - after_size) / before_size);
2308
2309 dump_assembly(p->store, disasm_info);
2310 }
2311 ralloc_free(disasm_info);
2312 assert(validated);
2313
2314 compiler->shader_debug_log(log_data,
2315 "%s SIMD%d shader: %d inst, %d loops, %u cycles, "
2316 "%d:%d spills:fills, Promoted %u constants, "
2317 "compacted %d to %d bytes.",
2318 _mesa_shader_stage_to_abbrev(stage),
2319 dispatch_width, before_size / 16,
2320 loop_count, cfg->cycle_count, spill_count,
2321 fill_count, promoted_constants, before_size,
2322 after_size);
2323
2324 return start_offset;
2325 }
2326
2327 const unsigned *
2328 fs_generator::get_assembly(unsigned int *assembly_size)
2329 {
2330 return brw_get_program(p, assembly_size);
2331 }