projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
intel/compiler/fs: Pass fs_inst to generate_ddx/ddy instead of opcode
[mesa.git] / src / intel / compiler / brw_fs_generator.cpp
1 /*
2 * Copyright © 2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 /** @file brw_fs_generator.cpp
25 *
26 * This file supports generating code from the FS LIR to the actual
27 * native instructions.
28 */
29
30 #include "brw_eu.h"
31 #include "brw_fs.h"
32 #include "brw_cfg.h"
33
34 static enum brw_reg_file
35 brw_file_from_reg(fs_reg *reg)
36 {
37 switch (reg->file) {
38 case ARF:
39 return BRW_ARCHITECTURE_REGISTER_FILE;
40 case FIXED_GRF:
41 case VGRF:
42 return BRW_GENERAL_REGISTER_FILE;
43 case MRF:
44 return BRW_MESSAGE_REGISTER_FILE;
45 case IMM:
46 return BRW_IMMEDIATE_VALUE;
47 case BAD_FILE:
48 case ATTR:
49 case UNIFORM:
50 unreachable("not reached");
51 }
52 return BRW_ARCHITECTURE_REGISTER_FILE;
53 }
54
55 static struct brw_reg
56 brw_reg_from_fs_reg(const struct gen_device_info *devinfo, fs_inst *inst,
57 fs_reg *reg, bool compressed)
58 {
59 struct brw_reg brw_reg;
60
61 switch (reg->file) {
62 case MRF:
63 assert((reg->nr & ~BRW_MRF_COMPR4) < BRW_MAX_MRF(devinfo->gen));
64 /* Fallthrough */
65 case VGRF:
66 if (reg->stride == 0) {
67 brw_reg = brw_vec1_reg(brw_file_from_reg(reg), reg->nr, 0);
68 } else {
69 /* From the Haswell PRM:
70 *
71 * "VertStride must be used to cross GRF register boundaries. This
72 * rule implies that elements within a 'Width' cannot cross GRF
73 * boundaries."
74 *
75 * The maximum width value that could satisfy this restriction is:
76 */
77 const unsigned reg_width = REG_SIZE / (reg->stride * type_sz(reg->type));
78
79 /* Because the hardware can only split source regions at a whole
80 * multiple of width during decompression (i.e. vertically), clamp
81 * the value obtained above to the physical execution size of a
82 * single decompressed chunk of the instruction:
83 */
84 const unsigned phys_width = compressed ? inst->exec_size / 2 :
85 inst->exec_size;
86
87 /* XXX - The equation above is strictly speaking not correct on
88 * hardware that supports unbalanced GRF writes -- On Gen9+
89 * each decompressed chunk of the instruction may have a
90 * different execution size when the number of components
91 * written to each destination GRF is not the same.
92 */
93 const unsigned width = MIN2(reg_width, phys_width);
94 brw_reg = brw_vecn_reg(width, brw_file_from_reg(reg), reg->nr, 0);
95 brw_reg = stride(brw_reg, width * reg->stride, width, reg->stride);
96
97 if (devinfo->gen == 7 && !devinfo->is_haswell) {
98 /* From the IvyBridge PRM (EU Changes by Processor Generation, page 13):
99 * "Each DF (Double Float) operand uses an element size of 4 rather
100 * than 8 and all regioning parameters are twice what the values
101 * would be based on the true element size: ExecSize, Width,
102 * HorzStride, and VertStride. Each DF operand uses a pair of
103 * channels and all masking and swizzing should be adjusted
104 * appropriately."
105 *
106 * From the IvyBridge PRM (Special Requirements for Handling Double
107 * Precision Data Types, page 71):
108 * "In Align1 mode, all regioning parameters like stride, execution
109 * size, and width must use the syntax of a pair of packed
110 * floats. The offsets for these data types must be 64-bit
111 * aligned. The execution size and regioning parameters are in terms
112 * of floats."
113 *
114 * Summarized: when handling DF-typed arguments, ExecSize,
115 * VertStride, and Width must be doubled.
116 *
117 * It applies to BayTrail too.
118 */
119 if (type_sz(reg->type) == 8) {
120 brw_reg.width++;
121 if (brw_reg.vstride > 0)
122 brw_reg.vstride++;
123 assert(brw_reg.hstride == BRW_HORIZONTAL_STRIDE_1);
124 }
125
126 /* When converting from DF->F, we set the destination stride to 2
127 * because each d2f conversion implicitly writes 2 floats, being
128 * the first one the converted value. IVB/BYT actually writes two
129 * F components per SIMD channel, and every other component is
130 * filled with garbage.
131 */
132 if (reg == &inst->dst && get_exec_type_size(inst) == 8 &&
133 type_sz(inst->dst.type) < 8) {
134 assert(brw_reg.hstride > BRW_HORIZONTAL_STRIDE_1);
135 brw_reg.hstride--;
136 }
137 }
138 }
139
140 brw_reg = retype(brw_reg, reg->type);
141 brw_reg = byte_offset(brw_reg, reg->offset);
142 brw_reg.abs = reg->abs;
143 brw_reg.negate = reg->negate;
144 break;
145 case ARF:
146 case FIXED_GRF:
147 case IMM:
148 assert(reg->offset == 0);
149 brw_reg = reg->as_brw_reg();
150 break;
151 case BAD_FILE:
152 /* Probably unused. */
153 brw_reg = brw_null_reg();
154 break;
155 case ATTR:
156 case UNIFORM:
157 unreachable("not reached");
158 }
159
160 /* On HSW+, scalar DF sources can be accessed using the normal <0,1,0>
161 * region, but on IVB and BYT DF regions must be programmed in terms of
162 * floats. A <0,2,1> region accomplishes this.
163 */
164 if (devinfo->gen == 7 && !devinfo->is_haswell &&
165 type_sz(reg->type) == 8 &&
166 brw_reg.vstride == BRW_VERTICAL_STRIDE_0 &&
167 brw_reg.width == BRW_WIDTH_1 &&
168 brw_reg.hstride == BRW_HORIZONTAL_STRIDE_0) {
169 brw_reg.width = BRW_WIDTH_2;
170 brw_reg.hstride = BRW_HORIZONTAL_STRIDE_1;
171 }
172
173 return brw_reg;
174 }
175
176 fs_generator::fs_generator(const struct brw_compiler *compiler, void *log_data,
177 void *mem_ctx,
178 const void *key,
179 struct brw_stage_prog_data *prog_data,
180 unsigned promoted_constants,
181 bool runtime_check_aads_emit,
182 gl_shader_stage stage)
183
184 : compiler(compiler), log_data(log_data),
185 devinfo(compiler->devinfo), key(key),
186 prog_data(prog_data),
187 promoted_constants(promoted_constants),
188 runtime_check_aads_emit(runtime_check_aads_emit), debug_flag(false),
189 stage(stage), mem_ctx(mem_ctx)
190 {
191 p = rzalloc(mem_ctx, struct brw_codegen);
192 brw_init_codegen(devinfo, p, mem_ctx);
193
194 /* In the FS code generator, we are very careful to ensure that we always
195 * set the right execution size so we don't need the EU code to "help" us
196 * by trying to infer it. Sometimes, it infers the wrong thing.
197 */
198 p->automatic_exec_sizes = false;
199 }
200
201 fs_generator::~fs_generator()
202 {
203 }
204
205 class ip_record : public exec_node {
206 public:
207 DECLARE_RALLOC_CXX_OPERATORS(ip_record)
208
209 ip_record(int ip)
210 {
211 this->ip = ip;
212 }
213
214 int ip;
215 };
216
217 bool
218 fs_generator::patch_discard_jumps_to_fb_writes()
219 {
220 if (devinfo->gen < 6 || this->discard_halt_patches.is_empty())
221 return false;
222
223 int scale = brw_jump_scale(p->devinfo);
224
225 /* There is a somewhat strange undocumented requirement of using
226 * HALT, according to the simulator. If some channel has HALTed to
227 * a particular UIP, then by the end of the program, every channel
228 * must have HALTed to that UIP. Furthermore, the tracking is a
229 * stack, so you can't do the final halt of a UIP after starting
230 * halting to a new UIP.
231 *
232 * Symptoms of not emitting this instruction on actual hardware
233 * included GPU hangs and sparkly rendering on the piglit discard
234 * tests.
235 */
236 brw_inst *last_halt = gen6_HALT(p);
237 brw_inst_set_uip(p->devinfo, last_halt, 1 * scale);
238 brw_inst_set_jip(p->devinfo, last_halt, 1 * scale);
239
240 int ip = p->nr_insn;
241
242 foreach_in_list(ip_record, patch_ip, &discard_halt_patches) {
243 brw_inst *patch = &p->store[patch_ip->ip];
244
245 assert(brw_inst_opcode(p->devinfo, patch) == BRW_OPCODE_HALT);
246 /* HALT takes a half-instruction distance from the pre-incremented IP. */
247 brw_inst_set_uip(p->devinfo, patch, (ip - patch_ip->ip) * scale);
248 }
249
250 this->discard_halt_patches.make_empty();
251 return true;
252 }
253
254 void
255 fs_generator::fire_fb_write(fs_inst *inst,
256 struct brw_reg payload,
257 struct brw_reg implied_header,
258 GLuint nr)
259 {
260 uint32_t msg_control;
261
262 struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
263
264 if (devinfo->gen < 6) {
265 brw_push_insn_state(p);
266 brw_set_default_exec_size(p, BRW_EXECUTE_8);
267 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
268 brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
269 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
270 brw_MOV(p, offset(payload, 1), brw_vec8_grf(1, 0));
271 brw_pop_insn_state(p);
272 }
273
274 if (inst->opcode == FS_OPCODE_REP_FB_WRITE)
275 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD16_SINGLE_SOURCE_REPLICATED;
276 else if (prog_data->dual_src_blend) {
277 if (!inst->group)
278 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_DUAL_SOURCE_SUBSPAN01;
279 else
280 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_DUAL_SOURCE_SUBSPAN23;
281 } else if (inst->exec_size == 16)
282 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD16_SINGLE_SOURCE;
283 else
284 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_SINGLE_SOURCE_SUBSPAN01;
285
286 /* We assume render targets start at 0, because headerless FB write
287 * messages set "Render Target Index" to 0. Using a different binding
288 * table index would make it impossible to use headerless messages.
289 */
290 const uint32_t surf_index = inst->target;
291
292 bool last_render_target = inst->eot ||
293 (prog_data->dual_src_blend && dispatch_width == 16);
294
295
296 brw_fb_WRITE(p,
297 payload,
298 implied_header,
299 msg_control,
300 surf_index,
301 nr,
302 0,
303 inst->eot,
304 last_render_target,
305 inst->header_size != 0);
306
307 brw_mark_surface_used(&prog_data->base, surf_index);
308 }
309
310 void
311 fs_generator::generate_fb_write(fs_inst *inst, struct brw_reg payload)
312 {
313 struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
314 const brw_wm_prog_key * const key = (brw_wm_prog_key * const) this->key;
315 struct brw_reg implied_header;
316
317 if (devinfo->gen < 8 && !devinfo->is_haswell) {
318 brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
319 }
320
321 if (inst->base_mrf >= 0)
322 payload = brw_message_reg(inst->base_mrf);
323
324 /* Header is 2 regs, g0 and g1 are the contents. g0 will be implied
325 * move, here's g1.
326 */
327 if (inst->header_size != 0) {
328 brw_push_insn_state(p);
329 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
330 brw_set_default_exec_size(p, BRW_EXECUTE_1);
331 brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
332 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
333 brw_set_default_flag_reg(p, 0, 0);
334
335 /* On HSW, the GPU will use the predicate on SENDC, unless the header is
336 * present.
337 */
338 if (prog_data->uses_kill) {
339 struct brw_reg pixel_mask;
340
341 if (devinfo->gen >= 6)
342 pixel_mask = retype(brw_vec1_grf(1, 7), BRW_REGISTER_TYPE_UW);
343 else
344 pixel_mask = retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UW);
345
346 brw_MOV(p, pixel_mask, brw_flag_reg(0, 1));
347 }
348
349 if (devinfo->gen >= 6) {
350 brw_push_insn_state(p);
351 brw_set_default_exec_size(p, BRW_EXECUTE_16);
352 brw_set_default_compression_control(p, BRW_COMPRESSION_COMPRESSED);
353 brw_MOV(p,
354 retype(payload, BRW_REGISTER_TYPE_UD),
355 retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));
356 brw_pop_insn_state(p);
357
358 if (inst->target > 0 && key->replicate_alpha) {
359 /* Set "Source0 Alpha Present to RenderTarget" bit in message
360 * header.
361 */
362 brw_OR(p,
363 vec1(retype(payload, BRW_REGISTER_TYPE_UD)),
364 vec1(retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD)),
365 brw_imm_ud(0x1 << 11));
366 }
367
368 if (inst->target > 0) {
369 /* Set the render target index for choosing BLEND_STATE. */
370 brw_MOV(p, retype(vec1(suboffset(payload, 2)),
371 BRW_REGISTER_TYPE_UD),
372 brw_imm_ud(inst->target));
373 }
374
375 /* Set computes stencil to render target */
376 if (prog_data->computed_stencil) {
377 brw_OR(p,
378 vec1(retype(payload, BRW_REGISTER_TYPE_UD)),
379 vec1(retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD)),
380 brw_imm_ud(0x1 << 14));
381 }
382
383 implied_header = brw_null_reg();
384 } else {
385 implied_header = retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UW);
386 }
387
388 brw_pop_insn_state(p);
389 } else {
390 implied_header = brw_null_reg();
391 }
392
393 if (!runtime_check_aads_emit) {
394 fire_fb_write(inst, payload, implied_header, inst->mlen);
395 } else {
396 /* This can only happen in gen < 6 */
397 assert(devinfo->gen < 6);
398
399 struct brw_reg v1_null_ud = vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_UD));
400
401 /* Check runtime bit to detect if we have to send AA data or not */
402 brw_push_insn_state(p);
403 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
404 brw_set_default_exec_size(p, BRW_EXECUTE_1);
405 brw_AND(p,
406 v1_null_ud,
407 retype(brw_vec1_grf(1, 6), BRW_REGISTER_TYPE_UD),
408 brw_imm_ud(1<<26));
409 brw_inst_set_cond_modifier(p->devinfo, brw_last_inst, BRW_CONDITIONAL_NZ);
410
411 int jmp = brw_JMPI(p, brw_imm_ud(0), BRW_PREDICATE_NORMAL) - p->store;
412 brw_pop_insn_state(p);
413 {
414 /* Don't send AA data */
415 fire_fb_write(inst, offset(payload, 1), implied_header, inst->mlen-1);
416 }
417 brw_land_fwd_jump(p, jmp);
418 fire_fb_write(inst, payload, implied_header, inst->mlen);
419 }
420 }
421
422 void
423 fs_generator::generate_fb_read(fs_inst *inst, struct brw_reg dst,
424 struct brw_reg payload)
425 {
426 assert(inst->size_written % REG_SIZE == 0);
427 struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
428 /* We assume that render targets start at binding table index 0. */
429 const unsigned surf_index = inst->target;
430
431 gen9_fb_READ(p, dst, payload, surf_index,
432 inst->header_size, inst->size_written / REG_SIZE,
433 prog_data->persample_dispatch);
434
435 brw_mark_surface_used(&prog_data->base, surf_index);
436 }
437
438 void
439 fs_generator::generate_mov_indirect(fs_inst *inst,
440 struct brw_reg dst,
441 struct brw_reg reg,
442 struct brw_reg indirect_byte_offset)
443 {
444 assert(indirect_byte_offset.type == BRW_REGISTER_TYPE_UD);
445 assert(indirect_byte_offset.file == BRW_GENERAL_REGISTER_FILE);
446 assert(!reg.abs && !reg.negate);
447 assert(reg.type == dst.type);
448
449 unsigned imm_byte_offset = reg.nr * REG_SIZE + reg.subnr;
450
451 if (indirect_byte_offset.file == BRW_IMMEDIATE_VALUE) {
452 imm_byte_offset += indirect_byte_offset.ud;
453
454 reg.nr = imm_byte_offset / REG_SIZE;
455 reg.subnr = imm_byte_offset % REG_SIZE;
456 brw_MOV(p, dst, reg);
457 } else {
458 /* Prior to Broadwell, there are only 8 address registers. */
459 assert(inst->exec_size <= 8 || devinfo->gen >= 8);
460
461 /* We use VxH indirect addressing, clobbering a0.0 through a0.7. */
462 struct brw_reg addr = vec8(brw_address_reg(0));
463
464 /* The destination stride of an instruction (in bytes) must be greater
465 * than or equal to the size of the rest of the instruction. Since the
466 * address register is of type UW, we can't use a D-type instruction.
467 * In order to get around this, re retype to UW and use a stride.
468 */
469 indirect_byte_offset =
470 retype(spread(indirect_byte_offset, 2), BRW_REGISTER_TYPE_UW);
471
472 /* There are a number of reasons why we don't use the base offset here.
473 * One reason is that the field is only 9 bits which means we can only
474 * use it to access the first 16 GRFs. Also, from the Haswell PRM
475 * section "Register Region Restrictions":
476 *
477 * "The lower bits of the AddressImmediate must not overflow to
478 * change the register address. The lower 5 bits of Address
479 * Immediate when added to lower 5 bits of address register gives
480 * the sub-register offset. The upper bits of Address Immediate
481 * when added to upper bits of address register gives the register
482 * address. Any overflow from sub-register offset is dropped."
483 *
484 * Since the indirect may cause us to cross a register boundary, this
485 * makes the base offset almost useless. We could try and do something
486 * clever where we use a actual base offset if base_offset % 32 == 0 but
487 * that would mean we were generating different code depending on the
488 * base offset. Instead, for the sake of consistency, we'll just do the
489 * add ourselves. This restriction is only listed in the Haswell PRM
490 * but empirical testing indicates that it applies on all older
491 * generations and is lifted on Broadwell.
492 *
493 * In the end, while base_offset is nice to look at in the generated
494 * code, using it saves us 0 instructions and would require quite a bit
495 * of case-by-case work. It's just not worth it.
496 */
497 brw_ADD(p, addr, indirect_byte_offset, brw_imm_uw(imm_byte_offset));
498
499 if (type_sz(reg.type) > 4 &&
500 ((devinfo->gen == 7 && !devinfo->is_haswell) ||
501 devinfo->is_cherryview || gen_device_info_is_9lp(devinfo))) {
502 /* IVB has an issue (which we found empirically) where it reads two
503 * address register components per channel for indirectly addressed
504 * 64-bit sources.
505 *
506 * From the Cherryview PRM Vol 7. "Register Region Restrictions":
507 *
508 * "When source or destination datatype is 64b or operation is
509 * integer DWord multiply, indirect addressing must not be used."
510 *
511 * To work around both of these, we do two integer MOVs insead of one
512 * 64-bit MOV. Because no double value should ever cross a register
513 * boundary, it's safe to use the immediate offset in the indirect
514 * here to handle adding 4 bytes to the offset and avoid the extra
515 * ADD to the register file.
516 */
517 brw_MOV(p, subscript(dst, BRW_REGISTER_TYPE_D, 0),
518 retype(brw_VxH_indirect(0, 0), BRW_REGISTER_TYPE_D));
519 brw_MOV(p, subscript(dst, BRW_REGISTER_TYPE_D, 1),
520 retype(brw_VxH_indirect(0, 4), BRW_REGISTER_TYPE_D));
521 } else {
522 struct brw_reg ind_src = brw_VxH_indirect(0, 0);
523
524 brw_inst *mov = brw_MOV(p, dst, retype(ind_src, reg.type));
525
526 if (devinfo->gen == 6 && dst.file == BRW_MESSAGE_REGISTER_FILE &&
527 !inst->get_next()->is_tail_sentinel() &&
528 ((fs_inst *)inst->get_next())->mlen > 0) {
529 /* From the Sandybridge PRM:
530 *
531 * "[Errata: DevSNB(SNB)] If MRF register is updated by any
532 * instruction that “indexed/indirect” source AND is followed
533 * by a send, the instruction requires a “Switch”. This is to
534 * avoid race condition where send may dispatch before MRF is
535 * updated."
536 */
537 brw_inst_set_thread_control(devinfo, mov, BRW_THREAD_SWITCH);
538 }
539 }
540 }
541 }
542
543 void
544 fs_generator::generate_urb_read(fs_inst *inst,
545 struct brw_reg dst,
546 struct brw_reg header)
547 {
548 assert(inst->size_written % REG_SIZE == 0);
549 assert(header.file == BRW_GENERAL_REGISTER_FILE);
550 assert(header.type == BRW_REGISTER_TYPE_UD);
551
552 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
553 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UD));
554 brw_set_src0(p, send, header);
555 brw_set_src1(p, send, brw_imm_ud(0u));
556
557 brw_inst_set_sfid(p->devinfo, send, BRW_SFID_URB);
558 brw_inst_set_urb_opcode(p->devinfo, send, GEN8_URB_OPCODE_SIMD8_READ);
559
560 if (inst->opcode == SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT)
561 brw_inst_set_urb_per_slot_offset(p->devinfo, send, true);
562
563 brw_inst_set_mlen(p->devinfo, send, inst->mlen);
564 brw_inst_set_rlen(p->devinfo, send, inst->size_written / REG_SIZE);
565 brw_inst_set_header_present(p->devinfo, send, true);
566 brw_inst_set_urb_global_offset(p->devinfo, send, inst->offset);
567 }
568
569 void
570 fs_generator::generate_urb_write(fs_inst *inst, struct brw_reg payload)
571 {
572 brw_inst *insn;
573
574 /* WaClearTDRRegBeforeEOTForNonPS.
575 *
576 * WA: Clear tdr register before send EOT in all non-PS shader kernels
577 *
578 * mov(8) tdr0:ud 0x0:ud {NoMask}"
579 */
580 if (inst->eot && p->devinfo->gen == 10) {
581 brw_push_insn_state(p);
582 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
583 brw_MOV(p, brw_tdr_reg(), brw_imm_uw(0));
584 brw_pop_insn_state(p);
585 }
586
587 insn = brw_next_insn(p, BRW_OPCODE_SEND);
588
589 brw_set_dest(p, insn, brw_null_reg());
590 brw_set_src0(p, insn, payload);
591 brw_set_src1(p, insn, brw_imm_d(0));
592
593 brw_inst_set_sfid(p->devinfo, insn, BRW_SFID_URB);
594 brw_inst_set_urb_opcode(p->devinfo, insn, GEN8_URB_OPCODE_SIMD8_WRITE);
595
596 if (inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT ||
597 inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT)
598 brw_inst_set_urb_per_slot_offset(p->devinfo, insn, true);
599
600 if (inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_MASKED ||
601 inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT)
602 brw_inst_set_urb_channel_mask_present(p->devinfo, insn, true);
603
604 brw_inst_set_mlen(p->devinfo, insn, inst->mlen);
605 brw_inst_set_rlen(p->devinfo, insn, 0);
606 brw_inst_set_eot(p->devinfo, insn, inst->eot);
607 brw_inst_set_header_present(p->devinfo, insn, true);
608 brw_inst_set_urb_global_offset(p->devinfo, insn, inst->offset);
609 }
610
611 void
612 fs_generator::generate_cs_terminate(fs_inst *inst, struct brw_reg payload)
613 {
614 struct brw_inst *insn;
615
616 insn = brw_next_insn(p, BRW_OPCODE_SEND);
617
618 brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW));
619 brw_set_src0(p, insn, retype(payload, BRW_REGISTER_TYPE_UW));
620 brw_set_src1(p, insn, brw_imm_d(0));
621
622 /* Terminate a compute shader by sending a message to the thread spawner.
623 */
624 brw_inst_set_sfid(devinfo, insn, BRW_SFID_THREAD_SPAWNER);
625 brw_inst_set_mlen(devinfo, insn, 1);
626 brw_inst_set_rlen(devinfo, insn, 0);
627 brw_inst_set_eot(devinfo, insn, inst->eot);
628 brw_inst_set_header_present(devinfo, insn, false);
629
630 brw_inst_set_ts_opcode(devinfo, insn, 0); /* Dereference resource */
631 brw_inst_set_ts_request_type(devinfo, insn, 0); /* Root thread */
632
633 /* Note that even though the thread has a URB resource associated with it,
634 * we set the "do not dereference URB" bit, because the URB resource is
635 * managed by the fixed-function unit, so it will free it automatically.
636 */
637 brw_inst_set_ts_resource_select(devinfo, insn, 1); /* Do not dereference URB */
638
639 brw_inst_set_mask_control(devinfo, insn, BRW_MASK_DISABLE);
640 }
641
642 void
643 fs_generator::generate_barrier(fs_inst *inst, struct brw_reg src)
644 {
645 brw_barrier(p, src);
646 brw_WAIT(p);
647 }
648
649 bool
650 fs_generator::generate_linterp(fs_inst *inst,
651 struct brw_reg dst, struct brw_reg *src)
652 {
653 /* PLN reads:
654 * / in SIMD16 \
655 * -----------------------------------
656 * | src1+0 | src1+1 | src1+2 | src1+3 |
657 * |-----------------------------------|
658 * |(x0, x1)|(y0, y1)|(x2, x3)|(y2, y3)|
659 * -----------------------------------
660 *
661 * but for the LINE/MAC pair, the LINE reads Xs and the MAC reads Ys:
662 *
663 * -----------------------------------
664 * | src1+0 | src1+1 | src1+2 | src1+3 |
665 * |-----------------------------------|
666 * |(x0, x1)|(y0, y1)| | | in SIMD8
667 * |-----------------------------------|
668 * |(x0, x1)|(x2, x3)|(y0, y1)|(y2, y3)| in SIMD16
669 * -----------------------------------
670 *
671 * See also: emit_interpolation_setup_gen4().
672 */
673 struct brw_reg delta_x = src[0];
674 struct brw_reg delta_y = offset(src[0], inst->exec_size / 8);
675 struct brw_reg interp = src[1];
676 brw_inst *i[4];
677
678 if (devinfo->gen >= 11) {
679 struct brw_reg acc = retype(brw_acc_reg(8), BRW_REGISTER_TYPE_NF);
680 struct brw_reg dwP = suboffset(interp, 0);
681 struct brw_reg dwQ = suboffset(interp, 1);
682 struct brw_reg dwR = suboffset(interp, 3);
683
684 brw_set_default_exec_size(p, BRW_EXECUTE_8);
685
686 if (inst->exec_size == 8) {
687 i[0] = brw_MAD(p, acc, dwR, offset(delta_x, 0), dwP);
688 i[1] = brw_MAD(p, offset(dst, 0), acc, offset(delta_y, 0), dwQ);
689
690 brw_inst_set_cond_modifier(p->devinfo, i[1], inst->conditional_mod);
691
692 /* brw_set_default_saturate() is called before emitting instructions,
693 * so the saturate bit is set in each instruction, so we need to unset
694 * it on the first instruction of each pair.
695 */
696 brw_inst_set_saturate(p->devinfo, i[0], false);
697 } else {
698 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
699 i[0] = brw_MAD(p, acc, dwR, offset(delta_x, 0), dwP);
700 i[1] = brw_MAD(p, offset(dst, 0), acc, offset(delta_x, 1), dwQ);
701
702 brw_set_default_compression_control(p, BRW_COMPRESSION_2NDHALF);
703 i[2] = brw_MAD(p, acc, dwR, offset(delta_y, 0), dwP);
704 i[3] = brw_MAD(p, offset(dst, 1), acc, offset(delta_y, 1), dwQ);
705
706 brw_set_default_compression_control(p, BRW_COMPRESSION_COMPRESSED);
707
708 brw_inst_set_cond_modifier(p->devinfo, i[1], inst->conditional_mod);
709 brw_inst_set_cond_modifier(p->devinfo, i[3], inst->conditional_mod);
710
711 /* brw_set_default_saturate() is called before emitting instructions,
712 * so the saturate bit is set in each instruction, so we need to unset
713 * it on the first instruction of each pair.
714 */
715 brw_inst_set_saturate(p->devinfo, i[0], false);
716 brw_inst_set_saturate(p->devinfo, i[2], false);
717 }
718
719 return true;
720 } else if (devinfo->has_pln &&
721 (devinfo->gen >= 7 || (delta_x.nr & 1) == 0)) {
722 brw_PLN(p, dst, interp, delta_x);
723
724 return false;
725 } else {
726 i[0] = brw_LINE(p, brw_null_reg(), interp, delta_x);
727 i[1] = brw_MAC(p, dst, suboffset(interp, 1), delta_y);
728
729 brw_inst_set_cond_modifier(p->devinfo, i[1], inst->conditional_mod);
730
731 /* brw_set_default_saturate() is called before emitting instructions, so
732 * the saturate bit is set in each instruction, so we need to unset it on
733 * the first instruction.
734 */
735 brw_inst_set_saturate(p->devinfo, i[0], false);
736
737 return true;
738 }
739 }
740
741 void
742 fs_generator::generate_get_buffer_size(fs_inst *inst,
743 struct brw_reg dst,
744 struct brw_reg src,
745 struct brw_reg surf_index)
746 {
747 assert(devinfo->gen >= 7);
748 assert(surf_index.file == BRW_IMMEDIATE_VALUE);
749
750 uint32_t simd_mode;
751 int rlen = 4;
752
753 switch (inst->exec_size) {
754 case 8:
755 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
756 break;
757 case 16:
758 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
759 break;
760 default:
761 unreachable("Invalid width for texture instruction");
762 }
763
764 if (simd_mode == BRW_SAMPLER_SIMD_MODE_SIMD16) {
765 rlen = 8;
766 dst = vec16(dst);
767 }
768
769 brw_SAMPLE(p,
770 retype(dst, BRW_REGISTER_TYPE_UW),
771 inst->base_mrf,
772 src,
773 surf_index.ud,
774 0,
775 GEN5_SAMPLER_MESSAGE_SAMPLE_RESINFO,
776 rlen, /* response length */
777 inst->mlen,
778 inst->header_size > 0,
779 simd_mode,
780 BRW_SAMPLER_RETURN_FORMAT_SINT32);
781
782 brw_mark_surface_used(prog_data, surf_index.ud);
783 }
784
785 void
786 fs_generator::generate_tex(fs_inst *inst, struct brw_reg dst, struct brw_reg src,
787 struct brw_reg surface_index,
788 struct brw_reg sampler_index)
789 {
790 assert(inst->size_written % REG_SIZE == 0);
791 int msg_type = -1;
792 uint32_t simd_mode;
793 uint32_t return_format;
794 bool is_combined_send = inst->eot;
795
796 switch (dst.type) {
797 case BRW_REGISTER_TYPE_D:
798 return_format = BRW_SAMPLER_RETURN_FORMAT_SINT32;
799 break;
800 case BRW_REGISTER_TYPE_UD:
801 return_format = BRW_SAMPLER_RETURN_FORMAT_UINT32;
802 break;
803 default:
804 return_format = BRW_SAMPLER_RETURN_FORMAT_FLOAT32;
805 break;
806 }
807
808 /* Stomp the resinfo output type to UINT32. On gens 4-5, the output type
809 * is set as part of the message descriptor. On gen4, the PRM seems to
810 * allow UINT32 and FLOAT32 (i965 PRM, Vol. 4 Section 4.8.1.1), but on
811 * later gens UINT32 is required. Once you hit Sandy Bridge, the bit is
812 * gone from the message descriptor entirely and you just get UINT32 all
813 * the time regasrdless. Since we can really only do non-UINT32 on gen4,
814 * just stomp it to UINT32 all the time.
815 */
816 if (inst->opcode == SHADER_OPCODE_TXS)
817 return_format = BRW_SAMPLER_RETURN_FORMAT_UINT32;
818
819 switch (inst->exec_size) {
820 case 8:
821 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
822 break;
823 case 16:
824 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
825 break;
826 default:
827 unreachable("Invalid width for texture instruction");
828 }
829
830 if (devinfo->gen >= 5) {
831 switch (inst->opcode) {
832 case SHADER_OPCODE_TEX:
833 if (inst->shadow_compare) {
834 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_COMPARE;
835 } else {
836 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE;
837 }
838 break;
839 case FS_OPCODE_TXB:
840 if (inst->shadow_compare) {
841 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_BIAS_COMPARE;
842 } else {
843 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_BIAS;
844 }
845 break;
846 case SHADER_OPCODE_TXL:
847 if (inst->shadow_compare) {
848 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LOD_COMPARE;
849 } else {
850 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LOD;
851 }
852 break;
853 case SHADER_OPCODE_TXL_LZ:
854 assert(devinfo->gen >= 9);
855 if (inst->shadow_compare) {
856 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_C_LZ;
857 } else {
858 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_LZ;
859 }
860 break;
861 case SHADER_OPCODE_TXS:
862 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_RESINFO;
863 break;
864 case SHADER_OPCODE_TXD:
865 if (inst->shadow_compare) {
866 /* Gen7.5+. Otherwise, lowered in NIR */
867 assert(devinfo->gen >= 8 || devinfo->is_haswell);
868 msg_type = HSW_SAMPLER_MESSAGE_SAMPLE_DERIV_COMPARE;
869 } else {
870 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_DERIVS;
871 }
872 break;
873 case SHADER_OPCODE_TXF:
874 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
875 break;
876 case SHADER_OPCODE_TXF_LZ:
877 assert(devinfo->gen >= 9);
878 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_LD_LZ;
879 break;
880 case SHADER_OPCODE_TXF_CMS_W:
881 assert(devinfo->gen >= 9);
882 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_LD2DMS_W;
883 break;
884 case SHADER_OPCODE_TXF_CMS:
885 if (devinfo->gen >= 7)
886 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD2DMS;
887 else
888 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
889 break;
890 case SHADER_OPCODE_TXF_UMS:
891 assert(devinfo->gen >= 7);
892 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD2DSS;
893 break;
894 case SHADER_OPCODE_TXF_MCS:
895 assert(devinfo->gen >= 7);
896 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD_MCS;
897 break;
898 case SHADER_OPCODE_LOD:
899 msg_type = GEN5_SAMPLER_MESSAGE_LOD;
900 break;
901 case SHADER_OPCODE_TG4:
902 if (inst->shadow_compare) {
903 assert(devinfo->gen >= 7);
904 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_C;
905 } else {
906 assert(devinfo->gen >= 6);
907 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4;
908 }
909 break;
910 case SHADER_OPCODE_TG4_OFFSET:
911 assert(devinfo->gen >= 7);
912 if (inst->shadow_compare) {
913 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_PO_C;
914 } else {
915 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_PO;
916 }
917 break;
918 case SHADER_OPCODE_SAMPLEINFO:
919 msg_type = GEN6_SAMPLER_MESSAGE_SAMPLE_SAMPLEINFO;
920 break;
921 default:
922 unreachable("not reached");
923 }
924 } else {
925 switch (inst->opcode) {
926 case SHADER_OPCODE_TEX:
927 /* Note that G45 and older determines shadow compare and dispatch width
928 * from message length for most messages.
929 */
930 if (inst->exec_size == 8) {
931 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE;
932 if (inst->shadow_compare) {
933 assert(inst->mlen == 6);
934 } else {
935 assert(inst->mlen <= 4);
936 }
937 } else {
938 if (inst->shadow_compare) {
939 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE_COMPARE;
940 assert(inst->mlen == 9);
941 } else {
942 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE;
943 assert(inst->mlen <= 7 && inst->mlen % 2 == 1);
944 }
945 }
946 break;
947 case FS_OPCODE_TXB:
948 if (inst->shadow_compare) {
949 assert(inst->exec_size == 8);
950 assert(inst->mlen == 6);
951 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE_BIAS_COMPARE;
952 } else {
953 assert(inst->mlen == 9);
954 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE_BIAS;
955 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
956 }
957 break;
958 case SHADER_OPCODE_TXL:
959 if (inst->shadow_compare) {
960 assert(inst->exec_size == 8);
961 assert(inst->mlen == 6);
962 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE_LOD_COMPARE;
963 } else {
964 assert(inst->mlen == 9);
965 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE_LOD;
966 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
967 }
968 break;
969 case SHADER_OPCODE_TXD:
970 /* There is no sample_d_c message; comparisons are done manually */
971 assert(inst->exec_size == 8);
972 assert(inst->mlen == 7 || inst->mlen == 10);
973 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE_GRADIENTS;
974 break;
975 case SHADER_OPCODE_TXF:
976 assert(inst->mlen <= 9 && inst->mlen % 2 == 1);
977 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_LD;
978 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
979 break;
980 case SHADER_OPCODE_TXS:
981 assert(inst->mlen == 3);
982 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_RESINFO;
983 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
984 break;
985 default:
986 unreachable("not reached");
987 }
988 }
989 assert(msg_type != -1);
990
991 if (simd_mode == BRW_SAMPLER_SIMD_MODE_SIMD16) {
992 dst = vec16(dst);
993 }
994
995 assert(devinfo->gen < 7 || inst->header_size == 0 ||
996 src.file == BRW_GENERAL_REGISTER_FILE);
997
998 assert(sampler_index.type == BRW_REGISTER_TYPE_UD);
999
1000 /* Load the message header if present. If there's a texture offset,
1001 * we need to set it up explicitly and load the offset bitfield.
1002 * Otherwise, we can use an implied move from g0 to the first message reg.
1003 */
1004 if (inst->header_size != 0) {
1005 if (devinfo->gen < 6 && !inst->offset) {
1006 /* Set up an implied move from g0 to the MRF. */
1007 src = retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UW);
1008 } else {
1009 struct brw_reg header_reg;
1010
1011 if (devinfo->gen >= 7) {
1012 header_reg = src;
1013 } else {
1014 assert(inst->base_mrf != -1);
1015 header_reg = brw_message_reg(inst->base_mrf);
1016 }
1017
1018 brw_push_insn_state(p);
1019 brw_set_default_exec_size(p, BRW_EXECUTE_8);
1020 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1021 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
1022 /* Explicitly set up the message header by copying g0 to the MRF. */
1023 brw_MOV(p, header_reg, brw_vec8_grf(0, 0));
1024
1025 brw_set_default_exec_size(p, BRW_EXECUTE_1);
1026 if (inst->offset) {
1027 /* Set the offset bits in DWord 2. */
1028 brw_MOV(p, get_element_ud(header_reg, 2),
1029 brw_imm_ud(inst->offset));
1030 } else if (stage != MESA_SHADER_VERTEX &&
1031 stage != MESA_SHADER_FRAGMENT) {
1032 /* The vertex and fragment stages have g0.2 set to 0, so
1033 * header0.2 is 0 when g0 is copied. Other stages may not, so we
1034 * must set it to 0 to avoid setting undesirable bits in the
1035 * message.
1036 */
1037 brw_MOV(p, get_element_ud(header_reg, 2), brw_imm_ud(0));
1038 }
1039
1040 brw_adjust_sampler_state_pointer(p, header_reg, sampler_index);
1041 brw_pop_insn_state(p);
1042 }
1043 }
1044
1045 uint32_t base_binding_table_index = (inst->opcode == SHADER_OPCODE_TG4 ||
1046 inst->opcode == SHADER_OPCODE_TG4_OFFSET)
1047 ? prog_data->binding_table.gather_texture_start
1048 : prog_data->binding_table.texture_start;
1049
1050 if (surface_index.file == BRW_IMMEDIATE_VALUE &&
1051 sampler_index.file == BRW_IMMEDIATE_VALUE) {
1052 uint32_t surface = surface_index.ud;
1053 uint32_t sampler = sampler_index.ud;
1054
1055 brw_SAMPLE(p,
1056 retype(dst, BRW_REGISTER_TYPE_UW),
1057 inst->base_mrf,
1058 src,
1059 surface + base_binding_table_index,
1060 sampler % 16,
1061 msg_type,
1062 inst->size_written / REG_SIZE,
1063 inst->mlen,
1064 inst->header_size != 0,
1065 simd_mode,
1066 return_format);
1067
1068 brw_mark_surface_used(prog_data, surface + base_binding_table_index);
1069 } else {
1070 /* Non-const sampler index */
1071
1072 struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
1073 struct brw_reg surface_reg = vec1(retype(surface_index, BRW_REGISTER_TYPE_UD));
1074 struct brw_reg sampler_reg = vec1(retype(sampler_index, BRW_REGISTER_TYPE_UD));
1075
1076 brw_push_insn_state(p);
1077 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1078 brw_set_default_access_mode(p, BRW_ALIGN_1);
1079 brw_set_default_exec_size(p, BRW_EXECUTE_1);
1080
1081 if (brw_regs_equal(&surface_reg, &sampler_reg)) {
1082 brw_MUL(p, addr, sampler_reg, brw_imm_uw(0x101));
1083 } else {
1084 if (sampler_reg.file == BRW_IMMEDIATE_VALUE) {
1085 brw_OR(p, addr, surface_reg, brw_imm_ud(sampler_reg.ud << 8));
1086 } else {
1087 brw_SHL(p, addr, sampler_reg, brw_imm_ud(8));
1088 brw_OR(p, addr, addr, surface_reg);
1089 }
1090 }
1091 if (base_binding_table_index)
1092 brw_ADD(p, addr, addr, brw_imm_ud(base_binding_table_index));
1093 brw_AND(p, addr, addr, brw_imm_ud(0xfff));
1094
1095 brw_pop_insn_state(p);
1096
1097 /* dst = send(offset, a0.0 | <descriptor>) */
1098 brw_inst *insn = brw_send_indirect_message(
1099 p, BRW_SFID_SAMPLER, dst, src, addr);
1100 brw_set_sampler_message(p, insn,
1101 0 /* surface */,
1102 0 /* sampler */,
1103 msg_type,
1104 inst->size_written / REG_SIZE,
1105 inst->mlen /* mlen */,
1106 inst->header_size != 0 /* header */,
1107 simd_mode,
1108 return_format);
1109
1110 /* visitor knows more than we do about the surface limit required,
1111 * so has already done marking.
1112 */
1113 }
1114
1115 if (is_combined_send) {
1116 brw_inst_set_eot(p->devinfo, brw_last_inst, true);
1117 brw_inst_set_opcode(p->devinfo, brw_last_inst, BRW_OPCODE_SENDC);
1118 }
1119 }
1120
1121
1122 /* For OPCODE_DDX and OPCODE_DDY, per channel of output we've got input
1123 * looking like:
1124 *
1125 * arg0: ss0.tl ss0.tr ss0.bl ss0.br ss1.tl ss1.tr ss1.bl ss1.br
1126 *
1127 * Ideally, we want to produce:
1128 *
1129 * DDX DDY
1130 * dst: (ss0.tr - ss0.tl) (ss0.tl - ss0.bl)
1131 * (ss0.tr - ss0.tl) (ss0.tr - ss0.br)
1132 * (ss0.br - ss0.bl) (ss0.tl - ss0.bl)
1133 * (ss0.br - ss0.bl) (ss0.tr - ss0.br)
1134 * (ss1.tr - ss1.tl) (ss1.tl - ss1.bl)
1135 * (ss1.tr - ss1.tl) (ss1.tr - ss1.br)
1136 * (ss1.br - ss1.bl) (ss1.tl - ss1.bl)
1137 * (ss1.br - ss1.bl) (ss1.tr - ss1.br)
1138 *
1139 * and add another set of two more subspans if in 16-pixel dispatch mode.
1140 *
1141 * For DDX, it ends up being easy: width = 2, horiz=0 gets us the same result
1142 * for each pair, and vertstride = 2 jumps us 2 elements after processing a
1143 * pair. But the ideal approximation may impose a huge performance cost on
1144 * sample_d. On at least Haswell, sample_d instruction does some
1145 * optimizations if the same LOD is used for all pixels in the subspan.
1146 *
1147 * For DDY, we need to use ALIGN16 mode since it's capable of doing the
1148 * appropriate swizzling.
1149 */
1150 void
1151 fs_generator::generate_ddx(const fs_inst *inst,
1152 struct brw_reg dst, struct brw_reg src)
1153 {
1154 unsigned vstride, width;
1155
1156 if (inst->opcode == FS_OPCODE_DDX_FINE) {
1157 /* produce accurate derivatives */
1158 vstride = BRW_VERTICAL_STRIDE_2;
1159 width = BRW_WIDTH_2;
1160 } else {
1161 /* replicate the derivative at the top-left pixel to other pixels */
1162 vstride = BRW_VERTICAL_STRIDE_4;
1163 width = BRW_WIDTH_4;
1164 }
1165
1166 struct brw_reg src0 = brw_reg(src.file, src.nr, 1,
1167 src.negate, src.abs,
1168 BRW_REGISTER_TYPE_F,
1169 vstride,
1170 width,
1171 BRW_HORIZONTAL_STRIDE_0,
1172 BRW_SWIZZLE_XYZW, WRITEMASK_XYZW);
1173 struct brw_reg src1 = brw_reg(src.file, src.nr, 0,
1174 src.negate, src.abs,
1175 BRW_REGISTER_TYPE_F,
1176 vstride,
1177 width,
1178 BRW_HORIZONTAL_STRIDE_0,
1179 BRW_SWIZZLE_XYZW, WRITEMASK_XYZW);
1180 brw_ADD(p, dst, src0, negate(src1));
1181 }
1182
1183 /* The negate_value boolean is used to negate the derivative computation for
1184 * FBOs, since they place the origin at the upper left instead of the lower
1185 * left.
1186 */
1187 void
1188 fs_generator::generate_ddy(const fs_inst *inst,
1189 struct brw_reg dst, struct brw_reg src)
1190 {
1191 if (inst->opcode == FS_OPCODE_DDY_FINE) {
1192 /* produce accurate derivatives */
1193 struct brw_reg src0 = brw_reg(src.file, src.nr, 0,
1194 src.negate, src.abs,
1195 BRW_REGISTER_TYPE_F,
1196 BRW_VERTICAL_STRIDE_4,
1197 BRW_WIDTH_4,
1198 BRW_HORIZONTAL_STRIDE_1,
1199 BRW_SWIZZLE_XYXY, WRITEMASK_XYZW);
1200 struct brw_reg src1 = brw_reg(src.file, src.nr, 0,
1201 src.negate, src.abs,
1202 BRW_REGISTER_TYPE_F,
1203 BRW_VERTICAL_STRIDE_4,
1204 BRW_WIDTH_4,
1205 BRW_HORIZONTAL_STRIDE_1,
1206 BRW_SWIZZLE_ZWZW, WRITEMASK_XYZW);
1207 brw_push_insn_state(p);
1208 brw_set_default_access_mode(p, BRW_ALIGN_16);
1209 brw_ADD(p, dst, negate(src0), src1);
1210 brw_pop_insn_state(p);
1211 } else {
1212 /* replicate the derivative at the top-left pixel to other pixels */
1213 struct brw_reg src0 = brw_reg(src.file, src.nr, 0,
1214 src.negate, src.abs,
1215 BRW_REGISTER_TYPE_F,
1216 BRW_VERTICAL_STRIDE_4,
1217 BRW_WIDTH_4,
1218 BRW_HORIZONTAL_STRIDE_0,
1219 BRW_SWIZZLE_XYZW, WRITEMASK_XYZW);
1220 struct brw_reg src1 = brw_reg(src.file, src.nr, 2,
1221 src.negate, src.abs,
1222 BRW_REGISTER_TYPE_F,
1223 BRW_VERTICAL_STRIDE_4,
1224 BRW_WIDTH_4,
1225 BRW_HORIZONTAL_STRIDE_0,
1226 BRW_SWIZZLE_XYZW, WRITEMASK_XYZW);
1227 brw_ADD(p, dst, negate(src0), src1);
1228 }
1229 }
1230
1231 void
1232 fs_generator::generate_discard_jump(fs_inst *inst)
1233 {
1234 assert(devinfo->gen >= 6);
1235
1236 /* This HALT will be patched up at FB write time to point UIP at the end of
1237 * the program, and at brw_uip_jip() JIP will be set to the end of the
1238 * current block (or the program).
1239 */
1240 this->discard_halt_patches.push_tail(new(mem_ctx) ip_record(p->nr_insn));
1241 gen6_HALT(p);
1242 }
1243
1244 void
1245 fs_generator::generate_scratch_write(fs_inst *inst, struct brw_reg src)
1246 {
1247 /* The 32-wide messages only respect the first 16-wide half of the channel
1248 * enable signals which are replicated identically for the second group of
1249 * 16 channels, so we cannot use them unless the write is marked
1250 * force_writemask_all.
1251 */
1252 const unsigned lower_size = inst->force_writemask_all ? inst->exec_size :
1253 MIN2(16, inst->exec_size);
1254 const unsigned block_size = 4 * lower_size / REG_SIZE;
1255 assert(inst->mlen != 0);
1256
1257 brw_push_insn_state(p);
1258 brw_set_default_exec_size(p, cvt(lower_size) - 1);
1259 brw_set_default_compression(p, lower_size > 8);
1260
1261 for (unsigned i = 0; i < inst->exec_size / lower_size; i++) {
1262 brw_set_default_group(p, inst->group + lower_size * i);
1263
1264 brw_MOV(p, brw_uvec_mrf(lower_size, inst->base_mrf + 1, 0),
1265 retype(offset(src, block_size * i), BRW_REGISTER_TYPE_UD));
1266
1267 brw_oword_block_write_scratch(p, brw_message_reg(inst->base_mrf),
1268 block_size,
1269 inst->offset + block_size * REG_SIZE * i);
1270 }
1271
1272 brw_pop_insn_state(p);
1273 }
1274
1275 void
1276 fs_generator::generate_scratch_read(fs_inst *inst, struct brw_reg dst)
1277 {
1278 assert(inst->exec_size <= 16 || inst->force_writemask_all);
1279 assert(inst->mlen != 0);
1280
1281 brw_oword_block_read_scratch(p, dst, brw_message_reg(inst->base_mrf),
1282 inst->exec_size / 8, inst->offset);
1283 }
1284
1285 void
1286 fs_generator::generate_scratch_read_gen7(fs_inst *inst, struct brw_reg dst)
1287 {
1288 assert(inst->exec_size <= 16 || inst->force_writemask_all);
1289
1290 gen7_block_read_scratch(p, dst, inst->exec_size / 8, inst->offset);
1291 }
1292
1293 void
1294 fs_generator::generate_uniform_pull_constant_load(fs_inst *inst,
1295 struct brw_reg dst,
1296 struct brw_reg index,
1297 struct brw_reg offset)
1298 {
1299 assert(type_sz(dst.type) == 4);
1300 assert(inst->mlen != 0);
1301
1302 assert(index.file == BRW_IMMEDIATE_VALUE &&
1303 index.type == BRW_REGISTER_TYPE_UD);
1304 uint32_t surf_index = index.ud;
1305
1306 assert(offset.file == BRW_IMMEDIATE_VALUE &&
1307 offset.type == BRW_REGISTER_TYPE_UD);
1308 uint32_t read_offset = offset.ud;
1309
1310 brw_oword_block_read(p, dst, brw_message_reg(inst->base_mrf),
1311 read_offset, surf_index);
1312 }
1313
1314 void
1315 fs_generator::generate_uniform_pull_constant_load_gen7(fs_inst *inst,
1316 struct brw_reg dst,
1317 struct brw_reg index,
1318 struct brw_reg payload)
1319 {
1320 assert(index.type == BRW_REGISTER_TYPE_UD);
1321 assert(payload.file == BRW_GENERAL_REGISTER_FILE);
1322 assert(type_sz(dst.type) == 4);
1323
1324 if (index.file == BRW_IMMEDIATE_VALUE) {
1325 const uint32_t surf_index = index.ud;
1326
1327 brw_push_insn_state(p);
1328 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1329 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
1330 brw_pop_insn_state(p);
1331
1332 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UD));
1333 brw_set_src0(p, send, retype(payload, BRW_REGISTER_TYPE_UD));
1334 brw_set_dp_read_message(p, send, surf_index,
1335 BRW_DATAPORT_OWORD_BLOCK_DWORDS(inst->exec_size),
1336 GEN7_DATAPORT_DC_OWORD_BLOCK_READ,
1337 GEN6_SFID_DATAPORT_CONSTANT_CACHE,
1338 1, /* mlen */
1339 true, /* header */
1340 DIV_ROUND_UP(inst->size_written, REG_SIZE));
1341
1342 } else {
1343 struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
1344
1345 brw_push_insn_state(p);
1346 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1347
1348 /* a0.0 = surf_index & 0xff */
1349 brw_inst *insn_and = brw_next_insn(p, BRW_OPCODE_AND);
1350 brw_inst_set_exec_size(p->devinfo, insn_and, BRW_EXECUTE_1);
1351 brw_set_dest(p, insn_and, addr);
1352 brw_set_src0(p, insn_and, vec1(retype(index, BRW_REGISTER_TYPE_UD)));
1353 brw_set_src1(p, insn_and, brw_imm_ud(0x0ff));
1354
1355 /* dst = send(payload, a0.0 | <descriptor>) */
1356 brw_inst *insn = brw_send_indirect_message(
1357 p, GEN6_SFID_DATAPORT_CONSTANT_CACHE,
1358 retype(dst, BRW_REGISTER_TYPE_UD),
1359 retype(payload, BRW_REGISTER_TYPE_UD), addr);
1360 brw_set_dp_read_message(p, insn, 0 /* surface */,
1361 BRW_DATAPORT_OWORD_BLOCK_DWORDS(inst->exec_size),
1362 GEN7_DATAPORT_DC_OWORD_BLOCK_READ,
1363 GEN6_SFID_DATAPORT_CONSTANT_CACHE,
1364 1, /* mlen */
1365 true, /* header */
1366 DIV_ROUND_UP(inst->size_written, REG_SIZE));
1367
1368 brw_pop_insn_state(p);
1369 }
1370 }
1371
1372 void
1373 fs_generator::generate_varying_pull_constant_load_gen4(fs_inst *inst,
1374 struct brw_reg dst,
1375 struct brw_reg index)
1376 {
1377 assert(devinfo->gen < 7); /* Should use the gen7 variant. */
1378 assert(inst->header_size != 0);
1379 assert(inst->mlen);
1380
1381 assert(index.file == BRW_IMMEDIATE_VALUE &&
1382 index.type == BRW_REGISTER_TYPE_UD);
1383 uint32_t surf_index = index.ud;
1384
1385 uint32_t simd_mode, rlen, msg_type;
1386 if (inst->exec_size == 16) {
1387 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1388 rlen = 8;
1389 } else {
1390 assert(inst->exec_size == 8);
1391 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
1392 rlen = 4;
1393 }
1394
1395 if (devinfo->gen >= 5)
1396 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
1397 else {
1398 /* We always use the SIMD16 message so that we only have to load U, and
1399 * not V or R.
1400 */
1401 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_LD;
1402 assert(inst->mlen == 3);
1403 assert(inst->size_written == 8 * REG_SIZE);
1404 rlen = 8;
1405 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1406 }
1407
1408 struct brw_reg header = brw_vec8_grf(0, 0);
1409 gen6_resolve_implied_move(p, &header, inst->base_mrf);
1410
1411 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
1412 brw_inst_set_compression(devinfo, send, false);
1413 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UW));
1414 brw_set_src0(p, send, header);
1415 if (devinfo->gen < 6)
1416 brw_inst_set_base_mrf(p->devinfo, send, inst->base_mrf);
1417
1418 /* Our surface is set up as floats, regardless of what actual data is
1419 * stored in it.
1420 */
1421 uint32_t return_format = BRW_SAMPLER_RETURN_FORMAT_FLOAT32;
1422 brw_set_sampler_message(p, send,
1423 surf_index,
1424 0, /* sampler (unused) */
1425 msg_type,
1426 rlen,
1427 inst->mlen,
1428 inst->header_size != 0,
1429 simd_mode,
1430 return_format);
1431 }
1432
1433 void
1434 fs_generator::generate_varying_pull_constant_load_gen7(fs_inst *inst,
1435 struct brw_reg dst,
1436 struct brw_reg index,
1437 struct brw_reg offset)
1438 {
1439 assert(devinfo->gen >= 7);
1440 /* Varying-offset pull constant loads are treated as a normal expression on
1441 * gen7, so the fact that it's a send message is hidden at the IR level.
1442 */
1443 assert(inst->header_size == 0);
1444 assert(!inst->mlen);
1445 assert(index.type == BRW_REGISTER_TYPE_UD);
1446
1447 uint32_t simd_mode, rlen, mlen;
1448 if (inst->exec_size == 16) {
1449 mlen = 2;
1450 rlen = 8;
1451 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1452 } else {
1453 assert(inst->exec_size == 8);
1454 mlen = 1;
1455 rlen = 4;
1456 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
1457 }
1458
1459 if (index.file == BRW_IMMEDIATE_VALUE) {
1460
1461 uint32_t surf_index = index.ud;
1462
1463 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
1464 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UW));
1465 brw_set_src0(p, send, offset);
1466 brw_set_sampler_message(p, send,
1467 surf_index,
1468 0, /* LD message ignores sampler unit */
1469 GEN5_SAMPLER_MESSAGE_SAMPLE_LD,
1470 rlen,
1471 mlen,
1472 false, /* no header */
1473 simd_mode,
1474 0);
1475
1476 } else {
1477
1478 struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
1479
1480 brw_push_insn_state(p);
1481 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1482
1483 /* a0.0 = surf_index & 0xff */
1484 brw_inst *insn_and = brw_next_insn(p, BRW_OPCODE_AND);
1485 brw_inst_set_exec_size(p->devinfo, insn_and, BRW_EXECUTE_1);
1486 brw_set_dest(p, insn_and, addr);
1487 brw_set_src0(p, insn_and, vec1(retype(index, BRW_REGISTER_TYPE_UD)));
1488 brw_set_src1(p, insn_and, brw_imm_ud(0x0ff));
1489
1490 brw_pop_insn_state(p);
1491
1492 /* dst = send(offset, a0.0 | <descriptor>) */
1493 brw_inst *insn = brw_send_indirect_message(
1494 p, BRW_SFID_SAMPLER, retype(dst, BRW_REGISTER_TYPE_UW),
1495 offset, addr);
1496 brw_set_sampler_message(p, insn,
1497 0 /* surface */,
1498 0 /* sampler */,
1499 GEN5_SAMPLER_MESSAGE_SAMPLE_LD,
1500 rlen /* rlen */,
1501 mlen /* mlen */,
1502 false /* header */,
1503 simd_mode,
1504 0);
1505 }
1506 }
1507
1508 /**
1509 * Cause the current pixel/sample mask (from R1.7 bits 15:0) to be transferred
1510 * into the flags register (f0.0).
1511 *
1512 * Used only on Gen6 and above.
1513 */
1514 void
1515 fs_generator::generate_mov_dispatch_to_flags(fs_inst *inst)
1516 {
1517 struct brw_reg flags = brw_flag_reg(0, inst->flag_subreg);
1518 struct brw_reg dispatch_mask;
1519
1520 if (devinfo->gen >= 6)
1521 dispatch_mask = retype(brw_vec1_grf(1, 7), BRW_REGISTER_TYPE_UW);
1522 else
1523 dispatch_mask = retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UW);
1524
1525 brw_push_insn_state(p);
1526 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1527 brw_set_default_exec_size(p, BRW_EXECUTE_1);
1528 brw_MOV(p, flags, dispatch_mask);
1529 brw_pop_insn_state(p);
1530 }
1531
1532 void
1533 fs_generator::generate_pixel_interpolator_query(fs_inst *inst,
1534 struct brw_reg dst,
1535 struct brw_reg src,
1536 struct brw_reg msg_data,
1537 unsigned msg_type)
1538 {
1539 assert(inst->size_written % REG_SIZE == 0);
1540 assert(msg_data.type == BRW_REGISTER_TYPE_UD);
1541
1542 brw_pixel_interpolator_query(p,
1543 retype(dst, BRW_REGISTER_TYPE_UW),
1544 src,
1545 inst->pi_noperspective,
1546 msg_type,
1547 msg_data,
1548 inst->mlen,
1549 inst->size_written / REG_SIZE);
1550 }
1551
1552 /* Sets vstride=1, width=4, hstride=0 of register src1 during
1553 * the ADD instruction.
1554 */
1555 void
1556 fs_generator::generate_set_sample_id(fs_inst *inst,
1557 struct brw_reg dst,
1558 struct brw_reg src0,
1559 struct brw_reg src1)
1560 {
1561 assert(dst.type == BRW_REGISTER_TYPE_D ||
1562 dst.type == BRW_REGISTER_TYPE_UD);
1563 assert(src0.type == BRW_REGISTER_TYPE_D ||
1564 src0.type == BRW_REGISTER_TYPE_UD);
1565
1566 struct brw_reg reg = stride(src1, 1, 4, 0);
1567 if (devinfo->gen >= 8 || inst->exec_size == 8) {
1568 brw_ADD(p, dst, src0, reg);
1569 } else if (inst->exec_size == 16) {
1570 brw_push_insn_state(p);
1571 brw_set_default_exec_size(p, BRW_EXECUTE_8);
1572 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
1573 brw_ADD(p, firsthalf(dst), firsthalf(src0), reg);
1574 brw_set_default_compression_control(p, BRW_COMPRESSION_2NDHALF);
1575 brw_ADD(p, sechalf(dst), sechalf(src0), suboffset(reg, 2));
1576 brw_pop_insn_state(p);
1577 }
1578 }
1579
1580 void
1581 fs_generator::generate_pack_half_2x16_split(fs_inst *inst,
1582 struct brw_reg dst,
1583 struct brw_reg x,
1584 struct brw_reg y)
1585 {
1586 assert(devinfo->gen >= 7);
1587 assert(dst.type == BRW_REGISTER_TYPE_UD);
1588 assert(x.type == BRW_REGISTER_TYPE_F);
1589 assert(y.type == BRW_REGISTER_TYPE_F);
1590
1591 /* From the Ivybridge PRM, Vol4, Part3, Section 6.27 f32to16:
1592 *
1593 * Because this instruction does not have a 16-bit floating-point type,
1594 * the destination data type must be Word (W).
1595 *
1596 * The destination must be DWord-aligned and specify a horizontal stride
1597 * (HorzStride) of 2. The 16-bit result is stored in the lower word of
1598 * each destination channel and the upper word is not modified.
1599 */
1600 struct brw_reg dst_w = spread(retype(dst, BRW_REGISTER_TYPE_W), 2);
1601
1602 /* Give each 32-bit channel of dst the form below, where "." means
1603 * unchanged.
1604 * 0x....hhhh
1605 */
1606 brw_F32TO16(p, dst_w, y);
1607
1608 /* Now the form:
1609 * 0xhhhh0000
1610 */
1611 brw_SHL(p, dst, dst, brw_imm_ud(16u));
1612
1613 /* And, finally the form of packHalf2x16's output:
1614 * 0xhhhhllll
1615 */
1616 brw_F32TO16(p, dst_w, x);
1617 }
1618
1619 void
1620 fs_generator::generate_unpack_half_2x16_split(fs_inst *inst,
1621 struct brw_reg dst,
1622 struct brw_reg src)
1623 {
1624 assert(devinfo->gen >= 7);
1625 assert(dst.type == BRW_REGISTER_TYPE_F);
1626 assert(src.type == BRW_REGISTER_TYPE_UD);
1627
1628 /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
1629 *
1630 * Because this instruction does not have a 16-bit floating-point type,
1631 * the source data type must be Word (W). The destination type must be
1632 * F (Float).
1633 */
1634 struct brw_reg src_w = spread(retype(src, BRW_REGISTER_TYPE_W), 2);
1635
1636 /* Each channel of src has the form of unpackHalf2x16's input: 0xhhhhllll.
1637 * For the Y case, we wish to access only the upper word; therefore
1638 * a 16-bit subregister offset is needed.
1639 */
1640 assert(inst->opcode == FS_OPCODE_UNPACK_HALF_2x16_SPLIT_X ||
1641 inst->opcode == FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y);
1642 if (inst->opcode == FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y)
1643 src_w.subnr += 2;
1644
1645 brw_F16TO32(p, dst, src_w);
1646 }
1647
1648 void
1649 fs_generator::generate_shader_time_add(fs_inst *inst,
1650 struct brw_reg payload,
1651 struct brw_reg offset,
1652 struct brw_reg value)
1653 {
1654 assert(devinfo->gen >= 7);
1655 brw_push_insn_state(p);
1656 brw_set_default_mask_control(p, true);
1657
1658 assert(payload.file == BRW_GENERAL_REGISTER_FILE);
1659 struct brw_reg payload_offset = retype(brw_vec1_grf(payload.nr, 0),
1660 offset.type);
1661 struct brw_reg payload_value = retype(brw_vec1_grf(payload.nr + 1, 0),
1662 value.type);
1663
1664 assert(offset.file == BRW_IMMEDIATE_VALUE);
1665 if (value.file == BRW_GENERAL_REGISTER_FILE) {
1666 value.width = BRW_WIDTH_1;
1667 value.hstride = BRW_HORIZONTAL_STRIDE_0;
1668 value.vstride = BRW_VERTICAL_STRIDE_0;
1669 } else {
1670 assert(value.file == BRW_IMMEDIATE_VALUE);
1671 }
1672
1673 /* Trying to deal with setup of the params from the IR is crazy in the FS8
1674 * case, and we don't really care about squeezing every bit of performance
1675 * out of this path, so we just emit the MOVs from here.
1676 */
1677 brw_MOV(p, payload_offset, offset);
1678 brw_MOV(p, payload_value, value);
1679 brw_shader_time_add(p, payload,
1680 prog_data->binding_table.shader_time_start);
1681 brw_pop_insn_state(p);
1682
1683 brw_mark_surface_used(prog_data,
1684 prog_data->binding_table.shader_time_start);
1685 }
1686
1687 void
1688 fs_generator::enable_debug(const char *shader_name)
1689 {
1690 debug_flag = true;
1691 this->shader_name = shader_name;
1692 }
1693
1694 int
1695 fs_generator::generate_code(const cfg_t *cfg, int dispatch_width)
1696 {
1697 /* align to 64 byte boundary. */
1698 while (p->next_insn_offset % 64)
1699 brw_NOP(p);
1700
1701 this->dispatch_width = dispatch_width;
1702
1703 int start_offset = p->next_insn_offset;
1704 int spill_count = 0, fill_count = 0;
1705 int loop_count = 0;
1706
1707 struct disasm_info *disasm_info = disasm_initialize(devinfo, cfg);
1708
1709 foreach_block_and_inst (block, fs_inst, inst, cfg) {
1710 struct brw_reg src[3], dst;
1711 unsigned int last_insn_offset = p->next_insn_offset;
1712 bool multiple_instructions_emitted = false;
1713
1714 /* From the Broadwell PRM, Volume 7, "3D-Media-GPGPU", in the
1715 * "Register Region Restrictions" section: for BDW, SKL:
1716 *
1717 * "A POW/FDIV operation must not be followed by an instruction
1718 * that requires two destination registers."
1719 *
1720 * The documentation is often lacking annotations for Atom parts,
1721 * and empirically this affects CHV as well.
1722 */
1723 if (devinfo->gen >= 8 &&
1724 devinfo->gen <= 9 &&
1725 p->nr_insn > 1 &&
1726 brw_inst_opcode(devinfo, brw_last_inst) == BRW_OPCODE_MATH &&
1727 brw_inst_math_function(devinfo, brw_last_inst) == BRW_MATH_FUNCTION_POW &&
1728 inst->dst.component_size(inst->exec_size) > REG_SIZE) {
1729 brw_NOP(p);
1730 last_insn_offset = p->next_insn_offset;
1731 }
1732
1733 if (unlikely(debug_flag))
1734 disasm_annotate(disasm_info, inst, p->next_insn_offset);
1735
1736 /* If the instruction writes to more than one register, it needs to be
1737 * explicitly marked as compressed on Gen <= 5. On Gen >= 6 the
1738 * hardware figures out by itself what the right compression mode is,
1739 * but we still need to know whether the instruction is compressed to
1740 * set up the source register regions appropriately.
1741 *
1742 * XXX - This is wrong for instructions that write a single register but
1743 * read more than one which should strictly speaking be treated as
1744 * compressed. For instructions that don't write any registers it
1745 * relies on the destination being a null register of the correct
1746 * type and regioning so the instruction is considered compressed
1747 * or not accordingly.
1748 */
1749 const bool compressed =
1750 inst->dst.component_size(inst->exec_size) > REG_SIZE;
1751 brw_set_default_compression(p, compressed);
1752 brw_set_default_group(p, inst->group);
1753
1754 for (unsigned int i = 0; i < inst->sources; i++) {
1755 src[i] = brw_reg_from_fs_reg(devinfo, inst,
1756 &inst->src[i], compressed);
1757 /* The accumulator result appears to get used for the
1758 * conditional modifier generation. When negating a UD
1759 * value, there is a 33rd bit generated for the sign in the
1760 * accumulator value, so now you can't check, for example,
1761 * equality with a 32-bit value. See piglit fs-op-neg-uvec4.
1762 */
1763 assert(!inst->conditional_mod ||
1764 inst->src[i].type != BRW_REGISTER_TYPE_UD ||
1765 !inst->src[i].negate);
1766 }
1767 dst = brw_reg_from_fs_reg(devinfo, inst,
1768 &inst->dst, compressed);
1769
1770 brw_set_default_access_mode(p, BRW_ALIGN_1);
1771 brw_set_default_predicate_control(p, inst->predicate);
1772 brw_set_default_predicate_inverse(p, inst->predicate_inverse);
1773 brw_set_default_flag_reg(p, 0, inst->flag_subreg);
1774 brw_set_default_saturate(p, inst->saturate);
1775 brw_set_default_mask_control(p, inst->force_writemask_all);
1776 brw_set_default_acc_write_control(p, inst->writes_accumulator);
1777
1778 unsigned exec_size = inst->exec_size;
1779 if (devinfo->gen == 7 && !devinfo->is_haswell &&
1780 (get_exec_type_size(inst) == 8 || type_sz(inst->dst.type) == 8)) {
1781 exec_size *= 2;
1782 }
1783
1784 brw_set_default_exec_size(p, cvt(exec_size) - 1);
1785
1786 assert(inst->force_writemask_all || inst->exec_size >= 4);
1787 assert(inst->force_writemask_all || inst->group % inst->exec_size == 0);
1788 assert(inst->base_mrf + inst->mlen <= BRW_MAX_MRF(devinfo->gen));
1789 assert(inst->mlen <= BRW_MAX_MSG_LENGTH);
1790
1791 switch (inst->opcode) {
1792 case BRW_OPCODE_MOV:
1793 brw_MOV(p, dst, src[0]);
1794 break;
1795 case BRW_OPCODE_ADD:
1796 brw_ADD(p, dst, src[0], src[1]);
1797 break;
1798 case BRW_OPCODE_MUL:
1799 brw_MUL(p, dst, src[0], src[1]);
1800 break;
1801 case BRW_OPCODE_AVG:
1802 brw_AVG(p, dst, src[0], src[1]);
1803 break;
1804 case BRW_OPCODE_MACH:
1805 brw_MACH(p, dst, src[0], src[1]);
1806 break;
1807
1808 case BRW_OPCODE_LINE:
1809 brw_LINE(p, dst, src[0], src[1]);
1810 break;
1811
1812 case BRW_OPCODE_MAD:
1813 assert(devinfo->gen >= 6);
1814 if (devinfo->gen < 10)
1815 brw_set_default_access_mode(p, BRW_ALIGN_16);
1816 brw_MAD(p, dst, src[0], src[1], src[2]);
1817 break;
1818
1819 case BRW_OPCODE_LRP:
1820 assert(devinfo->gen >= 6);
1821 if (devinfo->gen < 10)
1822 brw_set_default_access_mode(p, BRW_ALIGN_16);
1823 brw_LRP(p, dst, src[0], src[1], src[2]);
1824 break;
1825
1826 case BRW_OPCODE_FRC:
1827 brw_FRC(p, dst, src[0]);
1828 break;
1829 case BRW_OPCODE_RNDD:
1830 brw_RNDD(p, dst, src[0]);
1831 break;
1832 case BRW_OPCODE_RNDE:
1833 brw_RNDE(p, dst, src[0]);
1834 break;
1835 case BRW_OPCODE_RNDZ:
1836 brw_RNDZ(p, dst, src[0]);
1837 break;
1838
1839 case BRW_OPCODE_AND:
1840 brw_AND(p, dst, src[0], src[1]);
1841 break;
1842 case BRW_OPCODE_OR:
1843 brw_OR(p, dst, src[0], src[1]);
1844 break;
1845 case BRW_OPCODE_XOR:
1846 brw_XOR(p, dst, src[0], src[1]);
1847 break;
1848 case BRW_OPCODE_NOT:
1849 brw_NOT(p, dst, src[0]);
1850 break;
1851 case BRW_OPCODE_ASR:
1852 brw_ASR(p, dst, src[0], src[1]);
1853 break;
1854 case BRW_OPCODE_SHR:
1855 brw_SHR(p, dst, src[0], src[1]);
1856 break;
1857 case BRW_OPCODE_SHL:
1858 brw_SHL(p, dst, src[0], src[1]);
1859 break;
1860 case BRW_OPCODE_F32TO16:
1861 assert(devinfo->gen >= 7);
1862 brw_F32TO16(p, dst, src[0]);
1863 break;
1864 case BRW_OPCODE_F16TO32:
1865 assert(devinfo->gen >= 7);
1866 brw_F16TO32(p, dst, src[0]);
1867 break;
1868 case BRW_OPCODE_CMP:
1869 if (inst->exec_size >= 16 && devinfo->gen == 7 && !devinfo->is_haswell &&
1870 dst.file == BRW_ARCHITECTURE_REGISTER_FILE) {
1871 /* For unknown reasons the WaCMPInstFlagDepClearedEarly workaround
1872 * implemented in the compiler is not sufficient. Overriding the
1873 * type when the destination is the null register is necessary but
1874 * not sufficient by itself.
1875 */
1876 assert(dst.nr == BRW_ARF_NULL);
1877 dst.type = BRW_REGISTER_TYPE_D;
1878 }
1879 brw_CMP(p, dst, inst->conditional_mod, src[0], src[1]);
1880 break;
1881 case BRW_OPCODE_SEL:
1882 brw_SEL(p, dst, src[0], src[1]);
1883 break;
1884 case BRW_OPCODE_BFREV:
1885 assert(devinfo->gen >= 7);
1886 brw_BFREV(p, retype(dst, BRW_REGISTER_TYPE_UD),
1887 retype(src[0], BRW_REGISTER_TYPE_UD));
1888 break;
1889 case BRW_OPCODE_FBH:
1890 assert(devinfo->gen >= 7);
1891 brw_FBH(p, retype(dst, src[0].type), src[0]);
1892 break;
1893 case BRW_OPCODE_FBL:
1894 assert(devinfo->gen >= 7);
1895 brw_FBL(p, retype(dst, BRW_REGISTER_TYPE_UD),
1896 retype(src[0], BRW_REGISTER_TYPE_UD));
1897 break;
1898 case BRW_OPCODE_LZD:
1899 brw_LZD(p, dst, src[0]);
1900 break;
1901 case BRW_OPCODE_CBIT:
1902 assert(devinfo->gen >= 7);
1903 brw_CBIT(p, retype(dst, BRW_REGISTER_TYPE_UD),
1904 retype(src[0], BRW_REGISTER_TYPE_UD));
1905 break;
1906 case BRW_OPCODE_ADDC:
1907 assert(devinfo->gen >= 7);
1908 brw_ADDC(p, dst, src[0], src[1]);
1909 break;
1910 case BRW_OPCODE_SUBB:
1911 assert(devinfo->gen >= 7);
1912 brw_SUBB(p, dst, src[0], src[1]);
1913 break;
1914 case BRW_OPCODE_MAC:
1915 brw_MAC(p, dst, src[0], src[1]);
1916 break;
1917
1918 case BRW_OPCODE_BFE:
1919 assert(devinfo->gen >= 7);
1920 if (devinfo->gen < 10)
1921 brw_set_default_access_mode(p, BRW_ALIGN_16);
1922 brw_BFE(p, dst, src[0], src[1], src[2]);
1923 break;
1924
1925 case BRW_OPCODE_BFI1:
1926 assert(devinfo->gen >= 7);
1927 brw_BFI1(p, dst, src[0], src[1]);
1928 break;
1929 case BRW_OPCODE_BFI2:
1930 assert(devinfo->gen >= 7);
1931 if (devinfo->gen < 10)
1932 brw_set_default_access_mode(p, BRW_ALIGN_16);
1933 brw_BFI2(p, dst, src[0], src[1], src[2]);
1934 break;
1935
1936 case BRW_OPCODE_IF:
1937 if (inst->src[0].file != BAD_FILE) {
1938 /* The instruction has an embedded compare (only allowed on gen6) */
1939 assert(devinfo->gen == 6);
1940 gen6_IF(p, inst->conditional_mod, src[0], src[1]);
1941 } else {
1942 brw_IF(p, brw_inst_exec_size(devinfo, p->current));
1943 }
1944 break;
1945
1946 case BRW_OPCODE_ELSE:
1947 brw_ELSE(p);
1948 break;
1949 case BRW_OPCODE_ENDIF:
1950 brw_ENDIF(p);
1951 break;
1952
1953 case BRW_OPCODE_DO:
1954 brw_DO(p, brw_inst_exec_size(devinfo, p->current));
1955 break;
1956
1957 case BRW_OPCODE_BREAK:
1958 brw_BREAK(p);
1959 break;
1960 case BRW_OPCODE_CONTINUE:
1961 brw_CONT(p);
1962 break;
1963
1964 case BRW_OPCODE_WHILE:
1965 brw_WHILE(p);
1966 loop_count++;
1967 break;
1968
1969 case SHADER_OPCODE_RCP:
1970 case SHADER_OPCODE_RSQ:
1971 case SHADER_OPCODE_SQRT:
1972 case SHADER_OPCODE_EXP2:
1973 case SHADER_OPCODE_LOG2:
1974 case SHADER_OPCODE_SIN:
1975 case SHADER_OPCODE_COS:
1976 assert(inst->conditional_mod == BRW_CONDITIONAL_NONE);
1977 if (devinfo->gen >= 6) {
1978 assert(inst->mlen == 0);
1979 assert(devinfo->gen >= 7 || inst->exec_size == 8);
1980 gen6_math(p, dst, brw_math_function(inst->opcode),
1981 src[0], brw_null_reg());
1982 } else {
1983 assert(inst->mlen >= 1);
1984 assert(devinfo->gen == 5 || devinfo->is_g4x || inst->exec_size == 8);
1985 gen4_math(p, dst,
1986 brw_math_function(inst->opcode),
1987 inst->base_mrf, src[0],
1988 BRW_MATH_PRECISION_FULL);
1989 }
1990 break;
1991 case SHADER_OPCODE_INT_QUOTIENT:
1992 case SHADER_OPCODE_INT_REMAINDER:
1993 case SHADER_OPCODE_POW:
1994 assert(inst->conditional_mod == BRW_CONDITIONAL_NONE);
1995 if (devinfo->gen >= 6) {
1996 assert(inst->mlen == 0);
1997 assert((devinfo->gen >= 7 && inst->opcode == SHADER_OPCODE_POW) ||
1998 inst->exec_size == 8);
1999 gen6_math(p, dst, brw_math_function(inst->opcode), src[0], src[1]);
2000 } else {
2001 assert(inst->mlen >= 1);
2002 assert(inst->exec_size == 8);
2003 gen4_math(p, dst, brw_math_function(inst->opcode),
2004 inst->base_mrf, src[0],
2005 BRW_MATH_PRECISION_FULL);
2006 }
2007 break;
2008 case FS_OPCODE_CINTERP:
2009 brw_MOV(p, dst, src[0]);
2010 break;
2011 case FS_OPCODE_LINTERP:
2012 multiple_instructions_emitted = generate_linterp(inst, dst, src);
2013 break;
2014 case FS_OPCODE_PIXEL_X:
2015 assert(src[0].type == BRW_REGISTER_TYPE_UW);
2016 src[0].subnr = 0 * type_sz(src[0].type);
2017 brw_MOV(p, dst, stride(src[0], 8, 4, 1));
2018 break;
2019 case FS_OPCODE_PIXEL_Y:
2020 assert(src[0].type == BRW_REGISTER_TYPE_UW);
2021 src[0].subnr = 4 * type_sz(src[0].type);
2022 brw_MOV(p, dst, stride(src[0], 8, 4, 1));
2023 break;
2024 case SHADER_OPCODE_GET_BUFFER_SIZE:
2025 generate_get_buffer_size(inst, dst, src[0], src[1]);
2026 break;
2027 case SHADER_OPCODE_TEX:
2028 case FS_OPCODE_TXB:
2029 case SHADER_OPCODE_TXD:
2030 case SHADER_OPCODE_TXF:
2031 case SHADER_OPCODE_TXF_LZ:
2032 case SHADER_OPCODE_TXF_CMS:
2033 case SHADER_OPCODE_TXF_CMS_W:
2034 case SHADER_OPCODE_TXF_UMS:
2035 case SHADER_OPCODE_TXF_MCS:
2036 case SHADER_OPCODE_TXL:
2037 case SHADER_OPCODE_TXL_LZ:
2038 case SHADER_OPCODE_TXS:
2039 case SHADER_OPCODE_LOD:
2040 case SHADER_OPCODE_TG4:
2041 case SHADER_OPCODE_TG4_OFFSET:
2042 case SHADER_OPCODE_SAMPLEINFO:
2043 generate_tex(inst, dst, src[0], src[1], src[2]);
2044 break;
2045 case FS_OPCODE_DDX_COARSE:
2046 case FS_OPCODE_DDX_FINE:
2047 generate_ddx(inst, dst, src[0]);
2048 break;
2049 case FS_OPCODE_DDY_COARSE:
2050 case FS_OPCODE_DDY_FINE:
2051 generate_ddy(inst, dst, src[0]);
2052 break;
2053
2054 case SHADER_OPCODE_GEN4_SCRATCH_WRITE:
2055 generate_scratch_write(inst, src[0]);
2056 spill_count++;
2057 break;
2058
2059 case SHADER_OPCODE_GEN4_SCRATCH_READ:
2060 generate_scratch_read(inst, dst);
2061 fill_count++;
2062 break;
2063
2064 case SHADER_OPCODE_GEN7_SCRATCH_READ:
2065 generate_scratch_read_gen7(inst, dst);
2066 fill_count++;
2067 break;
2068
2069 case SHADER_OPCODE_MOV_INDIRECT:
2070 generate_mov_indirect(inst, dst, src[0], src[1]);
2071 break;
2072
2073 case SHADER_OPCODE_URB_READ_SIMD8:
2074 case SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT:
2075 generate_urb_read(inst, dst, src[0]);
2076 break;
2077
2078 case SHADER_OPCODE_URB_WRITE_SIMD8:
2079 case SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT:
2080 case SHADER_OPCODE_URB_WRITE_SIMD8_MASKED:
2081 case SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT:
2082 generate_urb_write(inst, src[0]);
2083 break;
2084
2085 case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD:
2086 assert(inst->force_writemask_all);
2087 generate_uniform_pull_constant_load(inst, dst, src[0], src[1]);
2088 break;
2089
2090 case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD_GEN7:
2091 assert(inst->force_writemask_all);
2092 generate_uniform_pull_constant_load_gen7(inst, dst, src[0], src[1]);
2093 break;
2094
2095 case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_GEN4:
2096 generate_varying_pull_constant_load_gen4(inst, dst, src[0]);
2097 break;
2098
2099 case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_GEN7:
2100 generate_varying_pull_constant_load_gen7(inst, dst, src[0], src[1]);
2101 break;
2102
2103 case FS_OPCODE_REP_FB_WRITE:
2104 case FS_OPCODE_FB_WRITE:
2105 generate_fb_write(inst, src[0]);
2106 break;
2107
2108 case FS_OPCODE_FB_READ:
2109 generate_fb_read(inst, dst, src[0]);
2110 break;
2111
2112 case FS_OPCODE_MOV_DISPATCH_TO_FLAGS:
2113 generate_mov_dispatch_to_flags(inst);
2114 break;
2115
2116 case FS_OPCODE_DISCARD_JUMP:
2117 generate_discard_jump(inst);
2118 break;
2119
2120 case SHADER_OPCODE_SHADER_TIME_ADD:
2121 generate_shader_time_add(inst, src[0], src[1], src[2]);
2122 break;
2123
2124 case SHADER_OPCODE_UNTYPED_ATOMIC:
2125 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2126 brw_untyped_atomic(p, dst, src[0], src[1], src[2].ud,
2127 inst->mlen, !inst->dst.is_null());
2128 break;
2129
2130 case SHADER_OPCODE_UNTYPED_SURFACE_READ:
2131 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2132 brw_untyped_surface_read(p, dst, src[0], src[1],
2133 inst->mlen, src[2].ud);
2134 break;
2135
2136 case SHADER_OPCODE_UNTYPED_SURFACE_WRITE:
2137 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2138 brw_untyped_surface_write(p, src[0], src[1],
2139 inst->mlen, src[2].ud);
2140 break;
2141
2142 case SHADER_OPCODE_BYTE_SCATTERED_READ:
2143 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2144 brw_byte_scattered_read(p, dst, src[0], src[1],
2145 inst->mlen, src[2].ud);
2146 break;
2147
2148 case SHADER_OPCODE_BYTE_SCATTERED_WRITE:
2149 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2150 brw_byte_scattered_write(p, src[0], src[1],
2151 inst->mlen, src[2].ud);
2152 break;
2153
2154 case SHADER_OPCODE_TYPED_ATOMIC:
2155 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2156 brw_typed_atomic(p, dst, src[0], src[1],
2157 src[2].ud, inst->mlen, !inst->dst.is_null());
2158 break;
2159
2160 case SHADER_OPCODE_TYPED_SURFACE_READ:
2161 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2162 brw_typed_surface_read(p, dst, src[0], src[1],
2163 inst->mlen, src[2].ud);
2164 break;
2165
2166 case SHADER_OPCODE_TYPED_SURFACE_WRITE:
2167 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2168 brw_typed_surface_write(p, src[0], src[1], inst->mlen, src[2].ud);
2169 break;
2170
2171 case SHADER_OPCODE_MEMORY_FENCE:
2172 brw_memory_fence(p, dst);
2173 break;
2174
2175 case SHADER_OPCODE_FIND_LIVE_CHANNEL: {
2176 const struct brw_reg mask =
2177 brw_stage_has_packed_dispatch(devinfo, stage,
2178 prog_data) ? brw_imm_ud(~0u) :
2179 stage == MESA_SHADER_FRAGMENT ? brw_vmask_reg() :
2180 brw_dmask_reg();
2181 brw_find_live_channel(p, dst, mask);
2182 break;
2183 }
2184
2185 case SHADER_OPCODE_BROADCAST:
2186 assert(inst->force_writemask_all);
2187 brw_broadcast(p, dst, src[0], src[1]);
2188 break;
2189
2190 case FS_OPCODE_SET_SAMPLE_ID:
2191 generate_set_sample_id(inst, dst, src[0], src[1]);
2192 break;
2193
2194 case FS_OPCODE_PACK_HALF_2x16_SPLIT:
2195 generate_pack_half_2x16_split(inst, dst, src[0], src[1]);
2196 break;
2197
2198 case FS_OPCODE_UNPACK_HALF_2x16_SPLIT_X:
2199 case FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y:
2200 generate_unpack_half_2x16_split(inst, dst, src[0]);
2201 break;
2202
2203 case FS_OPCODE_PLACEHOLDER_HALT:
2204 /* This is the place where the final HALT needs to be inserted if
2205 * we've emitted any discards. If not, this will emit no code.
2206 */
2207 if (!patch_discard_jumps_to_fb_writes()) {
2208 if (unlikely(debug_flag)) {
2209 disasm_info->use_tail = true;
2210 }
2211 }
2212 break;
2213
2214 case FS_OPCODE_INTERPOLATE_AT_SAMPLE:
2215 generate_pixel_interpolator_query(inst, dst, src[0], src[1],
2216 GEN7_PIXEL_INTERPOLATOR_LOC_SAMPLE);
2217 break;
2218
2219 case FS_OPCODE_INTERPOLATE_AT_SHARED_OFFSET:
2220 generate_pixel_interpolator_query(inst, dst, src[0], src[1],
2221 GEN7_PIXEL_INTERPOLATOR_LOC_SHARED_OFFSET);
2222 break;
2223
2224 case FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET:
2225 generate_pixel_interpolator_query(inst, dst, src[0], src[1],
2226 GEN7_PIXEL_INTERPOLATOR_LOC_PER_SLOT_OFFSET);
2227 break;
2228
2229 case CS_OPCODE_CS_TERMINATE:
2230 generate_cs_terminate(inst, src[0]);
2231 break;
2232
2233 case SHADER_OPCODE_BARRIER:
2234 generate_barrier(inst, src[0]);
2235 break;
2236
2237 case BRW_OPCODE_DIM:
2238 assert(devinfo->is_haswell);
2239 assert(src[0].type == BRW_REGISTER_TYPE_DF);
2240 assert(dst.type == BRW_REGISTER_TYPE_DF);
2241 brw_DIM(p, dst, retype(src[0], BRW_REGISTER_TYPE_F));
2242 break;
2243
2244 case SHADER_OPCODE_RND_MODE:
2245 assert(src[0].file == BRW_IMMEDIATE_VALUE);
2246 brw_rounding_mode(p, (brw_rnd_mode) src[0].d);
2247 break;
2248
2249 default:
2250 unreachable("Unsupported opcode");
2251
2252 case SHADER_OPCODE_LOAD_PAYLOAD:
2253 unreachable("Should be lowered by lower_load_payload()");
2254 }
2255
2256 if (multiple_instructions_emitted)
2257 continue;
2258
2259 if (inst->no_dd_clear || inst->no_dd_check || inst->conditional_mod) {
2260 assert(p->next_insn_offset == last_insn_offset + 16 ||
2261 !"conditional_mod, no_dd_check, or no_dd_clear set for IR "
2262 "emitting more than 1 instruction");
2263
2264 brw_inst *last = &p->store[last_insn_offset / 16];
2265
2266 if (inst->conditional_mod)
2267 brw_inst_set_cond_modifier(p->devinfo, last, inst->conditional_mod);
2268 brw_inst_set_no_dd_clear(p->devinfo, last, inst->no_dd_clear);
2269 brw_inst_set_no_dd_check(p->devinfo, last, inst->no_dd_check);
2270 }
2271 }
2272
2273 brw_set_uip_jip(p, start_offset);
2274
2275 /* end of program sentinel */
2276 disasm_new_inst_group(disasm_info, p->next_insn_offset);
2277
2278 #ifndef NDEBUG
2279 bool validated =
2280 #else
2281 if (unlikely(debug_flag))
2282 #endif
2283 brw_validate_instructions(devinfo, p->store,
2284 start_offset,
2285 p->next_insn_offset,
2286 disasm_info);
2287
2288 int before_size = p->next_insn_offset - start_offset;
2289 brw_compact_instructions(p, start_offset, disasm_info);
2290 int after_size = p->next_insn_offset - start_offset;
2291
2292 if (unlikely(debug_flag)) {
2293 fprintf(stderr, "Native code for %s\n"
2294 "SIMD%d shader: %d instructions. %d loops. %u cycles. %d:%d spills:fills. Promoted %u constants. Compacted %d to %d"
2295 " bytes (%.0f%%)\n",
2296 shader_name, dispatch_width, before_size / 16, loop_count, cfg->cycle_count,
2297 spill_count, fill_count, promoted_constants, before_size, after_size,
2298 100.0f * (before_size - after_size) / before_size);
2299
2300 dump_assembly(p->store, disasm_info);
2301 }
2302 ralloc_free(disasm_info);
2303 assert(validated);
2304
2305 compiler->shader_debug_log(log_data,
2306 "%s SIMD%d shader: %d inst, %d loops, %u cycles, "
2307 "%d:%d spills:fills, Promoted %u constants, "
2308 "compacted %d to %d bytes.",
2309 _mesa_shader_stage_to_abbrev(stage),
2310 dispatch_width, before_size / 16,
2311 loop_count, cfg->cycle_count, spill_count,
2312 fill_count, promoted_constants, before_size,
2313 after_size);
2314
2315 return start_offset;
2316 }
2317
2318 const unsigned *
2319 fs_generator::get_assembly(unsigned int *assembly_size)
2320 {
2321 return brw_get_program(p, assembly_size);
2322 }