projects / mesa.git / blob
? search:


aeed6a119776751f98ca3897518b86642d275090
[mesa.git] / src / mesa / drivers / dri / i965 / brw_fs_generator.cpp
1 /*
2 * Copyright © 2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 /** @file brw_fs_generator.cpp
25 *
26 * This file supports generating code from the FS LIR to the actual
27 * native instructions.
28 */
29
30 #include "brw_eu.h"
31 #include "brw_fs.h"
32 #include "brw_cfg.h"
33
34 static enum brw_reg_file
35 brw_file_from_reg(fs_reg *reg)
36 {
37 switch (reg->file) {
38 case ARF:
39 return BRW_ARCHITECTURE_REGISTER_FILE;
40 case FIXED_GRF:
41 case VGRF:
42 return BRW_GENERAL_REGISTER_FILE;
43 case MRF:
44 return BRW_MESSAGE_REGISTER_FILE;
45 case IMM:
46 return BRW_IMMEDIATE_VALUE;
47 case BAD_FILE:
48 case ATTR:
49 case UNIFORM:
50 unreachable("not reached");
51 }
52 return BRW_ARCHITECTURE_REGISTER_FILE;
53 }
54
55 static struct brw_reg
56 brw_reg_from_fs_reg(fs_inst *inst, fs_reg *reg, unsigned gen, bool compressed)
57 {
58 struct brw_reg brw_reg;
59
60 switch (reg->file) {
61 case MRF:
62 assert((reg->nr & ~BRW_MRF_COMPR4) < BRW_MAX_MRF(gen));
63 /* Fallthrough */
64 case VGRF:
65 if (reg->stride == 0) {
66 brw_reg = brw_vec1_reg(brw_file_from_reg(reg), reg->nr, 0);
67 } else {
68 /* From the Haswell PRM:
69 *
70 * "VertStride must be used to cross GRF register boundaries. This
71 * rule implies that elements within a 'Width' cannot cross GRF
72 * boundaries."
73 *
74 * The maximum width value that could satisfy this restriction is:
75 */
76 const unsigned reg_width = REG_SIZE / (reg->stride * type_sz(reg->type));
77
78 /* Because the hardware can only split source regions at a whole
79 * multiple of width during decompression (i.e. vertically), clamp
80 * the value obtained above to the physical execution size of a
81 * single decompressed chunk of the instruction:
82 */
83 const unsigned phys_width = compressed ? inst->exec_size / 2 :
84 inst->exec_size;
85
86 /* XXX - The equation above is strictly speaking not correct on
87 * hardware that supports unbalanced GRF writes -- On Gen9+
88 * each decompressed chunk of the instruction may have a
89 * different execution size when the number of components
90 * written to each destination GRF is not the same.
91 */
92 const unsigned width = MIN2(reg_width, phys_width);
93 brw_reg = brw_vecn_reg(width, brw_file_from_reg(reg), reg->nr, 0);
94 brw_reg = stride(brw_reg, width * reg->stride, width, reg->stride);
95 }
96
97 brw_reg = retype(brw_reg, reg->type);
98 brw_reg = byte_offset(brw_reg, reg->offset);
99 brw_reg.abs = reg->abs;
100 brw_reg.negate = reg->negate;
101 break;
102 case ARF:
103 case FIXED_GRF:
104 case IMM:
105 assert(reg->offset == 0);
106 brw_reg = reg->as_brw_reg();
107 break;
108 case BAD_FILE:
109 /* Probably unused. */
110 brw_reg = brw_null_reg();
111 break;
112 case ATTR:
113 case UNIFORM:
114 unreachable("not reached");
115 }
116
117 return brw_reg;
118 }
119
120 fs_generator::fs_generator(const struct brw_compiler *compiler, void *log_data,
121 void *mem_ctx,
122 const void *key,
123 struct brw_stage_prog_data *prog_data,
124 unsigned promoted_constants,
125 bool runtime_check_aads_emit,
126 gl_shader_stage stage)
127
128 : compiler(compiler), log_data(log_data),
129 devinfo(compiler->devinfo), key(key),
130 prog_data(prog_data),
131 promoted_constants(promoted_constants),
132 runtime_check_aads_emit(runtime_check_aads_emit), debug_flag(false),
133 stage(stage), mem_ctx(mem_ctx)
134 {
135 p = rzalloc(mem_ctx, struct brw_codegen);
136 brw_init_codegen(devinfo, p, mem_ctx);
137 }
138
139 fs_generator::~fs_generator()
140 {
141 }
142
143 class ip_record : public exec_node {
144 public:
145 DECLARE_RALLOC_CXX_OPERATORS(ip_record)
146
147 ip_record(int ip)
148 {
149 this->ip = ip;
150 }
151
152 int ip;
153 };
154
155 bool
156 fs_generator::patch_discard_jumps_to_fb_writes()
157 {
158 if (devinfo->gen < 6 || this->discard_halt_patches.is_empty())
159 return false;
160
161 int scale = brw_jump_scale(p->devinfo);
162
163 /* There is a somewhat strange undocumented requirement of using
164 * HALT, according to the simulator. If some channel has HALTed to
165 * a particular UIP, then by the end of the program, every channel
166 * must have HALTed to that UIP. Furthermore, the tracking is a
167 * stack, so you can't do the final halt of a UIP after starting
168 * halting to a new UIP.
169 *
170 * Symptoms of not emitting this instruction on actual hardware
171 * included GPU hangs and sparkly rendering on the piglit discard
172 * tests.
173 */
174 brw_inst *last_halt = gen6_HALT(p);
175 brw_inst_set_uip(p->devinfo, last_halt, 1 * scale);
176 brw_inst_set_jip(p->devinfo, last_halt, 1 * scale);
177
178 int ip = p->nr_insn;
179
180 foreach_in_list(ip_record, patch_ip, &discard_halt_patches) {
181 brw_inst *patch = &p->store[patch_ip->ip];
182
183 assert(brw_inst_opcode(p->devinfo, patch) == BRW_OPCODE_HALT);
184 /* HALT takes a half-instruction distance from the pre-incremented IP. */
185 brw_inst_set_uip(p->devinfo, patch, (ip - patch_ip->ip) * scale);
186 }
187
188 this->discard_halt_patches.make_empty();
189 return true;
190 }
191
192 void
193 fs_generator::fire_fb_write(fs_inst *inst,
194 struct brw_reg payload,
195 struct brw_reg implied_header,
196 GLuint nr)
197 {
198 uint32_t msg_control;
199
200 struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
201
202 if (devinfo->gen < 6) {
203 brw_push_insn_state(p);
204 brw_set_default_exec_size(p, BRW_EXECUTE_8);
205 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
206 brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
207 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
208 brw_MOV(p, offset(payload, 1), brw_vec8_grf(1, 0));
209 brw_pop_insn_state(p);
210 }
211
212 if (inst->opcode == FS_OPCODE_REP_FB_WRITE)
213 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD16_SINGLE_SOURCE_REPLICATED;
214 else if (prog_data->dual_src_blend) {
215 if (!inst->group)
216 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_DUAL_SOURCE_SUBSPAN01;
217 else
218 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_DUAL_SOURCE_SUBSPAN23;
219 } else if (inst->exec_size == 16)
220 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD16_SINGLE_SOURCE;
221 else
222 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_SINGLE_SOURCE_SUBSPAN01;
223
224 uint32_t surf_index =
225 prog_data->binding_table.render_target_start + inst->target;
226
227 bool last_render_target = inst->eot ||
228 (prog_data->dual_src_blend && dispatch_width == 16);
229
230
231 brw_fb_WRITE(p,
232 payload,
233 implied_header,
234 msg_control,
235 surf_index,
236 nr,
237 0,
238 inst->eot,
239 last_render_target,
240 inst->header_size != 0);
241
242 brw_mark_surface_used(&prog_data->base, surf_index);
243 }
244
245 void
246 fs_generator::generate_fb_write(fs_inst *inst, struct brw_reg payload)
247 {
248 struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
249 const brw_wm_prog_key * const key = (brw_wm_prog_key * const) this->key;
250 struct brw_reg implied_header;
251
252 if (devinfo->gen < 8 && !devinfo->is_haswell) {
253 brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
254 }
255
256 if (inst->base_mrf >= 0)
257 payload = brw_message_reg(inst->base_mrf);
258
259 /* Header is 2 regs, g0 and g1 are the contents. g0 will be implied
260 * move, here's g1.
261 */
262 if (inst->header_size != 0) {
263 brw_push_insn_state(p);
264 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
265 brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
266 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
267 brw_set_default_flag_reg(p, 0, 0);
268
269 /* On HSW, the GPU will use the predicate on SENDC, unless the header is
270 * present.
271 */
272 if (prog_data->uses_kill) {
273 struct brw_reg pixel_mask;
274
275 if (devinfo->gen >= 6)
276 pixel_mask = retype(brw_vec1_grf(1, 7), BRW_REGISTER_TYPE_UW);
277 else
278 pixel_mask = retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UW);
279
280 brw_MOV(p, pixel_mask, brw_flag_reg(0, 1));
281 }
282
283 if (devinfo->gen >= 6) {
284 brw_push_insn_state(p);
285 brw_set_default_exec_size(p, BRW_EXECUTE_16);
286 brw_set_default_compression_control(p, BRW_COMPRESSION_COMPRESSED);
287 brw_MOV(p,
288 retype(payload, BRW_REGISTER_TYPE_UD),
289 retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));
290 brw_pop_insn_state(p);
291
292 if (inst->target > 0 && key->replicate_alpha) {
293 /* Set "Source0 Alpha Present to RenderTarget" bit in message
294 * header.
295 */
296 brw_OR(p,
297 vec1(retype(payload, BRW_REGISTER_TYPE_UD)),
298 vec1(retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD)),
299 brw_imm_ud(0x1 << 11));
300 }
301
302 if (inst->target > 0) {
303 /* Set the render target index for choosing BLEND_STATE. */
304 brw_MOV(p, retype(vec1(suboffset(payload, 2)),
305 BRW_REGISTER_TYPE_UD),
306 brw_imm_ud(inst->target));
307 }
308
309 /* Set computes stencil to render target */
310 if (prog_data->computed_stencil) {
311 brw_OR(p,
312 vec1(retype(payload, BRW_REGISTER_TYPE_UD)),
313 vec1(retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD)),
314 brw_imm_ud(0x1 << 14));
315 }
316
317 implied_header = brw_null_reg();
318 } else {
319 implied_header = retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UW);
320 }
321
322 brw_pop_insn_state(p);
323 } else {
324 implied_header = brw_null_reg();
325 }
326
327 if (!runtime_check_aads_emit) {
328 fire_fb_write(inst, payload, implied_header, inst->mlen);
329 } else {
330 /* This can only happen in gen < 6 */
331 assert(devinfo->gen < 6);
332
333 struct brw_reg v1_null_ud = vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_UD));
334
335 /* Check runtime bit to detect if we have to send AA data or not */
336 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
337 brw_AND(p,
338 v1_null_ud,
339 retype(brw_vec1_grf(1, 6), BRW_REGISTER_TYPE_UD),
340 brw_imm_ud(1<<26));
341 brw_inst_set_cond_modifier(p->devinfo, brw_last_inst, BRW_CONDITIONAL_NZ);
342
343 int jmp = brw_JMPI(p, brw_imm_ud(0), BRW_PREDICATE_NORMAL) - p->store;
344 brw_inst_set_exec_size(p->devinfo, brw_last_inst, BRW_EXECUTE_1);
345 {
346 /* Don't send AA data */
347 fire_fb_write(inst, offset(payload, 1), implied_header, inst->mlen-1);
348 }
349 brw_land_fwd_jump(p, jmp);
350 fire_fb_write(inst, payload, implied_header, inst->mlen);
351 }
352 }
353
354 void
355 fs_generator::generate_fb_read(fs_inst *inst, struct brw_reg dst,
356 struct brw_reg payload)
357 {
358 assert(inst->size_written % REG_SIZE == 0);
359 struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
360 const unsigned surf_index =
361 prog_data->binding_table.render_target_start + inst->target;
362
363 gen9_fb_READ(p, dst, payload, surf_index,
364 inst->header_size, inst->size_written / REG_SIZE,
365 prog_data->persample_dispatch);
366
367 brw_mark_surface_used(&prog_data->base, surf_index);
368 }
369
370 void
371 fs_generator::generate_mov_indirect(fs_inst *inst,
372 struct brw_reg dst,
373 struct brw_reg reg,
374 struct brw_reg indirect_byte_offset)
375 {
376 assert(indirect_byte_offset.type == BRW_REGISTER_TYPE_UD);
377 assert(indirect_byte_offset.file == BRW_GENERAL_REGISTER_FILE);
378
379 unsigned imm_byte_offset = reg.nr * REG_SIZE + reg.subnr;
380
381 if (indirect_byte_offset.file == BRW_IMMEDIATE_VALUE) {
382 imm_byte_offset += indirect_byte_offset.ud;
383
384 reg.nr = imm_byte_offset / REG_SIZE;
385 reg.subnr = imm_byte_offset % REG_SIZE;
386 brw_MOV(p, dst, reg);
387 } else {
388 /* Prior to Broadwell, there are only 8 address registers. */
389 assert(inst->exec_size == 8 || devinfo->gen >= 8);
390
391 /* We use VxH indirect addressing, clobbering a0.0 through a0.7. */
392 struct brw_reg addr = vec8(brw_address_reg(0));
393
394 /* The destination stride of an instruction (in bytes) must be greater
395 * than or equal to the size of the rest of the instruction. Since the
396 * address register is of type UW, we can't use a D-type instruction.
397 * In order to get around this, re retype to UW and use a stride.
398 */
399 indirect_byte_offset =
400 retype(spread(indirect_byte_offset, 2), BRW_REGISTER_TYPE_UW);
401
402 /* There are a number of reasons why we don't use the base offset here.
403 * One reason is that the field is only 9 bits which means we can only
404 * use it to access the first 16 GRFs. Also, from the Haswell PRM
405 * section "Register Region Restrictions":
406 *
407 * "The lower bits of the AddressImmediate must not overflow to
408 * change the register address. The lower 5 bits of Address
409 * Immediate when added to lower 5 bits of address register gives
410 * the sub-register offset. The upper bits of Address Immediate
411 * when added to upper bits of address register gives the register
412 * address. Any overflow from sub-register offset is dropped."
413 *
414 * Since the indirect may cause us to cross a register boundary, this
415 * makes the base offset almost useless. We could try and do something
416 * clever where we use a actual base offset if base_offset % 32 == 0 but
417 * that would mean we were generating different code depending on the
418 * base offset. Instead, for the sake of consistency, we'll just do the
419 * add ourselves. This restriction is only listed in the Haswell PRM
420 * but empirical testing indicates that it applies on all older
421 * generations and is lifted on Broadwell.
422 *
423 * In the end, while base_offset is nice to look at in the generated
424 * code, using it saves us 0 instructions and would require quite a bit
425 * of case-by-case work. It's just not worth it.
426 */
427 brw_ADD(p, addr, indirect_byte_offset, brw_imm_uw(imm_byte_offset));
428 struct brw_reg ind_src = brw_VxH_indirect(0, 0);
429
430 brw_inst *mov = brw_MOV(p, dst, retype(ind_src, reg.type));
431
432 if (devinfo->gen == 6 && dst.file == BRW_MESSAGE_REGISTER_FILE &&
433 !inst->get_next()->is_tail_sentinel() &&
434 ((fs_inst *)inst->get_next())->mlen > 0) {
435 /* From the Sandybridge PRM:
436 *
437 * "[Errata: DevSNB(SNB)] If MRF register is updated by any
438 * instruction that “indexed/indirect” source AND is followed by a
439 * send, the instruction requires a “Switch”. This is to avoid
440 * race condition where send may dispatch before MRF is updated."
441 */
442 brw_inst_set_thread_control(devinfo, mov, BRW_THREAD_SWITCH);
443 }
444 }
445 }
446
447 void
448 fs_generator::generate_urb_read(fs_inst *inst,
449 struct brw_reg dst,
450 struct brw_reg header)
451 {
452 assert(inst->size_written % REG_SIZE == 0);
453 assert(header.file == BRW_GENERAL_REGISTER_FILE);
454 assert(header.type == BRW_REGISTER_TYPE_UD);
455
456 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
457 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UD));
458 brw_set_src0(p, send, header);
459 brw_set_src1(p, send, brw_imm_ud(0u));
460
461 brw_inst_set_sfid(p->devinfo, send, BRW_SFID_URB);
462 brw_inst_set_urb_opcode(p->devinfo, send, GEN8_URB_OPCODE_SIMD8_READ);
463
464 if (inst->opcode == SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT)
465 brw_inst_set_urb_per_slot_offset(p->devinfo, send, true);
466
467 brw_inst_set_mlen(p->devinfo, send, inst->mlen);
468 brw_inst_set_rlen(p->devinfo, send, inst->size_written / REG_SIZE);
469 brw_inst_set_header_present(p->devinfo, send, true);
470 brw_inst_set_urb_global_offset(p->devinfo, send, inst->offset);
471 }
472
473 void
474 fs_generator::generate_urb_write(fs_inst *inst, struct brw_reg payload)
475 {
476 brw_inst *insn;
477
478 insn = brw_next_insn(p, BRW_OPCODE_SEND);
479
480 brw_set_dest(p, insn, brw_null_reg());
481 brw_set_src0(p, insn, payload);
482 brw_set_src1(p, insn, brw_imm_d(0));
483
484 brw_inst_set_sfid(p->devinfo, insn, BRW_SFID_URB);
485 brw_inst_set_urb_opcode(p->devinfo, insn, GEN8_URB_OPCODE_SIMD8_WRITE);
486
487 if (inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT ||
488 inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT)
489 brw_inst_set_urb_per_slot_offset(p->devinfo, insn, true);
490
491 if (inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_MASKED ||
492 inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT)
493 brw_inst_set_urb_channel_mask_present(p->devinfo, insn, true);
494
495 brw_inst_set_mlen(p->devinfo, insn, inst->mlen);
496 brw_inst_set_rlen(p->devinfo, insn, 0);
497 brw_inst_set_eot(p->devinfo, insn, inst->eot);
498 brw_inst_set_header_present(p->devinfo, insn, true);
499 brw_inst_set_urb_global_offset(p->devinfo, insn, inst->offset);
500 }
501
502 void
503 fs_generator::generate_cs_terminate(fs_inst *inst, struct brw_reg payload)
504 {
505 struct brw_inst *insn;
506
507 insn = brw_next_insn(p, BRW_OPCODE_SEND);
508
509 brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW));
510 brw_set_src0(p, insn, retype(payload, BRW_REGISTER_TYPE_UW));
511 brw_set_src1(p, insn, brw_imm_d(0));
512
513 /* Terminate a compute shader by sending a message to the thread spawner.
514 */
515 brw_inst_set_sfid(devinfo, insn, BRW_SFID_THREAD_SPAWNER);
516 brw_inst_set_mlen(devinfo, insn, 1);
517 brw_inst_set_rlen(devinfo, insn, 0);
518 brw_inst_set_eot(devinfo, insn, inst->eot);
519 brw_inst_set_header_present(devinfo, insn, false);
520
521 brw_inst_set_ts_opcode(devinfo, insn, 0); /* Dereference resource */
522 brw_inst_set_ts_request_type(devinfo, insn, 0); /* Root thread */
523
524 /* Note that even though the thread has a URB resource associated with it,
525 * we set the "do not dereference URB" bit, because the URB resource is
526 * managed by the fixed-function unit, so it will free it automatically.
527 */
528 brw_inst_set_ts_resource_select(devinfo, insn, 1); /* Do not dereference URB */
529
530 brw_inst_set_mask_control(devinfo, insn, BRW_MASK_DISABLE);
531 }
532
533 void
534 fs_generator::generate_barrier(fs_inst *inst, struct brw_reg src)
535 {
536 brw_barrier(p, src);
537 brw_WAIT(p);
538 }
539
540 void
541 fs_generator::generate_linterp(fs_inst *inst,
542 struct brw_reg dst, struct brw_reg *src)
543 {
544 /* PLN reads:
545 * / in SIMD16 \
546 * -----------------------------------
547 * | src1+0 | src1+1 | src1+2 | src1+3 |
548 * |-----------------------------------|
549 * |(x0, x1)|(y0, y1)|(x2, x3)|(y2, y3)|
550 * -----------------------------------
551 *
552 * but for the LINE/MAC pair, the LINE reads Xs and the MAC reads Ys:
553 *
554 * -----------------------------------
555 * | src1+0 | src1+1 | src1+2 | src1+3 |
556 * |-----------------------------------|
557 * |(x0, x1)|(y0, y1)| | | in SIMD8
558 * |-----------------------------------|
559 * |(x0, x1)|(x2, x3)|(y0, y1)|(y2, y3)| in SIMD16
560 * -----------------------------------
561 *
562 * See also: emit_interpolation_setup_gen4().
563 */
564 struct brw_reg delta_x = src[0];
565 struct brw_reg delta_y = offset(src[0], inst->exec_size / 8);
566 struct brw_reg interp = src[1];
567
568 if (devinfo->has_pln &&
569 (devinfo->gen >= 7 || (delta_x.nr & 1) == 0)) {
570 brw_PLN(p, dst, interp, delta_x);
571 } else {
572 brw_LINE(p, brw_null_reg(), interp, delta_x);
573 brw_MAC(p, dst, suboffset(interp, 1), delta_y);
574 }
575 }
576
577 void
578 fs_generator::generate_get_buffer_size(fs_inst *inst,
579 struct brw_reg dst,
580 struct brw_reg src,
581 struct brw_reg surf_index)
582 {
583 assert(devinfo->gen >= 7);
584 assert(surf_index.file == BRW_IMMEDIATE_VALUE);
585
586 uint32_t simd_mode;
587 int rlen = 4;
588
589 switch (inst->exec_size) {
590 case 8:
591 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
592 break;
593 case 16:
594 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
595 break;
596 default:
597 unreachable("Invalid width for texture instruction");
598 }
599
600 if (simd_mode == BRW_SAMPLER_SIMD_MODE_SIMD16) {
601 rlen = 8;
602 dst = vec16(dst);
603 }
604
605 brw_SAMPLE(p,
606 retype(dst, BRW_REGISTER_TYPE_UW),
607 inst->base_mrf,
608 src,
609 surf_index.ud,
610 0,
611 GEN5_SAMPLER_MESSAGE_SAMPLE_RESINFO,
612 rlen, /* response length */
613 inst->mlen,
614 inst->header_size > 0,
615 simd_mode,
616 BRW_SAMPLER_RETURN_FORMAT_SINT32);
617
618 brw_mark_surface_used(prog_data, surf_index.ud);
619 }
620
621 void
622 fs_generator::generate_tex(fs_inst *inst, struct brw_reg dst, struct brw_reg src,
623 struct brw_reg surface_index,
624 struct brw_reg sampler_index)
625 {
626 assert(inst->size_written % REG_SIZE == 0);
627 int msg_type = -1;
628 uint32_t simd_mode;
629 uint32_t return_format;
630 bool is_combined_send = inst->eot;
631
632 switch (dst.type) {
633 case BRW_REGISTER_TYPE_D:
634 return_format = BRW_SAMPLER_RETURN_FORMAT_SINT32;
635 break;
636 case BRW_REGISTER_TYPE_UD:
637 return_format = BRW_SAMPLER_RETURN_FORMAT_UINT32;
638 break;
639 default:
640 return_format = BRW_SAMPLER_RETURN_FORMAT_FLOAT32;
641 break;
642 }
643
644 /* Stomp the resinfo output type to UINT32. On gens 4-5, the output type
645 * is set as part of the message descriptor. On gen4, the PRM seems to
646 * allow UINT32 and FLOAT32 (i965 PRM, Vol. 4 Section 4.8.1.1), but on
647 * later gens UINT32 is required. Once you hit Sandy Bridge, the bit is
648 * gone from the message descriptor entirely and you just get UINT32 all
649 * the time regasrdless. Since we can really only do non-UINT32 on gen4,
650 * just stomp it to UINT32 all the time.
651 */
652 if (inst->opcode == SHADER_OPCODE_TXS)
653 return_format = BRW_SAMPLER_RETURN_FORMAT_UINT32;
654
655 switch (inst->exec_size) {
656 case 8:
657 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
658 break;
659 case 16:
660 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
661 break;
662 default:
663 unreachable("Invalid width for texture instruction");
664 }
665
666 if (devinfo->gen >= 5) {
667 switch (inst->opcode) {
668 case SHADER_OPCODE_TEX:
669 if (inst->shadow_compare) {
670 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_COMPARE;
671 } else {
672 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE;
673 }
674 break;
675 case FS_OPCODE_TXB:
676 if (inst->shadow_compare) {
677 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_BIAS_COMPARE;
678 } else {
679 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_BIAS;
680 }
681 break;
682 case SHADER_OPCODE_TXL:
683 if (inst->shadow_compare) {
684 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LOD_COMPARE;
685 } else {
686 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LOD;
687 }
688 break;
689 case SHADER_OPCODE_TXL_LZ:
690 assert(devinfo->gen >= 9);
691 if (inst->shadow_compare) {
692 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_C_LZ;
693 } else {
694 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_LZ;
695 }
696 break;
697 case SHADER_OPCODE_TXS:
698 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_RESINFO;
699 break;
700 case SHADER_OPCODE_TXD:
701 if (inst->shadow_compare) {
702 /* Gen7.5+. Otherwise, lowered in NIR */
703 assert(devinfo->gen >= 8 || devinfo->is_haswell);
704 msg_type = HSW_SAMPLER_MESSAGE_SAMPLE_DERIV_COMPARE;
705 } else {
706 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_DERIVS;
707 }
708 break;
709 case SHADER_OPCODE_TXF:
710 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
711 break;
712 case SHADER_OPCODE_TXF_LZ:
713 assert(devinfo->gen >= 9);
714 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_LD_LZ;
715 break;
716 case SHADER_OPCODE_TXF_CMS_W:
717 assert(devinfo->gen >= 9);
718 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_LD2DMS_W;
719 break;
720 case SHADER_OPCODE_TXF_CMS:
721 if (devinfo->gen >= 7)
722 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD2DMS;
723 else
724 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
725 break;
726 case SHADER_OPCODE_TXF_UMS:
727 assert(devinfo->gen >= 7);
728 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD2DSS;
729 break;
730 case SHADER_OPCODE_TXF_MCS:
731 assert(devinfo->gen >= 7);
732 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD_MCS;
733 break;
734 case SHADER_OPCODE_LOD:
735 msg_type = GEN5_SAMPLER_MESSAGE_LOD;
736 break;
737 case SHADER_OPCODE_TG4:
738 if (inst->shadow_compare) {
739 assert(devinfo->gen >= 7);
740 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_C;
741 } else {
742 assert(devinfo->gen >= 6);
743 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4;
744 }
745 break;
746 case SHADER_OPCODE_TG4_OFFSET:
747 assert(devinfo->gen >= 7);
748 if (inst->shadow_compare) {
749 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_PO_C;
750 } else {
751 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_PO;
752 }
753 break;
754 case SHADER_OPCODE_SAMPLEINFO:
755 msg_type = GEN6_SAMPLER_MESSAGE_SAMPLE_SAMPLEINFO;
756 break;
757 default:
758 unreachable("not reached");
759 }
760 } else {
761 switch (inst->opcode) {
762 case SHADER_OPCODE_TEX:
763 /* Note that G45 and older determines shadow compare and dispatch width
764 * from message length for most messages.
765 */
766 if (inst->exec_size == 8) {
767 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE;
768 if (inst->shadow_compare) {
769 assert(inst->mlen == 6);
770 } else {
771 assert(inst->mlen <= 4);
772 }
773 } else {
774 if (inst->shadow_compare) {
775 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE_COMPARE;
776 assert(inst->mlen == 9);
777 } else {
778 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE;
779 assert(inst->mlen <= 7 && inst->mlen % 2 == 1);
780 }
781 }
782 break;
783 case FS_OPCODE_TXB:
784 if (inst->shadow_compare) {
785 assert(inst->exec_size == 8);
786 assert(inst->mlen == 6);
787 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE_BIAS_COMPARE;
788 } else {
789 assert(inst->mlen == 9);
790 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE_BIAS;
791 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
792 }
793 break;
794 case SHADER_OPCODE_TXL:
795 if (inst->shadow_compare) {
796 assert(inst->exec_size == 8);
797 assert(inst->mlen == 6);
798 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE_LOD_COMPARE;
799 } else {
800 assert(inst->mlen == 9);
801 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE_LOD;
802 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
803 }
804 break;
805 case SHADER_OPCODE_TXD:
806 /* There is no sample_d_c message; comparisons are done manually */
807 assert(inst->exec_size == 8);
808 assert(inst->mlen == 7 || inst->mlen == 10);
809 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE_GRADIENTS;
810 break;
811 case SHADER_OPCODE_TXF:
812 assert(inst->mlen <= 9 && inst->mlen % 2 == 1);
813 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_LD;
814 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
815 break;
816 case SHADER_OPCODE_TXS:
817 assert(inst->mlen == 3);
818 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_RESINFO;
819 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
820 break;
821 default:
822 unreachable("not reached");
823 }
824 }
825 assert(msg_type != -1);
826
827 if (simd_mode == BRW_SAMPLER_SIMD_MODE_SIMD16) {
828 dst = vec16(dst);
829 }
830
831 assert(devinfo->gen < 7 || inst->header_size == 0 ||
832 src.file == BRW_GENERAL_REGISTER_FILE);
833
834 assert(sampler_index.type == BRW_REGISTER_TYPE_UD);
835
836 /* Load the message header if present. If there's a texture offset,
837 * we need to set it up explicitly and load the offset bitfield.
838 * Otherwise, we can use an implied move from g0 to the first message reg.
839 */
840 if (inst->header_size != 0) {
841 if (devinfo->gen < 6 && !inst->offset) {
842 /* Set up an implied move from g0 to the MRF. */
843 src = retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UW);
844 } else {
845 struct brw_reg header_reg;
846
847 if (devinfo->gen >= 7) {
848 header_reg = src;
849 } else {
850 assert(inst->base_mrf != -1);
851 header_reg = brw_message_reg(inst->base_mrf);
852 }
853
854 brw_push_insn_state(p);
855 brw_set_default_exec_size(p, BRW_EXECUTE_8);
856 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
857 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
858 /* Explicitly set up the message header by copying g0 to the MRF. */
859 brw_MOV(p, header_reg, brw_vec8_grf(0, 0));
860
861 if (inst->offset) {
862 /* Set the offset bits in DWord 2. */
863 brw_MOV(p, get_element_ud(header_reg, 2),
864 brw_imm_ud(inst->offset));
865 } else if (stage != MESA_SHADER_VERTEX &&
866 stage != MESA_SHADER_FRAGMENT) {
867 /* The vertex and fragment stages have g0.2 set to 0, so
868 * header0.2 is 0 when g0 is copied. Other stages may not, so we
869 * must set it to 0 to avoid setting undesirable bits in the
870 * message.
871 */
872 brw_MOV(p, get_element_ud(header_reg, 2), brw_imm_ud(0));
873 }
874
875 brw_adjust_sampler_state_pointer(p, header_reg, sampler_index);
876 brw_pop_insn_state(p);
877 }
878 }
879
880 uint32_t base_binding_table_index = (inst->opcode == SHADER_OPCODE_TG4 ||
881 inst->opcode == SHADER_OPCODE_TG4_OFFSET)
882 ? prog_data->binding_table.gather_texture_start
883 : prog_data->binding_table.texture_start;
884
885 if (surface_index.file == BRW_IMMEDIATE_VALUE &&
886 sampler_index.file == BRW_IMMEDIATE_VALUE) {
887 uint32_t surface = surface_index.ud;
888 uint32_t sampler = sampler_index.ud;
889
890 brw_SAMPLE(p,
891 retype(dst, BRW_REGISTER_TYPE_UW),
892 inst->base_mrf,
893 src,
894 surface + base_binding_table_index,
895 sampler % 16,
896 msg_type,
897 inst->size_written / REG_SIZE,
898 inst->mlen,
899 inst->header_size != 0,
900 simd_mode,
901 return_format);
902
903 brw_mark_surface_used(prog_data, surface + base_binding_table_index);
904 } else {
905 /* Non-const sampler index */
906
907 struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
908 struct brw_reg surface_reg = vec1(retype(surface_index, BRW_REGISTER_TYPE_UD));
909 struct brw_reg sampler_reg = vec1(retype(sampler_index, BRW_REGISTER_TYPE_UD));
910
911 brw_push_insn_state(p);
912 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
913 brw_set_default_access_mode(p, BRW_ALIGN_1);
914
915 if (brw_regs_equal(&surface_reg, &sampler_reg)) {
916 brw_MUL(p, addr, sampler_reg, brw_imm_uw(0x101));
917 } else {
918 if (sampler_reg.file == BRW_IMMEDIATE_VALUE) {
919 brw_OR(p, addr, surface_reg, brw_imm_ud(sampler_reg.ud << 8));
920 } else {
921 brw_SHL(p, addr, sampler_reg, brw_imm_ud(8));
922 brw_OR(p, addr, addr, surface_reg);
923 }
924 }
925 if (base_binding_table_index)
926 brw_ADD(p, addr, addr, brw_imm_ud(base_binding_table_index));
927 brw_AND(p, addr, addr, brw_imm_ud(0xfff));
928
929 brw_pop_insn_state(p);
930
931 /* dst = send(offset, a0.0 | <descriptor>) */
932 brw_inst *insn = brw_send_indirect_message(
933 p, BRW_SFID_SAMPLER, dst, src, addr);
934 brw_set_sampler_message(p, insn,
935 0 /* surface */,
936 0 /* sampler */,
937 msg_type,
938 inst->size_written / REG_SIZE,
939 inst->mlen /* mlen */,
940 inst->header_size != 0 /* header */,
941 simd_mode,
942 return_format);
943
944 /* visitor knows more than we do about the surface limit required,
945 * so has already done marking.
946 */
947 }
948
949 if (is_combined_send) {
950 brw_inst_set_eot(p->devinfo, brw_last_inst, true);
951 brw_inst_set_opcode(p->devinfo, brw_last_inst, BRW_OPCODE_SENDC);
952 }
953 }
954
955
956 /* For OPCODE_DDX and OPCODE_DDY, per channel of output we've got input
957 * looking like:
958 *
959 * arg0: ss0.tl ss0.tr ss0.bl ss0.br ss1.tl ss1.tr ss1.bl ss1.br
960 *
961 * Ideally, we want to produce:
962 *
963 * DDX DDY
964 * dst: (ss0.tr - ss0.tl) (ss0.tl - ss0.bl)
965 * (ss0.tr - ss0.tl) (ss0.tr - ss0.br)
966 * (ss0.br - ss0.bl) (ss0.tl - ss0.bl)
967 * (ss0.br - ss0.bl) (ss0.tr - ss0.br)
968 * (ss1.tr - ss1.tl) (ss1.tl - ss1.bl)
969 * (ss1.tr - ss1.tl) (ss1.tr - ss1.br)
970 * (ss1.br - ss1.bl) (ss1.tl - ss1.bl)
971 * (ss1.br - ss1.bl) (ss1.tr - ss1.br)
972 *
973 * and add another set of two more subspans if in 16-pixel dispatch mode.
974 *
975 * For DDX, it ends up being easy: width = 2, horiz=0 gets us the same result
976 * for each pair, and vertstride = 2 jumps us 2 elements after processing a
977 * pair. But the ideal approximation may impose a huge performance cost on
978 * sample_d. On at least Haswell, sample_d instruction does some
979 * optimizations if the same LOD is used for all pixels in the subspan.
980 *
981 * For DDY, we need to use ALIGN16 mode since it's capable of doing the
982 * appropriate swizzling.
983 */
984 void
985 fs_generator::generate_ddx(enum opcode opcode,
986 struct brw_reg dst, struct brw_reg src)
987 {
988 unsigned vstride, width;
989
990 if (opcode == FS_OPCODE_DDX_FINE) {
991 /* produce accurate derivatives */
992 vstride = BRW_VERTICAL_STRIDE_2;
993 width = BRW_WIDTH_2;
994 } else {
995 /* replicate the derivative at the top-left pixel to other pixels */
996 vstride = BRW_VERTICAL_STRIDE_4;
997 width = BRW_WIDTH_4;
998 }
999
1000 struct brw_reg src0 = brw_reg(src.file, src.nr, 1,
1001 src.negate, src.abs,
1002 BRW_REGISTER_TYPE_F,
1003 vstride,
1004 width,
1005 BRW_HORIZONTAL_STRIDE_0,
1006 BRW_SWIZZLE_XYZW, WRITEMASK_XYZW);
1007 struct brw_reg src1 = brw_reg(src.file, src.nr, 0,
1008 src.negate, src.abs,
1009 BRW_REGISTER_TYPE_F,
1010 vstride,
1011 width,
1012 BRW_HORIZONTAL_STRIDE_0,
1013 BRW_SWIZZLE_XYZW, WRITEMASK_XYZW);
1014 brw_ADD(p, dst, src0, negate(src1));
1015 }
1016
1017 /* The negate_value boolean is used to negate the derivative computation for
1018 * FBOs, since they place the origin at the upper left instead of the lower
1019 * left.
1020 */
1021 void
1022 fs_generator::generate_ddy(enum opcode opcode,
1023 struct brw_reg dst, struct brw_reg src)
1024 {
1025 if (opcode == FS_OPCODE_DDY_FINE) {
1026 /* produce accurate derivatives */
1027 struct brw_reg src0 = brw_reg(src.file, src.nr, 0,
1028 src.negate, src.abs,
1029 BRW_REGISTER_TYPE_F,
1030 BRW_VERTICAL_STRIDE_4,
1031 BRW_WIDTH_4,
1032 BRW_HORIZONTAL_STRIDE_1,
1033 BRW_SWIZZLE_XYXY, WRITEMASK_XYZW);
1034 struct brw_reg src1 = brw_reg(src.file, src.nr, 0,
1035 src.negate, src.abs,
1036 BRW_REGISTER_TYPE_F,
1037 BRW_VERTICAL_STRIDE_4,
1038 BRW_WIDTH_4,
1039 BRW_HORIZONTAL_STRIDE_1,
1040 BRW_SWIZZLE_ZWZW, WRITEMASK_XYZW);
1041 brw_push_insn_state(p);
1042 brw_set_default_access_mode(p, BRW_ALIGN_16);
1043 brw_ADD(p, dst, negate(src0), src1);
1044 brw_pop_insn_state(p);
1045 } else {
1046 /* replicate the derivative at the top-left pixel to other pixels */
1047 struct brw_reg src0 = brw_reg(src.file, src.nr, 0,
1048 src.negate, src.abs,
1049 BRW_REGISTER_TYPE_F,
1050 BRW_VERTICAL_STRIDE_4,
1051 BRW_WIDTH_4,
1052 BRW_HORIZONTAL_STRIDE_0,
1053 BRW_SWIZZLE_XYZW, WRITEMASK_XYZW);
1054 struct brw_reg src1 = brw_reg(src.file, src.nr, 2,
1055 src.negate, src.abs,
1056 BRW_REGISTER_TYPE_F,
1057 BRW_VERTICAL_STRIDE_4,
1058 BRW_WIDTH_4,
1059 BRW_HORIZONTAL_STRIDE_0,
1060 BRW_SWIZZLE_XYZW, WRITEMASK_XYZW);
1061 brw_ADD(p, dst, negate(src0), src1);
1062 }
1063 }
1064
1065 void
1066 fs_generator::generate_discard_jump(fs_inst *inst)
1067 {
1068 assert(devinfo->gen >= 6);
1069
1070 /* This HALT will be patched up at FB write time to point UIP at the end of
1071 * the program, and at brw_uip_jip() JIP will be set to the end of the
1072 * current block (or the program).
1073 */
1074 this->discard_halt_patches.push_tail(new(mem_ctx) ip_record(p->nr_insn));
1075 gen6_HALT(p);
1076 }
1077
1078 void
1079 fs_generator::generate_scratch_write(fs_inst *inst, struct brw_reg src)
1080 {
1081 /* The 32-wide messages only respect the first 16-wide half of the channel
1082 * enable signals which are replicated identically for the second group of
1083 * 16 channels, so we cannot use them unless the write is marked
1084 * force_writemask_all.
1085 */
1086 const unsigned lower_size = inst->force_writemask_all ? inst->exec_size :
1087 MIN2(16, inst->exec_size);
1088 const unsigned block_size = 4 * lower_size / REG_SIZE;
1089 assert(inst->mlen != 0);
1090
1091 brw_push_insn_state(p);
1092 brw_set_default_exec_size(p, cvt(lower_size) - 1);
1093 brw_set_default_compression(p, lower_size > 8);
1094
1095 for (unsigned i = 0; i < inst->exec_size / lower_size; i++) {
1096 brw_set_default_group(p, inst->group + lower_size * i);
1097
1098 brw_MOV(p, brw_uvec_mrf(lower_size, inst->base_mrf + 1, 0),
1099 retype(offset(src, block_size * i), BRW_REGISTER_TYPE_UD));
1100
1101 brw_oword_block_write_scratch(p, brw_message_reg(inst->base_mrf),
1102 block_size,
1103 inst->offset + block_size * REG_SIZE * i);
1104 }
1105
1106 brw_pop_insn_state(p);
1107 }
1108
1109 void
1110 fs_generator::generate_scratch_read(fs_inst *inst, struct brw_reg dst)
1111 {
1112 assert(inst->exec_size <= 16 || inst->force_writemask_all);
1113 assert(inst->mlen != 0);
1114
1115 brw_oword_block_read_scratch(p, dst, brw_message_reg(inst->base_mrf),
1116 inst->exec_size / 8, inst->offset);
1117 }
1118
1119 void
1120 fs_generator::generate_scratch_read_gen7(fs_inst *inst, struct brw_reg dst)
1121 {
1122 assert(inst->exec_size <= 16 || inst->force_writemask_all);
1123
1124 gen7_block_read_scratch(p, dst, inst->exec_size / 8, inst->offset);
1125 }
1126
1127 void
1128 fs_generator::generate_uniform_pull_constant_load(fs_inst *inst,
1129 struct brw_reg dst,
1130 struct brw_reg index,
1131 struct brw_reg offset)
1132 {
1133 assert(type_sz(dst.type) == 4);
1134 assert(inst->mlen != 0);
1135
1136 assert(index.file == BRW_IMMEDIATE_VALUE &&
1137 index.type == BRW_REGISTER_TYPE_UD);
1138 uint32_t surf_index = index.ud;
1139
1140 assert(offset.file == BRW_IMMEDIATE_VALUE &&
1141 offset.type == BRW_REGISTER_TYPE_UD);
1142 uint32_t read_offset = offset.ud;
1143
1144 brw_oword_block_read(p, dst, brw_message_reg(inst->base_mrf),
1145 read_offset, surf_index);
1146 }
1147
1148 void
1149 fs_generator::generate_uniform_pull_constant_load_gen7(fs_inst *inst,
1150 struct brw_reg dst,
1151 struct brw_reg index,
1152 struct brw_reg payload)
1153 {
1154 assert(index.type == BRW_REGISTER_TYPE_UD);
1155 assert(payload.file == BRW_GENERAL_REGISTER_FILE);
1156 assert(type_sz(dst.type) == 4);
1157
1158 if (index.file == BRW_IMMEDIATE_VALUE) {
1159 const uint32_t surf_index = index.ud;
1160
1161 brw_push_insn_state(p);
1162 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1163 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
1164 brw_pop_insn_state(p);
1165
1166 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UD));
1167 brw_set_src0(p, send, retype(payload, BRW_REGISTER_TYPE_UD));
1168 brw_set_dp_read_message(p, send, surf_index,
1169 BRW_DATAPORT_OWORD_BLOCK_DWORDS(inst->exec_size),
1170 GEN7_DATAPORT_DC_OWORD_BLOCK_READ,
1171 GEN6_SFID_DATAPORT_CONSTANT_CACHE,
1172 1, /* mlen */
1173 true, /* header */
1174 DIV_ROUND_UP(inst->size_written, REG_SIZE));
1175
1176 } else {
1177 struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
1178
1179 brw_push_insn_state(p);
1180 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1181
1182 /* a0.0 = surf_index & 0xff */
1183 brw_inst *insn_and = brw_next_insn(p, BRW_OPCODE_AND);
1184 brw_inst_set_exec_size(p->devinfo, insn_and, BRW_EXECUTE_1);
1185 brw_set_dest(p, insn_and, addr);
1186 brw_set_src0(p, insn_and, vec1(retype(index, BRW_REGISTER_TYPE_UD)));
1187 brw_set_src1(p, insn_and, brw_imm_ud(0x0ff));
1188
1189 /* dst = send(payload, a0.0 | <descriptor>) */
1190 brw_inst *insn = brw_send_indirect_message(
1191 p, GEN6_SFID_DATAPORT_CONSTANT_CACHE,
1192 retype(dst, BRW_REGISTER_TYPE_UD),
1193 retype(payload, BRW_REGISTER_TYPE_UD), addr);
1194 brw_set_dp_read_message(p, insn, 0 /* surface */,
1195 BRW_DATAPORT_OWORD_BLOCK_DWORDS(inst->exec_size),
1196 GEN7_DATAPORT_DC_OWORD_BLOCK_READ,
1197 GEN6_SFID_DATAPORT_CONSTANT_CACHE,
1198 1, /* mlen */
1199 true, /* header */
1200 DIV_ROUND_UP(inst->size_written, REG_SIZE));
1201
1202 brw_pop_insn_state(p);
1203 }
1204 }
1205
1206 void
1207 fs_generator::generate_varying_pull_constant_load_gen4(fs_inst *inst,
1208 struct brw_reg dst,
1209 struct brw_reg index)
1210 {
1211 assert(devinfo->gen < 7); /* Should use the gen7 variant. */
1212 assert(inst->header_size != 0);
1213 assert(inst->mlen);
1214
1215 assert(index.file == BRW_IMMEDIATE_VALUE &&
1216 index.type == BRW_REGISTER_TYPE_UD);
1217 uint32_t surf_index = index.ud;
1218
1219 uint32_t simd_mode, rlen, msg_type;
1220 if (inst->exec_size == 16) {
1221 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1222 rlen = 8;
1223 } else {
1224 assert(inst->exec_size == 8);
1225 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
1226 rlen = 4;
1227 }
1228
1229 if (devinfo->gen >= 5)
1230 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
1231 else {
1232 /* We always use the SIMD16 message so that we only have to load U, and
1233 * not V or R.
1234 */
1235 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_LD;
1236 assert(inst->mlen == 3);
1237 assert(inst->size_written == 8 * REG_SIZE);
1238 rlen = 8;
1239 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1240 }
1241
1242 struct brw_reg header = brw_vec8_grf(0, 0);
1243 gen6_resolve_implied_move(p, &header, inst->base_mrf);
1244
1245 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
1246 brw_inst_set_compression(devinfo, send, false);
1247 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UW));
1248 brw_set_src0(p, send, header);
1249 if (devinfo->gen < 6)
1250 brw_inst_set_base_mrf(p->devinfo, send, inst->base_mrf);
1251
1252 /* Our surface is set up as floats, regardless of what actual data is
1253 * stored in it.
1254 */
1255 uint32_t return_format = BRW_SAMPLER_RETURN_FORMAT_FLOAT32;
1256 brw_set_sampler_message(p, send,
1257 surf_index,
1258 0, /* sampler (unused) */
1259 msg_type,
1260 rlen,
1261 inst->mlen,
1262 inst->header_size != 0,
1263 simd_mode,
1264 return_format);
1265 }
1266
1267 void
1268 fs_generator::generate_varying_pull_constant_load_gen7(fs_inst *inst,
1269 struct brw_reg dst,
1270 struct brw_reg index,
1271 struct brw_reg offset)
1272 {
1273 assert(devinfo->gen >= 7);
1274 /* Varying-offset pull constant loads are treated as a normal expression on
1275 * gen7, so the fact that it's a send message is hidden at the IR level.
1276 */
1277 assert(inst->header_size == 0);
1278 assert(!inst->mlen);
1279 assert(index.type == BRW_REGISTER_TYPE_UD);
1280
1281 uint32_t simd_mode, rlen, mlen;
1282 if (inst->exec_size == 16) {
1283 mlen = 2;
1284 rlen = 8;
1285 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1286 } else {
1287 assert(inst->exec_size == 8);
1288 mlen = 1;
1289 rlen = 4;
1290 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
1291 }
1292
1293 if (index.file == BRW_IMMEDIATE_VALUE) {
1294
1295 uint32_t surf_index = index.ud;
1296
1297 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
1298 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UW));
1299 brw_set_src0(p, send, offset);
1300 brw_set_sampler_message(p, send,
1301 surf_index,
1302 0, /* LD message ignores sampler unit */
1303 GEN5_SAMPLER_MESSAGE_SAMPLE_LD,
1304 rlen,
1305 mlen,
1306 false, /* no header */
1307 simd_mode,
1308 0);
1309
1310 } else {
1311
1312 struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
1313
1314 brw_push_insn_state(p);
1315 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1316
1317 /* a0.0 = surf_index & 0xff */
1318 brw_inst *insn_and = brw_next_insn(p, BRW_OPCODE_AND);
1319 brw_inst_set_exec_size(p->devinfo, insn_and, BRW_EXECUTE_1);
1320 brw_set_dest(p, insn_and, addr);
1321 brw_set_src0(p, insn_and, vec1(retype(index, BRW_REGISTER_TYPE_UD)));
1322 brw_set_src1(p, insn_and, brw_imm_ud(0x0ff));
1323
1324 brw_pop_insn_state(p);
1325
1326 /* dst = send(offset, a0.0 | <descriptor>) */
1327 brw_inst *insn = brw_send_indirect_message(
1328 p, BRW_SFID_SAMPLER, retype(dst, BRW_REGISTER_TYPE_UW),
1329 offset, addr);
1330 brw_set_sampler_message(p, insn,
1331 0 /* surface */,
1332 0 /* sampler */,
1333 GEN5_SAMPLER_MESSAGE_SAMPLE_LD,
1334 rlen /* rlen */,
1335 mlen /* mlen */,
1336 false /* header */,
1337 simd_mode,
1338 0);
1339 }
1340 }
1341
1342 /**
1343 * Cause the current pixel/sample mask (from R1.7 bits 15:0) to be transferred
1344 * into the flags register (f0.0).
1345 *
1346 * Used only on Gen6 and above.
1347 */
1348 void
1349 fs_generator::generate_mov_dispatch_to_flags(fs_inst *inst)
1350 {
1351 struct brw_reg flags = brw_flag_reg(0, inst->flag_subreg);
1352 struct brw_reg dispatch_mask;
1353
1354 if (devinfo->gen >= 6)
1355 dispatch_mask = retype(brw_vec1_grf(1, 7), BRW_REGISTER_TYPE_UW);
1356 else
1357 dispatch_mask = retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UW);
1358
1359 brw_push_insn_state(p);
1360 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1361 brw_MOV(p, flags, dispatch_mask);
1362 brw_pop_insn_state(p);
1363 }
1364
1365 void
1366 fs_generator::generate_pixel_interpolator_query(fs_inst *inst,
1367 struct brw_reg dst,
1368 struct brw_reg src,
1369 struct brw_reg msg_data,
1370 unsigned msg_type)
1371 {
1372 assert(inst->size_written % REG_SIZE == 0);
1373 assert(msg_data.type == BRW_REGISTER_TYPE_UD);
1374
1375 brw_pixel_interpolator_query(p,
1376 retype(dst, BRW_REGISTER_TYPE_UW),
1377 src,
1378 inst->pi_noperspective,
1379 msg_type,
1380 msg_data,
1381 inst->mlen,
1382 inst->size_written / REG_SIZE);
1383 }
1384
1385 /* Sets vstride=1, width=4, hstride=0 of register src1 during
1386 * the ADD instruction.
1387 */
1388 void
1389 fs_generator::generate_set_sample_id(fs_inst *inst,
1390 struct brw_reg dst,
1391 struct brw_reg src0,
1392 struct brw_reg src1)
1393 {
1394 assert(dst.type == BRW_REGISTER_TYPE_D ||
1395 dst.type == BRW_REGISTER_TYPE_UD);
1396 assert(src0.type == BRW_REGISTER_TYPE_D ||
1397 src0.type == BRW_REGISTER_TYPE_UD);
1398
1399 struct brw_reg reg = stride(src1, 1, 4, 0);
1400 if (devinfo->gen >= 8 || inst->exec_size == 8) {
1401 brw_ADD(p, dst, src0, reg);
1402 } else if (inst->exec_size == 16) {
1403 brw_push_insn_state(p);
1404 brw_set_default_exec_size(p, BRW_EXECUTE_8);
1405 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
1406 brw_ADD(p, firsthalf(dst), firsthalf(src0), reg);
1407 brw_set_default_compression_control(p, BRW_COMPRESSION_2NDHALF);
1408 brw_ADD(p, sechalf(dst), sechalf(src0), suboffset(reg, 2));
1409 brw_pop_insn_state(p);
1410 }
1411 }
1412
1413 void
1414 fs_generator::generate_pack_half_2x16_split(fs_inst *inst,
1415 struct brw_reg dst,
1416 struct brw_reg x,
1417 struct brw_reg y)
1418 {
1419 assert(devinfo->gen >= 7);
1420 assert(dst.type == BRW_REGISTER_TYPE_UD);
1421 assert(x.type == BRW_REGISTER_TYPE_F);
1422 assert(y.type == BRW_REGISTER_TYPE_F);
1423
1424 /* From the Ivybridge PRM, Vol4, Part3, Section 6.27 f32to16:
1425 *
1426 * Because this instruction does not have a 16-bit floating-point type,
1427 * the destination data type must be Word (W).
1428 *
1429 * The destination must be DWord-aligned and specify a horizontal stride
1430 * (HorzStride) of 2. The 16-bit result is stored in the lower word of
1431 * each destination channel and the upper word is not modified.
1432 */
1433 struct brw_reg dst_w = spread(retype(dst, BRW_REGISTER_TYPE_W), 2);
1434
1435 /* Give each 32-bit channel of dst the form below, where "." means
1436 * unchanged.
1437 * 0x....hhhh
1438 */
1439 brw_F32TO16(p, dst_w, y);
1440
1441 /* Now the form:
1442 * 0xhhhh0000
1443 */
1444 brw_SHL(p, dst, dst, brw_imm_ud(16u));
1445
1446 /* And, finally the form of packHalf2x16's output:
1447 * 0xhhhhllll
1448 */
1449 brw_F32TO16(p, dst_w, x);
1450 }
1451
1452 void
1453 fs_generator::generate_unpack_half_2x16_split(fs_inst *inst,
1454 struct brw_reg dst,
1455 struct brw_reg src)
1456 {
1457 assert(devinfo->gen >= 7);
1458 assert(dst.type == BRW_REGISTER_TYPE_F);
1459 assert(src.type == BRW_REGISTER_TYPE_UD);
1460
1461 /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
1462 *
1463 * Because this instruction does not have a 16-bit floating-point type,
1464 * the source data type must be Word (W). The destination type must be
1465 * F (Float).
1466 */
1467 struct brw_reg src_w = spread(retype(src, BRW_REGISTER_TYPE_W), 2);
1468
1469 /* Each channel of src has the form of unpackHalf2x16's input: 0xhhhhllll.
1470 * For the Y case, we wish to access only the upper word; therefore
1471 * a 16-bit subregister offset is needed.
1472 */
1473 assert(inst->opcode == FS_OPCODE_UNPACK_HALF_2x16_SPLIT_X ||
1474 inst->opcode == FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y);
1475 if (inst->opcode == FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y)
1476 src_w.subnr += 2;
1477
1478 brw_F16TO32(p, dst, src_w);
1479 }
1480
1481 void
1482 fs_generator::generate_shader_time_add(fs_inst *inst,
1483 struct brw_reg payload,
1484 struct brw_reg offset,
1485 struct brw_reg value)
1486 {
1487 assert(devinfo->gen >= 7);
1488 brw_push_insn_state(p);
1489 brw_set_default_mask_control(p, true);
1490
1491 assert(payload.file == BRW_GENERAL_REGISTER_FILE);
1492 struct brw_reg payload_offset = retype(brw_vec1_grf(payload.nr, 0),
1493 offset.type);
1494 struct brw_reg payload_value = retype(brw_vec1_grf(payload.nr + 1, 0),
1495 value.type);
1496
1497 assert(offset.file == BRW_IMMEDIATE_VALUE);
1498 if (value.file == BRW_GENERAL_REGISTER_FILE) {
1499 value.width = BRW_WIDTH_1;
1500 value.hstride = BRW_HORIZONTAL_STRIDE_0;
1501 value.vstride = BRW_VERTICAL_STRIDE_0;
1502 } else {
1503 assert(value.file == BRW_IMMEDIATE_VALUE);
1504 }
1505
1506 /* Trying to deal with setup of the params from the IR is crazy in the FS8
1507 * case, and we don't really care about squeezing every bit of performance
1508 * out of this path, so we just emit the MOVs from here.
1509 */
1510 brw_MOV(p, payload_offset, offset);
1511 brw_MOV(p, payload_value, value);
1512 brw_shader_time_add(p, payload,
1513 prog_data->binding_table.shader_time_start);
1514 brw_pop_insn_state(p);
1515
1516 brw_mark_surface_used(prog_data,
1517 prog_data->binding_table.shader_time_start);
1518 }
1519
1520 void
1521 fs_generator::enable_debug(const char *shader_name)
1522 {
1523 debug_flag = true;
1524 this->shader_name = shader_name;
1525 }
1526
1527 int
1528 fs_generator::generate_code(const cfg_t *cfg, int dispatch_width)
1529 {
1530 /* align to 64 byte boundary. */
1531 while (p->next_insn_offset % 64)
1532 brw_NOP(p);
1533
1534 this->dispatch_width = dispatch_width;
1535
1536 int start_offset = p->next_insn_offset;
1537 int spill_count = 0, fill_count = 0;
1538 int loop_count = 0;
1539
1540 struct annotation_info annotation;
1541 memset(&annotation, 0, sizeof(annotation));
1542
1543 foreach_block_and_inst (block, fs_inst, inst, cfg) {
1544 struct brw_reg src[3], dst;
1545 unsigned int last_insn_offset = p->next_insn_offset;
1546 bool multiple_instructions_emitted = false;
1547
1548 /* From the Broadwell PRM, Volume 7, "3D-Media-GPGPU", in the
1549 * "Register Region Restrictions" section: for BDW, SKL:
1550 *
1551 * "A POW/FDIV operation must not be followed by an instruction
1552 * that requires two destination registers."
1553 *
1554 * The documentation is often lacking annotations for Atom parts,
1555 * and empirically this affects CHV as well.
1556 */
1557 if (devinfo->gen >= 8 &&
1558 p->nr_insn > 1 &&
1559 brw_inst_opcode(devinfo, brw_last_inst) == BRW_OPCODE_MATH &&
1560 brw_inst_math_function(devinfo, brw_last_inst) == BRW_MATH_FUNCTION_POW &&
1561 inst->dst.component_size(inst->exec_size) > REG_SIZE) {
1562 brw_NOP(p);
1563 last_insn_offset = p->next_insn_offset;
1564 }
1565
1566 if (unlikely(debug_flag))
1567 annotate(p->devinfo, &annotation, cfg, inst, p->next_insn_offset);
1568
1569 /* If the instruction writes to more than one register, it needs to be
1570 * explicitly marked as compressed on Gen <= 5. On Gen >= 6 the
1571 * hardware figures out by itself what the right compression mode is,
1572 * but we still need to know whether the instruction is compressed to
1573 * set up the source register regions appropriately.
1574 *
1575 * XXX - This is wrong for instructions that write a single register but
1576 * read more than one which should strictly speaking be treated as
1577 * compressed. For instructions that don't write any registers it
1578 * relies on the destination being a null register of the correct
1579 * type and regioning so the instruction is considered compressed
1580 * or not accordingly.
1581 */
1582 const bool compressed =
1583 inst->dst.component_size(inst->exec_size) > REG_SIZE;
1584 brw_set_default_compression(p, compressed);
1585 brw_set_default_group(p, inst->group);
1586
1587 for (unsigned int i = 0; i < inst->sources; i++) {
1588 src[i] = brw_reg_from_fs_reg(inst, &inst->src[i], devinfo->gen,
1589 compressed);
1590
1591 /* The accumulator result appears to get used for the
1592 * conditional modifier generation. When negating a UD
1593 * value, there is a 33rd bit generated for the sign in the
1594 * accumulator value, so now you can't check, for example,
1595 * equality with a 32-bit value. See piglit fs-op-neg-uvec4.
1596 */
1597 assert(!inst->conditional_mod ||
1598 inst->src[i].type != BRW_REGISTER_TYPE_UD ||
1599 !inst->src[i].negate);
1600 }
1601 dst = brw_reg_from_fs_reg(inst, &inst->dst, devinfo->gen, compressed);
1602
1603 brw_set_default_access_mode(p, BRW_ALIGN_1);
1604 brw_set_default_predicate_control(p, inst->predicate);
1605 brw_set_default_predicate_inverse(p, inst->predicate_inverse);
1606 brw_set_default_flag_reg(p, 0, inst->flag_subreg);
1607 brw_set_default_saturate(p, inst->saturate);
1608 brw_set_default_mask_control(p, inst->force_writemask_all);
1609 brw_set_default_acc_write_control(p, inst->writes_accumulator);
1610 brw_set_default_exec_size(p, cvt(inst->exec_size) - 1);
1611
1612 assert(inst->force_writemask_all || inst->exec_size >= 4);
1613 assert(inst->force_writemask_all || inst->group % inst->exec_size == 0);
1614 assert(inst->base_mrf + inst->mlen <= BRW_MAX_MRF(devinfo->gen));
1615 assert(inst->mlen <= BRW_MAX_MSG_LENGTH);
1616
1617 switch (inst->opcode) {
1618 case BRW_OPCODE_MOV:
1619 brw_MOV(p, dst, src[0]);
1620 break;
1621 case BRW_OPCODE_ADD:
1622 brw_ADD(p, dst, src[0], src[1]);
1623 break;
1624 case BRW_OPCODE_MUL:
1625 brw_MUL(p, dst, src[0], src[1]);
1626 break;
1627 case BRW_OPCODE_AVG:
1628 brw_AVG(p, dst, src[0], src[1]);
1629 break;
1630 case BRW_OPCODE_MACH:
1631 brw_MACH(p, dst, src[0], src[1]);
1632 break;
1633
1634 case BRW_OPCODE_LINE:
1635 brw_LINE(p, dst, src[0], src[1]);
1636 break;
1637
1638 case BRW_OPCODE_MAD:
1639 assert(devinfo->gen >= 6);
1640 brw_set_default_access_mode(p, BRW_ALIGN_16);
1641 brw_MAD(p, dst, src[0], src[1], src[2]);
1642 break;
1643
1644 case BRW_OPCODE_LRP:
1645 assert(devinfo->gen >= 6);
1646 brw_set_default_access_mode(p, BRW_ALIGN_16);
1647 brw_LRP(p, dst, src[0], src[1], src[2]);
1648 break;
1649
1650 case BRW_OPCODE_FRC:
1651 brw_FRC(p, dst, src[0]);
1652 break;
1653 case BRW_OPCODE_RNDD:
1654 brw_RNDD(p, dst, src[0]);
1655 break;
1656 case BRW_OPCODE_RNDE:
1657 brw_RNDE(p, dst, src[0]);
1658 break;
1659 case BRW_OPCODE_RNDZ:
1660 brw_RNDZ(p, dst, src[0]);
1661 break;
1662
1663 case BRW_OPCODE_AND:
1664 brw_AND(p, dst, src[0], src[1]);
1665 break;
1666 case BRW_OPCODE_OR:
1667 brw_OR(p, dst, src[0], src[1]);
1668 break;
1669 case BRW_OPCODE_XOR:
1670 brw_XOR(p, dst, src[0], src[1]);
1671 break;
1672 case BRW_OPCODE_NOT:
1673 brw_NOT(p, dst, src[0]);
1674 break;
1675 case BRW_OPCODE_ASR:
1676 brw_ASR(p, dst, src[0], src[1]);
1677 break;
1678 case BRW_OPCODE_SHR:
1679 brw_SHR(p, dst, src[0], src[1]);
1680 break;
1681 case BRW_OPCODE_SHL:
1682 brw_SHL(p, dst, src[0], src[1]);
1683 break;
1684 case BRW_OPCODE_F32TO16:
1685 assert(devinfo->gen >= 7);
1686 brw_F32TO16(p, dst, src[0]);
1687 break;
1688 case BRW_OPCODE_F16TO32:
1689 assert(devinfo->gen >= 7);
1690 brw_F16TO32(p, dst, src[0]);
1691 break;
1692 case BRW_OPCODE_CMP:
1693 if (inst->exec_size >= 16 && devinfo->gen == 7 && !devinfo->is_haswell &&
1694 dst.file == BRW_ARCHITECTURE_REGISTER_FILE) {
1695 /* For unknown reasons the WaCMPInstFlagDepClearedEarly workaround
1696 * implemented in the compiler is not sufficient. Overriding the
1697 * type when the destination is the null register is necessary but
1698 * not sufficient by itself.
1699 */
1700 assert(dst.nr == BRW_ARF_NULL);
1701 dst.type = BRW_REGISTER_TYPE_D;
1702 }
1703 brw_CMP(p, dst, inst->conditional_mod, src[0], src[1]);
1704 break;
1705 case BRW_OPCODE_SEL:
1706 brw_SEL(p, dst, src[0], src[1]);
1707 break;
1708 case BRW_OPCODE_BFREV:
1709 assert(devinfo->gen >= 7);
1710 /* BFREV only supports UD type for src and dst. */
1711 brw_BFREV(p, retype(dst, BRW_REGISTER_TYPE_UD),
1712 retype(src[0], BRW_REGISTER_TYPE_UD));
1713 break;
1714 case BRW_OPCODE_FBH:
1715 assert(devinfo->gen >= 7);
1716 /* FBH only supports UD type for dst. */
1717 brw_FBH(p, retype(dst, BRW_REGISTER_TYPE_UD), src[0]);
1718 break;
1719 case BRW_OPCODE_FBL:
1720 assert(devinfo->gen >= 7);
1721 /* FBL only supports UD type for dst. */
1722 brw_FBL(p, retype(dst, BRW_REGISTER_TYPE_UD), src[0]);
1723 break;
1724 case BRW_OPCODE_LZD:
1725 brw_LZD(p, dst, src[0]);
1726 break;
1727 case BRW_OPCODE_CBIT:
1728 assert(devinfo->gen >= 7);
1729 /* CBIT only supports UD type for dst. */
1730 brw_CBIT(p, retype(dst, BRW_REGISTER_TYPE_UD), src[0]);
1731 break;
1732 case BRW_OPCODE_ADDC:
1733 assert(devinfo->gen >= 7);
1734 brw_ADDC(p, dst, src[0], src[1]);
1735 break;
1736 case BRW_OPCODE_SUBB:
1737 assert(devinfo->gen >= 7);
1738 brw_SUBB(p, dst, src[0], src[1]);
1739 break;
1740 case BRW_OPCODE_MAC:
1741 brw_MAC(p, dst, src[0], src[1]);
1742 break;
1743
1744 case BRW_OPCODE_BFE:
1745 assert(devinfo->gen >= 7);
1746 brw_set_default_access_mode(p, BRW_ALIGN_16);
1747 brw_BFE(p, dst, src[0], src[1], src[2]);
1748 break;
1749
1750 case BRW_OPCODE_BFI1:
1751 assert(devinfo->gen >= 7);
1752 brw_BFI1(p, dst, src[0], src[1]);
1753 break;
1754 case BRW_OPCODE_BFI2:
1755 assert(devinfo->gen >= 7);
1756 brw_set_default_access_mode(p, BRW_ALIGN_16);
1757 brw_BFI2(p, dst, src[0], src[1], src[2]);
1758 break;
1759
1760 case BRW_OPCODE_IF:
1761 if (inst->src[0].file != BAD_FILE) {
1762 /* The instruction has an embedded compare (only allowed on gen6) */
1763 assert(devinfo->gen == 6);
1764 gen6_IF(p, inst->conditional_mod, src[0], src[1]);
1765 } else {
1766 brw_IF(p, brw_inst_exec_size(devinfo, p->current));
1767 }
1768 break;
1769
1770 case BRW_OPCODE_ELSE:
1771 brw_ELSE(p);
1772 break;
1773 case BRW_OPCODE_ENDIF:
1774 brw_ENDIF(p);
1775 break;
1776
1777 case BRW_OPCODE_DO:
1778 brw_DO(p, brw_inst_exec_size(devinfo, p->current));
1779 break;
1780
1781 case BRW_OPCODE_BREAK:
1782 brw_BREAK(p);
1783 break;
1784 case BRW_OPCODE_CONTINUE:
1785 brw_CONT(p);
1786 break;
1787
1788 case BRW_OPCODE_WHILE:
1789 brw_WHILE(p);
1790 loop_count++;
1791 break;
1792
1793 case SHADER_OPCODE_RCP:
1794 case SHADER_OPCODE_RSQ:
1795 case SHADER_OPCODE_SQRT:
1796 case SHADER_OPCODE_EXP2:
1797 case SHADER_OPCODE_LOG2:
1798 case SHADER_OPCODE_SIN:
1799 case SHADER_OPCODE_COS:
1800 assert(inst->conditional_mod == BRW_CONDITIONAL_NONE);
1801 if (devinfo->gen >= 6) {
1802 assert(inst->mlen == 0);
1803 assert(devinfo->gen >= 7 || inst->exec_size == 8);
1804 gen6_math(p, dst, brw_math_function(inst->opcode),
1805 src[0], brw_null_reg());
1806 } else {
1807 assert(inst->mlen >= 1);
1808 assert(devinfo->gen == 5 || devinfo->is_g4x || inst->exec_size == 8);
1809 gen4_math(p, dst,
1810 brw_math_function(inst->opcode),
1811 inst->base_mrf, src[0],
1812 BRW_MATH_PRECISION_FULL);
1813 }
1814 break;
1815 case SHADER_OPCODE_INT_QUOTIENT:
1816 case SHADER_OPCODE_INT_REMAINDER:
1817 case SHADER_OPCODE_POW:
1818 assert(inst->conditional_mod == BRW_CONDITIONAL_NONE);
1819 if (devinfo->gen >= 6) {
1820 assert(inst->mlen == 0);
1821 assert((devinfo->gen >= 7 && inst->opcode == SHADER_OPCODE_POW) ||
1822 inst->exec_size == 8);
1823 gen6_math(p, dst, brw_math_function(inst->opcode), src[0], src[1]);
1824 } else {
1825 assert(inst->mlen >= 1);
1826 assert(inst->exec_size == 8);
1827 gen4_math(p, dst, brw_math_function(inst->opcode),
1828 inst->base_mrf, src[0],
1829 BRW_MATH_PRECISION_FULL);
1830 }
1831 break;
1832 case FS_OPCODE_CINTERP:
1833 brw_MOV(p, dst, src[0]);
1834 break;
1835 case FS_OPCODE_LINTERP:
1836 generate_linterp(inst, dst, src);
1837 break;
1838 case FS_OPCODE_PIXEL_X:
1839 assert(src[0].type == BRW_REGISTER_TYPE_UW);
1840 src[0].subnr = 0 * type_sz(src[0].type);
1841 brw_MOV(p, dst, stride(src[0], 8, 4, 1));
1842 break;
1843 case FS_OPCODE_PIXEL_Y:
1844 assert(src[0].type == BRW_REGISTER_TYPE_UW);
1845 src[0].subnr = 4 * type_sz(src[0].type);
1846 brw_MOV(p, dst, stride(src[0], 8, 4, 1));
1847 break;
1848 case FS_OPCODE_GET_BUFFER_SIZE:
1849 generate_get_buffer_size(inst, dst, src[0], src[1]);
1850 break;
1851 case SHADER_OPCODE_TEX:
1852 case FS_OPCODE_TXB:
1853 case SHADER_OPCODE_TXD:
1854 case SHADER_OPCODE_TXF:
1855 case SHADER_OPCODE_TXF_LZ:
1856 case SHADER_OPCODE_TXF_CMS:
1857 case SHADER_OPCODE_TXF_CMS_W:
1858 case SHADER_OPCODE_TXF_UMS:
1859 case SHADER_OPCODE_TXF_MCS:
1860 case SHADER_OPCODE_TXL:
1861 case SHADER_OPCODE_TXL_LZ:
1862 case SHADER_OPCODE_TXS:
1863 case SHADER_OPCODE_LOD:
1864 case SHADER_OPCODE_TG4:
1865 case SHADER_OPCODE_TG4_OFFSET:
1866 case SHADER_OPCODE_SAMPLEINFO:
1867 generate_tex(inst, dst, src[0], src[1], src[2]);
1868 break;
1869 case FS_OPCODE_DDX_COARSE:
1870 case FS_OPCODE_DDX_FINE:
1871 generate_ddx(inst->opcode, dst, src[0]);
1872 break;
1873 case FS_OPCODE_DDY_COARSE:
1874 case FS_OPCODE_DDY_FINE:
1875 generate_ddy(inst->opcode, dst, src[0]);
1876 break;
1877
1878 case SHADER_OPCODE_GEN4_SCRATCH_WRITE:
1879 generate_scratch_write(inst, src[0]);
1880 spill_count++;
1881 break;
1882
1883 case SHADER_OPCODE_GEN4_SCRATCH_READ:
1884 generate_scratch_read(inst, dst);
1885 fill_count++;
1886 break;
1887
1888 case SHADER_OPCODE_GEN7_SCRATCH_READ:
1889 generate_scratch_read_gen7(inst, dst);
1890 fill_count++;
1891 break;
1892
1893 case SHADER_OPCODE_MOV_INDIRECT:
1894 generate_mov_indirect(inst, dst, src[0], src[1]);
1895 break;
1896
1897 case SHADER_OPCODE_URB_READ_SIMD8:
1898 case SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT:
1899 generate_urb_read(inst, dst, src[0]);
1900 break;
1901
1902 case SHADER_OPCODE_URB_WRITE_SIMD8:
1903 case SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT:
1904 case SHADER_OPCODE_URB_WRITE_SIMD8_MASKED:
1905 case SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT:
1906 generate_urb_write(inst, src[0]);
1907 break;
1908
1909 case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD:
1910 assert(inst->force_writemask_all);
1911 generate_uniform_pull_constant_load(inst, dst, src[0], src[1]);
1912 break;
1913
1914 case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD_GEN7:
1915 assert(inst->force_writemask_all);
1916 generate_uniform_pull_constant_load_gen7(inst, dst, src[0], src[1]);
1917 break;
1918
1919 case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_GEN4:
1920 generate_varying_pull_constant_load_gen4(inst, dst, src[0]);
1921 break;
1922
1923 case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_GEN7:
1924 generate_varying_pull_constant_load_gen7(inst, dst, src[0], src[1]);
1925 break;
1926
1927 case FS_OPCODE_REP_FB_WRITE:
1928 case FS_OPCODE_FB_WRITE:
1929 generate_fb_write(inst, src[0]);
1930 break;
1931
1932 case FS_OPCODE_FB_READ:
1933 generate_fb_read(inst, dst, src[0]);
1934 break;
1935
1936 case FS_OPCODE_MOV_DISPATCH_TO_FLAGS:
1937 generate_mov_dispatch_to_flags(inst);
1938 break;
1939
1940 case FS_OPCODE_DISCARD_JUMP:
1941 generate_discard_jump(inst);
1942 break;
1943
1944 case SHADER_OPCODE_SHADER_TIME_ADD:
1945 generate_shader_time_add(inst, src[0], src[1], src[2]);
1946 break;
1947
1948 case SHADER_OPCODE_UNTYPED_ATOMIC:
1949 assert(src[2].file == BRW_IMMEDIATE_VALUE);
1950 brw_untyped_atomic(p, dst, src[0], src[1], src[2].ud,
1951 inst->mlen, !inst->dst.is_null());
1952 break;
1953
1954 case SHADER_OPCODE_UNTYPED_SURFACE_READ:
1955 assert(src[2].file == BRW_IMMEDIATE_VALUE);
1956 brw_untyped_surface_read(p, dst, src[0], src[1],
1957 inst->mlen, src[2].ud);
1958 break;
1959
1960 case SHADER_OPCODE_UNTYPED_SURFACE_WRITE:
1961 assert(src[2].file == BRW_IMMEDIATE_VALUE);
1962 brw_untyped_surface_write(p, src[0], src[1],
1963 inst->mlen, src[2].ud);
1964 break;
1965
1966 case SHADER_OPCODE_TYPED_ATOMIC:
1967 assert(src[2].file == BRW_IMMEDIATE_VALUE);
1968 brw_typed_atomic(p, dst, src[0], src[1],
1969 src[2].ud, inst->mlen, !inst->dst.is_null());
1970 break;
1971
1972 case SHADER_OPCODE_TYPED_SURFACE_READ:
1973 assert(src[2].file == BRW_IMMEDIATE_VALUE);
1974 brw_typed_surface_read(p, dst, src[0], src[1],
1975 inst->mlen, src[2].ud);
1976 break;
1977
1978 case SHADER_OPCODE_TYPED_SURFACE_WRITE:
1979 assert(src[2].file == BRW_IMMEDIATE_VALUE);
1980 brw_typed_surface_write(p, src[0], src[1], inst->mlen, src[2].ud);
1981 break;
1982
1983 case SHADER_OPCODE_MEMORY_FENCE:
1984 brw_memory_fence(p, dst);
1985 break;
1986
1987 case SHADER_OPCODE_FIND_LIVE_CHANNEL: {
1988 const struct brw_reg mask =
1989 brw_stage_has_packed_dispatch(devinfo, stage,
1990 prog_data) ? brw_imm_ud(~0u) :
1991 stage == MESA_SHADER_FRAGMENT ? brw_vmask_reg() :
1992 brw_dmask_reg();
1993 brw_find_live_channel(p, dst, mask);
1994 break;
1995 }
1996
1997 case SHADER_OPCODE_BROADCAST:
1998 assert(inst->force_writemask_all);
1999 brw_broadcast(p, dst, src[0], src[1]);
2000 break;
2001
2002 case FS_OPCODE_SET_SAMPLE_ID:
2003 generate_set_sample_id(inst, dst, src[0], src[1]);
2004 break;
2005
2006 case FS_OPCODE_PACK_HALF_2x16_SPLIT:
2007 generate_pack_half_2x16_split(inst, dst, src[0], src[1]);
2008 break;
2009
2010 case FS_OPCODE_UNPACK_HALF_2x16_SPLIT_X:
2011 case FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y:
2012 generate_unpack_half_2x16_split(inst, dst, src[0]);
2013 break;
2014
2015 case FS_OPCODE_PLACEHOLDER_HALT:
2016 /* This is the place where the final HALT needs to be inserted if
2017 * we've emitted any discards. If not, this will emit no code.
2018 */
2019 if (!patch_discard_jumps_to_fb_writes()) {
2020 if (unlikely(debug_flag)) {
2021 annotation.ann_count--;
2022 }
2023 }
2024 break;
2025
2026 case FS_OPCODE_INTERPOLATE_AT_SAMPLE:
2027 generate_pixel_interpolator_query(inst, dst, src[0], src[1],
2028 GEN7_PIXEL_INTERPOLATOR_LOC_SAMPLE);
2029 break;
2030
2031 case FS_OPCODE_INTERPOLATE_AT_SHARED_OFFSET:
2032 generate_pixel_interpolator_query(inst, dst, src[0], src[1],
2033 GEN7_PIXEL_INTERPOLATOR_LOC_SHARED_OFFSET);
2034 break;
2035
2036 case FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET:
2037 generate_pixel_interpolator_query(inst, dst, src[0], src[1],
2038 GEN7_PIXEL_INTERPOLATOR_LOC_PER_SLOT_OFFSET);
2039 break;
2040
2041 case CS_OPCODE_CS_TERMINATE:
2042 generate_cs_terminate(inst, src[0]);
2043 break;
2044
2045 case SHADER_OPCODE_BARRIER:
2046 generate_barrier(inst, src[0]);
2047 break;
2048
2049 case BRW_OPCODE_DIM:
2050 assert(devinfo->is_haswell);
2051 assert(src[0].type == BRW_REGISTER_TYPE_DF);
2052 assert(dst.type == BRW_REGISTER_TYPE_DF);
2053 brw_DIM(p, dst, retype(src[0], BRW_REGISTER_TYPE_F));
2054 break;
2055
2056 default:
2057 unreachable("Unsupported opcode");
2058
2059 case SHADER_OPCODE_LOAD_PAYLOAD:
2060 unreachable("Should be lowered by lower_load_payload()");
2061 }
2062
2063 if (multiple_instructions_emitted)
2064 continue;
2065
2066 if (inst->no_dd_clear || inst->no_dd_check || inst->conditional_mod) {
2067 assert(p->next_insn_offset == last_insn_offset + 16 ||
2068 !"conditional_mod, no_dd_check, or no_dd_clear set for IR "
2069 "emitting more than 1 instruction");
2070
2071 brw_inst *last = &p->store[last_insn_offset / 16];
2072
2073 if (inst->conditional_mod)
2074 brw_inst_set_cond_modifier(p->devinfo, last, inst->conditional_mod);
2075 brw_inst_set_no_dd_clear(p->devinfo, last, inst->no_dd_clear);
2076 brw_inst_set_no_dd_check(p->devinfo, last, inst->no_dd_check);
2077 }
2078 }
2079
2080 brw_set_uip_jip(p, start_offset);
2081 annotation_finalize(&annotation, p->next_insn_offset);
2082
2083 #ifndef NDEBUG
2084 bool validated = brw_validate_instructions(p, start_offset, &annotation);
2085 #else
2086 if (unlikely(debug_flag))
2087 brw_validate_instructions(p, start_offset, &annotation);
2088 #endif
2089
2090 int before_size = p->next_insn_offset - start_offset;
2091 brw_compact_instructions(p, start_offset, annotation.ann_count,
2092 annotation.ann);
2093 int after_size = p->next_insn_offset - start_offset;
2094
2095 if (unlikely(debug_flag)) {
2096 fprintf(stderr, "Native code for %s\n"
2097 "SIMD%d shader: %d instructions. %d loops. %u cycles. %d:%d spills:fills. Promoted %u constants. Compacted %d to %d"
2098 " bytes (%.0f%%)\n",
2099 shader_name, dispatch_width, before_size / 16, loop_count, cfg->cycle_count,
2100 spill_count, fill_count, promoted_constants, before_size, after_size,
2101 100.0f * (before_size - after_size) / before_size);
2102
2103 dump_assembly(p->store, annotation.ann_count, annotation.ann,
2104 p->devinfo);
2105 ralloc_free(annotation.mem_ctx);
2106 }
2107 assert(validated);
2108
2109 compiler->shader_debug_log(log_data,
2110 "%s SIMD%d shader: %d inst, %d loops, %u cycles, "
2111 "%d:%d spills:fills, Promoted %u constants, "
2112 "compacted %d to %d bytes.",
2113 _mesa_shader_stage_to_abbrev(stage),
2114 dispatch_width, before_size / 16,
2115 loop_count, cfg->cycle_count, spill_count,
2116 fill_count, promoted_constants, before_size,
2117 after_size);
2118
2119 return start_offset;
2120 }
2121
2122 const unsigned *
2123 fs_generator::get_assembly(unsigned int *assembly_size)
2124 {
2125 return brw_get_program(p, assembly_size);
2126 }