projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
i965/fs: Expose arbitrary pull constant load sizes to the IR.
[mesa.git] / src / mesa / drivers / dri / i965 / brw_fs_generator.cpp
1 /*
2 * Copyright © 2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 /** @file brw_fs_generator.cpp
25 *
26 * This file supports generating code from the FS LIR to the actual
27 * native instructions.
28 */
29
30 #include "brw_eu.h"
31 #include "brw_fs.h"
32 #include "brw_cfg.h"
33 #include "brw_program.h"
34
35 static enum brw_reg_file
36 brw_file_from_reg(fs_reg *reg)
37 {
38 switch (reg->file) {
39 case ARF:
40 return BRW_ARCHITECTURE_REGISTER_FILE;
41 case FIXED_GRF:
42 case VGRF:
43 return BRW_GENERAL_REGISTER_FILE;
44 case MRF:
45 return BRW_MESSAGE_REGISTER_FILE;
46 case IMM:
47 return BRW_IMMEDIATE_VALUE;
48 case BAD_FILE:
49 case ATTR:
50 case UNIFORM:
51 unreachable("not reached");
52 }
53 return BRW_ARCHITECTURE_REGISTER_FILE;
54 }
55
56 static struct brw_reg
57 brw_reg_from_fs_reg(fs_inst *inst, fs_reg *reg, unsigned gen, bool compressed)
58 {
59 struct brw_reg brw_reg;
60
61 switch (reg->file) {
62 case MRF:
63 assert((reg->nr & ~BRW_MRF_COMPR4) < BRW_MAX_MRF(gen));
64 /* Fallthrough */
65 case VGRF:
66 if (reg->stride == 0) {
67 brw_reg = brw_vec1_reg(brw_file_from_reg(reg), reg->nr, 0);
68 } else {
69 /* From the Haswell PRM:
70 *
71 * "VertStride must be used to cross GRF register boundaries. This
72 * rule implies that elements within a 'Width' cannot cross GRF
73 * boundaries."
74 *
75 * The maximum width value that could satisfy this restriction is:
76 */
77 const unsigned reg_width = REG_SIZE / (reg->stride * type_sz(reg->type));
78
79 /* Because the hardware can only split source regions at a whole
80 * multiple of width during decompression (i.e. vertically), clamp
81 * the value obtained above to the physical execution size of a
82 * single decompressed chunk of the instruction:
83 */
84 const unsigned phys_width = compressed ? inst->exec_size / 2 :
85 inst->exec_size;
86
87 /* XXX - The equation above is strictly speaking not correct on
88 * hardware that supports unbalanced GRF writes -- On Gen9+
89 * each decompressed chunk of the instruction may have a
90 * different execution size when the number of components
91 * written to each destination GRF is not the same.
92 */
93 const unsigned width = MIN2(reg_width, phys_width);
94 brw_reg = brw_vecn_reg(width, brw_file_from_reg(reg), reg->nr, 0);
95 brw_reg = stride(brw_reg, width * reg->stride, width, reg->stride);
96 }
97
98 brw_reg = retype(brw_reg, reg->type);
99 brw_reg = byte_offset(brw_reg, reg->offset);
100 brw_reg.abs = reg->abs;
101 brw_reg.negate = reg->negate;
102 break;
103 case ARF:
104 case FIXED_GRF:
105 case IMM:
106 assert(reg->offset == 0);
107 brw_reg = reg->as_brw_reg();
108 break;
109 case BAD_FILE:
110 /* Probably unused. */
111 brw_reg = brw_null_reg();
112 break;
113 case ATTR:
114 case UNIFORM:
115 unreachable("not reached");
116 }
117
118 return brw_reg;
119 }
120
121 fs_generator::fs_generator(const struct brw_compiler *compiler, void *log_data,
122 void *mem_ctx,
123 const void *key,
124 struct brw_stage_prog_data *prog_data,
125 unsigned promoted_constants,
126 bool runtime_check_aads_emit,
127 gl_shader_stage stage)
128
129 : compiler(compiler), log_data(log_data),
130 devinfo(compiler->devinfo), key(key),
131 prog_data(prog_data),
132 promoted_constants(promoted_constants),
133 runtime_check_aads_emit(runtime_check_aads_emit), debug_flag(false),
134 stage(stage), mem_ctx(mem_ctx)
135 {
136 p = rzalloc(mem_ctx, struct brw_codegen);
137 brw_init_codegen(devinfo, p, mem_ctx);
138 }
139
140 fs_generator::~fs_generator()
141 {
142 }
143
144 class ip_record : public exec_node {
145 public:
146 DECLARE_RALLOC_CXX_OPERATORS(ip_record)
147
148 ip_record(int ip)
149 {
150 this->ip = ip;
151 }
152
153 int ip;
154 };
155
156 bool
157 fs_generator::patch_discard_jumps_to_fb_writes()
158 {
159 if (devinfo->gen < 6 || this->discard_halt_patches.is_empty())
160 return false;
161
162 int scale = brw_jump_scale(p->devinfo);
163
164 /* There is a somewhat strange undocumented requirement of using
165 * HALT, according to the simulator. If some channel has HALTed to
166 * a particular UIP, then by the end of the program, every channel
167 * must have HALTed to that UIP. Furthermore, the tracking is a
168 * stack, so you can't do the final halt of a UIP after starting
169 * halting to a new UIP.
170 *
171 * Symptoms of not emitting this instruction on actual hardware
172 * included GPU hangs and sparkly rendering on the piglit discard
173 * tests.
174 */
175 brw_inst *last_halt = gen6_HALT(p);
176 brw_inst_set_uip(p->devinfo, last_halt, 1 * scale);
177 brw_inst_set_jip(p->devinfo, last_halt, 1 * scale);
178
179 int ip = p->nr_insn;
180
181 foreach_in_list(ip_record, patch_ip, &discard_halt_patches) {
182 brw_inst *patch = &p->store[patch_ip->ip];
183
184 assert(brw_inst_opcode(p->devinfo, patch) == BRW_OPCODE_HALT);
185 /* HALT takes a half-instruction distance from the pre-incremented IP. */
186 brw_inst_set_uip(p->devinfo, patch, (ip - patch_ip->ip) * scale);
187 }
188
189 this->discard_halt_patches.make_empty();
190 return true;
191 }
192
193 void
194 fs_generator::fire_fb_write(fs_inst *inst,
195 struct brw_reg payload,
196 struct brw_reg implied_header,
197 GLuint nr)
198 {
199 uint32_t msg_control;
200
201 struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
202
203 if (devinfo->gen < 6) {
204 brw_push_insn_state(p);
205 brw_set_default_exec_size(p, BRW_EXECUTE_8);
206 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
207 brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
208 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
209 brw_MOV(p, offset(payload, 1), brw_vec8_grf(1, 0));
210 brw_pop_insn_state(p);
211 }
212
213 if (inst->opcode == FS_OPCODE_REP_FB_WRITE)
214 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD16_SINGLE_SOURCE_REPLICATED;
215 else if (prog_data->dual_src_blend) {
216 if (!inst->group)
217 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_DUAL_SOURCE_SUBSPAN01;
218 else
219 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_DUAL_SOURCE_SUBSPAN23;
220 } else if (inst->exec_size == 16)
221 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD16_SINGLE_SOURCE;
222 else
223 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_SINGLE_SOURCE_SUBSPAN01;
224
225 uint32_t surf_index =
226 prog_data->binding_table.render_target_start + inst->target;
227
228 bool last_render_target = inst->eot ||
229 (prog_data->dual_src_blend && dispatch_width == 16);
230
231
232 brw_fb_WRITE(p,
233 payload,
234 implied_header,
235 msg_control,
236 surf_index,
237 nr,
238 0,
239 inst->eot,
240 last_render_target,
241 inst->header_size != 0);
242
243 brw_mark_surface_used(&prog_data->base, surf_index);
244 }
245
246 void
247 fs_generator::generate_fb_write(fs_inst *inst, struct brw_reg payload)
248 {
249 struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
250 const brw_wm_prog_key * const key = (brw_wm_prog_key * const) this->key;
251 struct brw_reg implied_header;
252
253 if (devinfo->gen < 8 && !devinfo->is_haswell) {
254 brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
255 }
256
257 if (inst->base_mrf >= 0)
258 payload = brw_message_reg(inst->base_mrf);
259
260 /* Header is 2 regs, g0 and g1 are the contents. g0 will be implied
261 * move, here's g1.
262 */
263 if (inst->header_size != 0) {
264 brw_push_insn_state(p);
265 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
266 brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
267 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
268 brw_set_default_flag_reg(p, 0, 0);
269
270 /* On HSW, the GPU will use the predicate on SENDC, unless the header is
271 * present.
272 */
273 if (prog_data->uses_kill) {
274 struct brw_reg pixel_mask;
275
276 if (devinfo->gen >= 6)
277 pixel_mask = retype(brw_vec1_grf(1, 7), BRW_REGISTER_TYPE_UW);
278 else
279 pixel_mask = retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UW);
280
281 brw_MOV(p, pixel_mask, brw_flag_reg(0, 1));
282 }
283
284 if (devinfo->gen >= 6) {
285 brw_push_insn_state(p);
286 brw_set_default_exec_size(p, BRW_EXECUTE_16);
287 brw_set_default_compression_control(p, BRW_COMPRESSION_COMPRESSED);
288 brw_MOV(p,
289 retype(payload, BRW_REGISTER_TYPE_UD),
290 retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));
291 brw_pop_insn_state(p);
292
293 if (inst->target > 0 && key->replicate_alpha) {
294 /* Set "Source0 Alpha Present to RenderTarget" bit in message
295 * header.
296 */
297 brw_OR(p,
298 vec1(retype(payload, BRW_REGISTER_TYPE_UD)),
299 vec1(retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD)),
300 brw_imm_ud(0x1 << 11));
301 }
302
303 if (inst->target > 0) {
304 /* Set the render target index for choosing BLEND_STATE. */
305 brw_MOV(p, retype(vec1(suboffset(payload, 2)),
306 BRW_REGISTER_TYPE_UD),
307 brw_imm_ud(inst->target));
308 }
309
310 /* Set computes stencil to render target */
311 if (prog_data->computed_stencil) {
312 brw_OR(p,
313 vec1(retype(payload, BRW_REGISTER_TYPE_UD)),
314 vec1(retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD)),
315 brw_imm_ud(0x1 << 14));
316 }
317
318 implied_header = brw_null_reg();
319 } else {
320 implied_header = retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UW);
321 }
322
323 brw_pop_insn_state(p);
324 } else {
325 implied_header = brw_null_reg();
326 }
327
328 if (!runtime_check_aads_emit) {
329 fire_fb_write(inst, payload, implied_header, inst->mlen);
330 } else {
331 /* This can only happen in gen < 6 */
332 assert(devinfo->gen < 6);
333
334 struct brw_reg v1_null_ud = vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_UD));
335
336 /* Check runtime bit to detect if we have to send AA data or not */
337 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
338 brw_AND(p,
339 v1_null_ud,
340 retype(brw_vec1_grf(1, 6), BRW_REGISTER_TYPE_UD),
341 brw_imm_ud(1<<26));
342 brw_inst_set_cond_modifier(p->devinfo, brw_last_inst, BRW_CONDITIONAL_NZ);
343
344 int jmp = brw_JMPI(p, brw_imm_ud(0), BRW_PREDICATE_NORMAL) - p->store;
345 brw_inst_set_exec_size(p->devinfo, brw_last_inst, BRW_EXECUTE_1);
346 {
347 /* Don't send AA data */
348 fire_fb_write(inst, offset(payload, 1), implied_header, inst->mlen-1);
349 }
350 brw_land_fwd_jump(p, jmp);
351 fire_fb_write(inst, payload, implied_header, inst->mlen);
352 }
353 }
354
355 void
356 fs_generator::generate_fb_read(fs_inst *inst, struct brw_reg dst,
357 struct brw_reg payload)
358 {
359 assert(inst->size_written % REG_SIZE == 0);
360 struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
361 const unsigned surf_index =
362 prog_data->binding_table.render_target_start + inst->target;
363
364 gen9_fb_READ(p, dst, payload, surf_index,
365 inst->header_size, inst->size_written / REG_SIZE,
366 prog_data->persample_dispatch);
367
368 brw_mark_surface_used(&prog_data->base, surf_index);
369 }
370
371 void
372 fs_generator::generate_mov_indirect(fs_inst *inst,
373 struct brw_reg dst,
374 struct brw_reg reg,
375 struct brw_reg indirect_byte_offset)
376 {
377 assert(indirect_byte_offset.type == BRW_REGISTER_TYPE_UD);
378 assert(indirect_byte_offset.file == BRW_GENERAL_REGISTER_FILE);
379
380 unsigned imm_byte_offset = reg.nr * REG_SIZE + reg.subnr;
381
382 if (indirect_byte_offset.file == BRW_IMMEDIATE_VALUE) {
383 imm_byte_offset += indirect_byte_offset.ud;
384
385 reg.nr = imm_byte_offset / REG_SIZE;
386 reg.subnr = imm_byte_offset % REG_SIZE;
387 brw_MOV(p, dst, reg);
388 } else {
389 /* Prior to Broadwell, there are only 8 address registers. */
390 assert(inst->exec_size == 8 || devinfo->gen >= 8);
391
392 /* We use VxH indirect addressing, clobbering a0.0 through a0.7. */
393 struct brw_reg addr = vec8(brw_address_reg(0));
394
395 /* The destination stride of an instruction (in bytes) must be greater
396 * than or equal to the size of the rest of the instruction. Since the
397 * address register is of type UW, we can't use a D-type instruction.
398 * In order to get around this, re retype to UW and use a stride.
399 */
400 indirect_byte_offset =
401 retype(spread(indirect_byte_offset, 2), BRW_REGISTER_TYPE_UW);
402
403 /* There are a number of reasons why we don't use the base offset here.
404 * One reason is that the field is only 9 bits which means we can only
405 * use it to access the first 16 GRFs. Also, from the Haswell PRM
406 * section "Register Region Restrictions":
407 *
408 * "The lower bits of the AddressImmediate must not overflow to
409 * change the register address. The lower 5 bits of Address
410 * Immediate when added to lower 5 bits of address register gives
411 * the sub-register offset. The upper bits of Address Immediate
412 * when added to upper bits of address register gives the register
413 * address. Any overflow from sub-register offset is dropped."
414 *
415 * Since the indirect may cause us to cross a register boundary, this
416 * makes the base offset almost useless. We could try and do something
417 * clever where we use a actual base offset if base_offset % 32 == 0 but
418 * that would mean we were generating different code depending on the
419 * base offset. Instead, for the sake of consistency, we'll just do the
420 * add ourselves. This restriction is only listed in the Haswell PRM
421 * but empirical testing indicates that it applies on all older
422 * generations and is lifted on Broadwell.
423 *
424 * In the end, while base_offset is nice to look at in the generated
425 * code, using it saves us 0 instructions and would require quite a bit
426 * of case-by-case work. It's just not worth it.
427 */
428 brw_ADD(p, addr, indirect_byte_offset, brw_imm_uw(imm_byte_offset));
429 struct brw_reg ind_src = brw_VxH_indirect(0, 0);
430
431 brw_inst *mov = brw_MOV(p, dst, retype(ind_src, dst.type));
432
433 if (devinfo->gen == 6 && dst.file == BRW_MESSAGE_REGISTER_FILE &&
434 !inst->get_next()->is_tail_sentinel() &&
435 ((fs_inst *)inst->get_next())->mlen > 0) {
436 /* From the Sandybridge PRM:
437 *
438 * "[Errata: DevSNB(SNB)] If MRF register is updated by any
439 * instruction that “indexed/indirect” source AND is followed by a
440 * send, the instruction requires a “Switch”. This is to avoid
441 * race condition where send may dispatch before MRF is updated."
442 */
443 brw_inst_set_thread_control(devinfo, mov, BRW_THREAD_SWITCH);
444 }
445 }
446 }
447
448 void
449 fs_generator::generate_urb_read(fs_inst *inst,
450 struct brw_reg dst,
451 struct brw_reg header)
452 {
453 assert(inst->size_written % REG_SIZE == 0);
454 assert(header.file == BRW_GENERAL_REGISTER_FILE);
455 assert(header.type == BRW_REGISTER_TYPE_UD);
456
457 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
458 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UD));
459 brw_set_src0(p, send, header);
460 brw_set_src1(p, send, brw_imm_ud(0u));
461
462 brw_inst_set_sfid(p->devinfo, send, BRW_SFID_URB);
463 brw_inst_set_urb_opcode(p->devinfo, send, GEN8_URB_OPCODE_SIMD8_READ);
464
465 if (inst->opcode == SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT)
466 brw_inst_set_urb_per_slot_offset(p->devinfo, send, true);
467
468 brw_inst_set_mlen(p->devinfo, send, inst->mlen);
469 brw_inst_set_rlen(p->devinfo, send, inst->size_written / REG_SIZE);
470 brw_inst_set_header_present(p->devinfo, send, true);
471 brw_inst_set_urb_global_offset(p->devinfo, send, inst->offset);
472 }
473
474 void
475 fs_generator::generate_urb_write(fs_inst *inst, struct brw_reg payload)
476 {
477 brw_inst *insn;
478
479 insn = brw_next_insn(p, BRW_OPCODE_SEND);
480
481 brw_set_dest(p, insn, brw_null_reg());
482 brw_set_src0(p, insn, payload);
483 brw_set_src1(p, insn, brw_imm_d(0));
484
485 brw_inst_set_sfid(p->devinfo, insn, BRW_SFID_URB);
486 brw_inst_set_urb_opcode(p->devinfo, insn, GEN8_URB_OPCODE_SIMD8_WRITE);
487
488 if (inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT ||
489 inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT)
490 brw_inst_set_urb_per_slot_offset(p->devinfo, insn, true);
491
492 if (inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_MASKED ||
493 inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT)
494 brw_inst_set_urb_channel_mask_present(p->devinfo, insn, true);
495
496 brw_inst_set_mlen(p->devinfo, insn, inst->mlen);
497 brw_inst_set_rlen(p->devinfo, insn, 0);
498 brw_inst_set_eot(p->devinfo, insn, inst->eot);
499 brw_inst_set_header_present(p->devinfo, insn, true);
500 brw_inst_set_urb_global_offset(p->devinfo, insn, inst->offset);
501 }
502
503 void
504 fs_generator::generate_cs_terminate(fs_inst *inst, struct brw_reg payload)
505 {
506 struct brw_inst *insn;
507
508 insn = brw_next_insn(p, BRW_OPCODE_SEND);
509
510 brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW));
511 brw_set_src0(p, insn, payload);
512 brw_set_src1(p, insn, brw_imm_d(0));
513
514 /* Terminate a compute shader by sending a message to the thread spawner.
515 */
516 brw_inst_set_sfid(devinfo, insn, BRW_SFID_THREAD_SPAWNER);
517 brw_inst_set_mlen(devinfo, insn, 1);
518 brw_inst_set_rlen(devinfo, insn, 0);
519 brw_inst_set_eot(devinfo, insn, inst->eot);
520 brw_inst_set_header_present(devinfo, insn, false);
521
522 brw_inst_set_ts_opcode(devinfo, insn, 0); /* Dereference resource */
523 brw_inst_set_ts_request_type(devinfo, insn, 0); /* Root thread */
524
525 /* Note that even though the thread has a URB resource associated with it,
526 * we set the "do not dereference URB" bit, because the URB resource is
527 * managed by the fixed-function unit, so it will free it automatically.
528 */
529 brw_inst_set_ts_resource_select(devinfo, insn, 1); /* Do not dereference URB */
530
531 brw_inst_set_mask_control(devinfo, insn, BRW_MASK_DISABLE);
532 }
533
534 void
535 fs_generator::generate_barrier(fs_inst *inst, struct brw_reg src)
536 {
537 brw_barrier(p, src);
538 brw_WAIT(p);
539 }
540
541 void
542 fs_generator::generate_linterp(fs_inst *inst,
543 struct brw_reg dst, struct brw_reg *src)
544 {
545 /* PLN reads:
546 * / in SIMD16 \
547 * -----------------------------------
548 * | src1+0 | src1+1 | src1+2 | src1+3 |
549 * |-----------------------------------|
550 * |(x0, x1)|(y0, y1)|(x2, x3)|(y2, y3)|
551 * -----------------------------------
552 *
553 * but for the LINE/MAC pair, the LINE reads Xs and the MAC reads Ys:
554 *
555 * -----------------------------------
556 * | src1+0 | src1+1 | src1+2 | src1+3 |
557 * |-----------------------------------|
558 * |(x0, x1)|(y0, y1)| | | in SIMD8
559 * |-----------------------------------|
560 * |(x0, x1)|(x2, x3)|(y0, y1)|(y2, y3)| in SIMD16
561 * -----------------------------------
562 *
563 * See also: emit_interpolation_setup_gen4().
564 */
565 struct brw_reg delta_x = src[0];
566 struct brw_reg delta_y = offset(src[0], inst->exec_size / 8);
567 struct brw_reg interp = src[1];
568
569 if (devinfo->has_pln &&
570 (devinfo->gen >= 7 || (delta_x.nr & 1) == 0)) {
571 brw_PLN(p, dst, interp, delta_x);
572 } else {
573 brw_LINE(p, brw_null_reg(), interp, delta_x);
574 brw_MAC(p, dst, suboffset(interp, 1), delta_y);
575 }
576 }
577
578 void
579 fs_generator::generate_get_buffer_size(fs_inst *inst,
580 struct brw_reg dst,
581 struct brw_reg src,
582 struct brw_reg surf_index)
583 {
584 assert(devinfo->gen >= 7);
585 assert(surf_index.file == BRW_IMMEDIATE_VALUE);
586
587 uint32_t simd_mode;
588 int rlen = 4;
589
590 switch (inst->exec_size) {
591 case 8:
592 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
593 break;
594 case 16:
595 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
596 break;
597 default:
598 unreachable("Invalid width for texture instruction");
599 }
600
601 if (simd_mode == BRW_SAMPLER_SIMD_MODE_SIMD16) {
602 rlen = 8;
603 dst = vec16(dst);
604 }
605
606 brw_SAMPLE(p,
607 retype(dst, BRW_REGISTER_TYPE_UW),
608 inst->base_mrf,
609 src,
610 surf_index.ud,
611 0,
612 GEN5_SAMPLER_MESSAGE_SAMPLE_RESINFO,
613 rlen, /* response length */
614 inst->mlen,
615 inst->header_size > 0,
616 simd_mode,
617 BRW_SAMPLER_RETURN_FORMAT_SINT32);
618
619 brw_mark_surface_used(prog_data, surf_index.ud);
620 }
621
622 void
623 fs_generator::generate_tex(fs_inst *inst, struct brw_reg dst, struct brw_reg src,
624 struct brw_reg surface_index,
625 struct brw_reg sampler_index)
626 {
627 assert(inst->size_written % REG_SIZE == 0);
628 int msg_type = -1;
629 uint32_t simd_mode;
630 uint32_t return_format;
631 bool is_combined_send = inst->eot;
632
633 switch (dst.type) {
634 case BRW_REGISTER_TYPE_D:
635 return_format = BRW_SAMPLER_RETURN_FORMAT_SINT32;
636 break;
637 case BRW_REGISTER_TYPE_UD:
638 return_format = BRW_SAMPLER_RETURN_FORMAT_UINT32;
639 break;
640 default:
641 return_format = BRW_SAMPLER_RETURN_FORMAT_FLOAT32;
642 break;
643 }
644
645 /* Stomp the resinfo output type to UINT32. On gens 4-5, the output type
646 * is set as part of the message descriptor. On gen4, the PRM seems to
647 * allow UINT32 and FLOAT32 (i965 PRM, Vol. 4 Section 4.8.1.1), but on
648 * later gens UINT32 is required. Once you hit Sandy Bridge, the bit is
649 * gone from the message descriptor entirely and you just get UINT32 all
650 * the time regasrdless. Since we can really only do non-UINT32 on gen4,
651 * just stomp it to UINT32 all the time.
652 */
653 if (inst->opcode == SHADER_OPCODE_TXS)
654 return_format = BRW_SAMPLER_RETURN_FORMAT_UINT32;
655
656 switch (inst->exec_size) {
657 case 8:
658 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
659 break;
660 case 16:
661 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
662 break;
663 default:
664 unreachable("Invalid width for texture instruction");
665 }
666
667 if (devinfo->gen >= 5) {
668 switch (inst->opcode) {
669 case SHADER_OPCODE_TEX:
670 if (inst->shadow_compare) {
671 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_COMPARE;
672 } else {
673 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE;
674 }
675 break;
676 case FS_OPCODE_TXB:
677 if (inst->shadow_compare) {
678 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_BIAS_COMPARE;
679 } else {
680 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_BIAS;
681 }
682 break;
683 case SHADER_OPCODE_TXL:
684 if (inst->shadow_compare) {
685 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LOD_COMPARE;
686 } else {
687 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LOD;
688 }
689 break;
690 case SHADER_OPCODE_TXL_LZ:
691 assert(devinfo->gen >= 9);
692 if (inst->shadow_compare) {
693 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_C_LZ;
694 } else {
695 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_LZ;
696 }
697 break;
698 case SHADER_OPCODE_TXS:
699 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_RESINFO;
700 break;
701 case SHADER_OPCODE_TXD:
702 if (inst->shadow_compare) {
703 /* Gen7.5+. Otherwise, lowered in NIR */
704 assert(devinfo->gen >= 8 || devinfo->is_haswell);
705 msg_type = HSW_SAMPLER_MESSAGE_SAMPLE_DERIV_COMPARE;
706 } else {
707 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_DERIVS;
708 }
709 break;
710 case SHADER_OPCODE_TXF:
711 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
712 break;
713 case SHADER_OPCODE_TXF_LZ:
714 assert(devinfo->gen >= 9);
715 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_LD_LZ;
716 break;
717 case SHADER_OPCODE_TXF_CMS_W:
718 assert(devinfo->gen >= 9);
719 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_LD2DMS_W;
720 break;
721 case SHADER_OPCODE_TXF_CMS:
722 if (devinfo->gen >= 7)
723 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD2DMS;
724 else
725 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
726 break;
727 case SHADER_OPCODE_TXF_UMS:
728 assert(devinfo->gen >= 7);
729 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD2DSS;
730 break;
731 case SHADER_OPCODE_TXF_MCS:
732 assert(devinfo->gen >= 7);
733 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD_MCS;
734 break;
735 case SHADER_OPCODE_LOD:
736 msg_type = GEN5_SAMPLER_MESSAGE_LOD;
737 break;
738 case SHADER_OPCODE_TG4:
739 if (inst->shadow_compare) {
740 assert(devinfo->gen >= 7);
741 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_C;
742 } else {
743 assert(devinfo->gen >= 6);
744 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4;
745 }
746 break;
747 case SHADER_OPCODE_TG4_OFFSET:
748 assert(devinfo->gen >= 7);
749 if (inst->shadow_compare) {
750 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_PO_C;
751 } else {
752 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_PO;
753 }
754 break;
755 case SHADER_OPCODE_SAMPLEINFO:
756 msg_type = GEN6_SAMPLER_MESSAGE_SAMPLE_SAMPLEINFO;
757 break;
758 default:
759 unreachable("not reached");
760 }
761 } else {
762 switch (inst->opcode) {
763 case SHADER_OPCODE_TEX:
764 /* Note that G45 and older determines shadow compare and dispatch width
765 * from message length for most messages.
766 */
767 if (inst->exec_size == 8) {
768 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE;
769 if (inst->shadow_compare) {
770 assert(inst->mlen == 6);
771 } else {
772 assert(inst->mlen <= 4);
773 }
774 } else {
775 if (inst->shadow_compare) {
776 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE_COMPARE;
777 assert(inst->mlen == 9);
778 } else {
779 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE;
780 assert(inst->mlen <= 7 && inst->mlen % 2 == 1);
781 }
782 }
783 break;
784 case FS_OPCODE_TXB:
785 if (inst->shadow_compare) {
786 assert(inst->exec_size == 8);
787 assert(inst->mlen == 6);
788 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE_BIAS_COMPARE;
789 } else {
790 assert(inst->mlen == 9);
791 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE_BIAS;
792 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
793 }
794 break;
795 case SHADER_OPCODE_TXL:
796 if (inst->shadow_compare) {
797 assert(inst->exec_size == 8);
798 assert(inst->mlen == 6);
799 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE_LOD_COMPARE;
800 } else {
801 assert(inst->mlen == 9);
802 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE_LOD;
803 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
804 }
805 break;
806 case SHADER_OPCODE_TXD:
807 /* There is no sample_d_c message; comparisons are done manually */
808 assert(inst->exec_size == 8);
809 assert(inst->mlen == 7 || inst->mlen == 10);
810 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE_GRADIENTS;
811 break;
812 case SHADER_OPCODE_TXF:
813 assert(inst->mlen <= 9 && inst->mlen % 2 == 1);
814 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_LD;
815 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
816 break;
817 case SHADER_OPCODE_TXS:
818 assert(inst->mlen == 3);
819 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_RESINFO;
820 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
821 break;
822 default:
823 unreachable("not reached");
824 }
825 }
826 assert(msg_type != -1);
827
828 if (simd_mode == BRW_SAMPLER_SIMD_MODE_SIMD16) {
829 dst = vec16(dst);
830 }
831
832 assert(devinfo->gen < 7 || inst->header_size == 0 ||
833 src.file == BRW_GENERAL_REGISTER_FILE);
834
835 assert(sampler_index.type == BRW_REGISTER_TYPE_UD);
836
837 /* Load the message header if present. If there's a texture offset,
838 * we need to set it up explicitly and load the offset bitfield.
839 * Otherwise, we can use an implied move from g0 to the first message reg.
840 */
841 if (inst->header_size != 0) {
842 if (devinfo->gen < 6 && !inst->offset) {
843 /* Set up an implied move from g0 to the MRF. */
844 src = retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UW);
845 } else {
846 struct brw_reg header_reg;
847
848 if (devinfo->gen >= 7) {
849 header_reg = src;
850 } else {
851 assert(inst->base_mrf != -1);
852 header_reg = brw_message_reg(inst->base_mrf);
853 }
854
855 brw_push_insn_state(p);
856 brw_set_default_exec_size(p, BRW_EXECUTE_8);
857 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
858 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
859 /* Explicitly set up the message header by copying g0 to the MRF. */
860 brw_MOV(p, header_reg, brw_vec8_grf(0, 0));
861
862 if (inst->offset) {
863 /* Set the offset bits in DWord 2. */
864 brw_MOV(p, get_element_ud(header_reg, 2),
865 brw_imm_ud(inst->offset));
866 } else if (stage != MESA_SHADER_VERTEX &&
867 stage != MESA_SHADER_FRAGMENT) {
868 /* The vertex and fragment stages have g0.2 set to 0, so
869 * header0.2 is 0 when g0 is copied. Other stages may not, so we
870 * must set it to 0 to avoid setting undesirable bits in the
871 * message.
872 */
873 brw_MOV(p, get_element_ud(header_reg, 2), brw_imm_ud(0));
874 }
875
876 brw_adjust_sampler_state_pointer(p, header_reg, sampler_index);
877 brw_pop_insn_state(p);
878 }
879 }
880
881 uint32_t base_binding_table_index = (inst->opcode == SHADER_OPCODE_TG4 ||
882 inst->opcode == SHADER_OPCODE_TG4_OFFSET)
883 ? prog_data->binding_table.gather_texture_start
884 : prog_data->binding_table.texture_start;
885
886 if (surface_index.file == BRW_IMMEDIATE_VALUE &&
887 sampler_index.file == BRW_IMMEDIATE_VALUE) {
888 uint32_t surface = surface_index.ud;
889 uint32_t sampler = sampler_index.ud;
890
891 brw_SAMPLE(p,
892 retype(dst, BRW_REGISTER_TYPE_UW),
893 inst->base_mrf,
894 src,
895 surface + base_binding_table_index,
896 sampler % 16,
897 msg_type,
898 inst->size_written / REG_SIZE,
899 inst->mlen,
900 inst->header_size != 0,
901 simd_mode,
902 return_format);
903
904 brw_mark_surface_used(prog_data, surface + base_binding_table_index);
905 } else {
906 /* Non-const sampler index */
907
908 struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
909 struct brw_reg surface_reg = vec1(retype(surface_index, BRW_REGISTER_TYPE_UD));
910 struct brw_reg sampler_reg = vec1(retype(sampler_index, BRW_REGISTER_TYPE_UD));
911
912 brw_push_insn_state(p);
913 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
914 brw_set_default_access_mode(p, BRW_ALIGN_1);
915
916 if (brw_regs_equal(&surface_reg, &sampler_reg)) {
917 brw_MUL(p, addr, sampler_reg, brw_imm_uw(0x101));
918 } else {
919 brw_SHL(p, addr, sampler_reg, brw_imm_ud(8));
920 brw_OR(p, addr, addr, surface_reg);
921 }
922 if (base_binding_table_index)
923 brw_ADD(p, addr, addr, brw_imm_ud(base_binding_table_index));
924 brw_AND(p, addr, addr, brw_imm_ud(0xfff));
925
926 brw_pop_insn_state(p);
927
928 /* dst = send(offset, a0.0 | <descriptor>) */
929 brw_inst *insn = brw_send_indirect_message(
930 p, BRW_SFID_SAMPLER, dst, src, addr);
931 brw_set_sampler_message(p, insn,
932 0 /* surface */,
933 0 /* sampler */,
934 msg_type,
935 inst->size_written / REG_SIZE,
936 inst->mlen /* mlen */,
937 inst->header_size != 0 /* header */,
938 simd_mode,
939 return_format);
940
941 /* visitor knows more than we do about the surface limit required,
942 * so has already done marking.
943 */
944 }
945
946 if (is_combined_send) {
947 brw_inst_set_eot(p->devinfo, brw_last_inst, true);
948 brw_inst_set_opcode(p->devinfo, brw_last_inst, BRW_OPCODE_SENDC);
949 }
950 }
951
952
953 /* For OPCODE_DDX and OPCODE_DDY, per channel of output we've got input
954 * looking like:
955 *
956 * arg0: ss0.tl ss0.tr ss0.bl ss0.br ss1.tl ss1.tr ss1.bl ss1.br
957 *
958 * Ideally, we want to produce:
959 *
960 * DDX DDY
961 * dst: (ss0.tr - ss0.tl) (ss0.tl - ss0.bl)
962 * (ss0.tr - ss0.tl) (ss0.tr - ss0.br)
963 * (ss0.br - ss0.bl) (ss0.tl - ss0.bl)
964 * (ss0.br - ss0.bl) (ss0.tr - ss0.br)
965 * (ss1.tr - ss1.tl) (ss1.tl - ss1.bl)
966 * (ss1.tr - ss1.tl) (ss1.tr - ss1.br)
967 * (ss1.br - ss1.bl) (ss1.tl - ss1.bl)
968 * (ss1.br - ss1.bl) (ss1.tr - ss1.br)
969 *
970 * and add another set of two more subspans if in 16-pixel dispatch mode.
971 *
972 * For DDX, it ends up being easy: width = 2, horiz=0 gets us the same result
973 * for each pair, and vertstride = 2 jumps us 2 elements after processing a
974 * pair. But the ideal approximation may impose a huge performance cost on
975 * sample_d. On at least Haswell, sample_d instruction does some
976 * optimizations if the same LOD is used for all pixels in the subspan.
977 *
978 * For DDY, we need to use ALIGN16 mode since it's capable of doing the
979 * appropriate swizzling.
980 */
981 void
982 fs_generator::generate_ddx(enum opcode opcode,
983 struct brw_reg dst, struct brw_reg src)
984 {
985 unsigned vstride, width;
986
987 if (opcode == FS_OPCODE_DDX_FINE) {
988 /* produce accurate derivatives */
989 vstride = BRW_VERTICAL_STRIDE_2;
990 width = BRW_WIDTH_2;
991 } else {
992 /* replicate the derivative at the top-left pixel to other pixels */
993 vstride = BRW_VERTICAL_STRIDE_4;
994 width = BRW_WIDTH_4;
995 }
996
997 struct brw_reg src0 = brw_reg(src.file, src.nr, 1,
998 src.negate, src.abs,
999 BRW_REGISTER_TYPE_F,
1000 vstride,
1001 width,
1002 BRW_HORIZONTAL_STRIDE_0,
1003 BRW_SWIZZLE_XYZW, WRITEMASK_XYZW);
1004 struct brw_reg src1 = brw_reg(src.file, src.nr, 0,
1005 src.negate, src.abs,
1006 BRW_REGISTER_TYPE_F,
1007 vstride,
1008 width,
1009 BRW_HORIZONTAL_STRIDE_0,
1010 BRW_SWIZZLE_XYZW, WRITEMASK_XYZW);
1011 brw_ADD(p, dst, src0, negate(src1));
1012 }
1013
1014 /* The negate_value boolean is used to negate the derivative computation for
1015 * FBOs, since they place the origin at the upper left instead of the lower
1016 * left.
1017 */
1018 void
1019 fs_generator::generate_ddy(enum opcode opcode,
1020 struct brw_reg dst, struct brw_reg src)
1021 {
1022 if (opcode == FS_OPCODE_DDY_FINE) {
1023 /* produce accurate derivatives */
1024 struct brw_reg src0 = brw_reg(src.file, src.nr, 0,
1025 src.negate, src.abs,
1026 BRW_REGISTER_TYPE_F,
1027 BRW_VERTICAL_STRIDE_4,
1028 BRW_WIDTH_4,
1029 BRW_HORIZONTAL_STRIDE_1,
1030 BRW_SWIZZLE_XYXY, WRITEMASK_XYZW);
1031 struct brw_reg src1 = brw_reg(src.file, src.nr, 0,
1032 src.negate, src.abs,
1033 BRW_REGISTER_TYPE_F,
1034 BRW_VERTICAL_STRIDE_4,
1035 BRW_WIDTH_4,
1036 BRW_HORIZONTAL_STRIDE_1,
1037 BRW_SWIZZLE_ZWZW, WRITEMASK_XYZW);
1038 brw_push_insn_state(p);
1039 brw_set_default_access_mode(p, BRW_ALIGN_16);
1040 brw_ADD(p, dst, negate(src0), src1);
1041 brw_pop_insn_state(p);
1042 } else {
1043 /* replicate the derivative at the top-left pixel to other pixels */
1044 struct brw_reg src0 = brw_reg(src.file, src.nr, 0,
1045 src.negate, src.abs,
1046 BRW_REGISTER_TYPE_F,
1047 BRW_VERTICAL_STRIDE_4,
1048 BRW_WIDTH_4,
1049 BRW_HORIZONTAL_STRIDE_0,
1050 BRW_SWIZZLE_XYZW, WRITEMASK_XYZW);
1051 struct brw_reg src1 = brw_reg(src.file, src.nr, 2,
1052 src.negate, src.abs,
1053 BRW_REGISTER_TYPE_F,
1054 BRW_VERTICAL_STRIDE_4,
1055 BRW_WIDTH_4,
1056 BRW_HORIZONTAL_STRIDE_0,
1057 BRW_SWIZZLE_XYZW, WRITEMASK_XYZW);
1058 brw_ADD(p, dst, negate(src0), src1);
1059 }
1060 }
1061
1062 void
1063 fs_generator::generate_discard_jump(fs_inst *inst)
1064 {
1065 assert(devinfo->gen >= 6);
1066
1067 /* This HALT will be patched up at FB write time to point UIP at the end of
1068 * the program, and at brw_uip_jip() JIP will be set to the end of the
1069 * current block (or the program).
1070 */
1071 this->discard_halt_patches.push_tail(new(mem_ctx) ip_record(p->nr_insn));
1072 gen6_HALT(p);
1073 }
1074
1075 void
1076 fs_generator::generate_scratch_write(fs_inst *inst, struct brw_reg src)
1077 {
1078 /* The 32-wide messages only respect the first 16-wide half of the channel
1079 * enable signals which are replicated identically for the second group of
1080 * 16 channels, so we cannot use them unless the write is marked
1081 * force_writemask_all.
1082 */
1083 const unsigned lower_size = inst->force_writemask_all ? inst->exec_size :
1084 MIN2(16, inst->exec_size);
1085 const unsigned block_size = 4 * lower_size / REG_SIZE;
1086 assert(inst->mlen != 0);
1087
1088 brw_push_insn_state(p);
1089 brw_set_default_exec_size(p, cvt(lower_size) - 1);
1090 brw_set_default_compression(p, lower_size > 8);
1091
1092 for (unsigned i = 0; i < inst->exec_size / lower_size; i++) {
1093 brw_set_default_group(p, inst->group + lower_size * i);
1094
1095 brw_MOV(p, brw_uvec_mrf(lower_size, inst->base_mrf + 1, 0),
1096 retype(offset(src, block_size * i), BRW_REGISTER_TYPE_UD));
1097
1098 brw_oword_block_write_scratch(p, brw_message_reg(inst->base_mrf),
1099 block_size,
1100 inst->offset + block_size * REG_SIZE * i);
1101 }
1102
1103 brw_pop_insn_state(p);
1104 }
1105
1106 void
1107 fs_generator::generate_scratch_read(fs_inst *inst, struct brw_reg dst)
1108 {
1109 assert(inst->exec_size <= 16 || inst->force_writemask_all);
1110 assert(inst->mlen != 0);
1111
1112 brw_oword_block_read_scratch(p, dst, brw_message_reg(inst->base_mrf),
1113 inst->exec_size / 8, inst->offset);
1114 }
1115
1116 void
1117 fs_generator::generate_scratch_read_gen7(fs_inst *inst, struct brw_reg dst)
1118 {
1119 assert(inst->exec_size <= 16 || inst->force_writemask_all);
1120
1121 gen7_block_read_scratch(p, dst, inst->exec_size / 8, inst->offset);
1122 }
1123
1124 void
1125 fs_generator::generate_uniform_pull_constant_load(fs_inst *inst,
1126 struct brw_reg dst,
1127 struct brw_reg index,
1128 struct brw_reg offset)
1129 {
1130 assert(type_sz(dst.type) == 4);
1131 assert(inst->mlen != 0);
1132
1133 assert(index.file == BRW_IMMEDIATE_VALUE &&
1134 index.type == BRW_REGISTER_TYPE_UD);
1135 uint32_t surf_index = index.ud;
1136
1137 assert(offset.file == BRW_IMMEDIATE_VALUE &&
1138 offset.type == BRW_REGISTER_TYPE_UD);
1139 uint32_t read_offset = offset.ud;
1140
1141 brw_oword_block_read(p, dst, brw_message_reg(inst->base_mrf),
1142 read_offset, surf_index);
1143 }
1144
1145 void
1146 fs_generator::generate_uniform_pull_constant_load_gen7(fs_inst *inst,
1147 struct brw_reg dst,
1148 struct brw_reg index,
1149 struct brw_reg payload)
1150 {
1151 assert(index.type == BRW_REGISTER_TYPE_UD);
1152 assert(payload.file == BRW_GENERAL_REGISTER_FILE);
1153 assert(type_sz(dst.type) == 4);
1154
1155 if (index.file == BRW_IMMEDIATE_VALUE) {
1156 const uint32_t surf_index = index.ud;
1157
1158 brw_push_insn_state(p);
1159 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1160 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
1161 brw_pop_insn_state(p);
1162
1163 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UD));
1164 brw_set_src0(p, send, retype(payload, BRW_REGISTER_TYPE_UD));
1165 brw_set_dp_read_message(p, send, surf_index,
1166 BRW_DATAPORT_OWORD_BLOCK_DWORDS(inst->exec_size),
1167 GEN7_DATAPORT_DC_OWORD_BLOCK_READ,
1168 GEN6_SFID_DATAPORT_CONSTANT_CACHE,
1169 1, /* mlen */
1170 true, /* header */
1171 DIV_ROUND_UP(inst->size_written, REG_SIZE));
1172
1173 } else {
1174 struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
1175
1176 brw_push_insn_state(p);
1177 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1178 brw_set_default_access_mode(p, BRW_ALIGN_1);
1179
1180 /* a0.0 = surf_index & 0xff */
1181 brw_inst *insn_and = brw_next_insn(p, BRW_OPCODE_AND);
1182 brw_inst_set_exec_size(p->devinfo, insn_and, BRW_EXECUTE_1);
1183 brw_set_dest(p, insn_and, addr);
1184 brw_set_src0(p, insn_and, vec1(retype(index, BRW_REGISTER_TYPE_UD)));
1185 brw_set_src1(p, insn_and, brw_imm_ud(0x0ff));
1186
1187 /* dst = send(payload, a0.0 | <descriptor>) */
1188 brw_inst *insn = brw_send_indirect_message(
1189 p, GEN6_SFID_DATAPORT_CONSTANT_CACHE,
1190 retype(dst, BRW_REGISTER_TYPE_UD),
1191 retype(payload, BRW_REGISTER_TYPE_UD), addr);
1192 brw_set_dp_read_message(p, insn, 0 /* surface */,
1193 BRW_DATAPORT_OWORD_BLOCK_DWORDS(inst->exec_size),
1194 GEN7_DATAPORT_DC_OWORD_BLOCK_READ,
1195 GEN6_SFID_DATAPORT_CONSTANT_CACHE,
1196 1, /* mlen */
1197 true, /* header */
1198 DIV_ROUND_UP(inst->size_written, REG_SIZE));
1199
1200 brw_pop_insn_state(p);
1201 }
1202 }
1203
1204 void
1205 fs_generator::generate_varying_pull_constant_load_gen4(fs_inst *inst,
1206 struct brw_reg dst,
1207 struct brw_reg index)
1208 {
1209 assert(devinfo->gen < 7); /* Should use the gen7 variant. */
1210 assert(inst->header_size != 0);
1211 assert(inst->mlen);
1212
1213 assert(index.file == BRW_IMMEDIATE_VALUE &&
1214 index.type == BRW_REGISTER_TYPE_UD);
1215 uint32_t surf_index = index.ud;
1216
1217 uint32_t simd_mode, rlen, msg_type;
1218 if (inst->exec_size == 16) {
1219 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1220 rlen = 8;
1221 } else {
1222 assert(inst->exec_size == 8);
1223 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
1224 rlen = 4;
1225 }
1226
1227 if (devinfo->gen >= 5)
1228 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
1229 else {
1230 /* We always use the SIMD16 message so that we only have to load U, and
1231 * not V or R.
1232 */
1233 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_LD;
1234 assert(inst->mlen == 3);
1235 assert(inst->size_written == 8 * REG_SIZE);
1236 rlen = 8;
1237 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1238 }
1239
1240 struct brw_reg header = brw_vec8_grf(0, 0);
1241 gen6_resolve_implied_move(p, &header, inst->base_mrf);
1242
1243 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
1244 brw_inst_set_compression(devinfo, send, false);
1245 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UW));
1246 brw_set_src0(p, send, header);
1247 if (devinfo->gen < 6)
1248 brw_inst_set_base_mrf(p->devinfo, send, inst->base_mrf);
1249
1250 /* Our surface is set up as floats, regardless of what actual data is
1251 * stored in it.
1252 */
1253 uint32_t return_format = BRW_SAMPLER_RETURN_FORMAT_FLOAT32;
1254 brw_set_sampler_message(p, send,
1255 surf_index,
1256 0, /* sampler (unused) */
1257 msg_type,
1258 rlen,
1259 inst->mlen,
1260 inst->header_size != 0,
1261 simd_mode,
1262 return_format);
1263 }
1264
1265 void
1266 fs_generator::generate_varying_pull_constant_load_gen7(fs_inst *inst,
1267 struct brw_reg dst,
1268 struct brw_reg index,
1269 struct brw_reg offset)
1270 {
1271 assert(devinfo->gen >= 7);
1272 /* Varying-offset pull constant loads are treated as a normal expression on
1273 * gen7, so the fact that it's a send message is hidden at the IR level.
1274 */
1275 assert(inst->header_size == 0);
1276 assert(!inst->mlen);
1277 assert(index.type == BRW_REGISTER_TYPE_UD);
1278
1279 uint32_t simd_mode, rlen, mlen;
1280 if (inst->exec_size == 16) {
1281 mlen = 2;
1282 rlen = 8;
1283 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1284 } else {
1285 assert(inst->exec_size == 8);
1286 mlen = 1;
1287 rlen = 4;
1288 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
1289 }
1290
1291 if (index.file == BRW_IMMEDIATE_VALUE) {
1292
1293 uint32_t surf_index = index.ud;
1294
1295 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
1296 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UW));
1297 brw_set_src0(p, send, offset);
1298 brw_set_sampler_message(p, send,
1299 surf_index,
1300 0, /* LD message ignores sampler unit */
1301 GEN5_SAMPLER_MESSAGE_SAMPLE_LD,
1302 rlen,
1303 mlen,
1304 false, /* no header */
1305 simd_mode,
1306 0);
1307
1308 } else {
1309
1310 struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
1311
1312 brw_push_insn_state(p);
1313 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1314 brw_set_default_access_mode(p, BRW_ALIGN_1);
1315
1316 /* a0.0 = surf_index & 0xff */
1317 brw_inst *insn_and = brw_next_insn(p, BRW_OPCODE_AND);
1318 brw_inst_set_exec_size(p->devinfo, insn_and, BRW_EXECUTE_1);
1319 brw_set_dest(p, insn_and, addr);
1320 brw_set_src0(p, insn_and, vec1(retype(index, BRW_REGISTER_TYPE_UD)));
1321 brw_set_src1(p, insn_and, brw_imm_ud(0x0ff));
1322
1323 brw_pop_insn_state(p);
1324
1325 /* dst = send(offset, a0.0 | <descriptor>) */
1326 brw_inst *insn = brw_send_indirect_message(
1327 p, BRW_SFID_SAMPLER, retype(dst, BRW_REGISTER_TYPE_UW),
1328 offset, addr);
1329 brw_set_sampler_message(p, insn,
1330 0 /* surface */,
1331 0 /* sampler */,
1332 GEN5_SAMPLER_MESSAGE_SAMPLE_LD,
1333 rlen /* rlen */,
1334 mlen /* mlen */,
1335 false /* header */,
1336 simd_mode,
1337 0);
1338 }
1339 }
1340
1341 /**
1342 * Cause the current pixel/sample mask (from R1.7 bits 15:0) to be transferred
1343 * into the flags register (f0.0).
1344 *
1345 * Used only on Gen6 and above.
1346 */
1347 void
1348 fs_generator::generate_mov_dispatch_to_flags(fs_inst *inst)
1349 {
1350 struct brw_reg flags = brw_flag_reg(0, inst->flag_subreg);
1351 struct brw_reg dispatch_mask;
1352
1353 if (devinfo->gen >= 6)
1354 dispatch_mask = retype(brw_vec1_grf(1, 7), BRW_REGISTER_TYPE_UW);
1355 else
1356 dispatch_mask = retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UW);
1357
1358 brw_push_insn_state(p);
1359 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1360 brw_MOV(p, flags, dispatch_mask);
1361 brw_pop_insn_state(p);
1362 }
1363
1364 void
1365 fs_generator::generate_pixel_interpolator_query(fs_inst *inst,
1366 struct brw_reg dst,
1367 struct brw_reg src,
1368 struct brw_reg msg_data,
1369 unsigned msg_type)
1370 {
1371 assert(inst->size_written % REG_SIZE == 0);
1372 assert(msg_data.type == BRW_REGISTER_TYPE_UD);
1373
1374 brw_pixel_interpolator_query(p,
1375 retype(dst, BRW_REGISTER_TYPE_UW),
1376 src,
1377 inst->pi_noperspective,
1378 msg_type,
1379 msg_data,
1380 inst->mlen,
1381 inst->size_written / REG_SIZE);
1382 }
1383
1384
1385 /**
1386 * Sets the first word of a vgrf for gen7+ simd4x2 uniform pull constant
1387 * sampler LD messages.
1388 *
1389 * We don't want to bake it into the send message's code generation because
1390 * that means we don't get a chance to schedule the instructions.
1391 */
1392 void
1393 fs_generator::generate_set_simd4x2_offset(fs_inst *inst,
1394 struct brw_reg dst,
1395 struct brw_reg value)
1396 {
1397 assert(value.file == BRW_IMMEDIATE_VALUE);
1398
1399 brw_push_insn_state(p);
1400 brw_set_default_exec_size(p, BRW_EXECUTE_8);
1401 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
1402 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1403 brw_MOV(p, retype(brw_vec1_reg(dst.file, dst.nr, 0), value.type), value);
1404 brw_pop_insn_state(p);
1405 }
1406
1407 /* Sets vstride=1, width=4, hstride=0 of register src1 during
1408 * the ADD instruction.
1409 */
1410 void
1411 fs_generator::generate_set_sample_id(fs_inst *inst,
1412 struct brw_reg dst,
1413 struct brw_reg src0,
1414 struct brw_reg src1)
1415 {
1416 assert(dst.type == BRW_REGISTER_TYPE_D ||
1417 dst.type == BRW_REGISTER_TYPE_UD);
1418 assert(src0.type == BRW_REGISTER_TYPE_D ||
1419 src0.type == BRW_REGISTER_TYPE_UD);
1420
1421 struct brw_reg reg = stride(src1, 1, 4, 0);
1422 if (devinfo->gen >= 8 || inst->exec_size == 8) {
1423 brw_ADD(p, dst, src0, reg);
1424 } else if (inst->exec_size == 16) {
1425 brw_push_insn_state(p);
1426 brw_set_default_exec_size(p, BRW_EXECUTE_8);
1427 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
1428 brw_ADD(p, firsthalf(dst), firsthalf(src0), reg);
1429 brw_set_default_compression_control(p, BRW_COMPRESSION_2NDHALF);
1430 brw_ADD(p, sechalf(dst), sechalf(src0), suboffset(reg, 2));
1431 brw_pop_insn_state(p);
1432 }
1433 }
1434
1435 void
1436 fs_generator::generate_pack_half_2x16_split(fs_inst *inst,
1437 struct brw_reg dst,
1438 struct brw_reg x,
1439 struct brw_reg y)
1440 {
1441 assert(devinfo->gen >= 7);
1442 assert(dst.type == BRW_REGISTER_TYPE_UD);
1443 assert(x.type == BRW_REGISTER_TYPE_F);
1444 assert(y.type == BRW_REGISTER_TYPE_F);
1445
1446 /* From the Ivybridge PRM, Vol4, Part3, Section 6.27 f32to16:
1447 *
1448 * Because this instruction does not have a 16-bit floating-point type,
1449 * the destination data type must be Word (W).
1450 *
1451 * The destination must be DWord-aligned and specify a horizontal stride
1452 * (HorzStride) of 2. The 16-bit result is stored in the lower word of
1453 * each destination channel and the upper word is not modified.
1454 */
1455 struct brw_reg dst_w = spread(retype(dst, BRW_REGISTER_TYPE_W), 2);
1456
1457 /* Give each 32-bit channel of dst the form below, where "." means
1458 * unchanged.
1459 * 0x....hhhh
1460 */
1461 brw_F32TO16(p, dst_w, y);
1462
1463 /* Now the form:
1464 * 0xhhhh0000
1465 */
1466 brw_SHL(p, dst, dst, brw_imm_ud(16u));
1467
1468 /* And, finally the form of packHalf2x16's output:
1469 * 0xhhhhllll
1470 */
1471 brw_F32TO16(p, dst_w, x);
1472 }
1473
1474 void
1475 fs_generator::generate_unpack_half_2x16_split(fs_inst *inst,
1476 struct brw_reg dst,
1477 struct brw_reg src)
1478 {
1479 assert(devinfo->gen >= 7);
1480 assert(dst.type == BRW_REGISTER_TYPE_F);
1481 assert(src.type == BRW_REGISTER_TYPE_UD);
1482
1483 /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
1484 *
1485 * Because this instruction does not have a 16-bit floating-point type,
1486 * the source data type must be Word (W). The destination type must be
1487 * F (Float).
1488 */
1489 struct brw_reg src_w = spread(retype(src, BRW_REGISTER_TYPE_W), 2);
1490
1491 /* Each channel of src has the form of unpackHalf2x16's input: 0xhhhhllll.
1492 * For the Y case, we wish to access only the upper word; therefore
1493 * a 16-bit subregister offset is needed.
1494 */
1495 assert(inst->opcode == FS_OPCODE_UNPACK_HALF_2x16_SPLIT_X ||
1496 inst->opcode == FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y);
1497 if (inst->opcode == FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y)
1498 src_w.subnr += 2;
1499
1500 brw_F16TO32(p, dst, src_w);
1501 }
1502
1503 void
1504 fs_generator::generate_shader_time_add(fs_inst *inst,
1505 struct brw_reg payload,
1506 struct brw_reg offset,
1507 struct brw_reg value)
1508 {
1509 assert(devinfo->gen >= 7);
1510 brw_push_insn_state(p);
1511 brw_set_default_mask_control(p, true);
1512
1513 assert(payload.file == BRW_GENERAL_REGISTER_FILE);
1514 struct brw_reg payload_offset = retype(brw_vec1_grf(payload.nr, 0),
1515 offset.type);
1516 struct brw_reg payload_value = retype(brw_vec1_grf(payload.nr + 1, 0),
1517 value.type);
1518
1519 assert(offset.file == BRW_IMMEDIATE_VALUE);
1520 if (value.file == BRW_GENERAL_REGISTER_FILE) {
1521 value.width = BRW_WIDTH_1;
1522 value.hstride = BRW_HORIZONTAL_STRIDE_0;
1523 value.vstride = BRW_VERTICAL_STRIDE_0;
1524 } else {
1525 assert(value.file == BRW_IMMEDIATE_VALUE);
1526 }
1527
1528 /* Trying to deal with setup of the params from the IR is crazy in the FS8
1529 * case, and we don't really care about squeezing every bit of performance
1530 * out of this path, so we just emit the MOVs from here.
1531 */
1532 brw_MOV(p, payload_offset, offset);
1533 brw_MOV(p, payload_value, value);
1534 brw_shader_time_add(p, payload,
1535 prog_data->binding_table.shader_time_start);
1536 brw_pop_insn_state(p);
1537
1538 brw_mark_surface_used(prog_data,
1539 prog_data->binding_table.shader_time_start);
1540 }
1541
1542 void
1543 fs_generator::enable_debug(const char *shader_name)
1544 {
1545 debug_flag = true;
1546 this->shader_name = shader_name;
1547 }
1548
1549 int
1550 fs_generator::generate_code(const cfg_t *cfg, int dispatch_width)
1551 {
1552 /* align to 64 byte boundary. */
1553 while (p->next_insn_offset % 64)
1554 brw_NOP(p);
1555
1556 this->dispatch_width = dispatch_width;
1557
1558 int start_offset = p->next_insn_offset;
1559 int spill_count = 0, fill_count = 0;
1560 int loop_count = 0;
1561
1562 struct annotation_info annotation;
1563 memset(&annotation, 0, sizeof(annotation));
1564
1565 foreach_block_and_inst (block, fs_inst, inst, cfg) {
1566 struct brw_reg src[3], dst;
1567 unsigned int last_insn_offset = p->next_insn_offset;
1568 bool multiple_instructions_emitted = false;
1569
1570 /* From the Broadwell PRM, Volume 7, "3D-Media-GPGPU", in the
1571 * "Register Region Restrictions" section: for BDW, SKL:
1572 *
1573 * "A POW/FDIV operation must not be followed by an instruction
1574 * that requires two destination registers."
1575 *
1576 * The documentation is often lacking annotations for Atom parts,
1577 * and empirically this affects CHV as well.
1578 */
1579 if (devinfo->gen >= 8 &&
1580 p->nr_insn > 1 &&
1581 brw_inst_opcode(devinfo, brw_last_inst) == BRW_OPCODE_MATH &&
1582 brw_inst_math_function(devinfo, brw_last_inst) == BRW_MATH_FUNCTION_POW &&
1583 inst->dst.component_size(inst->exec_size) > REG_SIZE) {
1584 brw_NOP(p);
1585 last_insn_offset = p->next_insn_offset;
1586 }
1587
1588 if (unlikely(debug_flag))
1589 annotate(p->devinfo, &annotation, cfg, inst, p->next_insn_offset);
1590
1591 /* If the instruction writes to more than one register, it needs to be
1592 * explicitly marked as compressed on Gen <= 5. On Gen >= 6 the
1593 * hardware figures out by itself what the right compression mode is,
1594 * but we still need to know whether the instruction is compressed to
1595 * set up the source register regions appropriately.
1596 *
1597 * XXX - This is wrong for instructions that write a single register but
1598 * read more than one which should strictly speaking be treated as
1599 * compressed. For instructions that don't write any registers it
1600 * relies on the destination being a null register of the correct
1601 * type and regioning so the instruction is considered compressed
1602 * or not accordingly.
1603 */
1604 const bool compressed =
1605 inst->dst.component_size(inst->exec_size) > REG_SIZE;
1606 brw_set_default_compression(p, compressed);
1607 brw_set_default_group(p, inst->group);
1608
1609 for (unsigned int i = 0; i < inst->sources; i++) {
1610 src[i] = brw_reg_from_fs_reg(inst, &inst->src[i], devinfo->gen,
1611 compressed);
1612
1613 /* The accumulator result appears to get used for the
1614 * conditional modifier generation. When negating a UD
1615 * value, there is a 33rd bit generated for the sign in the
1616 * accumulator value, so now you can't check, for example,
1617 * equality with a 32-bit value. See piglit fs-op-neg-uvec4.
1618 */
1619 assert(!inst->conditional_mod ||
1620 inst->src[i].type != BRW_REGISTER_TYPE_UD ||
1621 !inst->src[i].negate);
1622 }
1623 dst = brw_reg_from_fs_reg(inst, &inst->dst, devinfo->gen, compressed);
1624
1625 brw_set_default_access_mode(p, BRW_ALIGN_1);
1626 brw_set_default_predicate_control(p, inst->predicate);
1627 brw_set_default_predicate_inverse(p, inst->predicate_inverse);
1628 brw_set_default_flag_reg(p, 0, inst->flag_subreg);
1629 brw_set_default_saturate(p, inst->saturate);
1630 brw_set_default_mask_control(p, inst->force_writemask_all);
1631 brw_set_default_acc_write_control(p, inst->writes_accumulator);
1632 brw_set_default_exec_size(p, cvt(inst->exec_size) - 1);
1633
1634 assert(inst->force_writemask_all || inst->exec_size >= 4);
1635 assert(inst->force_writemask_all || inst->group % inst->exec_size == 0);
1636 assert(inst->base_mrf + inst->mlen <= BRW_MAX_MRF(devinfo->gen));
1637 assert(inst->mlen <= BRW_MAX_MSG_LENGTH);
1638
1639 switch (inst->opcode) {
1640 case BRW_OPCODE_MOV:
1641 brw_MOV(p, dst, src[0]);
1642 break;
1643 case BRW_OPCODE_ADD:
1644 brw_ADD(p, dst, src[0], src[1]);
1645 break;
1646 case BRW_OPCODE_MUL:
1647 brw_MUL(p, dst, src[0], src[1]);
1648 break;
1649 case BRW_OPCODE_AVG:
1650 brw_AVG(p, dst, src[0], src[1]);
1651 break;
1652 case BRW_OPCODE_MACH:
1653 brw_MACH(p, dst, src[0], src[1]);
1654 break;
1655
1656 case BRW_OPCODE_LINE:
1657 brw_LINE(p, dst, src[0], src[1]);
1658 break;
1659
1660 case BRW_OPCODE_MAD:
1661 assert(devinfo->gen >= 6);
1662 brw_set_default_access_mode(p, BRW_ALIGN_16);
1663 brw_MAD(p, dst, src[0], src[1], src[2]);
1664 break;
1665
1666 case BRW_OPCODE_LRP:
1667 assert(devinfo->gen >= 6);
1668 brw_set_default_access_mode(p, BRW_ALIGN_16);
1669 brw_LRP(p, dst, src[0], src[1], src[2]);
1670 break;
1671
1672 case BRW_OPCODE_FRC:
1673 brw_FRC(p, dst, src[0]);
1674 break;
1675 case BRW_OPCODE_RNDD:
1676 brw_RNDD(p, dst, src[0]);
1677 break;
1678 case BRW_OPCODE_RNDE:
1679 brw_RNDE(p, dst, src[0]);
1680 break;
1681 case BRW_OPCODE_RNDZ:
1682 brw_RNDZ(p, dst, src[0]);
1683 break;
1684
1685 case BRW_OPCODE_AND:
1686 brw_AND(p, dst, src[0], src[1]);
1687 break;
1688 case BRW_OPCODE_OR:
1689 brw_OR(p, dst, src[0], src[1]);
1690 break;
1691 case BRW_OPCODE_XOR:
1692 brw_XOR(p, dst, src[0], src[1]);
1693 break;
1694 case BRW_OPCODE_NOT:
1695 brw_NOT(p, dst, src[0]);
1696 break;
1697 case BRW_OPCODE_ASR:
1698 brw_ASR(p, dst, src[0], src[1]);
1699 break;
1700 case BRW_OPCODE_SHR:
1701 brw_SHR(p, dst, src[0], src[1]);
1702 break;
1703 case BRW_OPCODE_SHL:
1704 brw_SHL(p, dst, src[0], src[1]);
1705 break;
1706 case BRW_OPCODE_F32TO16:
1707 assert(devinfo->gen >= 7);
1708 brw_F32TO16(p, dst, src[0]);
1709 break;
1710 case BRW_OPCODE_F16TO32:
1711 assert(devinfo->gen >= 7);
1712 brw_F16TO32(p, dst, src[0]);
1713 break;
1714 case BRW_OPCODE_CMP:
1715 if (inst->exec_size >= 16 && devinfo->gen == 7 && !devinfo->is_haswell &&
1716 dst.file == BRW_ARCHITECTURE_REGISTER_FILE) {
1717 /* For unknown reasons the WaCMPInstFlagDepClearedEarly workaround
1718 * implemented in the compiler is not sufficient. Overriding the
1719 * type when the destination is the null register is necessary but
1720 * not sufficient by itself.
1721 */
1722 assert(dst.nr == BRW_ARF_NULL);
1723 dst.type = BRW_REGISTER_TYPE_D;
1724 }
1725 brw_CMP(p, dst, inst->conditional_mod, src[0], src[1]);
1726 break;
1727 case BRW_OPCODE_SEL:
1728 brw_SEL(p, dst, src[0], src[1]);
1729 break;
1730 case BRW_OPCODE_BFREV:
1731 assert(devinfo->gen >= 7);
1732 /* BFREV only supports UD type for src and dst. */
1733 brw_BFREV(p, retype(dst, BRW_REGISTER_TYPE_UD),
1734 retype(src[0], BRW_REGISTER_TYPE_UD));
1735 break;
1736 case BRW_OPCODE_FBH:
1737 assert(devinfo->gen >= 7);
1738 /* FBH only supports UD type for dst. */
1739 brw_FBH(p, retype(dst, BRW_REGISTER_TYPE_UD), src[0]);
1740 break;
1741 case BRW_OPCODE_FBL:
1742 assert(devinfo->gen >= 7);
1743 /* FBL only supports UD type for dst. */
1744 brw_FBL(p, retype(dst, BRW_REGISTER_TYPE_UD), src[0]);
1745 break;
1746 case BRW_OPCODE_LZD:
1747 brw_LZD(p, dst, src[0]);
1748 break;
1749 case BRW_OPCODE_CBIT:
1750 assert(devinfo->gen >= 7);
1751 /* CBIT only supports UD type for dst. */
1752 brw_CBIT(p, retype(dst, BRW_REGISTER_TYPE_UD), src[0]);
1753 break;
1754 case BRW_OPCODE_ADDC:
1755 assert(devinfo->gen >= 7);
1756 brw_ADDC(p, dst, src[0], src[1]);
1757 break;
1758 case BRW_OPCODE_SUBB:
1759 assert(devinfo->gen >= 7);
1760 brw_SUBB(p, dst, src[0], src[1]);
1761 break;
1762 case BRW_OPCODE_MAC:
1763 brw_MAC(p, dst, src[0], src[1]);
1764 break;
1765
1766 case BRW_OPCODE_BFE:
1767 assert(devinfo->gen >= 7);
1768 brw_set_default_access_mode(p, BRW_ALIGN_16);
1769 brw_BFE(p, dst, src[0], src[1], src[2]);
1770 break;
1771
1772 case BRW_OPCODE_BFI1:
1773 assert(devinfo->gen >= 7);
1774 brw_BFI1(p, dst, src[0], src[1]);
1775 break;
1776 case BRW_OPCODE_BFI2:
1777 assert(devinfo->gen >= 7);
1778 brw_set_default_access_mode(p, BRW_ALIGN_16);
1779 brw_BFI2(p, dst, src[0], src[1], src[2]);
1780 break;
1781
1782 case BRW_OPCODE_IF:
1783 if (inst->src[0].file != BAD_FILE) {
1784 /* The instruction has an embedded compare (only allowed on gen6) */
1785 assert(devinfo->gen == 6);
1786 gen6_IF(p, inst->conditional_mod, src[0], src[1]);
1787 } else {
1788 brw_IF(p, brw_inst_exec_size(devinfo, p->current));
1789 }
1790 break;
1791
1792 case BRW_OPCODE_ELSE:
1793 brw_ELSE(p);
1794 break;
1795 case BRW_OPCODE_ENDIF:
1796 brw_ENDIF(p);
1797 break;
1798
1799 case BRW_OPCODE_DO:
1800 brw_DO(p, brw_inst_exec_size(devinfo, p->current));
1801 break;
1802
1803 case BRW_OPCODE_BREAK:
1804 brw_BREAK(p);
1805 break;
1806 case BRW_OPCODE_CONTINUE:
1807 brw_CONT(p);
1808 break;
1809
1810 case BRW_OPCODE_WHILE:
1811 brw_WHILE(p);
1812 loop_count++;
1813 break;
1814
1815 case SHADER_OPCODE_RCP:
1816 case SHADER_OPCODE_RSQ:
1817 case SHADER_OPCODE_SQRT:
1818 case SHADER_OPCODE_EXP2:
1819 case SHADER_OPCODE_LOG2:
1820 case SHADER_OPCODE_SIN:
1821 case SHADER_OPCODE_COS:
1822 assert(inst->conditional_mod == BRW_CONDITIONAL_NONE);
1823 if (devinfo->gen >= 6) {
1824 assert(inst->mlen == 0);
1825 assert(devinfo->gen >= 7 || inst->exec_size == 8);
1826 gen6_math(p, dst, brw_math_function(inst->opcode),
1827 src[0], brw_null_reg());
1828 } else {
1829 assert(inst->mlen >= 1);
1830 assert(devinfo->gen == 5 || devinfo->is_g4x || inst->exec_size == 8);
1831 gen4_math(p, dst,
1832 brw_math_function(inst->opcode),
1833 inst->base_mrf, src[0],
1834 BRW_MATH_PRECISION_FULL);
1835 }
1836 break;
1837 case SHADER_OPCODE_INT_QUOTIENT:
1838 case SHADER_OPCODE_INT_REMAINDER:
1839 case SHADER_OPCODE_POW:
1840 assert(inst->conditional_mod == BRW_CONDITIONAL_NONE);
1841 if (devinfo->gen >= 6) {
1842 assert(inst->mlen == 0);
1843 assert((devinfo->gen >= 7 && inst->opcode == SHADER_OPCODE_POW) ||
1844 inst->exec_size == 8);
1845 gen6_math(p, dst, brw_math_function(inst->opcode), src[0], src[1]);
1846 } else {
1847 assert(inst->mlen >= 1);
1848 assert(inst->exec_size == 8);
1849 gen4_math(p, dst, brw_math_function(inst->opcode),
1850 inst->base_mrf, src[0],
1851 BRW_MATH_PRECISION_FULL);
1852 }
1853 break;
1854 case FS_OPCODE_CINTERP:
1855 brw_MOV(p, dst, src[0]);
1856 break;
1857 case FS_OPCODE_LINTERP:
1858 generate_linterp(inst, dst, src);
1859 break;
1860 case FS_OPCODE_PIXEL_X:
1861 assert(src[0].type == BRW_REGISTER_TYPE_UW);
1862 src[0].subnr = 0 * type_sz(src[0].type);
1863 brw_MOV(p, dst, stride(src[0], 8, 4, 1));
1864 break;
1865 case FS_OPCODE_PIXEL_Y:
1866 assert(src[0].type == BRW_REGISTER_TYPE_UW);
1867 src[0].subnr = 4 * type_sz(src[0].type);
1868 brw_MOV(p, dst, stride(src[0], 8, 4, 1));
1869 break;
1870 case FS_OPCODE_GET_BUFFER_SIZE:
1871 generate_get_buffer_size(inst, dst, src[0], src[1]);
1872 break;
1873 case SHADER_OPCODE_TEX:
1874 case FS_OPCODE_TXB:
1875 case SHADER_OPCODE_TXD:
1876 case SHADER_OPCODE_TXF:
1877 case SHADER_OPCODE_TXF_LZ:
1878 case SHADER_OPCODE_TXF_CMS:
1879 case SHADER_OPCODE_TXF_CMS_W:
1880 case SHADER_OPCODE_TXF_UMS:
1881 case SHADER_OPCODE_TXF_MCS:
1882 case SHADER_OPCODE_TXL:
1883 case SHADER_OPCODE_TXL_LZ:
1884 case SHADER_OPCODE_TXS:
1885 case SHADER_OPCODE_LOD:
1886 case SHADER_OPCODE_TG4:
1887 case SHADER_OPCODE_TG4_OFFSET:
1888 case SHADER_OPCODE_SAMPLEINFO:
1889 generate_tex(inst, dst, src[0], src[1], src[2]);
1890 break;
1891 case FS_OPCODE_DDX_COARSE:
1892 case FS_OPCODE_DDX_FINE:
1893 generate_ddx(inst->opcode, dst, src[0]);
1894 break;
1895 case FS_OPCODE_DDY_COARSE:
1896 case FS_OPCODE_DDY_FINE:
1897 generate_ddy(inst->opcode, dst, src[0]);
1898 break;
1899
1900 case SHADER_OPCODE_GEN4_SCRATCH_WRITE:
1901 generate_scratch_write(inst, src[0]);
1902 spill_count++;
1903 break;
1904
1905 case SHADER_OPCODE_GEN4_SCRATCH_READ:
1906 generate_scratch_read(inst, dst);
1907 fill_count++;
1908 break;
1909
1910 case SHADER_OPCODE_GEN7_SCRATCH_READ:
1911 generate_scratch_read_gen7(inst, dst);
1912 fill_count++;
1913 break;
1914
1915 case SHADER_OPCODE_MOV_INDIRECT:
1916 generate_mov_indirect(inst, dst, src[0], src[1]);
1917 break;
1918
1919 case SHADER_OPCODE_URB_READ_SIMD8:
1920 case SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT:
1921 generate_urb_read(inst, dst, src[0]);
1922 break;
1923
1924 case SHADER_OPCODE_URB_WRITE_SIMD8:
1925 case SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT:
1926 case SHADER_OPCODE_URB_WRITE_SIMD8_MASKED:
1927 case SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT:
1928 generate_urb_write(inst, src[0]);
1929 break;
1930
1931 case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD:
1932 assert(inst->force_writemask_all);
1933 generate_uniform_pull_constant_load(inst, dst, src[0], src[1]);
1934 break;
1935
1936 case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD_GEN7:
1937 assert(inst->force_writemask_all);
1938 generate_uniform_pull_constant_load_gen7(inst, dst, src[0], src[1]);
1939 break;
1940
1941 case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_GEN4:
1942 generate_varying_pull_constant_load_gen4(inst, dst, src[0]);
1943 break;
1944
1945 case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_GEN7:
1946 generate_varying_pull_constant_load_gen7(inst, dst, src[0], src[1]);
1947 break;
1948
1949 case FS_OPCODE_REP_FB_WRITE:
1950 case FS_OPCODE_FB_WRITE:
1951 generate_fb_write(inst, src[0]);
1952 break;
1953
1954 case FS_OPCODE_FB_READ:
1955 generate_fb_read(inst, dst, src[0]);
1956 break;
1957
1958 case FS_OPCODE_MOV_DISPATCH_TO_FLAGS:
1959 generate_mov_dispatch_to_flags(inst);
1960 break;
1961
1962 case FS_OPCODE_DISCARD_JUMP:
1963 generate_discard_jump(inst);
1964 break;
1965
1966 case SHADER_OPCODE_SHADER_TIME_ADD:
1967 generate_shader_time_add(inst, src[0], src[1], src[2]);
1968 break;
1969
1970 case SHADER_OPCODE_UNTYPED_ATOMIC:
1971 assert(src[2].file == BRW_IMMEDIATE_VALUE);
1972 brw_untyped_atomic(p, dst, src[0], src[1], src[2].ud,
1973 inst->mlen, !inst->dst.is_null());
1974 break;
1975
1976 case SHADER_OPCODE_UNTYPED_SURFACE_READ:
1977 assert(src[2].file == BRW_IMMEDIATE_VALUE);
1978 brw_untyped_surface_read(p, dst, src[0], src[1],
1979 inst->mlen, src[2].ud);
1980 break;
1981
1982 case SHADER_OPCODE_UNTYPED_SURFACE_WRITE:
1983 assert(src[2].file == BRW_IMMEDIATE_VALUE);
1984 brw_untyped_surface_write(p, src[0], src[1],
1985 inst->mlen, src[2].ud);
1986 break;
1987
1988 case SHADER_OPCODE_TYPED_ATOMIC:
1989 assert(src[2].file == BRW_IMMEDIATE_VALUE);
1990 brw_typed_atomic(p, dst, src[0], src[1],
1991 src[2].ud, inst->mlen, !inst->dst.is_null());
1992 break;
1993
1994 case SHADER_OPCODE_TYPED_SURFACE_READ:
1995 assert(src[2].file == BRW_IMMEDIATE_VALUE);
1996 brw_typed_surface_read(p, dst, src[0], src[1],
1997 inst->mlen, src[2].ud);
1998 break;
1999
2000 case SHADER_OPCODE_TYPED_SURFACE_WRITE:
2001 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2002 brw_typed_surface_write(p, src[0], src[1], inst->mlen, src[2].ud);
2003 break;
2004
2005 case SHADER_OPCODE_MEMORY_FENCE:
2006 brw_memory_fence(p, dst);
2007 break;
2008
2009 case FS_OPCODE_SET_SIMD4X2_OFFSET:
2010 generate_set_simd4x2_offset(inst, dst, src[0]);
2011 break;
2012
2013 case SHADER_OPCODE_FIND_LIVE_CHANNEL: {
2014 const struct brw_reg mask =
2015 brw_stage_has_packed_dispatch(devinfo, stage,
2016 prog_data) ? brw_imm_ud(~0u) :
2017 stage == MESA_SHADER_FRAGMENT ? brw_vmask_reg() :
2018 brw_dmask_reg();
2019 brw_find_live_channel(p, dst, mask);
2020 break;
2021 }
2022
2023 case SHADER_OPCODE_BROADCAST:
2024 assert(inst->force_writemask_all);
2025 brw_broadcast(p, dst, src[0], src[1]);
2026 break;
2027
2028 case FS_OPCODE_SET_SAMPLE_ID:
2029 generate_set_sample_id(inst, dst, src[0], src[1]);
2030 break;
2031
2032 case FS_OPCODE_PACK_HALF_2x16_SPLIT:
2033 generate_pack_half_2x16_split(inst, dst, src[0], src[1]);
2034 break;
2035
2036 case FS_OPCODE_UNPACK_HALF_2x16_SPLIT_X:
2037 case FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y:
2038 generate_unpack_half_2x16_split(inst, dst, src[0]);
2039 break;
2040
2041 case FS_OPCODE_PLACEHOLDER_HALT:
2042 /* This is the place where the final HALT needs to be inserted if
2043 * we've emitted any discards. If not, this will emit no code.
2044 */
2045 if (!patch_discard_jumps_to_fb_writes()) {
2046 if (unlikely(debug_flag)) {
2047 annotation.ann_count--;
2048 }
2049 }
2050 break;
2051
2052 case FS_OPCODE_INTERPOLATE_AT_SAMPLE:
2053 generate_pixel_interpolator_query(inst, dst, src[0], src[1],
2054 GEN7_PIXEL_INTERPOLATOR_LOC_SAMPLE);
2055 break;
2056
2057 case FS_OPCODE_INTERPOLATE_AT_SHARED_OFFSET:
2058 generate_pixel_interpolator_query(inst, dst, src[0], src[1],
2059 GEN7_PIXEL_INTERPOLATOR_LOC_SHARED_OFFSET);
2060 break;
2061
2062 case FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET:
2063 generate_pixel_interpolator_query(inst, dst, src[0], src[1],
2064 GEN7_PIXEL_INTERPOLATOR_LOC_PER_SLOT_OFFSET);
2065 break;
2066
2067 case CS_OPCODE_CS_TERMINATE:
2068 generate_cs_terminate(inst, src[0]);
2069 break;
2070
2071 case SHADER_OPCODE_BARRIER:
2072 generate_barrier(inst, src[0]);
2073 break;
2074
2075 case BRW_OPCODE_DIM:
2076 assert(devinfo->is_haswell);
2077 assert(src[0].type == BRW_REGISTER_TYPE_DF);
2078 assert(dst.type == BRW_REGISTER_TYPE_DF);
2079 brw_DIM(p, dst, retype(src[0], BRW_REGISTER_TYPE_F));
2080 break;
2081
2082 default:
2083 unreachable("Unsupported opcode");
2084
2085 case SHADER_OPCODE_LOAD_PAYLOAD:
2086 unreachable("Should be lowered by lower_load_payload()");
2087 }
2088
2089 if (multiple_instructions_emitted)
2090 continue;
2091
2092 if (inst->no_dd_clear || inst->no_dd_check || inst->conditional_mod) {
2093 assert(p->next_insn_offset == last_insn_offset + 16 ||
2094 !"conditional_mod, no_dd_check, or no_dd_clear set for IR "
2095 "emitting more than 1 instruction");
2096
2097 brw_inst *last = &p->store[last_insn_offset / 16];
2098
2099 if (inst->conditional_mod)
2100 brw_inst_set_cond_modifier(p->devinfo, last, inst->conditional_mod);
2101 brw_inst_set_no_dd_clear(p->devinfo, last, inst->no_dd_clear);
2102 brw_inst_set_no_dd_check(p->devinfo, last, inst->no_dd_check);
2103 }
2104 }
2105
2106 brw_set_uip_jip(p, start_offset);
2107 annotation_finalize(&annotation, p->next_insn_offset);
2108
2109 #ifndef NDEBUG
2110 bool validated = brw_validate_instructions(p, start_offset, &annotation);
2111 #else
2112 if (unlikely(debug_flag))
2113 brw_validate_instructions(p, start_offset, &annotation);
2114 #endif
2115
2116 int before_size = p->next_insn_offset - start_offset;
2117 brw_compact_instructions(p, start_offset, annotation.ann_count,
2118 annotation.ann);
2119 int after_size = p->next_insn_offset - start_offset;
2120
2121 if (unlikely(debug_flag)) {
2122 fprintf(stderr, "Native code for %s\n"
2123 "SIMD%d shader: %d instructions. %d loops. %u cycles. %d:%d spills:fills. Promoted %u constants. Compacted %d to %d"
2124 " bytes (%.0f%%)\n",
2125 shader_name, dispatch_width, before_size / 16, loop_count, cfg->cycle_count,
2126 spill_count, fill_count, promoted_constants, before_size, after_size,
2127 100.0f * (before_size - after_size) / before_size);
2128
2129 dump_assembly(p->store, annotation.ann_count, annotation.ann,
2130 p->devinfo);
2131 ralloc_free(annotation.mem_ctx);
2132 }
2133 assert(validated);
2134
2135 compiler->shader_debug_log(log_data,
2136 "%s SIMD%d shader: %d inst, %d loops, %u cycles, "
2137 "%d:%d spills:fills, Promoted %u constants, "
2138 "compacted %d to %d bytes.",
2139 _mesa_shader_stage_to_abbrev(stage),
2140 dispatch_width, before_size / 16,
2141 loop_count, cfg->cycle_count, spill_count,
2142 fill_count, promoted_constants, before_size,
2143 after_size);
2144
2145 return start_offset;
2146 }
2147
2148 const unsigned *
2149 fs_generator::get_assembly(unsigned int *assembly_size)
2150 {
2151 return brw_get_program(p, assembly_size);
2152 }