projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
intel/fs: Remove program key argument from generator.
[mesa.git] / src / intel / compiler / brw_fs_generator.cpp
1 /*
2 * Copyright © 2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 /** @file brw_fs_generator.cpp
25 *
26 * This file supports generating code from the FS LIR to the actual
27 * native instructions.
28 */
29
30 #include "brw_eu.h"
31 #include "brw_fs.h"
32 #include "brw_cfg.h"
33
34 static enum brw_reg_file
35 brw_file_from_reg(fs_reg *reg)
36 {
37 switch (reg->file) {
38 case ARF:
39 return BRW_ARCHITECTURE_REGISTER_FILE;
40 case FIXED_GRF:
41 case VGRF:
42 return BRW_GENERAL_REGISTER_FILE;
43 case MRF:
44 return BRW_MESSAGE_REGISTER_FILE;
45 case IMM:
46 return BRW_IMMEDIATE_VALUE;
47 case BAD_FILE:
48 case ATTR:
49 case UNIFORM:
50 unreachable("not reached");
51 }
52 return BRW_ARCHITECTURE_REGISTER_FILE;
53 }
54
55 static struct brw_reg
56 brw_reg_from_fs_reg(const struct gen_device_info *devinfo, fs_inst *inst,
57 fs_reg *reg, bool compressed)
58 {
59 struct brw_reg brw_reg;
60
61 switch (reg->file) {
62 case MRF:
63 assert((reg->nr & ~BRW_MRF_COMPR4) < BRW_MAX_MRF(devinfo->gen));
64 /* Fallthrough */
65 case VGRF:
66 if (reg->stride == 0) {
67 brw_reg = brw_vec1_reg(brw_file_from_reg(reg), reg->nr, 0);
68 } else {
69 /* From the Haswell PRM:
70 *
71 * "VertStride must be used to cross GRF register boundaries. This
72 * rule implies that elements within a 'Width' cannot cross GRF
73 * boundaries."
74 *
75 * The maximum width value that could satisfy this restriction is:
76 */
77 const unsigned reg_width = REG_SIZE / (reg->stride * type_sz(reg->type));
78
79 /* Because the hardware can only split source regions at a whole
80 * multiple of width during decompression (i.e. vertically), clamp
81 * the value obtained above to the physical execution size of a
82 * single decompressed chunk of the instruction:
83 */
84 const unsigned phys_width = compressed ? inst->exec_size / 2 :
85 inst->exec_size;
86
87 /* XXX - The equation above is strictly speaking not correct on
88 * hardware that supports unbalanced GRF writes -- On Gen9+
89 * each decompressed chunk of the instruction may have a
90 * different execution size when the number of components
91 * written to each destination GRF is not the same.
92 */
93 const unsigned width = MIN2(reg_width, phys_width);
94 brw_reg = brw_vecn_reg(width, brw_file_from_reg(reg), reg->nr, 0);
95 brw_reg = stride(brw_reg, width * reg->stride, width, reg->stride);
96
97 if (devinfo->gen == 7 && !devinfo->is_haswell) {
98 /* From the IvyBridge PRM (EU Changes by Processor Generation, page 13):
99 * "Each DF (Double Float) operand uses an element size of 4 rather
100 * than 8 and all regioning parameters are twice what the values
101 * would be based on the true element size: ExecSize, Width,
102 * HorzStride, and VertStride. Each DF operand uses a pair of
103 * channels and all masking and swizzing should be adjusted
104 * appropriately."
105 *
106 * From the IvyBridge PRM (Special Requirements for Handling Double
107 * Precision Data Types, page 71):
108 * "In Align1 mode, all regioning parameters like stride, execution
109 * size, and width must use the syntax of a pair of packed
110 * floats. The offsets for these data types must be 64-bit
111 * aligned. The execution size and regioning parameters are in terms
112 * of floats."
113 *
114 * Summarized: when handling DF-typed arguments, ExecSize,
115 * VertStride, and Width must be doubled.
116 *
117 * It applies to BayTrail too.
118 */
119 if (type_sz(reg->type) == 8) {
120 brw_reg.width++;
121 if (brw_reg.vstride > 0)
122 brw_reg.vstride++;
123 assert(brw_reg.hstride == BRW_HORIZONTAL_STRIDE_1);
124 }
125
126 /* When converting from DF->F, we set the destination stride to 2
127 * because each d2f conversion implicitly writes 2 floats, being
128 * the first one the converted value. IVB/BYT actually writes two
129 * F components per SIMD channel, and every other component is
130 * filled with garbage.
131 */
132 if (reg == &inst->dst && get_exec_type_size(inst) == 8 &&
133 type_sz(inst->dst.type) < 8) {
134 assert(brw_reg.hstride > BRW_HORIZONTAL_STRIDE_1);
135 brw_reg.hstride--;
136 }
137 }
138 }
139
140 brw_reg = retype(brw_reg, reg->type);
141 brw_reg = byte_offset(brw_reg, reg->offset);
142 brw_reg.abs = reg->abs;
143 brw_reg.negate = reg->negate;
144 break;
145 case ARF:
146 case FIXED_GRF:
147 case IMM:
148 assert(reg->offset == 0);
149 brw_reg = reg->as_brw_reg();
150 break;
151 case BAD_FILE:
152 /* Probably unused. */
153 brw_reg = brw_null_reg();
154 break;
155 case ATTR:
156 case UNIFORM:
157 unreachable("not reached");
158 }
159
160 /* On HSW+, scalar DF sources can be accessed using the normal <0,1,0>
161 * region, but on IVB and BYT DF regions must be programmed in terms of
162 * floats. A <0,2,1> region accomplishes this.
163 */
164 if (devinfo->gen == 7 && !devinfo->is_haswell &&
165 type_sz(reg->type) == 8 &&
166 brw_reg.vstride == BRW_VERTICAL_STRIDE_0 &&
167 brw_reg.width == BRW_WIDTH_1 &&
168 brw_reg.hstride == BRW_HORIZONTAL_STRIDE_0) {
169 brw_reg.width = BRW_WIDTH_2;
170 brw_reg.hstride = BRW_HORIZONTAL_STRIDE_1;
171 }
172
173 return brw_reg;
174 }
175
176 fs_generator::fs_generator(const struct brw_compiler *compiler, void *log_data,
177 void *mem_ctx,
178 struct brw_stage_prog_data *prog_data,
179 unsigned promoted_constants,
180 bool runtime_check_aads_emit,
181 gl_shader_stage stage)
182
183 : compiler(compiler), log_data(log_data),
184 devinfo(compiler->devinfo),
185 prog_data(prog_data),
186 promoted_constants(promoted_constants),
187 runtime_check_aads_emit(runtime_check_aads_emit), debug_flag(false),
188 stage(stage), mem_ctx(mem_ctx)
189 {
190 p = rzalloc(mem_ctx, struct brw_codegen);
191 brw_init_codegen(devinfo, p, mem_ctx);
192
193 /* In the FS code generator, we are very careful to ensure that we always
194 * set the right execution size so we don't need the EU code to "help" us
195 * by trying to infer it. Sometimes, it infers the wrong thing.
196 */
197 p->automatic_exec_sizes = false;
198 }
199
200 fs_generator::~fs_generator()
201 {
202 }
203
204 class ip_record : public exec_node {
205 public:
206 DECLARE_RALLOC_CXX_OPERATORS(ip_record)
207
208 ip_record(int ip)
209 {
210 this->ip = ip;
211 }
212
213 int ip;
214 };
215
216 bool
217 fs_generator::patch_discard_jumps_to_fb_writes()
218 {
219 if (devinfo->gen < 6 || this->discard_halt_patches.is_empty())
220 return false;
221
222 int scale = brw_jump_scale(p->devinfo);
223
224 /* There is a somewhat strange undocumented requirement of using
225 * HALT, according to the simulator. If some channel has HALTed to
226 * a particular UIP, then by the end of the program, every channel
227 * must have HALTed to that UIP. Furthermore, the tracking is a
228 * stack, so you can't do the final halt of a UIP after starting
229 * halting to a new UIP.
230 *
231 * Symptoms of not emitting this instruction on actual hardware
232 * included GPU hangs and sparkly rendering on the piglit discard
233 * tests.
234 */
235 brw_inst *last_halt = gen6_HALT(p);
236 brw_inst_set_uip(p->devinfo, last_halt, 1 * scale);
237 brw_inst_set_jip(p->devinfo, last_halt, 1 * scale);
238
239 int ip = p->nr_insn;
240
241 foreach_in_list(ip_record, patch_ip, &discard_halt_patches) {
242 brw_inst *patch = &p->store[patch_ip->ip];
243
244 assert(brw_inst_opcode(p->devinfo, patch) == BRW_OPCODE_HALT);
245 /* HALT takes a half-instruction distance from the pre-incremented IP. */
246 brw_inst_set_uip(p->devinfo, patch, (ip - patch_ip->ip) * scale);
247 }
248
249 this->discard_halt_patches.make_empty();
250 return true;
251 }
252
253 void
254 fs_generator::fire_fb_write(fs_inst *inst,
255 struct brw_reg payload,
256 struct brw_reg implied_header,
257 GLuint nr)
258 {
259 uint32_t msg_control;
260
261 struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
262
263 if (devinfo->gen < 6) {
264 brw_push_insn_state(p);
265 brw_set_default_exec_size(p, BRW_EXECUTE_8);
266 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
267 brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
268 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
269 brw_MOV(p, offset(retype(payload, BRW_REGISTER_TYPE_UD), 1),
270 offset(retype(implied_header, BRW_REGISTER_TYPE_UD), 1));
271 brw_pop_insn_state(p);
272 }
273
274 if (inst->opcode == FS_OPCODE_REP_FB_WRITE)
275 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD16_SINGLE_SOURCE_REPLICATED;
276 else if (prog_data->dual_src_blend) {
277 if (!inst->group)
278 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_DUAL_SOURCE_SUBSPAN01;
279 else
280 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_DUAL_SOURCE_SUBSPAN23;
281 } else if (inst->exec_size == 16)
282 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD16_SINGLE_SOURCE;
283 else
284 msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_SINGLE_SOURCE_SUBSPAN01;
285
286 /* We assume render targets start at 0, because headerless FB write
287 * messages set "Render Target Index" to 0. Using a different binding
288 * table index would make it impossible to use headerless messages.
289 */
290 const uint32_t surf_index = inst->target;
291
292 brw_fb_WRITE(p,
293 payload,
294 retype(implied_header, BRW_REGISTER_TYPE_UW),
295 msg_control,
296 surf_index,
297 nr,
298 0,
299 inst->eot,
300 inst->last_rt,
301 inst->header_size != 0);
302
303 brw_mark_surface_used(&prog_data->base, surf_index);
304 }
305
306 void
307 fs_generator::generate_fb_write(fs_inst *inst, struct brw_reg payload)
308 {
309 if (devinfo->gen < 8 && !devinfo->is_haswell) {
310 brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
311 }
312
313 const struct brw_reg implied_header =
314 devinfo->gen < 6 ? payload : brw_null_reg();
315
316 if (inst->base_mrf >= 0)
317 payload = brw_message_reg(inst->base_mrf);
318
319 if (!runtime_check_aads_emit) {
320 fire_fb_write(inst, payload, implied_header, inst->mlen);
321 } else {
322 /* This can only happen in gen < 6 */
323 assert(devinfo->gen < 6);
324
325 struct brw_reg v1_null_ud = vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_UD));
326
327 /* Check runtime bit to detect if we have to send AA data or not */
328 brw_push_insn_state(p);
329 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
330 brw_set_default_exec_size(p, BRW_EXECUTE_1);
331 brw_AND(p,
332 v1_null_ud,
333 retype(brw_vec1_grf(1, 6), BRW_REGISTER_TYPE_UD),
334 brw_imm_ud(1<<26));
335 brw_inst_set_cond_modifier(p->devinfo, brw_last_inst, BRW_CONDITIONAL_NZ);
336
337 int jmp = brw_JMPI(p, brw_imm_ud(0), BRW_PREDICATE_NORMAL) - p->store;
338 brw_pop_insn_state(p);
339 {
340 /* Don't send AA data */
341 fire_fb_write(inst, offset(payload, 1), implied_header, inst->mlen-1);
342 }
343 brw_land_fwd_jump(p, jmp);
344 fire_fb_write(inst, payload, implied_header, inst->mlen);
345 }
346 }
347
348 void
349 fs_generator::generate_fb_read(fs_inst *inst, struct brw_reg dst,
350 struct brw_reg payload)
351 {
352 assert(inst->size_written % REG_SIZE == 0);
353 struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
354 /* We assume that render targets start at binding table index 0. */
355 const unsigned surf_index = inst->target;
356
357 gen9_fb_READ(p, dst, payload, surf_index,
358 inst->header_size, inst->size_written / REG_SIZE,
359 prog_data->persample_dispatch);
360
361 brw_mark_surface_used(&prog_data->base, surf_index);
362 }
363
364 void
365 fs_generator::generate_mov_indirect(fs_inst *inst,
366 struct brw_reg dst,
367 struct brw_reg reg,
368 struct brw_reg indirect_byte_offset)
369 {
370 assert(indirect_byte_offset.type == BRW_REGISTER_TYPE_UD);
371 assert(indirect_byte_offset.file == BRW_GENERAL_REGISTER_FILE);
372 assert(!reg.abs && !reg.negate);
373 assert(reg.type == dst.type);
374
375 unsigned imm_byte_offset = reg.nr * REG_SIZE + reg.subnr;
376
377 if (indirect_byte_offset.file == BRW_IMMEDIATE_VALUE) {
378 imm_byte_offset += indirect_byte_offset.ud;
379
380 reg.nr = imm_byte_offset / REG_SIZE;
381 reg.subnr = imm_byte_offset % REG_SIZE;
382 brw_MOV(p, dst, reg);
383 } else {
384 /* Prior to Broadwell, there are only 8 address registers. */
385 assert(inst->exec_size <= 8 || devinfo->gen >= 8);
386
387 /* We use VxH indirect addressing, clobbering a0.0 through a0.7. */
388 struct brw_reg addr = vec8(brw_address_reg(0));
389
390 /* The destination stride of an instruction (in bytes) must be greater
391 * than or equal to the size of the rest of the instruction. Since the
392 * address register is of type UW, we can't use a D-type instruction.
393 * In order to get around this, re retype to UW and use a stride.
394 */
395 indirect_byte_offset =
396 retype(spread(indirect_byte_offset, 2), BRW_REGISTER_TYPE_UW);
397
398 /* There are a number of reasons why we don't use the base offset here.
399 * One reason is that the field is only 9 bits which means we can only
400 * use it to access the first 16 GRFs. Also, from the Haswell PRM
401 * section "Register Region Restrictions":
402 *
403 * "The lower bits of the AddressImmediate must not overflow to
404 * change the register address. The lower 5 bits of Address
405 * Immediate when added to lower 5 bits of address register gives
406 * the sub-register offset. The upper bits of Address Immediate
407 * when added to upper bits of address register gives the register
408 * address. Any overflow from sub-register offset is dropped."
409 *
410 * Since the indirect may cause us to cross a register boundary, this
411 * makes the base offset almost useless. We could try and do something
412 * clever where we use a actual base offset if base_offset % 32 == 0 but
413 * that would mean we were generating different code depending on the
414 * base offset. Instead, for the sake of consistency, we'll just do the
415 * add ourselves. This restriction is only listed in the Haswell PRM
416 * but empirical testing indicates that it applies on all older
417 * generations and is lifted on Broadwell.
418 *
419 * In the end, while base_offset is nice to look at in the generated
420 * code, using it saves us 0 instructions and would require quite a bit
421 * of case-by-case work. It's just not worth it.
422 */
423 brw_ADD(p, addr, indirect_byte_offset, brw_imm_uw(imm_byte_offset));
424
425 if (type_sz(reg.type) > 4 &&
426 ((devinfo->gen == 7 && !devinfo->is_haswell) ||
427 devinfo->is_cherryview || gen_device_info_is_9lp(devinfo))) {
428 /* IVB has an issue (which we found empirically) where it reads two
429 * address register components per channel for indirectly addressed
430 * 64-bit sources.
431 *
432 * From the Cherryview PRM Vol 7. "Register Region Restrictions":
433 *
434 * "When source or destination datatype is 64b or operation is
435 * integer DWord multiply, indirect addressing must not be used."
436 *
437 * To work around both of these, we do two integer MOVs insead of one
438 * 64-bit MOV. Because no double value should ever cross a register
439 * boundary, it's safe to use the immediate offset in the indirect
440 * here to handle adding 4 bytes to the offset and avoid the extra
441 * ADD to the register file.
442 */
443 brw_MOV(p, subscript(dst, BRW_REGISTER_TYPE_D, 0),
444 retype(brw_VxH_indirect(0, 0), BRW_REGISTER_TYPE_D));
445 brw_MOV(p, subscript(dst, BRW_REGISTER_TYPE_D, 1),
446 retype(brw_VxH_indirect(0, 4), BRW_REGISTER_TYPE_D));
447 } else {
448 struct brw_reg ind_src = brw_VxH_indirect(0, 0);
449
450 brw_inst *mov = brw_MOV(p, dst, retype(ind_src, reg.type));
451
452 if (devinfo->gen == 6 && dst.file == BRW_MESSAGE_REGISTER_FILE &&
453 !inst->get_next()->is_tail_sentinel() &&
454 ((fs_inst *)inst->get_next())->mlen > 0) {
455 /* From the Sandybridge PRM:
456 *
457 * "[Errata: DevSNB(SNB)] If MRF register is updated by any
458 * instruction that “indexed/indirect” source AND is followed
459 * by a send, the instruction requires a “Switch”. This is to
460 * avoid race condition where send may dispatch before MRF is
461 * updated."
462 */
463 brw_inst_set_thread_control(devinfo, mov, BRW_THREAD_SWITCH);
464 }
465 }
466 }
467 }
468
469 void
470 fs_generator::generate_shuffle(fs_inst *inst,
471 struct brw_reg dst,
472 struct brw_reg src,
473 struct brw_reg idx)
474 {
475 /* Ivy bridge has some strange behavior that makes this a real pain to
476 * implement for 64-bit values so we just don't bother.
477 */
478 assert(devinfo->gen >= 8 || devinfo->is_haswell || type_sz(src.type) <= 4);
479
480 /* Because we're using the address register, we're limited to 8-wide
481 * execution on gen7. On gen8, we're limited to 16-wide by the address
482 * register file and 8-wide for 64-bit types. We could try and make this
483 * instruction splittable higher up in the compiler but that gets weird
484 * because it reads all of the channels regardless of execution size. It's
485 * easier just to split it here.
486 */
487 const unsigned lower_width =
488 (devinfo->gen <= 7 || type_sz(src.type) > 4) ?
489 8 : MIN2(16, inst->exec_size);
490
491 brw_set_default_exec_size(p, cvt(lower_width) - 1);
492 for (unsigned group = 0; group < inst->exec_size; group += lower_width) {
493 brw_set_default_group(p, group);
494
495 if ((src.vstride == 0 && src.hstride == 0) ||
496 idx.file == BRW_IMMEDIATE_VALUE) {
497 /* Trivial, the source is already uniform or the index is a constant.
498 * We will typically not get here if the optimizer is doing its job,
499 * but asserting would be mean.
500 */
501 const unsigned i = idx.file == BRW_IMMEDIATE_VALUE ? idx.ud : 0;
502 brw_MOV(p, suboffset(dst, group), stride(suboffset(src, i), 0, 1, 0));
503 } else {
504 /* We use VxH indirect addressing, clobbering a0.0 through a0.7. */
505 struct brw_reg addr = vec8(brw_address_reg(0));
506
507 struct brw_reg group_idx = suboffset(idx, group);
508
509 if (lower_width == 8 && group_idx.width == BRW_WIDTH_16) {
510 /* Things get grumpy if the register is too wide. */
511 group_idx.width--;
512 group_idx.vstride--;
513 }
514
515 assert(type_sz(group_idx.type) <= 4);
516 if (type_sz(group_idx.type) == 4) {
517 /* The destination stride of an instruction (in bytes) must be
518 * greater than or equal to the size of the rest of the
519 * instruction. Since the address register is of type UW, we
520 * can't use a D-type instruction. In order to get around this,
521 * re retype to UW and use a stride.
522 */
523 group_idx = retype(spread(group_idx, 2), BRW_REGISTER_TYPE_W);
524 }
525
526 /* Take into account the component size and horizontal stride. */
527 assert(src.vstride == src.hstride + src.width);
528 brw_SHL(p, addr, group_idx,
529 brw_imm_uw(_mesa_logbase2(type_sz(src.type)) +
530 src.hstride - 1));
531
532 /* Add on the register start offset */
533 brw_ADD(p, addr, addr, brw_imm_uw(src.nr * REG_SIZE + src.subnr));
534
535 if (type_sz(src.type) > 4 &&
536 ((devinfo->gen == 7 && !devinfo->is_haswell) ||
537 devinfo->is_cherryview || gen_device_info_is_9lp(devinfo))) {
538 /* IVB has an issue (which we found empirically) where it reads
539 * two address register components per channel for indirectly
540 * addressed 64-bit sources.
541 *
542 * From the Cherryview PRM Vol 7. "Register Region Restrictions":
543 *
544 * "When source or destination datatype is 64b or operation is
545 * integer DWord multiply, indirect addressing must not be
546 * used."
547 *
548 * To work around both of these, we do two integer MOVs insead of
549 * one 64-bit MOV. Because no double value should ever cross a
550 * register boundary, it's safe to use the immediate offset in the
551 * indirect here to handle adding 4 bytes to the offset and avoid
552 * the extra ADD to the register file.
553 */
554 struct brw_reg gdst = suboffset(dst, group);
555 struct brw_reg dst_d = retype(spread(gdst, 2),
556 BRW_REGISTER_TYPE_D);
557 brw_MOV(p, dst_d,
558 retype(brw_VxH_indirect(0, 0), BRW_REGISTER_TYPE_D));
559 brw_MOV(p, byte_offset(dst_d, 4),
560 retype(brw_VxH_indirect(0, 4), BRW_REGISTER_TYPE_D));
561 } else {
562 brw_MOV(p, suboffset(dst, group),
563 retype(brw_VxH_indirect(0, 0), src.type));
564 }
565 }
566 }
567 }
568
569 void
570 fs_generator::generate_urb_read(fs_inst *inst,
571 struct brw_reg dst,
572 struct brw_reg header)
573 {
574 assert(inst->size_written % REG_SIZE == 0);
575 assert(header.file == BRW_GENERAL_REGISTER_FILE);
576 assert(header.type == BRW_REGISTER_TYPE_UD);
577
578 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
579 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UD));
580 brw_set_src0(p, send, header);
581 brw_set_src1(p, send, brw_imm_ud(0u));
582
583 brw_inst_set_sfid(p->devinfo, send, BRW_SFID_URB);
584 brw_inst_set_urb_opcode(p->devinfo, send, GEN8_URB_OPCODE_SIMD8_READ);
585
586 if (inst->opcode == SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT)
587 brw_inst_set_urb_per_slot_offset(p->devinfo, send, true);
588
589 brw_inst_set_mlen(p->devinfo, send, inst->mlen);
590 brw_inst_set_rlen(p->devinfo, send, inst->size_written / REG_SIZE);
591 brw_inst_set_header_present(p->devinfo, send, true);
592 brw_inst_set_urb_global_offset(p->devinfo, send, inst->offset);
593 }
594
595 void
596 fs_generator::generate_urb_write(fs_inst *inst, struct brw_reg payload)
597 {
598 brw_inst *insn;
599
600 /* WaClearTDRRegBeforeEOTForNonPS.
601 *
602 * WA: Clear tdr register before send EOT in all non-PS shader kernels
603 *
604 * mov(8) tdr0:ud 0x0:ud {NoMask}"
605 */
606 if (inst->eot && p->devinfo->gen == 10) {
607 brw_push_insn_state(p);
608 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
609 brw_MOV(p, brw_tdr_reg(), brw_imm_uw(0));
610 brw_pop_insn_state(p);
611 }
612
613 insn = brw_next_insn(p, BRW_OPCODE_SEND);
614
615 brw_set_dest(p, insn, brw_null_reg());
616 brw_set_src0(p, insn, payload);
617 brw_set_src1(p, insn, brw_imm_d(0));
618
619 brw_inst_set_sfid(p->devinfo, insn, BRW_SFID_URB);
620 brw_inst_set_urb_opcode(p->devinfo, insn, GEN8_URB_OPCODE_SIMD8_WRITE);
621
622 if (inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT ||
623 inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT)
624 brw_inst_set_urb_per_slot_offset(p->devinfo, insn, true);
625
626 if (inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_MASKED ||
627 inst->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT)
628 brw_inst_set_urb_channel_mask_present(p->devinfo, insn, true);
629
630 brw_inst_set_mlen(p->devinfo, insn, inst->mlen);
631 brw_inst_set_rlen(p->devinfo, insn, 0);
632 brw_inst_set_eot(p->devinfo, insn, inst->eot);
633 brw_inst_set_header_present(p->devinfo, insn, true);
634 brw_inst_set_urb_global_offset(p->devinfo, insn, inst->offset);
635 }
636
637 void
638 fs_generator::generate_cs_terminate(fs_inst *inst, struct brw_reg payload)
639 {
640 struct brw_inst *insn;
641
642 insn = brw_next_insn(p, BRW_OPCODE_SEND);
643
644 brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW));
645 brw_set_src0(p, insn, retype(payload, BRW_REGISTER_TYPE_UW));
646 brw_set_src1(p, insn, brw_imm_d(0));
647
648 /* Terminate a compute shader by sending a message to the thread spawner.
649 */
650 brw_inst_set_sfid(devinfo, insn, BRW_SFID_THREAD_SPAWNER);
651 brw_inst_set_mlen(devinfo, insn, 1);
652 brw_inst_set_rlen(devinfo, insn, 0);
653 brw_inst_set_eot(devinfo, insn, inst->eot);
654 brw_inst_set_header_present(devinfo, insn, false);
655
656 brw_inst_set_ts_opcode(devinfo, insn, 0); /* Dereference resource */
657 brw_inst_set_ts_request_type(devinfo, insn, 0); /* Root thread */
658
659 /* Note that even though the thread has a URB resource associated with it,
660 * we set the "do not dereference URB" bit, because the URB resource is
661 * managed by the fixed-function unit, so it will free it automatically.
662 */
663 brw_inst_set_ts_resource_select(devinfo, insn, 1); /* Do not dereference URB */
664
665 brw_inst_set_mask_control(devinfo, insn, BRW_MASK_DISABLE);
666 }
667
668 void
669 fs_generator::generate_barrier(fs_inst *, struct brw_reg src)
670 {
671 brw_barrier(p, src);
672 brw_WAIT(p);
673 }
674
675 bool
676 fs_generator::generate_linterp(fs_inst *inst,
677 struct brw_reg dst, struct brw_reg *src)
678 {
679 /* PLN reads:
680 * / in SIMD16 \
681 * -----------------------------------
682 * | src1+0 | src1+1 | src1+2 | src1+3 |
683 * |-----------------------------------|
684 * |(x0, x1)|(y0, y1)|(x2, x3)|(y2, y3)|
685 * -----------------------------------
686 *
687 * but for the LINE/MAC pair, the LINE reads Xs and the MAC reads Ys:
688 *
689 * -----------------------------------
690 * | src1+0 | src1+1 | src1+2 | src1+3 |
691 * |-----------------------------------|
692 * |(x0, x1)|(y0, y1)| | | in SIMD8
693 * |-----------------------------------|
694 * |(x0, x1)|(x2, x3)|(y0, y1)|(y2, y3)| in SIMD16
695 * -----------------------------------
696 *
697 * See also: emit_interpolation_setup_gen4().
698 */
699 struct brw_reg delta_x = src[0];
700 struct brw_reg delta_y = offset(src[0], inst->exec_size / 8);
701 struct brw_reg interp = src[1];
702 brw_inst *i[4];
703
704 if (devinfo->gen >= 11) {
705 struct brw_reg acc = retype(brw_acc_reg(8), BRW_REGISTER_TYPE_NF);
706 struct brw_reg dwP = suboffset(interp, 0);
707 struct brw_reg dwQ = suboffset(interp, 1);
708 struct brw_reg dwR = suboffset(interp, 3);
709
710 brw_push_insn_state(p);
711 brw_set_default_exec_size(p, BRW_EXECUTE_8);
712
713 if (inst->exec_size == 8) {
714 i[0] = brw_MAD(p, acc, dwR, offset(delta_x, 0), dwP);
715 i[1] = brw_MAD(p, offset(dst, 0), acc, offset(delta_y, 0), dwQ);
716
717 brw_inst_set_cond_modifier(p->devinfo, i[1], inst->conditional_mod);
718
719 /* brw_set_default_saturate() is called before emitting instructions,
720 * so the saturate bit is set in each instruction, so we need to unset
721 * it on the first instruction of each pair.
722 */
723 brw_inst_set_saturate(p->devinfo, i[0], false);
724 } else {
725 brw_set_default_group(p, inst->group);
726 i[0] = brw_MAD(p, acc, dwR, offset(delta_x, 0), dwP);
727 i[1] = brw_MAD(p, offset(dst, 0), acc, offset(delta_x, 1), dwQ);
728
729 brw_set_default_group(p, inst->group + 8);
730 i[2] = brw_MAD(p, acc, dwR, offset(delta_y, 0), dwP);
731 i[3] = brw_MAD(p, offset(dst, 1), acc, offset(delta_y, 1), dwQ);
732
733 brw_inst_set_cond_modifier(p->devinfo, i[1], inst->conditional_mod);
734 brw_inst_set_cond_modifier(p->devinfo, i[3], inst->conditional_mod);
735
736 /* brw_set_default_saturate() is called before emitting instructions,
737 * so the saturate bit is set in each instruction, so we need to unset
738 * it on the first instruction of each pair.
739 */
740 brw_inst_set_saturate(p->devinfo, i[0], false);
741 brw_inst_set_saturate(p->devinfo, i[2], false);
742 }
743
744 brw_pop_insn_state(p);
745
746 return true;
747 } else if (devinfo->has_pln) {
748 /* From the Sandy Bridge PRM Vol. 4, Pt. 2, Section 8.3.53, "Plane":
749 *
750 * "[DevSNB]:<src1> must be even register aligned.
751 *
752 * This restriction is lifted on Ivy Bridge.
753 */
754 assert(devinfo->gen >= 7 || (delta_x.nr & 1) == 0);
755 brw_PLN(p, dst, interp, delta_x);
756
757 return false;
758 } else {
759 i[0] = brw_LINE(p, brw_null_reg(), interp, delta_x);
760 i[1] = brw_MAC(p, dst, suboffset(interp, 1), delta_y);
761
762 brw_inst_set_cond_modifier(p->devinfo, i[1], inst->conditional_mod);
763
764 /* brw_set_default_saturate() is called before emitting instructions, so
765 * the saturate bit is set in each instruction, so we need to unset it on
766 * the first instruction.
767 */
768 brw_inst_set_saturate(p->devinfo, i[0], false);
769
770 return true;
771 }
772 }
773
774 void
775 fs_generator::generate_get_buffer_size(fs_inst *inst,
776 struct brw_reg dst,
777 struct brw_reg src,
778 struct brw_reg surf_index)
779 {
780 assert(devinfo->gen >= 7);
781 assert(surf_index.file == BRW_IMMEDIATE_VALUE);
782
783 uint32_t simd_mode;
784 int rlen = 4;
785
786 switch (inst->exec_size) {
787 case 8:
788 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
789 break;
790 case 16:
791 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
792 break;
793 default:
794 unreachable("Invalid width for texture instruction");
795 }
796
797 if (simd_mode == BRW_SAMPLER_SIMD_MODE_SIMD16) {
798 rlen = 8;
799 dst = vec16(dst);
800 }
801
802 brw_SAMPLE(p,
803 retype(dst, BRW_REGISTER_TYPE_UW),
804 inst->base_mrf,
805 src,
806 surf_index.ud,
807 0,
808 GEN5_SAMPLER_MESSAGE_SAMPLE_RESINFO,
809 rlen, /* response length */
810 inst->mlen,
811 inst->header_size > 0,
812 simd_mode,
813 BRW_SAMPLER_RETURN_FORMAT_SINT32);
814
815 brw_mark_surface_used(prog_data, surf_index.ud);
816 }
817
818 void
819 fs_generator::generate_tex(fs_inst *inst, struct brw_reg dst, struct brw_reg src,
820 struct brw_reg surface_index,
821 struct brw_reg sampler_index)
822 {
823 assert(inst->size_written % REG_SIZE == 0);
824 int msg_type = -1;
825 uint32_t simd_mode;
826 uint32_t return_format;
827 bool is_combined_send = inst->eot;
828
829 switch (dst.type) {
830 case BRW_REGISTER_TYPE_D:
831 return_format = BRW_SAMPLER_RETURN_FORMAT_SINT32;
832 break;
833 case BRW_REGISTER_TYPE_UD:
834 return_format = BRW_SAMPLER_RETURN_FORMAT_UINT32;
835 break;
836 default:
837 return_format = BRW_SAMPLER_RETURN_FORMAT_FLOAT32;
838 break;
839 }
840
841 /* Stomp the resinfo output type to UINT32. On gens 4-5, the output type
842 * is set as part of the message descriptor. On gen4, the PRM seems to
843 * allow UINT32 and FLOAT32 (i965 PRM, Vol. 4 Section 4.8.1.1), but on
844 * later gens UINT32 is required. Once you hit Sandy Bridge, the bit is
845 * gone from the message descriptor entirely and you just get UINT32 all
846 * the time regasrdless. Since we can really only do non-UINT32 on gen4,
847 * just stomp it to UINT32 all the time.
848 */
849 if (inst->opcode == SHADER_OPCODE_TXS)
850 return_format = BRW_SAMPLER_RETURN_FORMAT_UINT32;
851
852 switch (inst->exec_size) {
853 case 8:
854 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
855 break;
856 case 16:
857 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
858 break;
859 default:
860 unreachable("Invalid width for texture instruction");
861 }
862
863 if (devinfo->gen >= 5) {
864 switch (inst->opcode) {
865 case SHADER_OPCODE_TEX:
866 if (inst->shadow_compare) {
867 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_COMPARE;
868 } else {
869 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE;
870 }
871 break;
872 case FS_OPCODE_TXB:
873 if (inst->shadow_compare) {
874 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_BIAS_COMPARE;
875 } else {
876 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_BIAS;
877 }
878 break;
879 case SHADER_OPCODE_TXL:
880 if (inst->shadow_compare) {
881 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LOD_COMPARE;
882 } else {
883 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LOD;
884 }
885 break;
886 case SHADER_OPCODE_TXL_LZ:
887 assert(devinfo->gen >= 9);
888 if (inst->shadow_compare) {
889 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_C_LZ;
890 } else {
891 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_LZ;
892 }
893 break;
894 case SHADER_OPCODE_TXS:
895 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_RESINFO;
896 break;
897 case SHADER_OPCODE_TXD:
898 if (inst->shadow_compare) {
899 /* Gen7.5+. Otherwise, lowered in NIR */
900 assert(devinfo->gen >= 8 || devinfo->is_haswell);
901 msg_type = HSW_SAMPLER_MESSAGE_SAMPLE_DERIV_COMPARE;
902 } else {
903 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_DERIVS;
904 }
905 break;
906 case SHADER_OPCODE_TXF:
907 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
908 break;
909 case SHADER_OPCODE_TXF_LZ:
910 assert(devinfo->gen >= 9);
911 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_LD_LZ;
912 break;
913 case SHADER_OPCODE_TXF_CMS_W:
914 assert(devinfo->gen >= 9);
915 msg_type = GEN9_SAMPLER_MESSAGE_SAMPLE_LD2DMS_W;
916 break;
917 case SHADER_OPCODE_TXF_CMS:
918 if (devinfo->gen >= 7)
919 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD2DMS;
920 else
921 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
922 break;
923 case SHADER_OPCODE_TXF_UMS:
924 assert(devinfo->gen >= 7);
925 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD2DSS;
926 break;
927 case SHADER_OPCODE_TXF_MCS:
928 assert(devinfo->gen >= 7);
929 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD_MCS;
930 break;
931 case SHADER_OPCODE_LOD:
932 msg_type = GEN5_SAMPLER_MESSAGE_LOD;
933 break;
934 case SHADER_OPCODE_TG4:
935 if (inst->shadow_compare) {
936 assert(devinfo->gen >= 7);
937 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_C;
938 } else {
939 assert(devinfo->gen >= 6);
940 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4;
941 }
942 break;
943 case SHADER_OPCODE_TG4_OFFSET:
944 assert(devinfo->gen >= 7);
945 if (inst->shadow_compare) {
946 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_PO_C;
947 } else {
948 msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_PO;
949 }
950 break;
951 case SHADER_OPCODE_SAMPLEINFO:
952 msg_type = GEN6_SAMPLER_MESSAGE_SAMPLE_SAMPLEINFO;
953 break;
954 default:
955 unreachable("not reached");
956 }
957 } else {
958 switch (inst->opcode) {
959 case SHADER_OPCODE_TEX:
960 /* Note that G45 and older determines shadow compare and dispatch width
961 * from message length for most messages.
962 */
963 if (inst->exec_size == 8) {
964 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE;
965 if (inst->shadow_compare) {
966 assert(inst->mlen == 6);
967 } else {
968 assert(inst->mlen <= 4);
969 }
970 } else {
971 if (inst->shadow_compare) {
972 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE_COMPARE;
973 assert(inst->mlen == 9);
974 } else {
975 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE;
976 assert(inst->mlen <= 7 && inst->mlen % 2 == 1);
977 }
978 }
979 break;
980 case FS_OPCODE_TXB:
981 if (inst->shadow_compare) {
982 assert(inst->exec_size == 8);
983 assert(inst->mlen == 6);
984 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE_BIAS_COMPARE;
985 } else {
986 assert(inst->mlen == 9);
987 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE_BIAS;
988 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
989 }
990 break;
991 case SHADER_OPCODE_TXL:
992 if (inst->shadow_compare) {
993 assert(inst->exec_size == 8);
994 assert(inst->mlen == 6);
995 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE_LOD_COMPARE;
996 } else {
997 assert(inst->mlen == 9);
998 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE_LOD;
999 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1000 }
1001 break;
1002 case SHADER_OPCODE_TXD:
1003 /* There is no sample_d_c message; comparisons are done manually */
1004 assert(inst->exec_size == 8);
1005 assert(inst->mlen == 7 || inst->mlen == 10);
1006 msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE_GRADIENTS;
1007 break;
1008 case SHADER_OPCODE_TXF:
1009 assert(inst->mlen <= 9 && inst->mlen % 2 == 1);
1010 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_LD;
1011 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1012 break;
1013 case SHADER_OPCODE_TXS:
1014 assert(inst->mlen == 3);
1015 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_RESINFO;
1016 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1017 break;
1018 default:
1019 unreachable("not reached");
1020 }
1021 }
1022 assert(msg_type != -1);
1023
1024 if (simd_mode == BRW_SAMPLER_SIMD_MODE_SIMD16) {
1025 dst = vec16(dst);
1026 }
1027
1028 assert(devinfo->gen < 7 || inst->header_size == 0 ||
1029 src.file == BRW_GENERAL_REGISTER_FILE);
1030
1031 assert(sampler_index.type == BRW_REGISTER_TYPE_UD);
1032
1033 /* Load the message header if present. If there's a texture offset,
1034 * we need to set it up explicitly and load the offset bitfield.
1035 * Otherwise, we can use an implied move from g0 to the first message reg.
1036 */
1037 if (inst->header_size != 0 && devinfo->gen < 7) {
1038 if (devinfo->gen < 6 && !inst->offset) {
1039 /* Set up an implied move from g0 to the MRF. */
1040 src = retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UW);
1041 } else {
1042 assert(inst->base_mrf != -1);
1043 struct brw_reg header_reg = brw_message_reg(inst->base_mrf);
1044
1045 brw_push_insn_state(p);
1046 brw_set_default_exec_size(p, BRW_EXECUTE_8);
1047 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1048 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
1049 /* Explicitly set up the message header by copying g0 to the MRF. */
1050 brw_MOV(p, header_reg, brw_vec8_grf(0, 0));
1051
1052 brw_set_default_exec_size(p, BRW_EXECUTE_1);
1053 if (inst->offset) {
1054 /* Set the offset bits in DWord 2. */
1055 brw_MOV(p, get_element_ud(header_reg, 2),
1056 brw_imm_ud(inst->offset));
1057 }
1058
1059 brw_pop_insn_state(p);
1060 }
1061 }
1062
1063 uint32_t base_binding_table_index = (inst->opcode == SHADER_OPCODE_TG4 ||
1064 inst->opcode == SHADER_OPCODE_TG4_OFFSET)
1065 ? prog_data->binding_table.gather_texture_start
1066 : prog_data->binding_table.texture_start;
1067
1068 if (surface_index.file == BRW_IMMEDIATE_VALUE &&
1069 sampler_index.file == BRW_IMMEDIATE_VALUE) {
1070 uint32_t surface = surface_index.ud;
1071 uint32_t sampler = sampler_index.ud;
1072
1073 brw_SAMPLE(p,
1074 retype(dst, BRW_REGISTER_TYPE_UW),
1075 inst->base_mrf,
1076 src,
1077 surface + base_binding_table_index,
1078 sampler % 16,
1079 msg_type,
1080 inst->size_written / REG_SIZE,
1081 inst->mlen,
1082 inst->header_size != 0,
1083 simd_mode,
1084 return_format);
1085
1086 brw_mark_surface_used(prog_data, surface + base_binding_table_index);
1087 } else {
1088 /* Non-const sampler index */
1089
1090 struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
1091 struct brw_reg surface_reg = vec1(retype(surface_index, BRW_REGISTER_TYPE_UD));
1092 struct brw_reg sampler_reg = vec1(retype(sampler_index, BRW_REGISTER_TYPE_UD));
1093
1094 brw_push_insn_state(p);
1095 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1096 brw_set_default_access_mode(p, BRW_ALIGN_1);
1097 brw_set_default_exec_size(p, BRW_EXECUTE_1);
1098
1099 if (brw_regs_equal(&surface_reg, &sampler_reg)) {
1100 brw_MUL(p, addr, sampler_reg, brw_imm_uw(0x101));
1101 } else {
1102 if (sampler_reg.file == BRW_IMMEDIATE_VALUE) {
1103 brw_OR(p, addr, surface_reg, brw_imm_ud(sampler_reg.ud << 8));
1104 } else {
1105 brw_SHL(p, addr, sampler_reg, brw_imm_ud(8));
1106 brw_OR(p, addr, addr, surface_reg);
1107 }
1108 }
1109 if (base_binding_table_index)
1110 brw_ADD(p, addr, addr, brw_imm_ud(base_binding_table_index));
1111 brw_AND(p, addr, addr, brw_imm_ud(0xfff));
1112
1113 brw_pop_insn_state(p);
1114
1115 /* dst = send(offset, a0.0 | <descriptor>) */
1116 brw_inst *insn = brw_send_indirect_message(
1117 p, BRW_SFID_SAMPLER, dst, src, addr);
1118 brw_set_sampler_message(p, insn,
1119 0 /* surface */,
1120 0 /* sampler */,
1121 msg_type,
1122 inst->size_written / REG_SIZE,
1123 inst->mlen /* mlen */,
1124 inst->header_size != 0 /* header */,
1125 simd_mode,
1126 return_format);
1127
1128 /* visitor knows more than we do about the surface limit required,
1129 * so has already done marking.
1130 */
1131 }
1132
1133 if (is_combined_send) {
1134 brw_inst_set_eot(p->devinfo, brw_last_inst, true);
1135 brw_inst_set_opcode(p->devinfo, brw_last_inst, BRW_OPCODE_SENDC);
1136 }
1137 }
1138
1139
1140 /* For OPCODE_DDX and OPCODE_DDY, per channel of output we've got input
1141 * looking like:
1142 *
1143 * arg0: ss0.tl ss0.tr ss0.bl ss0.br ss1.tl ss1.tr ss1.bl ss1.br
1144 *
1145 * Ideally, we want to produce:
1146 *
1147 * DDX DDY
1148 * dst: (ss0.tr - ss0.tl) (ss0.tl - ss0.bl)
1149 * (ss0.tr - ss0.tl) (ss0.tr - ss0.br)
1150 * (ss0.br - ss0.bl) (ss0.tl - ss0.bl)
1151 * (ss0.br - ss0.bl) (ss0.tr - ss0.br)
1152 * (ss1.tr - ss1.tl) (ss1.tl - ss1.bl)
1153 * (ss1.tr - ss1.tl) (ss1.tr - ss1.br)
1154 * (ss1.br - ss1.bl) (ss1.tl - ss1.bl)
1155 * (ss1.br - ss1.bl) (ss1.tr - ss1.br)
1156 *
1157 * and add another set of two more subspans if in 16-pixel dispatch mode.
1158 *
1159 * For DDX, it ends up being easy: width = 2, horiz=0 gets us the same result
1160 * for each pair, and vertstride = 2 jumps us 2 elements after processing a
1161 * pair. But the ideal approximation may impose a huge performance cost on
1162 * sample_d. On at least Haswell, sample_d instruction does some
1163 * optimizations if the same LOD is used for all pixels in the subspan.
1164 *
1165 * For DDY, we need to use ALIGN16 mode since it's capable of doing the
1166 * appropriate swizzling.
1167 */
1168 void
1169 fs_generator::generate_ddx(const fs_inst *inst,
1170 struct brw_reg dst, struct brw_reg src)
1171 {
1172 unsigned vstride, width;
1173
1174 if (inst->opcode == FS_OPCODE_DDX_FINE) {
1175 /* produce accurate derivatives */
1176 vstride = BRW_VERTICAL_STRIDE_2;
1177 width = BRW_WIDTH_2;
1178 } else {
1179 /* replicate the derivative at the top-left pixel to other pixels */
1180 vstride = BRW_VERTICAL_STRIDE_4;
1181 width = BRW_WIDTH_4;
1182 }
1183
1184 struct brw_reg src0 = src;
1185 struct brw_reg src1 = src;
1186
1187 src0.subnr = sizeof(float);
1188 src0.vstride = vstride;
1189 src0.width = width;
1190 src0.hstride = BRW_HORIZONTAL_STRIDE_0;
1191 src1.vstride = vstride;
1192 src1.width = width;
1193 src1.hstride = BRW_HORIZONTAL_STRIDE_0;
1194
1195 brw_ADD(p, dst, src0, negate(src1));
1196 }
1197
1198 /* The negate_value boolean is used to negate the derivative computation for
1199 * FBOs, since they place the origin at the upper left instead of the lower
1200 * left.
1201 */
1202 void
1203 fs_generator::generate_ddy(const fs_inst *inst,
1204 struct brw_reg dst, struct brw_reg src)
1205 {
1206 if (inst->opcode == FS_OPCODE_DDY_FINE) {
1207 /* produce accurate derivatives */
1208 if (devinfo->gen >= 11) {
1209 src = stride(src, 0, 2, 1);
1210 struct brw_reg src_0 = byte_offset(src, 0 * sizeof(float));
1211 struct brw_reg src_2 = byte_offset(src, 2 * sizeof(float));
1212 struct brw_reg src_4 = byte_offset(src, 4 * sizeof(float));
1213 struct brw_reg src_6 = byte_offset(src, 6 * sizeof(float));
1214 struct brw_reg src_8 = byte_offset(src, 8 * sizeof(float));
1215 struct brw_reg src_10 = byte_offset(src, 10 * sizeof(float));
1216 struct brw_reg src_12 = byte_offset(src, 12 * sizeof(float));
1217 struct brw_reg src_14 = byte_offset(src, 14 * sizeof(float));
1218
1219 struct brw_reg dst_0 = byte_offset(dst, 0 * sizeof(float));
1220 struct brw_reg dst_4 = byte_offset(dst, 4 * sizeof(float));
1221 struct brw_reg dst_8 = byte_offset(dst, 8 * sizeof(float));
1222 struct brw_reg dst_12 = byte_offset(dst, 12 * sizeof(float));
1223
1224 brw_push_insn_state(p);
1225 brw_set_default_exec_size(p, BRW_EXECUTE_4);
1226
1227 brw_ADD(p, dst_0, negate(src_0), src_2);
1228 brw_ADD(p, dst_4, negate(src_4), src_6);
1229
1230 if (inst->exec_size == 16) {
1231 brw_ADD(p, dst_8, negate(src_8), src_10);
1232 brw_ADD(p, dst_12, negate(src_12), src_14);
1233 }
1234
1235 brw_pop_insn_state(p);
1236 } else {
1237 struct brw_reg src0 = stride(src, 4, 4, 1);
1238 struct brw_reg src1 = stride(src, 4, 4, 1);
1239 src0.swizzle = BRW_SWIZZLE_XYXY;
1240 src1.swizzle = BRW_SWIZZLE_ZWZW;
1241
1242 brw_push_insn_state(p);
1243 brw_set_default_access_mode(p, BRW_ALIGN_16);
1244 brw_ADD(p, dst, negate(src0), src1);
1245 brw_pop_insn_state(p);
1246 }
1247 } else {
1248 /* replicate the derivative at the top-left pixel to other pixels */
1249 struct brw_reg src0 = stride(src, 4, 4, 0);
1250 struct brw_reg src1 = stride(src, 4, 4, 0);
1251 src0.subnr = 0 * sizeof(float);
1252 src1.subnr = 2 * sizeof(float);
1253
1254 brw_ADD(p, dst, negate(src0), src1);
1255 }
1256 }
1257
1258 void
1259 fs_generator::generate_discard_jump(fs_inst *)
1260 {
1261 assert(devinfo->gen >= 6);
1262
1263 /* This HALT will be patched up at FB write time to point UIP at the end of
1264 * the program, and at brw_uip_jip() JIP will be set to the end of the
1265 * current block (or the program).
1266 */
1267 this->discard_halt_patches.push_tail(new(mem_ctx) ip_record(p->nr_insn));
1268 gen6_HALT(p);
1269 }
1270
1271 void
1272 fs_generator::generate_scratch_write(fs_inst *inst, struct brw_reg src)
1273 {
1274 /* The 32-wide messages only respect the first 16-wide half of the channel
1275 * enable signals which are replicated identically for the second group of
1276 * 16 channels, so we cannot use them unless the write is marked
1277 * force_writemask_all.
1278 */
1279 const unsigned lower_size = inst->force_writemask_all ? inst->exec_size :
1280 MIN2(16, inst->exec_size);
1281 const unsigned block_size = 4 * lower_size / REG_SIZE;
1282 assert(inst->mlen != 0);
1283
1284 brw_push_insn_state(p);
1285 brw_set_default_exec_size(p, cvt(lower_size) - 1);
1286 brw_set_default_compression(p, lower_size > 8);
1287
1288 for (unsigned i = 0; i < inst->exec_size / lower_size; i++) {
1289 brw_set_default_group(p, inst->group + lower_size * i);
1290
1291 brw_MOV(p, brw_uvec_mrf(lower_size, inst->base_mrf + 1, 0),
1292 retype(offset(src, block_size * i), BRW_REGISTER_TYPE_UD));
1293
1294 brw_oword_block_write_scratch(p, brw_message_reg(inst->base_mrf),
1295 block_size,
1296 inst->offset + block_size * REG_SIZE * i);
1297 }
1298
1299 brw_pop_insn_state(p);
1300 }
1301
1302 void
1303 fs_generator::generate_scratch_read(fs_inst *inst, struct brw_reg dst)
1304 {
1305 assert(inst->exec_size <= 16 || inst->force_writemask_all);
1306 assert(inst->mlen != 0);
1307
1308 brw_oword_block_read_scratch(p, dst, brw_message_reg(inst->base_mrf),
1309 inst->exec_size / 8, inst->offset);
1310 }
1311
1312 void
1313 fs_generator::generate_scratch_read_gen7(fs_inst *inst, struct brw_reg dst)
1314 {
1315 assert(inst->exec_size <= 16 || inst->force_writemask_all);
1316
1317 gen7_block_read_scratch(p, dst, inst->exec_size / 8, inst->offset);
1318 }
1319
1320 void
1321 fs_generator::generate_uniform_pull_constant_load(fs_inst *inst,
1322 struct brw_reg dst,
1323 struct brw_reg index,
1324 struct brw_reg offset)
1325 {
1326 assert(type_sz(dst.type) == 4);
1327 assert(inst->mlen != 0);
1328
1329 assert(index.file == BRW_IMMEDIATE_VALUE &&
1330 index.type == BRW_REGISTER_TYPE_UD);
1331 uint32_t surf_index = index.ud;
1332
1333 assert(offset.file == BRW_IMMEDIATE_VALUE &&
1334 offset.type == BRW_REGISTER_TYPE_UD);
1335 uint32_t read_offset = offset.ud;
1336
1337 brw_oword_block_read(p, dst, brw_message_reg(inst->base_mrf),
1338 read_offset, surf_index);
1339 }
1340
1341 void
1342 fs_generator::generate_uniform_pull_constant_load_gen7(fs_inst *inst,
1343 struct brw_reg dst,
1344 struct brw_reg index,
1345 struct brw_reg payload)
1346 {
1347 assert(index.type == BRW_REGISTER_TYPE_UD);
1348 assert(payload.file == BRW_GENERAL_REGISTER_FILE);
1349 assert(type_sz(dst.type) == 4);
1350
1351 if (index.file == BRW_IMMEDIATE_VALUE) {
1352 const uint32_t surf_index = index.ud;
1353
1354 brw_push_insn_state(p);
1355 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1356 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
1357 brw_pop_insn_state(p);
1358
1359 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UD));
1360 brw_set_src0(p, send, retype(payload, BRW_REGISTER_TYPE_UD));
1361 brw_set_dp_read_message(p, send, surf_index,
1362 BRW_DATAPORT_OWORD_BLOCK_DWORDS(inst->exec_size),
1363 GEN7_DATAPORT_DC_OWORD_BLOCK_READ,
1364 GEN6_SFID_DATAPORT_CONSTANT_CACHE,
1365 1, /* mlen */
1366 true, /* header */
1367 DIV_ROUND_UP(inst->size_written, REG_SIZE));
1368
1369 } else {
1370 struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
1371
1372 brw_push_insn_state(p);
1373 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1374
1375 /* a0.0 = surf_index & 0xff */
1376 brw_inst *insn_and = brw_next_insn(p, BRW_OPCODE_AND);
1377 brw_inst_set_exec_size(p->devinfo, insn_and, BRW_EXECUTE_1);
1378 brw_set_dest(p, insn_and, addr);
1379 brw_set_src0(p, insn_and, vec1(retype(index, BRW_REGISTER_TYPE_UD)));
1380 brw_set_src1(p, insn_and, brw_imm_ud(0x0ff));
1381
1382 /* dst = send(payload, a0.0 | <descriptor>) */
1383 brw_inst *insn = brw_send_indirect_message(
1384 p, GEN6_SFID_DATAPORT_CONSTANT_CACHE,
1385 retype(dst, BRW_REGISTER_TYPE_UD),
1386 retype(payload, BRW_REGISTER_TYPE_UD), addr);
1387 brw_set_dp_read_message(p, insn, 0 /* surface */,
1388 BRW_DATAPORT_OWORD_BLOCK_DWORDS(inst->exec_size),
1389 GEN7_DATAPORT_DC_OWORD_BLOCK_READ,
1390 GEN6_SFID_DATAPORT_CONSTANT_CACHE,
1391 1, /* mlen */
1392 true, /* header */
1393 DIV_ROUND_UP(inst->size_written, REG_SIZE));
1394
1395 brw_pop_insn_state(p);
1396 }
1397 }
1398
1399 void
1400 fs_generator::generate_varying_pull_constant_load_gen4(fs_inst *inst,
1401 struct brw_reg dst,
1402 struct brw_reg index)
1403 {
1404 assert(devinfo->gen < 7); /* Should use the gen7 variant. */
1405 assert(inst->header_size != 0);
1406 assert(inst->mlen);
1407
1408 assert(index.file == BRW_IMMEDIATE_VALUE &&
1409 index.type == BRW_REGISTER_TYPE_UD);
1410 uint32_t surf_index = index.ud;
1411
1412 uint32_t simd_mode, rlen, msg_type;
1413 if (inst->exec_size == 16) {
1414 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1415 rlen = 8;
1416 } else {
1417 assert(inst->exec_size == 8);
1418 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
1419 rlen = 4;
1420 }
1421
1422 if (devinfo->gen >= 5)
1423 msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
1424 else {
1425 /* We always use the SIMD16 message so that we only have to load U, and
1426 * not V or R.
1427 */
1428 msg_type = BRW_SAMPLER_MESSAGE_SIMD16_LD;
1429 assert(inst->mlen == 3);
1430 assert(inst->size_written == 8 * REG_SIZE);
1431 rlen = 8;
1432 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1433 }
1434
1435 struct brw_reg header = brw_vec8_grf(0, 0);
1436 gen6_resolve_implied_move(p, &header, inst->base_mrf);
1437
1438 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
1439 brw_inst_set_compression(devinfo, send, false);
1440 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UW));
1441 brw_set_src0(p, send, header);
1442 if (devinfo->gen < 6)
1443 brw_inst_set_base_mrf(p->devinfo, send, inst->base_mrf);
1444
1445 /* Our surface is set up as floats, regardless of what actual data is
1446 * stored in it.
1447 */
1448 uint32_t return_format = BRW_SAMPLER_RETURN_FORMAT_FLOAT32;
1449 brw_set_sampler_message(p, send,
1450 surf_index,
1451 0, /* sampler (unused) */
1452 msg_type,
1453 rlen,
1454 inst->mlen,
1455 inst->header_size != 0,
1456 simd_mode,
1457 return_format);
1458 }
1459
1460 void
1461 fs_generator::generate_varying_pull_constant_load_gen7(fs_inst *inst,
1462 struct brw_reg dst,
1463 struct brw_reg index,
1464 struct brw_reg offset)
1465 {
1466 assert(devinfo->gen >= 7);
1467 /* Varying-offset pull constant loads are treated as a normal expression on
1468 * gen7, so the fact that it's a send message is hidden at the IR level.
1469 */
1470 assert(inst->header_size == 0);
1471 assert(!inst->mlen);
1472 assert(index.type == BRW_REGISTER_TYPE_UD);
1473
1474 uint32_t simd_mode, rlen, mlen;
1475 if (inst->exec_size == 16) {
1476 mlen = 2;
1477 rlen = 8;
1478 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1479 } else {
1480 assert(inst->exec_size == 8);
1481 mlen = 1;
1482 rlen = 4;
1483 simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
1484 }
1485
1486 if (index.file == BRW_IMMEDIATE_VALUE) {
1487
1488 uint32_t surf_index = index.ud;
1489
1490 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
1491 brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UW));
1492 brw_set_src0(p, send, offset);
1493 brw_set_sampler_message(p, send,
1494 surf_index,
1495 0, /* LD message ignores sampler unit */
1496 GEN5_SAMPLER_MESSAGE_SAMPLE_LD,
1497 rlen,
1498 mlen,
1499 false, /* no header */
1500 simd_mode,
1501 0);
1502
1503 } else {
1504
1505 struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
1506
1507 brw_push_insn_state(p);
1508 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1509
1510 /* a0.0 = surf_index & 0xff */
1511 brw_inst *insn_and = brw_next_insn(p, BRW_OPCODE_AND);
1512 brw_inst_set_exec_size(p->devinfo, insn_and, BRW_EXECUTE_1);
1513 brw_set_dest(p, insn_and, addr);
1514 brw_set_src0(p, insn_and, vec1(retype(index, BRW_REGISTER_TYPE_UD)));
1515 brw_set_src1(p, insn_and, brw_imm_ud(0x0ff));
1516
1517 brw_pop_insn_state(p);
1518
1519 /* dst = send(offset, a0.0 | <descriptor>) */
1520 brw_inst *insn = brw_send_indirect_message(
1521 p, BRW_SFID_SAMPLER, retype(dst, BRW_REGISTER_TYPE_UW),
1522 offset, addr);
1523 brw_set_sampler_message(p, insn,
1524 0 /* surface */,
1525 0 /* sampler */,
1526 GEN5_SAMPLER_MESSAGE_SAMPLE_LD,
1527 rlen /* rlen */,
1528 mlen /* mlen */,
1529 false /* header */,
1530 simd_mode,
1531 0);
1532 }
1533 }
1534
1535 /**
1536 * Cause the current pixel/sample mask (from R1.7 bits 15:0) to be transferred
1537 * into the flags register (f0.0).
1538 *
1539 * Used only on Gen6 and above.
1540 */
1541 void
1542 fs_generator::generate_mov_dispatch_to_flags(fs_inst *inst)
1543 {
1544 struct brw_reg flags = brw_flag_subreg(inst->flag_subreg);
1545 struct brw_reg dispatch_mask;
1546
1547 if (devinfo->gen >= 6)
1548 dispatch_mask = retype(brw_vec1_grf(1, 7), BRW_REGISTER_TYPE_UW);
1549 else
1550 dispatch_mask = retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UW);
1551
1552 brw_push_insn_state(p);
1553 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1554 brw_set_default_exec_size(p, BRW_EXECUTE_1);
1555 brw_MOV(p, flags, dispatch_mask);
1556 brw_pop_insn_state(p);
1557 }
1558
1559 void
1560 fs_generator::generate_pixel_interpolator_query(fs_inst *inst,
1561 struct brw_reg dst,
1562 struct brw_reg src,
1563 struct brw_reg msg_data,
1564 unsigned msg_type)
1565 {
1566 assert(inst->size_written % REG_SIZE == 0);
1567 assert(msg_data.type == BRW_REGISTER_TYPE_UD);
1568
1569 brw_pixel_interpolator_query(p,
1570 retype(dst, BRW_REGISTER_TYPE_UW),
1571 src,
1572 inst->pi_noperspective,
1573 msg_type,
1574 msg_data,
1575 inst->mlen,
1576 inst->size_written / REG_SIZE);
1577 }
1578
1579 /* Sets vstride=1, width=4, hstride=0 of register src1 during
1580 * the ADD instruction.
1581 */
1582 void
1583 fs_generator::generate_set_sample_id(fs_inst *inst,
1584 struct brw_reg dst,
1585 struct brw_reg src0,
1586 struct brw_reg src1)
1587 {
1588 assert(dst.type == BRW_REGISTER_TYPE_D ||
1589 dst.type == BRW_REGISTER_TYPE_UD);
1590 assert(src0.type == BRW_REGISTER_TYPE_D ||
1591 src0.type == BRW_REGISTER_TYPE_UD);
1592
1593 struct brw_reg reg = stride(src1, 1, 4, 0);
1594 if (devinfo->gen >= 8 || inst->exec_size == 8) {
1595 brw_ADD(p, dst, src0, reg);
1596 } else if (inst->exec_size == 16) {
1597 brw_push_insn_state(p);
1598 brw_set_default_exec_size(p, BRW_EXECUTE_8);
1599 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
1600 brw_ADD(p, firsthalf(dst), firsthalf(src0), reg);
1601 brw_set_default_compression_control(p, BRW_COMPRESSION_2NDHALF);
1602 brw_ADD(p, sechalf(dst), sechalf(src0), suboffset(reg, 2));
1603 brw_pop_insn_state(p);
1604 }
1605 }
1606
1607 void
1608 fs_generator::generate_pack_half_2x16_split(fs_inst *,
1609 struct brw_reg dst,
1610 struct brw_reg x,
1611 struct brw_reg y)
1612 {
1613 assert(devinfo->gen >= 7);
1614 assert(dst.type == BRW_REGISTER_TYPE_UD);
1615 assert(x.type == BRW_REGISTER_TYPE_F);
1616 assert(y.type == BRW_REGISTER_TYPE_F);
1617
1618 /* From the Ivybridge PRM, Vol4, Part3, Section 6.27 f32to16:
1619 *
1620 * Because this instruction does not have a 16-bit floating-point type,
1621 * the destination data type must be Word (W).
1622 *
1623 * The destination must be DWord-aligned and specify a horizontal stride
1624 * (HorzStride) of 2. The 16-bit result is stored in the lower word of
1625 * each destination channel and the upper word is not modified.
1626 */
1627 struct brw_reg dst_w = spread(retype(dst, BRW_REGISTER_TYPE_W), 2);
1628
1629 /* Give each 32-bit channel of dst the form below, where "." means
1630 * unchanged.
1631 * 0x....hhhh
1632 */
1633 brw_F32TO16(p, dst_w, y);
1634
1635 /* Now the form:
1636 * 0xhhhh0000
1637 */
1638 brw_SHL(p, dst, dst, brw_imm_ud(16u));
1639
1640 /* And, finally the form of packHalf2x16's output:
1641 * 0xhhhhllll
1642 */
1643 brw_F32TO16(p, dst_w, x);
1644 }
1645
1646 void
1647 fs_generator::generate_unpack_half_2x16_split(fs_inst *inst,
1648 struct brw_reg dst,
1649 struct brw_reg src)
1650 {
1651 assert(devinfo->gen >= 7);
1652 assert(dst.type == BRW_REGISTER_TYPE_F);
1653 assert(src.type == BRW_REGISTER_TYPE_UD);
1654
1655 /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
1656 *
1657 * Because this instruction does not have a 16-bit floating-point type,
1658 * the source data type must be Word (W). The destination type must be
1659 * F (Float).
1660 */
1661 struct brw_reg src_w = spread(retype(src, BRW_REGISTER_TYPE_W), 2);
1662
1663 /* Each channel of src has the form of unpackHalf2x16's input: 0xhhhhllll.
1664 * For the Y case, we wish to access only the upper word; therefore
1665 * a 16-bit subregister offset is needed.
1666 */
1667 assert(inst->opcode == FS_OPCODE_UNPACK_HALF_2x16_SPLIT_X ||
1668 inst->opcode == FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y);
1669 if (inst->opcode == FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y)
1670 src_w.subnr += 2;
1671
1672 brw_F16TO32(p, dst, src_w);
1673 }
1674
1675 void
1676 fs_generator::generate_shader_time_add(fs_inst *,
1677 struct brw_reg payload,
1678 struct brw_reg offset,
1679 struct brw_reg value)
1680 {
1681 assert(devinfo->gen >= 7);
1682 brw_push_insn_state(p);
1683 brw_set_default_mask_control(p, true);
1684
1685 assert(payload.file == BRW_GENERAL_REGISTER_FILE);
1686 struct brw_reg payload_offset = retype(brw_vec1_grf(payload.nr, 0),
1687 offset.type);
1688 struct brw_reg payload_value = retype(brw_vec1_grf(payload.nr + 1, 0),
1689 value.type);
1690
1691 assert(offset.file == BRW_IMMEDIATE_VALUE);
1692 if (value.file == BRW_GENERAL_REGISTER_FILE) {
1693 value.width = BRW_WIDTH_1;
1694 value.hstride = BRW_HORIZONTAL_STRIDE_0;
1695 value.vstride = BRW_VERTICAL_STRIDE_0;
1696 } else {
1697 assert(value.file == BRW_IMMEDIATE_VALUE);
1698 }
1699
1700 /* Trying to deal with setup of the params from the IR is crazy in the FS8
1701 * case, and we don't really care about squeezing every bit of performance
1702 * out of this path, so we just emit the MOVs from here.
1703 */
1704 brw_MOV(p, payload_offset, offset);
1705 brw_MOV(p, payload_value, value);
1706 brw_shader_time_add(p, payload,
1707 prog_data->binding_table.shader_time_start);
1708 brw_pop_insn_state(p);
1709
1710 brw_mark_surface_used(prog_data,
1711 prog_data->binding_table.shader_time_start);
1712 }
1713
1714 void
1715 fs_generator::enable_debug(const char *shader_name)
1716 {
1717 debug_flag = true;
1718 this->shader_name = shader_name;
1719 }
1720
1721 int
1722 fs_generator::generate_code(const cfg_t *cfg, int dispatch_width)
1723 {
1724 /* align to 64 byte boundary. */
1725 while (p->next_insn_offset % 64)
1726 brw_NOP(p);
1727
1728 this->dispatch_width = dispatch_width;
1729
1730 int start_offset = p->next_insn_offset;
1731 int spill_count = 0, fill_count = 0;
1732 int loop_count = 0;
1733
1734 struct disasm_info *disasm_info = disasm_initialize(devinfo, cfg);
1735
1736 foreach_block_and_inst (block, fs_inst, inst, cfg) {
1737 struct brw_reg src[3], dst;
1738 unsigned int last_insn_offset = p->next_insn_offset;
1739 bool multiple_instructions_emitted = false;
1740
1741 /* From the Broadwell PRM, Volume 7, "3D-Media-GPGPU", in the
1742 * "Register Region Restrictions" section: for BDW, SKL:
1743 *
1744 * "A POW/FDIV operation must not be followed by an instruction
1745 * that requires two destination registers."
1746 *
1747 * The documentation is often lacking annotations for Atom parts,
1748 * and empirically this affects CHV as well.
1749 */
1750 if (devinfo->gen >= 8 &&
1751 devinfo->gen <= 9 &&
1752 p->nr_insn > 1 &&
1753 brw_inst_opcode(devinfo, brw_last_inst) == BRW_OPCODE_MATH &&
1754 brw_inst_math_function(devinfo, brw_last_inst) == BRW_MATH_FUNCTION_POW &&
1755 inst->dst.component_size(inst->exec_size) > REG_SIZE) {
1756 brw_NOP(p);
1757 last_insn_offset = p->next_insn_offset;
1758 }
1759
1760 if (unlikely(debug_flag))
1761 disasm_annotate(disasm_info, inst, p->next_insn_offset);
1762
1763 /* If the instruction writes to more than one register, it needs to be
1764 * explicitly marked as compressed on Gen <= 5. On Gen >= 6 the
1765 * hardware figures out by itself what the right compression mode is,
1766 * but we still need to know whether the instruction is compressed to
1767 * set up the source register regions appropriately.
1768 *
1769 * XXX - This is wrong for instructions that write a single register but
1770 * read more than one which should strictly speaking be treated as
1771 * compressed. For instructions that don't write any registers it
1772 * relies on the destination being a null register of the correct
1773 * type and regioning so the instruction is considered compressed
1774 * or not accordingly.
1775 */
1776 const bool compressed =
1777 inst->dst.component_size(inst->exec_size) > REG_SIZE;
1778 brw_set_default_compression(p, compressed);
1779 brw_set_default_group(p, inst->group);
1780
1781 for (unsigned int i = 0; i < inst->sources; i++) {
1782 src[i] = brw_reg_from_fs_reg(devinfo, inst,
1783 &inst->src[i], compressed);
1784 /* The accumulator result appears to get used for the
1785 * conditional modifier generation. When negating a UD
1786 * value, there is a 33rd bit generated for the sign in the
1787 * accumulator value, so now you can't check, for example,
1788 * equality with a 32-bit value. See piglit fs-op-neg-uvec4.
1789 */
1790 assert(!inst->conditional_mod ||
1791 inst->src[i].type != BRW_REGISTER_TYPE_UD ||
1792 !inst->src[i].negate);
1793 }
1794 dst = brw_reg_from_fs_reg(devinfo, inst,
1795 &inst->dst, compressed);
1796
1797 brw_set_default_access_mode(p, BRW_ALIGN_1);
1798 brw_set_default_predicate_control(p, inst->predicate);
1799 brw_set_default_predicate_inverse(p, inst->predicate_inverse);
1800 brw_set_default_flag_reg(p, inst->flag_subreg / 2, inst->flag_subreg % 2);
1801 brw_set_default_saturate(p, inst->saturate);
1802 brw_set_default_mask_control(p, inst->force_writemask_all);
1803 brw_set_default_acc_write_control(p, inst->writes_accumulator);
1804
1805 unsigned exec_size = inst->exec_size;
1806 if (devinfo->gen == 7 && !devinfo->is_haswell &&
1807 (get_exec_type_size(inst) == 8 || type_sz(inst->dst.type) == 8)) {
1808 exec_size *= 2;
1809 }
1810
1811 brw_set_default_exec_size(p, cvt(exec_size) - 1);
1812
1813 assert(inst->force_writemask_all || inst->exec_size >= 4);
1814 assert(inst->force_writemask_all || inst->group % inst->exec_size == 0);
1815 assert(inst->base_mrf + inst->mlen <= BRW_MAX_MRF(devinfo->gen));
1816 assert(inst->mlen <= BRW_MAX_MSG_LENGTH);
1817
1818 switch (inst->opcode) {
1819 case BRW_OPCODE_MOV:
1820 brw_MOV(p, dst, src[0]);
1821 break;
1822 case BRW_OPCODE_ADD:
1823 brw_ADD(p, dst, src[0], src[1]);
1824 break;
1825 case BRW_OPCODE_MUL:
1826 brw_MUL(p, dst, src[0], src[1]);
1827 break;
1828 case BRW_OPCODE_AVG:
1829 brw_AVG(p, dst, src[0], src[1]);
1830 break;
1831 case BRW_OPCODE_MACH:
1832 brw_MACH(p, dst, src[0], src[1]);
1833 break;
1834
1835 case BRW_OPCODE_LINE:
1836 brw_LINE(p, dst, src[0], src[1]);
1837 break;
1838
1839 case BRW_OPCODE_MAD:
1840 assert(devinfo->gen >= 6);
1841 if (devinfo->gen < 10)
1842 brw_set_default_access_mode(p, BRW_ALIGN_16);
1843 brw_MAD(p, dst, src[0], src[1], src[2]);
1844 break;
1845
1846 case BRW_OPCODE_LRP:
1847 assert(devinfo->gen >= 6 && devinfo->gen <= 10);
1848 if (devinfo->gen < 10)
1849 brw_set_default_access_mode(p, BRW_ALIGN_16);
1850 brw_LRP(p, dst, src[0], src[1], src[2]);
1851 break;
1852
1853 case BRW_OPCODE_FRC:
1854 brw_FRC(p, dst, src[0]);
1855 break;
1856 case BRW_OPCODE_RNDD:
1857 brw_RNDD(p, dst, src[0]);
1858 break;
1859 case BRW_OPCODE_RNDE:
1860 brw_RNDE(p, dst, src[0]);
1861 break;
1862 case BRW_OPCODE_RNDZ:
1863 brw_RNDZ(p, dst, src[0]);
1864 break;
1865
1866 case BRW_OPCODE_AND:
1867 brw_AND(p, dst, src[0], src[1]);
1868 break;
1869 case BRW_OPCODE_OR:
1870 brw_OR(p, dst, src[0], src[1]);
1871 break;
1872 case BRW_OPCODE_XOR:
1873 brw_XOR(p, dst, src[0], src[1]);
1874 break;
1875 case BRW_OPCODE_NOT:
1876 brw_NOT(p, dst, src[0]);
1877 break;
1878 case BRW_OPCODE_ASR:
1879 brw_ASR(p, dst, src[0], src[1]);
1880 break;
1881 case BRW_OPCODE_SHR:
1882 brw_SHR(p, dst, src[0], src[1]);
1883 break;
1884 case BRW_OPCODE_SHL:
1885 brw_SHL(p, dst, src[0], src[1]);
1886 break;
1887 case BRW_OPCODE_F32TO16:
1888 assert(devinfo->gen >= 7);
1889 brw_F32TO16(p, dst, src[0]);
1890 break;
1891 case BRW_OPCODE_F16TO32:
1892 assert(devinfo->gen >= 7);
1893 brw_F16TO32(p, dst, src[0]);
1894 break;
1895 case BRW_OPCODE_CMP:
1896 if (inst->exec_size >= 16 && devinfo->gen == 7 && !devinfo->is_haswell &&
1897 dst.file == BRW_ARCHITECTURE_REGISTER_FILE) {
1898 /* For unknown reasons the WaCMPInstFlagDepClearedEarly workaround
1899 * implemented in the compiler is not sufficient. Overriding the
1900 * type when the destination is the null register is necessary but
1901 * not sufficient by itself.
1902 */
1903 assert(dst.nr == BRW_ARF_NULL);
1904 dst.type = BRW_REGISTER_TYPE_D;
1905 }
1906 brw_CMP(p, dst, inst->conditional_mod, src[0], src[1]);
1907 break;
1908 case BRW_OPCODE_SEL:
1909 brw_SEL(p, dst, src[0], src[1]);
1910 break;
1911 case BRW_OPCODE_CSEL:
1912 assert(devinfo->gen >= 8);
1913 if (devinfo->gen < 10)
1914 brw_set_default_access_mode(p, BRW_ALIGN_16);
1915 brw_CSEL(p, dst, src[0], src[1], src[2]);
1916 break;
1917 case BRW_OPCODE_BFREV:
1918 assert(devinfo->gen >= 7);
1919 brw_BFREV(p, retype(dst, BRW_REGISTER_TYPE_UD),
1920 retype(src[0], BRW_REGISTER_TYPE_UD));
1921 break;
1922 case BRW_OPCODE_FBH:
1923 assert(devinfo->gen >= 7);
1924 brw_FBH(p, retype(dst, src[0].type), src[0]);
1925 break;
1926 case BRW_OPCODE_FBL:
1927 assert(devinfo->gen >= 7);
1928 brw_FBL(p, retype(dst, BRW_REGISTER_TYPE_UD),
1929 retype(src[0], BRW_REGISTER_TYPE_UD));
1930 break;
1931 case BRW_OPCODE_LZD:
1932 brw_LZD(p, dst, src[0]);
1933 break;
1934 case BRW_OPCODE_CBIT:
1935 assert(devinfo->gen >= 7);
1936 brw_CBIT(p, retype(dst, BRW_REGISTER_TYPE_UD),
1937 retype(src[0], BRW_REGISTER_TYPE_UD));
1938 break;
1939 case BRW_OPCODE_ADDC:
1940 assert(devinfo->gen >= 7);
1941 brw_ADDC(p, dst, src[0], src[1]);
1942 break;
1943 case BRW_OPCODE_SUBB:
1944 assert(devinfo->gen >= 7);
1945 brw_SUBB(p, dst, src[0], src[1]);
1946 break;
1947 case BRW_OPCODE_MAC:
1948 brw_MAC(p, dst, src[0], src[1]);
1949 break;
1950
1951 case BRW_OPCODE_BFE:
1952 assert(devinfo->gen >= 7);
1953 if (devinfo->gen < 10)
1954 brw_set_default_access_mode(p, BRW_ALIGN_16);
1955 brw_BFE(p, dst, src[0], src[1], src[2]);
1956 break;
1957
1958 case BRW_OPCODE_BFI1:
1959 assert(devinfo->gen >= 7);
1960 brw_BFI1(p, dst, src[0], src[1]);
1961 break;
1962 case BRW_OPCODE_BFI2:
1963 assert(devinfo->gen >= 7);
1964 if (devinfo->gen < 10)
1965 brw_set_default_access_mode(p, BRW_ALIGN_16);
1966 brw_BFI2(p, dst, src[0], src[1], src[2]);
1967 break;
1968
1969 case BRW_OPCODE_IF:
1970 if (inst->src[0].file != BAD_FILE) {
1971 /* The instruction has an embedded compare (only allowed on gen6) */
1972 assert(devinfo->gen == 6);
1973 gen6_IF(p, inst->conditional_mod, src[0], src[1]);
1974 } else {
1975 brw_IF(p, brw_get_default_exec_size(p));
1976 }
1977 break;
1978
1979 case BRW_OPCODE_ELSE:
1980 brw_ELSE(p);
1981 break;
1982 case BRW_OPCODE_ENDIF:
1983 brw_ENDIF(p);
1984 break;
1985
1986 case BRW_OPCODE_DO:
1987 brw_DO(p, brw_get_default_exec_size(p));
1988 break;
1989
1990 case BRW_OPCODE_BREAK:
1991 brw_BREAK(p);
1992 break;
1993 case BRW_OPCODE_CONTINUE:
1994 brw_CONT(p);
1995 break;
1996
1997 case BRW_OPCODE_WHILE:
1998 brw_WHILE(p);
1999 loop_count++;
2000 break;
2001
2002 case SHADER_OPCODE_RCP:
2003 case SHADER_OPCODE_RSQ:
2004 case SHADER_OPCODE_SQRT:
2005 case SHADER_OPCODE_EXP2:
2006 case SHADER_OPCODE_LOG2:
2007 case SHADER_OPCODE_SIN:
2008 case SHADER_OPCODE_COS:
2009 assert(inst->conditional_mod == BRW_CONDITIONAL_NONE);
2010 if (devinfo->gen >= 6) {
2011 assert(inst->mlen == 0);
2012 assert(devinfo->gen >= 7 || inst->exec_size == 8);
2013 gen6_math(p, dst, brw_math_function(inst->opcode),
2014 src[0], brw_null_reg());
2015 } else {
2016 assert(inst->mlen >= 1);
2017 assert(devinfo->gen == 5 || devinfo->is_g4x || inst->exec_size == 8);
2018 gen4_math(p, dst,
2019 brw_math_function(inst->opcode),
2020 inst->base_mrf, src[0],
2021 BRW_MATH_PRECISION_FULL);
2022 }
2023 break;
2024 case SHADER_OPCODE_INT_QUOTIENT:
2025 case SHADER_OPCODE_INT_REMAINDER:
2026 case SHADER_OPCODE_POW:
2027 assert(inst->conditional_mod == BRW_CONDITIONAL_NONE);
2028 if (devinfo->gen >= 6) {
2029 assert(inst->mlen == 0);
2030 assert((devinfo->gen >= 7 && inst->opcode == SHADER_OPCODE_POW) ||
2031 inst->exec_size == 8);
2032 gen6_math(p, dst, brw_math_function(inst->opcode), src[0], src[1]);
2033 } else {
2034 assert(inst->mlen >= 1);
2035 assert(inst->exec_size == 8);
2036 gen4_math(p, dst, brw_math_function(inst->opcode),
2037 inst->base_mrf, src[0],
2038 BRW_MATH_PRECISION_FULL);
2039 }
2040 break;
2041 case FS_OPCODE_LINTERP:
2042 multiple_instructions_emitted = generate_linterp(inst, dst, src);
2043 break;
2044 case FS_OPCODE_PIXEL_X:
2045 assert(src[0].type == BRW_REGISTER_TYPE_UW);
2046 src[0].subnr = 0 * type_sz(src[0].type);
2047 brw_MOV(p, dst, stride(src[0], 8, 4, 1));
2048 break;
2049 case FS_OPCODE_PIXEL_Y:
2050 assert(src[0].type == BRW_REGISTER_TYPE_UW);
2051 src[0].subnr = 4 * type_sz(src[0].type);
2052 brw_MOV(p, dst, stride(src[0], 8, 4, 1));
2053 break;
2054 case SHADER_OPCODE_GET_BUFFER_SIZE:
2055 generate_get_buffer_size(inst, dst, src[0], src[1]);
2056 break;
2057 case SHADER_OPCODE_TEX:
2058 case FS_OPCODE_TXB:
2059 case SHADER_OPCODE_TXD:
2060 case SHADER_OPCODE_TXF:
2061 case SHADER_OPCODE_TXF_LZ:
2062 case SHADER_OPCODE_TXF_CMS:
2063 case SHADER_OPCODE_TXF_CMS_W:
2064 case SHADER_OPCODE_TXF_UMS:
2065 case SHADER_OPCODE_TXF_MCS:
2066 case SHADER_OPCODE_TXL:
2067 case SHADER_OPCODE_TXL_LZ:
2068 case SHADER_OPCODE_TXS:
2069 case SHADER_OPCODE_LOD:
2070 case SHADER_OPCODE_TG4:
2071 case SHADER_OPCODE_TG4_OFFSET:
2072 case SHADER_OPCODE_SAMPLEINFO:
2073 generate_tex(inst, dst, src[0], src[1], src[2]);
2074 break;
2075 case FS_OPCODE_DDX_COARSE:
2076 case FS_OPCODE_DDX_FINE:
2077 generate_ddx(inst, dst, src[0]);
2078 break;
2079 case FS_OPCODE_DDY_COARSE:
2080 case FS_OPCODE_DDY_FINE:
2081 generate_ddy(inst, dst, src[0]);
2082 break;
2083
2084 case SHADER_OPCODE_GEN4_SCRATCH_WRITE:
2085 generate_scratch_write(inst, src[0]);
2086 spill_count++;
2087 break;
2088
2089 case SHADER_OPCODE_GEN4_SCRATCH_READ:
2090 generate_scratch_read(inst, dst);
2091 fill_count++;
2092 break;
2093
2094 case SHADER_OPCODE_GEN7_SCRATCH_READ:
2095 generate_scratch_read_gen7(inst, dst);
2096 fill_count++;
2097 break;
2098
2099 case SHADER_OPCODE_MOV_INDIRECT:
2100 generate_mov_indirect(inst, dst, src[0], src[1]);
2101 break;
2102
2103 case SHADER_OPCODE_URB_READ_SIMD8:
2104 case SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT:
2105 generate_urb_read(inst, dst, src[0]);
2106 break;
2107
2108 case SHADER_OPCODE_URB_WRITE_SIMD8:
2109 case SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT:
2110 case SHADER_OPCODE_URB_WRITE_SIMD8_MASKED:
2111 case SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT:
2112 generate_urb_write(inst, src[0]);
2113 break;
2114
2115 case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD:
2116 assert(inst->force_writemask_all);
2117 generate_uniform_pull_constant_load(inst, dst, src[0], src[1]);
2118 break;
2119
2120 case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD_GEN7:
2121 assert(inst->force_writemask_all);
2122 generate_uniform_pull_constant_load_gen7(inst, dst, src[0], src[1]);
2123 break;
2124
2125 case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_GEN4:
2126 generate_varying_pull_constant_load_gen4(inst, dst, src[0]);
2127 break;
2128
2129 case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_GEN7:
2130 generate_varying_pull_constant_load_gen7(inst, dst, src[0], src[1]);
2131 break;
2132
2133 case FS_OPCODE_REP_FB_WRITE:
2134 case FS_OPCODE_FB_WRITE:
2135 generate_fb_write(inst, src[0]);
2136 break;
2137
2138 case FS_OPCODE_FB_READ:
2139 generate_fb_read(inst, dst, src[0]);
2140 break;
2141
2142 case FS_OPCODE_MOV_DISPATCH_TO_FLAGS:
2143 generate_mov_dispatch_to_flags(inst);
2144 break;
2145
2146 case FS_OPCODE_DISCARD_JUMP:
2147 generate_discard_jump(inst);
2148 break;
2149
2150 case SHADER_OPCODE_SHADER_TIME_ADD:
2151 generate_shader_time_add(inst, src[0], src[1], src[2]);
2152 break;
2153
2154 case SHADER_OPCODE_UNTYPED_ATOMIC:
2155 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2156 brw_untyped_atomic(p, dst, src[0], src[1], src[2].ud,
2157 inst->mlen, !inst->dst.is_null(),
2158 inst->header_size);
2159 break;
2160
2161 case SHADER_OPCODE_UNTYPED_SURFACE_READ:
2162 assert(!inst->header_size);
2163 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2164 brw_untyped_surface_read(p, dst, src[0], src[1],
2165 inst->mlen, src[2].ud);
2166 break;
2167
2168 case SHADER_OPCODE_UNTYPED_SURFACE_WRITE:
2169 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2170 brw_untyped_surface_write(p, src[0], src[1],
2171 inst->mlen, src[2].ud,
2172 inst->header_size);
2173 break;
2174
2175 case SHADER_OPCODE_BYTE_SCATTERED_READ:
2176 assert(!inst->header_size);
2177 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2178 brw_byte_scattered_read(p, dst, src[0], src[1],
2179 inst->mlen, src[2].ud);
2180 break;
2181
2182 case SHADER_OPCODE_BYTE_SCATTERED_WRITE:
2183 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2184 brw_byte_scattered_write(p, src[0], src[1],
2185 inst->mlen, src[2].ud,
2186 inst->header_size);
2187 break;
2188
2189 case SHADER_OPCODE_TYPED_ATOMIC:
2190 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2191 brw_typed_atomic(p, dst, src[0], src[1],
2192 src[2].ud, inst->mlen, !inst->dst.is_null(),
2193 inst->header_size);
2194 break;
2195
2196 case SHADER_OPCODE_TYPED_SURFACE_READ:
2197 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2198 brw_typed_surface_read(p, dst, src[0], src[1],
2199 inst->mlen, src[2].ud,
2200 inst->header_size);
2201 break;
2202
2203 case SHADER_OPCODE_TYPED_SURFACE_WRITE:
2204 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2205 brw_typed_surface_write(p, src[0], src[1], inst->mlen, src[2].ud,
2206 inst->header_size);
2207 break;
2208
2209 case SHADER_OPCODE_MEMORY_FENCE:
2210 brw_memory_fence(p, dst, BRW_OPCODE_SEND);
2211 break;
2212
2213 case SHADER_OPCODE_INTERLOCK:
2214 /* The interlock is basically a memory fence issued via sendc */
2215 brw_memory_fence(p, dst, BRW_OPCODE_SENDC);
2216 break;
2217
2218 case SHADER_OPCODE_FIND_LIVE_CHANNEL: {
2219 const struct brw_reg mask =
2220 brw_stage_has_packed_dispatch(devinfo, stage,
2221 prog_data) ? brw_imm_ud(~0u) :
2222 stage == MESA_SHADER_FRAGMENT ? brw_vmask_reg() :
2223 brw_dmask_reg();
2224 brw_find_live_channel(p, dst, mask);
2225 break;
2226 }
2227
2228 case SHADER_OPCODE_BROADCAST:
2229 assert(inst->force_writemask_all);
2230 brw_broadcast(p, dst, src[0], src[1]);
2231 break;
2232
2233 case SHADER_OPCODE_SHUFFLE:
2234 generate_shuffle(inst, dst, src[0], src[1]);
2235 break;
2236
2237 case SHADER_OPCODE_SEL_EXEC:
2238 assert(inst->force_writemask_all);
2239 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
2240 brw_MOV(p, dst, src[1]);
2241 brw_set_default_mask_control(p, BRW_MASK_ENABLE);
2242 brw_MOV(p, dst, src[0]);
2243 break;
2244
2245 case SHADER_OPCODE_QUAD_SWIZZLE:
2246 /* This only works on 8-wide 32-bit values */
2247 assert(inst->exec_size == 8);
2248 assert(type_sz(src[0].type) == 4);
2249 assert(inst->force_writemask_all);
2250 assert(src[1].file == BRW_IMMEDIATE_VALUE);
2251 assert(src[1].type == BRW_REGISTER_TYPE_UD);
2252
2253 if (src[0].file == BRW_IMMEDIATE_VALUE ||
2254 (src[0].vstride == 0 && src[0].hstride == 0)) {
2255 /* The value is uniform across all channels */
2256 brw_MOV(p, dst, src[0]);
2257 } else {
2258 brw_set_default_access_mode(p, BRW_ALIGN_16);
2259 struct brw_reg swiz_src = stride(src[0], 4, 4, 1);
2260 swiz_src.swizzle = inst->src[1].ud;
2261 brw_MOV(p, dst, swiz_src);
2262 }
2263 break;
2264
2265 case SHADER_OPCODE_CLUSTER_BROADCAST: {
2266 assert(src[0].type == dst.type);
2267 assert(!src[0].negate && !src[0].abs);
2268 assert(src[1].file == BRW_IMMEDIATE_VALUE);
2269 assert(src[1].type == BRW_REGISTER_TYPE_UD);
2270 assert(src[2].file == BRW_IMMEDIATE_VALUE);
2271 assert(src[2].type == BRW_REGISTER_TYPE_UD);
2272 const unsigned component = src[1].ud;
2273 const unsigned cluster_size = src[2].ud;
2274 struct brw_reg strided = stride(suboffset(src[0], component),
2275 cluster_size, cluster_size, 0);
2276 if (type_sz(src[0].type) > 4 &&
2277 (devinfo->is_cherryview || gen_device_info_is_9lp(devinfo))) {
2278 /* IVB has an issue (which we found empirically) where it reads
2279 * two address register components per channel for indirectly
2280 * addressed 64-bit sources.
2281 *
2282 * From the Cherryview PRM Vol 7. "Register Region Restrictions":
2283 *
2284 * "When source or destination datatype is 64b or operation is
2285 * integer DWord multiply, indirect addressing must not be
2286 * used."
2287 *
2288 * To work around both of these, we do two integer MOVs insead of
2289 * one 64-bit MOV. Because no double value should ever cross a
2290 * register boundary, it's safe to use the immediate offset in the
2291 * indirect here to handle adding 4 bytes to the offset and avoid
2292 * the extra ADD to the register file.
2293 */
2294 brw_MOV(p, subscript(dst, BRW_REGISTER_TYPE_D, 0),
2295 subscript(strided, BRW_REGISTER_TYPE_D, 0));
2296 brw_MOV(p, subscript(dst, BRW_REGISTER_TYPE_D, 1),
2297 subscript(strided, BRW_REGISTER_TYPE_D, 1));
2298 } else {
2299 brw_MOV(p, dst, strided);
2300 }
2301 break;
2302 }
2303
2304 case FS_OPCODE_SET_SAMPLE_ID:
2305 generate_set_sample_id(inst, dst, src[0], src[1]);
2306 break;
2307
2308 case FS_OPCODE_PACK_HALF_2x16_SPLIT:
2309 generate_pack_half_2x16_split(inst, dst, src[0], src[1]);
2310 break;
2311
2312 case FS_OPCODE_UNPACK_HALF_2x16_SPLIT_X:
2313 case FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y:
2314 generate_unpack_half_2x16_split(inst, dst, src[0]);
2315 break;
2316
2317 case FS_OPCODE_PLACEHOLDER_HALT:
2318 /* This is the place where the final HALT needs to be inserted if
2319 * we've emitted any discards. If not, this will emit no code.
2320 */
2321 if (!patch_discard_jumps_to_fb_writes()) {
2322 if (unlikely(debug_flag)) {
2323 disasm_info->use_tail = true;
2324 }
2325 }
2326 break;
2327
2328 case FS_OPCODE_INTERPOLATE_AT_SAMPLE:
2329 generate_pixel_interpolator_query(inst, dst, src[0], src[1],
2330 GEN7_PIXEL_INTERPOLATOR_LOC_SAMPLE);
2331 break;
2332
2333 case FS_OPCODE_INTERPOLATE_AT_SHARED_OFFSET:
2334 generate_pixel_interpolator_query(inst, dst, src[0], src[1],
2335 GEN7_PIXEL_INTERPOLATOR_LOC_SHARED_OFFSET);
2336 break;
2337
2338 case FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET:
2339 generate_pixel_interpolator_query(inst, dst, src[0], src[1],
2340 GEN7_PIXEL_INTERPOLATOR_LOC_PER_SLOT_OFFSET);
2341 break;
2342
2343 case CS_OPCODE_CS_TERMINATE:
2344 generate_cs_terminate(inst, src[0]);
2345 break;
2346
2347 case SHADER_OPCODE_BARRIER:
2348 generate_barrier(inst, src[0]);
2349 break;
2350
2351 case BRW_OPCODE_DIM:
2352 assert(devinfo->is_haswell);
2353 assert(src[0].type == BRW_REGISTER_TYPE_DF);
2354 assert(dst.type == BRW_REGISTER_TYPE_DF);
2355 brw_DIM(p, dst, retype(src[0], BRW_REGISTER_TYPE_F));
2356 break;
2357
2358 case SHADER_OPCODE_RND_MODE:
2359 assert(src[0].file == BRW_IMMEDIATE_VALUE);
2360 brw_rounding_mode(p, (brw_rnd_mode) src[0].d);
2361 break;
2362
2363 default:
2364 unreachable("Unsupported opcode");
2365
2366 case SHADER_OPCODE_LOAD_PAYLOAD:
2367 unreachable("Should be lowered by lower_load_payload()");
2368 }
2369
2370 if (multiple_instructions_emitted)
2371 continue;
2372
2373 if (inst->no_dd_clear || inst->no_dd_check || inst->conditional_mod) {
2374 assert(p->next_insn_offset == last_insn_offset + 16 ||
2375 !"conditional_mod, no_dd_check, or no_dd_clear set for IR "
2376 "emitting more than 1 instruction");
2377
2378 brw_inst *last = &p->store[last_insn_offset / 16];
2379
2380 if (inst->conditional_mod)
2381 brw_inst_set_cond_modifier(p->devinfo, last, inst->conditional_mod);
2382 brw_inst_set_no_dd_clear(p->devinfo, last, inst->no_dd_clear);
2383 brw_inst_set_no_dd_check(p->devinfo, last, inst->no_dd_check);
2384 }
2385 }
2386
2387 brw_set_uip_jip(p, start_offset);
2388
2389 /* end of program sentinel */
2390 disasm_new_inst_group(disasm_info, p->next_insn_offset);
2391
2392 #ifndef NDEBUG
2393 bool validated =
2394 #else
2395 if (unlikely(debug_flag))
2396 #endif
2397 brw_validate_instructions(devinfo, p->store,
2398 start_offset,
2399 p->next_insn_offset,
2400 disasm_info);
2401
2402 int before_size = p->next_insn_offset - start_offset;
2403 brw_compact_instructions(p, start_offset, disasm_info);
2404 int after_size = p->next_insn_offset - start_offset;
2405
2406 if (unlikely(debug_flag)) {
2407 fprintf(stderr, "Native code for %s\n"
2408 "SIMD%d shader: %d instructions. %d loops. %u cycles. %d:%d spills:fills. Promoted %u constants. Compacted %d to %d"
2409 " bytes (%.0f%%)\n",
2410 shader_name, dispatch_width, before_size / 16, loop_count, cfg->cycle_count,
2411 spill_count, fill_count, promoted_constants, before_size, after_size,
2412 100.0f * (before_size - after_size) / before_size);
2413
2414 dump_assembly(p->store, disasm_info);
2415 }
2416 ralloc_free(disasm_info);
2417 assert(validated);
2418
2419 compiler->shader_debug_log(log_data,
2420 "%s SIMD%d shader: %d inst, %d loops, %u cycles, "
2421 "%d:%d spills:fills, Promoted %u constants, "
2422 "compacted %d to %d bytes.",
2423 _mesa_shader_stage_to_abbrev(stage),
2424 dispatch_width, before_size / 16,
2425 loop_count, cfg->cycle_count, spill_count,
2426 fill_count, promoted_constants, before_size,
2427 after_size);
2428
2429 return start_offset;
2430 }
2431
2432 const unsigned *
2433 fs_generator::get_assembly()
2434 {
2435 return brw_get_program(p, &prog_data->program_size);
2436 }