projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
meson: inline `inc_common`
[mesa.git] / src / intel / common / gen_mi_builder.h
1 /*
2 * Copyright © 2019 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #ifndef GEN_MI_BUILDER_H
25 #define GEN_MI_BUILDER_H
26
27 #include "util/bitscan.h"
28 #include "util/fast_idiv_by_const.h"
29 #include "util/u_math.h"
30
31 #ifndef GEN_MI_BUILDER_NUM_ALLOC_GPRS
32 /** The number of GPRs the MI builder is allowed to allocate
33 *
34 * This may be set by a user of this API so that it can reserve some GPRs at
35 * the top end for its own use.
36 */
37 #define GEN_MI_BUILDER_NUM_ALLOC_GPRS 16
38 #endif
39
40 /** These must be defined by the user of the builder
41 *
42 * void *__gen_get_batch_dwords(__gen_user_data *user_data,
43 * unsigned num_dwords);
44 *
45 * __gen_address_type
46 * __gen_address_offset(__gen_address_type addr, uint64_t offset);
47 *
48 */
49
50 /*
51 * Start of the actual MI builder
52 */
53
54 #define __genxml_cmd_length(cmd) cmd ## _length
55 #define __genxml_cmd_header(cmd) cmd ## _header
56 #define __genxml_cmd_pack(cmd) cmd ## _pack
57
58 #define gen_mi_builder_pack(b, cmd, dst, name) \
59 for (struct cmd name = { __genxml_cmd_header(cmd) }, \
60 *_dst = (struct cmd *)(dst); __builtin_expect(_dst != NULL, 1); \
61 __genxml_cmd_pack(cmd)((b)->user_data, (void *)_dst, &name), \
62 _dst = NULL)
63
64 #define gen_mi_builder_emit(b, cmd, name) \
65 gen_mi_builder_pack((b), cmd, __gen_get_batch_dwords((b)->user_data, __genxml_cmd_length(cmd)), name)
66
67
68 enum gen_mi_value_type {
69 GEN_MI_VALUE_TYPE_IMM,
70 GEN_MI_VALUE_TYPE_MEM32,
71 GEN_MI_VALUE_TYPE_MEM64,
72 GEN_MI_VALUE_TYPE_REG32,
73 GEN_MI_VALUE_TYPE_REG64,
74 };
75
76 struct gen_mi_value {
77 enum gen_mi_value_type type;
78
79 union {
80 uint64_t imm;
81 __gen_address_type addr;
82 uint32_t reg;
83 };
84
85 #if GEN_GEN >= 7 || GEN_IS_HASWELL
86 bool invert;
87 #endif
88 };
89
90 #if GEN_GEN >= 9
91 #define GEN_MI_BUILDER_MAX_MATH_DWORDS 256
92 #else
93 #define GEN_MI_BUILDER_MAX_MATH_DWORDS 64
94 #endif
95
96 struct gen_mi_builder {
97 __gen_user_data *user_data;
98
99 #if GEN_GEN >= 8 || GEN_IS_HASWELL
100 uint32_t gprs;
101 uint8_t gpr_refs[GEN_MI_BUILDER_NUM_ALLOC_GPRS];
102
103 unsigned num_math_dwords;
104 uint32_t math_dwords[GEN_MI_BUILDER_MAX_MATH_DWORDS];
105 #endif
106 };
107
108 static inline void
109 gen_mi_builder_init(struct gen_mi_builder *b, __gen_user_data *user_data)
110 {
111 memset(b, 0, sizeof(*b));
112 b->user_data = user_data;
113
114 #if GEN_GEN >= 8 || GEN_IS_HASWELL
115 b->gprs = 0;
116 b->num_math_dwords = 0;
117 #endif
118 }
119
120 static inline void
121 gen_mi_builder_flush_math(struct gen_mi_builder *b)
122 {
123 #if GEN_GEN >= 8 || GEN_IS_HASWELL
124 if (b->num_math_dwords == 0)
125 return;
126
127 uint32_t *dw = (uint32_t *)__gen_get_batch_dwords(b->user_data,
128 1 + b->num_math_dwords);
129 gen_mi_builder_pack(b, GENX(MI_MATH), dw, math) {
130 math.DWordLength = 1 + b->num_math_dwords - GENX(MI_MATH_length_bias);
131 }
132 memcpy(dw + 1, b->math_dwords, b->num_math_dwords * sizeof(uint32_t));
133 b->num_math_dwords = 0;
134 #endif
135 }
136
137 #define _GEN_MI_BUILDER_GPR_BASE 0x2600
138 /* The actual hardware limit on GPRs */
139 #define _GEN_MI_BUILDER_NUM_HW_GPRS 16
140
141 #if GEN_GEN >= 8 || GEN_IS_HASWELL
142
143 static inline bool
144 gen_mi_value_is_gpr(struct gen_mi_value val)
145 {
146 return (val.type == GEN_MI_VALUE_TYPE_REG32 ||
147 val.type == GEN_MI_VALUE_TYPE_REG64) &&
148 val.reg >= _GEN_MI_BUILDER_GPR_BASE &&
149 val.reg < _GEN_MI_BUILDER_GPR_BASE +
150 _GEN_MI_BUILDER_NUM_HW_GPRS * 8;
151 }
152
153 static inline bool
154 _gen_mi_value_is_allocated_gpr(struct gen_mi_value val)
155 {
156 return (val.type == GEN_MI_VALUE_TYPE_REG32 ||
157 val.type == GEN_MI_VALUE_TYPE_REG64) &&
158 val.reg >= _GEN_MI_BUILDER_GPR_BASE &&
159 val.reg < _GEN_MI_BUILDER_GPR_BASE +
160 GEN_MI_BUILDER_NUM_ALLOC_GPRS * 8;
161 }
162
163 static inline uint32_t
164 _gen_mi_value_as_gpr(struct gen_mi_value val)
165 {
166 assert(gen_mi_value_is_gpr(val));
167 assert(val.reg % 8 == 0);
168 return (val.reg - _GEN_MI_BUILDER_GPR_BASE) / 8;
169 }
170
171 static inline struct gen_mi_value
172 gen_mi_new_gpr(struct gen_mi_builder *b)
173 {
174 unsigned gpr = ffs(~b->gprs) - 1;
175 assert(gpr < GEN_MI_BUILDER_NUM_ALLOC_GPRS);
176 assert(b->gpr_refs[gpr] == 0);
177 b->gprs |= (1u << gpr);
178 b->gpr_refs[gpr] = 1;
179
180 return (struct gen_mi_value) {
181 .type = GEN_MI_VALUE_TYPE_REG64,
182 .reg = _GEN_MI_BUILDER_GPR_BASE + gpr * 8,
183 };
184 }
185 #endif /* GEN_GEN >= 8 || GEN_IS_HASWELL */
186
187 /** Take a reference to a gen_mi_value
188 *
189 * The MI builder uses reference counting to automatically free ALU GPRs for
190 * re-use in calculations. All gen_mi_* math functions consume the reference
191 * they are handed for each source and return a reference to a value which the
192 * caller must consume. In particular, if you pas the same value into a
193 * single gen_mi_* math function twice (say to add a number to itself), you
194 * are responsible for calling gen_mi_value_ref() to get a second reference
195 * because the gen_mi_* math function will consume it twice.
196 */
197 static inline struct gen_mi_value
198 gen_mi_value_ref(struct gen_mi_builder *b, struct gen_mi_value val)
199 {
200 #if GEN_GEN >= 8 || GEN_IS_HASWELL
201 if (_gen_mi_value_is_allocated_gpr(val)) {
202 unsigned gpr = _gen_mi_value_as_gpr(val);
203 assert(gpr < GEN_MI_BUILDER_NUM_ALLOC_GPRS);
204 assert(b->gprs & (1u << gpr));
205 assert(b->gpr_refs[gpr] < UINT8_MAX);
206 b->gpr_refs[gpr]++;
207 }
208 #endif /* GEN_GEN >= 8 || GEN_IS_HASWELL */
209
210 return val;
211 }
212
213 /** Drop a reference to a gen_mi_value
214 *
215 * See also gen_mi_value_ref.
216 */
217 static inline void
218 gen_mi_value_unref(struct gen_mi_builder *b, struct gen_mi_value val)
219 {
220 #if GEN_GEN >= 8 || GEN_IS_HASWELL
221 if (_gen_mi_value_is_allocated_gpr(val)) {
222 unsigned gpr = _gen_mi_value_as_gpr(val);
223 assert(gpr < GEN_MI_BUILDER_NUM_ALLOC_GPRS);
224 assert(b->gprs & (1u << gpr));
225 assert(b->gpr_refs[gpr] > 0);
226 if (--b->gpr_refs[gpr] == 0)
227 b->gprs &= ~(1u << gpr);
228 }
229 #endif /* GEN_GEN >= 8 || GEN_IS_HASWELL */
230 }
231
232 static inline struct gen_mi_value
233 gen_mi_imm(uint64_t imm)
234 {
235 return (struct gen_mi_value) {
236 .type = GEN_MI_VALUE_TYPE_IMM,
237 .imm = imm,
238 };
239 }
240
241 static inline struct gen_mi_value
242 gen_mi_reg32(uint32_t reg)
243 {
244 struct gen_mi_value val = {
245 .type = GEN_MI_VALUE_TYPE_REG32,
246 .reg = reg,
247 };
248 #if GEN_GEN >= 8 || GEN_IS_HASWELL
249 assert(!_gen_mi_value_is_allocated_gpr(val));
250 #endif
251 return val;
252 }
253
254 static inline struct gen_mi_value
255 gen_mi_reg64(uint32_t reg)
256 {
257 struct gen_mi_value val = {
258 .type = GEN_MI_VALUE_TYPE_REG64,
259 .reg = reg,
260 };
261 #if GEN_GEN >= 8 || GEN_IS_HASWELL
262 assert(!_gen_mi_value_is_allocated_gpr(val));
263 #endif
264 return val;
265 }
266
267 static inline struct gen_mi_value
268 gen_mi_mem32(__gen_address_type addr)
269 {
270 return (struct gen_mi_value) {
271 .type = GEN_MI_VALUE_TYPE_MEM32,
272 .addr = addr,
273 };
274 }
275
276 static inline struct gen_mi_value
277 gen_mi_mem64(__gen_address_type addr)
278 {
279 return (struct gen_mi_value) {
280 .type = GEN_MI_VALUE_TYPE_MEM64,
281 .addr = addr,
282 };
283 }
284
285 static inline struct gen_mi_value
286 gen_mi_value_half(struct gen_mi_value value, bool top_32_bits)
287 {
288 switch (value.type) {
289 case GEN_MI_VALUE_TYPE_IMM:
290 if (top_32_bits)
291 value.imm >>= 32;
292 else
293 value.imm &= 0xffffffffu;
294 return value;
295
296 case GEN_MI_VALUE_TYPE_MEM32:
297 assert(!top_32_bits);
298 return value;
299
300 case GEN_MI_VALUE_TYPE_MEM64:
301 if (top_32_bits)
302 value.addr = __gen_address_offset(value.addr, 4);
303 value.type = GEN_MI_VALUE_TYPE_MEM32;
304 return value;
305
306 case GEN_MI_VALUE_TYPE_REG32:
307 assert(!top_32_bits);
308 return value;
309
310 case GEN_MI_VALUE_TYPE_REG64:
311 if (top_32_bits)
312 value.reg += 4;
313 value.type = GEN_MI_VALUE_TYPE_REG32;
314 return value;
315 }
316
317 unreachable("Invalid gen_mi_value type");
318 }
319
320 static inline void
321 _gen_mi_copy_no_unref(struct gen_mi_builder *b,
322 struct gen_mi_value dst, struct gen_mi_value src)
323 {
324 #if GEN_GEN >= 7 || GEN_IS_HASWELL
325 /* TODO: We could handle src.invert by emitting a bit of math if we really
326 * wanted to.
327 */
328 assert(!dst.invert && !src.invert);
329 #endif
330 gen_mi_builder_flush_math(b);
331
332 switch (dst.type) {
333 case GEN_MI_VALUE_TYPE_IMM:
334 unreachable("Cannot copy to an immediate");
335
336 case GEN_MI_VALUE_TYPE_MEM64:
337 case GEN_MI_VALUE_TYPE_REG64:
338 /* If the destination is 64 bits, we have to copy in two halves */
339 _gen_mi_copy_no_unref(b, gen_mi_value_half(dst, false),
340 gen_mi_value_half(src, false));
341 switch (src.type) {
342 case GEN_MI_VALUE_TYPE_IMM:
343 case GEN_MI_VALUE_TYPE_MEM64:
344 case GEN_MI_VALUE_TYPE_REG64:
345 /* TODO: Use MI_STORE_DATA_IMM::StoreQWord when we have it */
346 _gen_mi_copy_no_unref(b, gen_mi_value_half(dst, true),
347 gen_mi_value_half(src, true));
348 break;
349 default:
350 _gen_mi_copy_no_unref(b, gen_mi_value_half(dst, true),
351 gen_mi_imm(0));
352 break;
353 }
354 break;
355
356 case GEN_MI_VALUE_TYPE_MEM32:
357 switch (src.type) {
358 case GEN_MI_VALUE_TYPE_IMM:
359 gen_mi_builder_emit(b, GENX(MI_STORE_DATA_IMM), sdi) {
360 sdi.Address = dst.addr;
361 #if GEN_GEN >= 12
362 sdi.ForceWriteCompletionCheck = true;
363 #endif
364 sdi.ImmediateData = src.imm;
365 }
366 break;
367
368 case GEN_MI_VALUE_TYPE_MEM32:
369 case GEN_MI_VALUE_TYPE_MEM64:
370 #if GEN_GEN >= 8
371 gen_mi_builder_emit(b, GENX(MI_COPY_MEM_MEM), cmm) {
372 cmm.DestinationMemoryAddress = dst.addr;
373 cmm.SourceMemoryAddress = src.addr;
374 }
375 #elif GEN_IS_HASWELL
376 {
377 struct gen_mi_value tmp = gen_mi_new_gpr(b);
378 _gen_mi_copy_no_unref(b, tmp, src);
379 _gen_mi_copy_no_unref(b, dst, tmp);
380 gen_mi_value_unref(b, tmp);
381 }
382 #else
383 unreachable("Cannot do mem <-> mem copy on IVB and earlier");
384 #endif
385 break;
386
387 case GEN_MI_VALUE_TYPE_REG32:
388 case GEN_MI_VALUE_TYPE_REG64:
389 gen_mi_builder_emit(b, GENX(MI_STORE_REGISTER_MEM), srm) {
390 srm.RegisterAddress = src.reg;
391 srm.MemoryAddress = dst.addr;
392 }
393 break;
394
395 default:
396 unreachable("Invalid gen_mi_value type");
397 }
398 break;
399
400 case GEN_MI_VALUE_TYPE_REG32:
401 switch (src.type) {
402 case GEN_MI_VALUE_TYPE_IMM:
403 gen_mi_builder_emit(b, GENX(MI_LOAD_REGISTER_IMM), lri) {
404 lri.RegisterOffset = dst.reg;
405 lri.DataDWord = src.imm;
406 }
407 break;
408
409 case GEN_MI_VALUE_TYPE_MEM32:
410 case GEN_MI_VALUE_TYPE_MEM64:
411 gen_mi_builder_emit(b, GENX(MI_LOAD_REGISTER_MEM), lrm) {
412 lrm.RegisterAddress = dst.reg;
413 lrm.MemoryAddress = src.addr;
414 }
415 break;
416
417 case GEN_MI_VALUE_TYPE_REG32:
418 case GEN_MI_VALUE_TYPE_REG64:
419 #if GEN_GEN >= 8 || GEN_IS_HASWELL
420 if (src.reg != dst.reg) {
421 gen_mi_builder_emit(b, GENX(MI_LOAD_REGISTER_REG), lrr) {
422 lrr.SourceRegisterAddress = src.reg;
423 lrr.DestinationRegisterAddress = dst.reg;
424 }
425 }
426 #else
427 unreachable("Cannot do reg <-> reg copy on IVB and earlier");
428 #endif
429 break;
430
431 default:
432 unreachable("Invalid gen_mi_value type");
433 }
434 break;
435
436 default:
437 unreachable("Invalid gen_mi_value type");
438 }
439 }
440
441 /** Store the value in src to the value represented by dst
442 *
443 * If the bit size of src and dst mismatch, this function does an unsigned
444 * integer cast. If src has more bits than dst, it takes the bottom bits. If
445 * src has fewer bits then dst, it fills the top bits with zeros.
446 *
447 * This function consumes one reference for each of src and dst.
448 */
449 static inline void
450 gen_mi_store(struct gen_mi_builder *b,
451 struct gen_mi_value dst, struct gen_mi_value src)
452 {
453 _gen_mi_copy_no_unref(b, dst, src);
454 gen_mi_value_unref(b, src);
455 gen_mi_value_unref(b, dst);
456 }
457
458 static inline void
459 gen_mi_memset(struct gen_mi_builder *b, __gen_address_type dst,
460 uint32_t value, uint32_t size)
461 {
462 #if GEN_GEN >= 8 || GEN_IS_HASWELL
463 assert(b->num_math_dwords == 0);
464 #endif
465
466 /* This memset operates in units of dwords. */
467 assert(size % 4 == 0);
468
469 for (uint32_t i = 0; i < size; i += 4) {
470 gen_mi_store(b, gen_mi_mem32(__gen_address_offset(dst, i)),
471 gen_mi_imm(value));
472 }
473 }
474
475 /* NOTE: On IVB, this function stomps GEN7_3DPRIM_BASE_VERTEX */
476 static inline void
477 gen_mi_memcpy(struct gen_mi_builder *b, __gen_address_type dst,
478 __gen_address_type src, uint32_t size)
479 {
480 #if GEN_GEN >= 8 || GEN_IS_HASWELL
481 assert(b->num_math_dwords == 0);
482 #endif
483
484 /* This memcpy operates in units of dwords. */
485 assert(size % 4 == 0);
486
487 for (uint32_t i = 0; i < size; i += 4) {
488 struct gen_mi_value dst_val = gen_mi_mem32(__gen_address_offset(dst, i));
489 struct gen_mi_value src_val = gen_mi_mem32(__gen_address_offset(src, i));
490 #if GEN_GEN >= 8 || GEN_IS_HASWELL
491 gen_mi_store(b, dst_val, src_val);
492 #else
493 /* IVB does not have a general purpose register for command streamer
494 * commands. Therefore, we use an alternate temporary register.
495 */
496 struct gen_mi_value tmp_reg = gen_mi_reg32(0x2440); /* GEN7_3DPRIM_BASE_VERTEX */
497 gen_mi_store(b, tmp_reg, src_val);
498 gen_mi_store(b, dst_val, tmp_reg);
499 #endif
500 }
501 }
502
503 /*
504 * MI_MATH Section. Only available on Haswell+
505 */
506
507 #if GEN_GEN >= 8 || GEN_IS_HASWELL
508
509 /**
510 * Perform a predicated store (assuming the condition is already loaded
511 * in the MI_PREDICATE_RESULT register) of the value in src to the memory
512 * location specified by dst. Non-memory destinations are not supported.
513 *
514 * This function consumes one reference for each of src and dst.
515 */
516 static inline void
517 gen_mi_store_if(struct gen_mi_builder *b,
518 struct gen_mi_value dst,
519 struct gen_mi_value src)
520 {
521 assert(!dst.invert && !src.invert);
522
523 gen_mi_builder_flush_math(b);
524
525 /* We can only predicate MI_STORE_REGISTER_MEM, so restrict the
526 * destination to be memory, and resolve the source to a temporary
527 * register if it isn't in one already.
528 */
529 assert(dst.type == GEN_MI_VALUE_TYPE_MEM64 ||
530 dst.type == GEN_MI_VALUE_TYPE_MEM32);
531
532 if (src.type != GEN_MI_VALUE_TYPE_REG32 &&
533 src.type != GEN_MI_VALUE_TYPE_REG64) {
534 struct gen_mi_value tmp = gen_mi_new_gpr(b);
535 _gen_mi_copy_no_unref(b, tmp, src);
536 src = tmp;
537 }
538
539 if (dst.type == GEN_MI_VALUE_TYPE_MEM64) {
540 gen_mi_builder_emit(b, GENX(MI_STORE_REGISTER_MEM), srm) {
541 srm.RegisterAddress = src.reg;
542 srm.MemoryAddress = dst.addr;
543 srm.PredicateEnable = true;
544 }
545 gen_mi_builder_emit(b, GENX(MI_STORE_REGISTER_MEM), srm) {
546 srm.RegisterAddress = src.reg + 4;
547 srm.MemoryAddress = __gen_address_offset(dst.addr, 4);
548 srm.PredicateEnable = true;
549 }
550 } else {
551 gen_mi_builder_emit(b, GENX(MI_STORE_REGISTER_MEM), srm) {
552 srm.RegisterAddress = src.reg;
553 srm.MemoryAddress = dst.addr;
554 srm.PredicateEnable = true;
555 }
556 }
557
558 gen_mi_value_unref(b, src);
559 gen_mi_value_unref(b, dst);
560 }
561
562 static inline void
563 _gen_mi_builder_push_math(struct gen_mi_builder *b,
564 const uint32_t *dwords,
565 unsigned num_dwords)
566 {
567 assert(num_dwords < GEN_MI_BUILDER_MAX_MATH_DWORDS);
568 if (b->num_math_dwords + num_dwords > GEN_MI_BUILDER_MAX_MATH_DWORDS)
569 gen_mi_builder_flush_math(b);
570
571 memcpy(&b->math_dwords[b->num_math_dwords],
572 dwords, num_dwords * sizeof(*dwords));
573 b->num_math_dwords += num_dwords;
574 }
575
576 static inline uint32_t
577 _gen_mi_pack_alu(uint32_t opcode, uint32_t operand1, uint32_t operand2)
578 {
579 struct GENX(MI_MATH_ALU_INSTRUCTION) instr = {
580 .Operand2 = operand2,
581 .Operand1 = operand1,
582 .ALUOpcode = opcode,
583 };
584
585 uint32_t dw;
586 GENX(MI_MATH_ALU_INSTRUCTION_pack)(NULL, &dw, &instr);
587
588 return dw;
589 }
590
591 static inline struct gen_mi_value
592 gen_mi_value_to_gpr(struct gen_mi_builder *b, struct gen_mi_value val)
593 {
594 if (gen_mi_value_is_gpr(val))
595 return val;
596
597 /* Save off the invert flag because it makes copy() grumpy */
598 bool invert = val.invert;
599 val.invert = false;
600
601 struct gen_mi_value tmp = gen_mi_new_gpr(b);
602 _gen_mi_copy_no_unref(b, tmp, val);
603 tmp.invert = invert;
604
605 return tmp;
606 }
607
608 static inline uint32_t
609 _gen_mi_math_load_src(struct gen_mi_builder *b,
610 unsigned src, struct gen_mi_value *val)
611 {
612 if (val->type == GEN_MI_VALUE_TYPE_IMM &&
613 (val->imm == 0 || val->imm == UINT64_MAX)) {
614 uint64_t imm = val->invert ? ~val->imm : val->imm;
615 return _gen_mi_pack_alu(imm ? MI_ALU_LOAD1 : MI_ALU_LOAD0, src, 0);
616 } else {
617 *val = gen_mi_value_to_gpr(b, *val);
618 return _gen_mi_pack_alu(val->invert ? MI_ALU_LOADINV : MI_ALU_LOAD,
619 src, _gen_mi_value_as_gpr(*val));
620 }
621 }
622
623 static inline struct gen_mi_value
624 gen_mi_math_binop(struct gen_mi_builder *b, uint32_t opcode,
625 struct gen_mi_value src0, struct gen_mi_value src1,
626 uint32_t store_op, uint32_t store_src)
627 {
628 struct gen_mi_value dst = gen_mi_new_gpr(b);
629
630 uint32_t dw[4];
631 dw[0] = _gen_mi_math_load_src(b, MI_ALU_SRCA, &src0);
632 dw[1] = _gen_mi_math_load_src(b, MI_ALU_SRCB, &src1);
633 dw[2] = _gen_mi_pack_alu(opcode, 0, 0);
634 dw[3] = _gen_mi_pack_alu(store_op, _gen_mi_value_as_gpr(dst), store_src);
635 _gen_mi_builder_push_math(b, dw, 4);
636
637 gen_mi_value_unref(b, src0);
638 gen_mi_value_unref(b, src1);
639
640 return dst;
641 }
642
643 static inline struct gen_mi_value
644 gen_mi_inot(struct gen_mi_builder *b, struct gen_mi_value val)
645 {
646 /* TODO These currently can't be passed into gen_mi_copy */
647 val.invert = !val.invert;
648 return val;
649 }
650
651 static inline struct gen_mi_value
652 gen_mi_iadd(struct gen_mi_builder *b,
653 struct gen_mi_value src0, struct gen_mi_value src1)
654 {
655 return gen_mi_math_binop(b, MI_ALU_ADD, src0, src1,
656 MI_ALU_STORE, MI_ALU_ACCU);
657 }
658
659 static inline struct gen_mi_value
660 gen_mi_iadd_imm(struct gen_mi_builder *b,
661 struct gen_mi_value src, uint64_t N)
662 {
663 if (N == 0)
664 return src;
665
666 return gen_mi_iadd(b, src, gen_mi_imm(N));
667 }
668
669 static inline struct gen_mi_value
670 gen_mi_isub(struct gen_mi_builder *b,
671 struct gen_mi_value src0, struct gen_mi_value src1)
672 {
673 return gen_mi_math_binop(b, MI_ALU_SUB, src0, src1,
674 MI_ALU_STORE, MI_ALU_ACCU);
675 }
676
677 static inline struct gen_mi_value
678 gen_mi_ult(struct gen_mi_builder *b,
679 struct gen_mi_value src0, struct gen_mi_value src1)
680 {
681 /* Compute "less than" by subtracting and storing the carry bit */
682 return gen_mi_math_binop(b, MI_ALU_SUB, src0, src1,
683 MI_ALU_STORE, MI_ALU_CF);
684 }
685
686 static inline struct gen_mi_value
687 gen_mi_uge(struct gen_mi_builder *b,
688 struct gen_mi_value src0, struct gen_mi_value src1)
689 {
690 /* Compute "less than" by subtracting and storing the carry bit */
691 return gen_mi_math_binop(b, MI_ALU_SUB, src0, src1,
692 MI_ALU_STOREINV, MI_ALU_CF);
693 }
694
695 static inline struct gen_mi_value
696 gen_mi_iand(struct gen_mi_builder *b,
697 struct gen_mi_value src0, struct gen_mi_value src1)
698 {
699 return gen_mi_math_binop(b, MI_ALU_AND, src0, src1,
700 MI_ALU_STORE, MI_ALU_ACCU);
701 }
702
703 /**
704 * Returns (src != 0) ? 1 : 0.
705 */
706 static inline struct gen_mi_value
707 gen_mi_nz(struct gen_mi_builder *b, struct gen_mi_value src)
708 {
709 return gen_mi_math_binop(b, MI_ALU_ADD, src, gen_mi_imm(0),
710 MI_ALU_STOREINV, MI_ALU_ZF);
711 }
712
713 /**
714 * Returns (src == 0) ? 1 : 0.
715 */
716 static inline struct gen_mi_value
717 gen_mi_z(struct gen_mi_builder *b, struct gen_mi_value src)
718 {
719 return gen_mi_math_binop(b, MI_ALU_ADD, src, gen_mi_imm(0),
720 MI_ALU_STORE, MI_ALU_ZF);
721 }
722
723 static inline struct gen_mi_value
724 gen_mi_ior(struct gen_mi_builder *b,
725 struct gen_mi_value src0, struct gen_mi_value src1)
726 {
727 return gen_mi_math_binop(b, MI_ALU_OR, src0, src1,
728 MI_ALU_STORE, MI_ALU_ACCU);
729 }
730
731 static inline struct gen_mi_value
732 gen_mi_imul_imm(struct gen_mi_builder *b,
733 struct gen_mi_value src, uint32_t N)
734 {
735 if (N == 0) {
736 gen_mi_value_unref(b, src);
737 return gen_mi_imm(0);
738 }
739
740 if (N == 1)
741 return src;
742
743 src = gen_mi_value_to_gpr(b, src);
744
745 struct gen_mi_value res = gen_mi_value_ref(b, src);
746
747 unsigned top_bit = 31 - __builtin_clz(N);
748 for (int i = top_bit - 1; i >= 0; i--) {
749 res = gen_mi_iadd(b, res, gen_mi_value_ref(b, res));
750 if (N & (1 << i))
751 res = gen_mi_iadd(b, res, gen_mi_value_ref(b, src));
752 }
753
754 gen_mi_value_unref(b, src);
755
756 return res;
757 }
758
759 static inline struct gen_mi_value
760 gen_mi_ishl_imm(struct gen_mi_builder *b,
761 struct gen_mi_value src, uint32_t shift)
762 {
763 struct gen_mi_value res = gen_mi_value_to_gpr(b, src);
764
765 for (unsigned i = 0; i < shift; i++)
766 res = gen_mi_iadd(b, res, gen_mi_value_ref(b, res));
767
768 return res;
769 }
770
771 static inline struct gen_mi_value
772 gen_mi_ushr32_imm(struct gen_mi_builder *b,
773 struct gen_mi_value src, uint32_t shift)
774 {
775 /* We right-shift by left-shifting by 32 - shift and taking the top 32 bits
776 * of the result. This assumes the top 32 bits are zero.
777 */
778 if (shift > 64)
779 return gen_mi_imm(0);
780
781 if (shift > 32) {
782 struct gen_mi_value tmp = gen_mi_new_gpr(b);
783 _gen_mi_copy_no_unref(b, gen_mi_value_half(tmp, false),
784 gen_mi_value_half(src, true));
785 _gen_mi_copy_no_unref(b, gen_mi_value_half(tmp, true), gen_mi_imm(0));
786 gen_mi_value_unref(b, src);
787 src = tmp;
788 shift -= 32;
789 }
790 assert(shift <= 32);
791 struct gen_mi_value tmp = gen_mi_ishl_imm(b, src, 32 - shift);
792 struct gen_mi_value dst = gen_mi_new_gpr(b);
793 _gen_mi_copy_no_unref(b, gen_mi_value_half(dst, false),
794 gen_mi_value_half(tmp, true));
795 _gen_mi_copy_no_unref(b, gen_mi_value_half(dst, true), gen_mi_imm(0));
796 gen_mi_value_unref(b, tmp);
797 return dst;
798 }
799
800 static inline struct gen_mi_value
801 gen_mi_udiv32_imm(struct gen_mi_builder *b,
802 struct gen_mi_value N, uint32_t D)
803 {
804 /* We implicitly assume that N is only a 32-bit value */
805 if (D == 0) {
806 /* This is invalid but we should do something */
807 return gen_mi_imm(0);
808 } else if (util_is_power_of_two_or_zero(D)) {
809 return gen_mi_ushr32_imm(b, N, util_logbase2(D));
810 } else {
811 struct util_fast_udiv_info m = util_compute_fast_udiv_info(D, 32, 32);
812 assert(m.multiplier <= UINT32_MAX);
813
814 if (m.pre_shift)
815 N = gen_mi_ushr32_imm(b, N, m.pre_shift);
816
817 /* Do the 32x32 multiply into gpr0 */
818 N = gen_mi_imul_imm(b, N, m.multiplier);
819
820 if (m.increment)
821 N = gen_mi_iadd(b, N, gen_mi_imm(m.multiplier));
822
823 N = gen_mi_ushr32_imm(b, N, 32);
824
825 if (m.post_shift)
826 N = gen_mi_ushr32_imm(b, N, m.post_shift);
827
828 return N;
829 }
830 }
831
832 #endif /* MI_MATH section */
833
834 #endif /* GEN_MI_BUILDER_H */