src/freedreno/vulkan/tu_cs.h

   1 /*
   2  * Copyright © 2019 Google LLC
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  */
  23 #ifndef TU_CS_H
  24 #define TU_CS_H
  25
  26 #include "tu_private.h"
  27
  28 #include "registers/adreno_pm4.xml.h"
  29
  30 void
  31 tu_cs_init(struct tu_cs *cs, enum tu_cs_mode mode, uint32_t initial_size);
  32
  33 void
  34 tu_cs_init_external(struct tu_cs *cs, uint32_t *start, uint32_t *end);
  35
  36 void
  37 tu_cs_finish(struct tu_device *dev, struct tu_cs *cs);
  38
  39 void
  40 tu_cs_begin(struct tu_cs *cs);
  41
  42 void
  43 tu_cs_end(struct tu_cs *cs);
  44
  45 VkResult
  46 tu_cs_begin_sub_stream(struct tu_device *dev,
  47                        struct tu_cs *cs,
  48                        uint32_t size,
  49                        struct tu_cs *sub_cs);
  50
  51 VkResult
  52 tu_cs_alloc(struct tu_device *dev,
  53             struct tu_cs *cs,
  54             uint32_t count,
  55             uint32_t size,
  56             struct ts_cs_memory *memory);
  57
  58 struct tu_cs_entry
  59 tu_cs_end_sub_stream(struct tu_cs *cs, struct tu_cs *sub_cs);
  60
  61 VkResult
  62 tu_cs_reserve_space(struct tu_device *dev,
  63                     struct tu_cs *cs,
  64                     uint32_t reserved_size);
  65
  66 void
  67 tu_cs_reset(struct tu_device *dev, struct tu_cs *cs);
  68
  69 VkResult
  70 tu_cs_add_entries(struct tu_cs *cs, struct tu_cs *target);
  71
  72 /**
  73  * Discard all entries.  This allows \a cs to be reused while keeping the
  74  * existing BOs and command packets intact.
  75  */
  76 static inline void
  77 tu_cs_discard_entries(struct tu_cs *cs)
  78 {
  79    assert(cs->mode == TU_CS_MODE_GROW);
  80    cs->entry_count = 0;
  81 }
  82
  83 /**
  84  * Get the size needed for tu_cs_emit_call.
  85  */
  86 static inline uint32_t
  87 tu_cs_get_call_size(const struct tu_cs *cs)
  88 {
  89    assert(cs->mode == TU_CS_MODE_GROW);
  90    /* each CP_INDIRECT_BUFFER needs 4 dwords */
  91    return cs->entry_count * 4;
  92 }
  93
  94 /**
  95  * Assert that we did not exceed the reserved space.
  96  */
  97 static inline void
  98 tu_cs_sanity_check(const struct tu_cs *cs)
  99 {
 100    assert(cs->start <= cs->cur);
 101    assert(cs->cur <= cs->reserved_end);
 102    assert(cs->reserved_end <= cs->end);
 103 }
 104
 105 /**
 106  * Emit a uint32_t value into a command stream, without boundary checking.
 107  */
 108 static inline void
 109 tu_cs_emit(struct tu_cs *cs, uint32_t value)
 110 {
 111    assert(cs->cur < cs->reserved_end);
 112    *cs->cur = value;
 113    ++cs->cur;
 114 }
 115
 116 /**
 117  * Emit an array of uint32_t into a command stream, without boundary checking.
 118  */
 119 static inline void
 120 tu_cs_emit_array(struct tu_cs *cs, const uint32_t *values, uint32_t length)
 121 {
 122    assert(cs->cur + length <= cs->reserved_end);
 123    memcpy(cs->cur, values, sizeof(uint32_t) * length);
 124    cs->cur += length;
 125 }
 126
 127 static inline unsigned
 128 tu_odd_parity_bit(unsigned val)
 129 {
 130    /* See: http://graphics.stanford.edu/~seander/bithacks.html#ParityParallel
 131     * note that we want odd parity so 0x6996 is inverted.
 132     */
 133    val ^= val >> 16;
 134    val ^= val >> 8;
 135    val ^= val >> 4;
 136    val &= 0xf;
 137    return (~0x6996 >> val) & 1;
 138 }
 139
 140 /**
 141  * Emit a type-4 command packet header into a command stream.
 142  */
 143 static inline void
 144 tu_cs_emit_pkt4(struct tu_cs *cs, uint16_t regindx, uint16_t cnt)
 145 {
 146    tu_cs_emit(cs, CP_TYPE4_PKT | cnt | (tu_odd_parity_bit(cnt) << 7) |
 147                      ((regindx & 0x3ffff) << 8) |
 148                      ((tu_odd_parity_bit(regindx) << 27)));
 149 }
 150
 151 /**
 152  * Emit a type-7 command packet header into a command stream.
 153  */
 154 static inline void
 155 tu_cs_emit_pkt7(struct tu_cs *cs, uint8_t opcode, uint16_t cnt)
 156 {
 157    tu_cs_emit(cs, CP_TYPE7_PKT | cnt | (tu_odd_parity_bit(cnt) << 15) |
 158                      ((opcode & 0x7f) << 16) |
 159                      ((tu_odd_parity_bit(opcode) << 23)));
 160 }
 161
 162 static inline void
 163 tu_cs_emit_wfi(struct tu_cs *cs)
 164 {
 165    tu_cs_emit_pkt7(cs, CP_WAIT_FOR_IDLE, 0);
 166 }
 167
 168 static inline void
 169 tu_cs_emit_qw(struct tu_cs *cs, uint64_t value)
 170 {
 171    tu_cs_emit(cs, (uint32_t) value);
 172    tu_cs_emit(cs, (uint32_t) (value >> 32));
 173 }
 174
 175 static inline void
 176 tu_cs_emit_write_reg(struct tu_cs *cs, uint16_t reg, uint32_t value)
 177 {
 178    tu_cs_emit_pkt4(cs, reg, 1);
 179    tu_cs_emit(cs, value);
 180 }
 181
 182 /**
 183  * Emit a CP_INDIRECT_BUFFER command packet.
 184  */
 185 static inline void
 186 tu_cs_emit_ib(struct tu_cs *cs, const struct tu_cs_entry *entry)
 187 {
 188    assert(entry->bo);
 189    assert(entry->size && entry->offset + entry->size <= entry->bo->size);
 190    assert(entry->size % sizeof(uint32_t) == 0);
 191    assert(entry->offset % sizeof(uint32_t) == 0);
 192
 193    tu_cs_emit_pkt7(cs, CP_INDIRECT_BUFFER, 3);
 194    tu_cs_emit_qw(cs, entry->bo->iova + entry->offset);
 195    tu_cs_emit(cs, entry->size / sizeof(uint32_t));
 196 }
 197
 198 /**
 199  * Emit a CP_INDIRECT_BUFFER command packet for each entry in the target
 200  * command stream.
 201  */
 202 static inline void
 203 tu_cs_emit_call(struct tu_cs *cs, const struct tu_cs *target)
 204 {
 205    assert(target->mode == TU_CS_MODE_GROW);
 206    for (uint32_t i = 0; i < target->entry_count; i++)
 207       tu_cs_emit_ib(cs, target->entries + i);
 208 }
 209
 210 /* Helpers for bracketing a large sequence of commands of unknown size inside
 211  * a CP_COND_REG_EXEC packet.
 212  */
 213
 214 struct tu_cond_exec_state {
 215    uint32_t *dword_ptr;
 216    uint32_t max_dwords;
 217 };
 218
 219 static inline VkResult
 220 tu_cond_exec_start(struct tu_device *dev, struct tu_cs *cs,
 221                    struct tu_cond_exec_state *state,
 222                    uint32_t condition, uint32_t max_dwords)
 223 {
 224    /* Reserve enough space so that both the condition packet and the actual
 225     * condition will fit in the same IB.
 226     */
 227    VkResult result = tu_cs_reserve_space(dev, cs, max_dwords + 3);
 228    if (result != VK_SUCCESS)
 229       return result;
 230
 231    state->max_dwords = max_dwords;
 232    tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
 233    tu_cs_emit(cs, condition);
 234    state->dword_ptr = cs->cur;
 235    /* Emit dummy DWORD field here */
 236    tu_cs_emit(cs, CP_COND_REG_EXEC_1_DWORDS(0));
 237
 238    return VK_SUCCESS;
 239 }
 240
 241 static inline void
 242 tu_cond_exec_end(struct tu_cs *cs, struct tu_cond_exec_state *state)
 243 {
 244    /* Subtract one here to account for the DWORD field itself. */
 245    uint32_t actual_dwords = cs->cur - state->dword_ptr - 1;
 246    assert(actual_dwords <= state->max_dwords);
 247    *state->dword_ptr = actual_dwords;
 248 }
 249
 250 #define fd_reg_pair tu_reg_value
 251 #define __bo_type struct tu_bo *
 252
 253 #include "a6xx.xml.h"
 254 #include "a6xx-pack.xml.h"
 255
 256 #define __assert_eq(a, b)                                               \
 257    do {                                                                 \
 258       if ((a) != (b)) {                                                 \
 259          fprintf(stderr, "assert failed: " #a " (0x%x) != " #b " (0x%x)\n", a, b); \
 260          assert((a) == (b));                                            \
 261       }                                                                 \
 262    } while (0)
 263
 264 #define __ONE_REG(i, regs)                                      \
 265    do {                                                         \
 266       if (i < ARRAY_SIZE(regs) && regs[i].reg > 0) {            \
 267          __assert_eq(regs[0].reg + i, regs[i].reg);             \
 268          if (regs[i].bo) {                                      \
 269             uint64_t v = regs[i].bo->iova + regs[i].bo_offset;  \
 270             v >>= regs[i].bo_shift;                             \
 271             v |= regs[i].value;                                 \
 272                                                                 \
 273             *p++ = v;                                           \
 274             *p++ = v >> 32;                                     \
 275          } else {                                               \
 276             *p++ = regs[i].value;                               \
 277             if (regs[i].is_address)                             \
 278                *p++ = regs[i].value >> 32;                      \
 279          }                                                      \
 280       }                                                         \
 281    } while (0)
 282
 283 /* Emits a sequence of register writes in order using a pkt4.  This will check
 284  * (at runtime on a !NDEBUG build) that the registers were actually set up in
 285  * order in the code.
 286  *
 287  * Note that references to buffers aren't automatically added to the CS,
 288  * unlike in freedreno.  We are clever in various places to avoid duplicating
 289  * the reference add work.
 290  *
 291  * Also, 64-bit address registers don't have a way (currently) to set a 64-bit
 292  * address without having a reference to a BO, since the .dword field in the
 293  * register's struct is only 32-bit wide.  We should fix this in the pack
 294  * codegen later.
 295  */
 296 #define tu_cs_emit_regs(cs, ...) do {                   \
 297    const struct fd_reg_pair regs[] = { __VA_ARGS__ };   \
 298    unsigned count = ARRAY_SIZE(regs);                   \
 299                                                         \
 300    STATIC_ASSERT(count > 0);                            \
 301    STATIC_ASSERT(count <= 16);                          \
 302                                                         \
 303    uint32_t *p = cs->cur;                               \
 304    *p++ = CP_TYPE4_PKT | count |                        \
 305       (tu_odd_parity_bit(count) << 7) |                 \
 306       ((regs[0].reg & 0x3ffff) << 8) |                  \
 307       ((tu_odd_parity_bit(regs[0].reg) << 27));         \
 308                                                         \
 309    __ONE_REG( 0, regs);                                 \
 310    __ONE_REG( 1, regs);                                 \
 311    __ONE_REG( 2, regs);                                 \
 312    __ONE_REG( 3, regs);                                 \
 313    __ONE_REG( 4, regs);                                 \
 314    __ONE_REG( 5, regs);                                 \
 315    __ONE_REG( 6, regs);                                 \
 316    __ONE_REG( 7, regs);                                 \
 317    __ONE_REG( 8, regs);                                 \
 318    __ONE_REG( 9, regs);                                 \
 319    __ONE_REG(10, regs);                                 \
 320    __ONE_REG(11, regs);                                 \
 321    __ONE_REG(12, regs);                                 \
 322    __ONE_REG(13, regs);                                 \
 323    __ONE_REG(14, regs);                                 \
 324    __ONE_REG(15, regs);                                 \
 325    cs->cur = p;                                         \
 326    } while (0)
 327
 328 #endif /* TU_CS_H */