src/freedreno/vulkan/tu_cs.h

   1 /*
   2  * Copyright © 2019 Google LLC
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  */
  23 #ifndef TU_CS_H
  24 #define TU_CS_H
  25
  26 #include "tu_private.h"
  27
  28 #include "registers/adreno_pm4.xml.h"
  29
  30 void
  31 tu_cs_init(struct tu_cs *cs,
  32            struct tu_device *device,
  33            enum tu_cs_mode mode,
  34            uint32_t initial_size);
  35
  36 void
  37 tu_cs_init_external(struct tu_cs *cs, uint32_t *start, uint32_t *end);
  38
  39 void
  40 tu_cs_finish(struct tu_cs *cs);
  41
  42 void
  43 tu_cs_begin(struct tu_cs *cs);
  44
  45 void
  46 tu_cs_end(struct tu_cs *cs);
  47
  48 VkResult
  49 tu_cs_begin_sub_stream(struct tu_cs *cs, uint32_t size, struct tu_cs *sub_cs);
  50
  51 VkResult
  52 tu_cs_alloc(struct tu_cs *cs,
  53             uint32_t count,
  54             uint32_t size,
  55             struct ts_cs_memory *memory);
  56
  57 struct tu_cs_entry
  58 tu_cs_end_sub_stream(struct tu_cs *cs, struct tu_cs *sub_cs);
  59
  60 VkResult
  61 tu_cs_reserve_space(struct tu_cs *cs, uint32_t reserved_size);
  62
  63 void
  64 tu_cs_reset(struct tu_cs *cs);
  65
  66 VkResult
  67 tu_cs_add_entries(struct tu_cs *cs, struct tu_cs *target);
  68
  69 /**
  70  * Discard all entries.  This allows \a cs to be reused while keeping the
  71  * existing BOs and command packets intact.
  72  */
  73 static inline void
  74 tu_cs_discard_entries(struct tu_cs *cs)
  75 {
  76    assert(cs->mode == TU_CS_MODE_GROW);
  77    cs->entry_count = 0;
  78 }
  79
  80 /**
  81  * Get the size needed for tu_cs_emit_call.
  82  */
  83 static inline uint32_t
  84 tu_cs_get_call_size(const struct tu_cs *cs)
  85 {
  86    assert(cs->mode == TU_CS_MODE_GROW);
  87    /* each CP_INDIRECT_BUFFER needs 4 dwords */
  88    return cs->entry_count * 4;
  89 }
  90
  91 /**
  92  * Assert that we did not exceed the reserved space.
  93  */
  94 static inline void
  95 tu_cs_sanity_check(const struct tu_cs *cs)
  96 {
  97    assert(cs->start <= cs->cur);
  98    assert(cs->cur <= cs->reserved_end);
  99    assert(cs->reserved_end <= cs->end);
 100 }
 101
 102 /**
 103  * Emit a uint32_t value into a command stream, without boundary checking.
 104  */
 105 static inline void
 106 tu_cs_emit(struct tu_cs *cs, uint32_t value)
 107 {
 108    assert(cs->cur < cs->reserved_end);
 109    *cs->cur = value;
 110    ++cs->cur;
 111 }
 112
 113 /**
 114  * Emit an array of uint32_t into a command stream, without boundary checking.
 115  */
 116 static inline void
 117 tu_cs_emit_array(struct tu_cs *cs, const uint32_t *values, uint32_t length)
 118 {
 119    assert(cs->cur + length <= cs->reserved_end);
 120    memcpy(cs->cur, values, sizeof(uint32_t) * length);
 121    cs->cur += length;
 122 }
 123
 124 static inline unsigned
 125 tu_odd_parity_bit(unsigned val)
 126 {
 127    /* See: http://graphics.stanford.edu/~seander/bithacks.html#ParityParallel
 128     * note that we want odd parity so 0x6996 is inverted.
 129     */
 130    val ^= val >> 16;
 131    val ^= val >> 8;
 132    val ^= val >> 4;
 133    val &= 0xf;
 134    return (~0x6996 >> val) & 1;
 135 }
 136
 137 /**
 138  * Get the size of the remaining space in the current BO.
 139  */
 140 static inline uint32_t
 141 tu_cs_get_space(const struct tu_cs *cs)
 142 {
 143    return cs->end - cs->cur;
 144 }
 145
 146 static inline void
 147 tu_cs_reserve(struct tu_cs *cs, uint32_t reserved_size)
 148 {
 149    if (cs->mode != TU_CS_MODE_GROW) {
 150       assert(tu_cs_get_space(cs) >= reserved_size);
 151       assert(cs->reserved_end == cs->end);
 152       return;
 153    }
 154
 155    if (tu_cs_get_space(cs) >= reserved_size &&
 156        cs->entry_count < cs->entry_capacity) {
 157       cs->reserved_end = cs->cur + reserved_size;
 158       return;
 159    }
 160
 161    VkResult result = tu_cs_reserve_space(cs, reserved_size);
 162    /* TODO: set this error in tu_cs and use it */
 163    assert(result == VK_SUCCESS);
 164 }
 165
 166 /**
 167  * Emit a type-4 command packet header into a command stream.
 168  */
 169 static inline void
 170 tu_cs_emit_pkt4(struct tu_cs *cs, uint16_t regindx, uint16_t cnt)
 171 {
 172    tu_cs_reserve(cs, cnt + 1);
 173    tu_cs_emit(cs, CP_TYPE4_PKT | cnt | (tu_odd_parity_bit(cnt) << 7) |
 174                      ((regindx & 0x3ffff) << 8) |
 175                      ((tu_odd_parity_bit(regindx) << 27)));
 176 }
 177
 178 /**
 179  * Emit a type-7 command packet header into a command stream.
 180  */
 181 static inline void
 182 tu_cs_emit_pkt7(struct tu_cs *cs, uint8_t opcode, uint16_t cnt)
 183 {
 184    tu_cs_reserve(cs, cnt + 1);
 185    tu_cs_emit(cs, CP_TYPE7_PKT | cnt | (tu_odd_parity_bit(cnt) << 15) |
 186                      ((opcode & 0x7f) << 16) |
 187                      ((tu_odd_parity_bit(opcode) << 23)));
 188 }
 189
 190 static inline void
 191 tu_cs_emit_wfi(struct tu_cs *cs)
 192 {
 193    tu_cs_emit_pkt7(cs, CP_WAIT_FOR_IDLE, 0);
 194 }
 195
 196 static inline void
 197 tu_cs_emit_qw(struct tu_cs *cs, uint64_t value)
 198 {
 199    tu_cs_emit(cs, (uint32_t) value);
 200    tu_cs_emit(cs, (uint32_t) (value >> 32));
 201 }
 202
 203 static inline void
 204 tu_cs_emit_write_reg(struct tu_cs *cs, uint16_t reg, uint32_t value)
 205 {
 206    tu_cs_emit_pkt4(cs, reg, 1);
 207    tu_cs_emit(cs, value);
 208 }
 209
 210 /**
 211  * Emit a CP_INDIRECT_BUFFER command packet.
 212  */
 213 static inline void
 214 tu_cs_emit_ib(struct tu_cs *cs, const struct tu_cs_entry *entry)
 215 {
 216    assert(entry->bo);
 217    assert(entry->size && entry->offset + entry->size <= entry->bo->size);
 218    assert(entry->size % sizeof(uint32_t) == 0);
 219    assert(entry->offset % sizeof(uint32_t) == 0);
 220
 221    tu_cs_emit_pkt7(cs, CP_INDIRECT_BUFFER, 3);
 222    tu_cs_emit_qw(cs, entry->bo->iova + entry->offset);
 223    tu_cs_emit(cs, entry->size / sizeof(uint32_t));
 224 }
 225
 226 /**
 227  * Emit a CP_INDIRECT_BUFFER command packet for each entry in the target
 228  * command stream.
 229  */
 230 static inline void
 231 tu_cs_emit_call(struct tu_cs *cs, const struct tu_cs *target)
 232 {
 233    assert(target->mode == TU_CS_MODE_GROW);
 234    for (uint32_t i = 0; i < target->entry_count; i++)
 235       tu_cs_emit_ib(cs, target->entries + i);
 236 }
 237
 238 /* Helpers for bracketing a large sequence of commands of unknown size inside
 239  * a CP_COND_REG_EXEC packet.
 240  */
 241 static inline void
 242 tu_cond_exec_start(struct tu_cs *cs, uint32_t cond_flags)
 243 {
 244    assert(cs->mode == TU_CS_MODE_GROW);
 245    assert(!cs->cond_flags && cond_flags);
 246
 247    tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
 248    tu_cs_emit(cs, cond_flags);
 249
 250    cs->cond_flags = cond_flags;
 251    cs->cond_dwords = cs->cur;
 252
 253    /* Emit dummy DWORD field here */
 254    tu_cs_emit(cs, CP_COND_REG_EXEC_1_DWORDS(0));
 255 }
 256 #define CP_COND_EXEC_0_RENDER_MODE_GMEM \
 257    (CP_COND_REG_EXEC_0_MODE(RENDER_MODE) | CP_COND_REG_EXEC_0_GMEM)
 258 #define CP_COND_EXEC_0_RENDER_MODE_SYSMEM \
 259    (CP_COND_REG_EXEC_0_MODE(RENDER_MODE) | CP_COND_REG_EXEC_0_SYSMEM)
 260
 261 static inline void
 262 tu_cond_exec_end(struct tu_cs *cs)
 263 {
 264    assert(cs->cond_flags);
 265
 266    cs->cond_flags = 0;
 267    /* Subtract one here to account for the DWORD field itself. */
 268    *cs->cond_dwords = cs->cur - cs->cond_dwords - 1;
 269 }
 270
 271 #define fd_reg_pair tu_reg_value
 272 #define __bo_type struct tu_bo *
 273
 274 #include "a6xx.xml.h"
 275 #include "a6xx-pack.xml.h"
 276
 277 #define __assert_eq(a, b)                                               \
 278    do {                                                                 \
 279       if ((a) != (b)) {                                                 \
 280          fprintf(stderr, "assert failed: " #a " (0x%x) != " #b " (0x%x)\n", a, b); \
 281          assert((a) == (b));                                            \
 282       }                                                                 \
 283    } while (0)
 284
 285 #define __ONE_REG(i, regs)                                      \
 286    do {                                                         \
 287       if (i < ARRAY_SIZE(regs) && regs[i].reg > 0) {            \
 288          __assert_eq(regs[0].reg + i, regs[i].reg);             \
 289          if (regs[i].bo) {                                      \
 290             uint64_t v = regs[i].bo->iova + regs[i].bo_offset;  \
 291             v >>= regs[i].bo_shift;                             \
 292             v |= regs[i].value;                                 \
 293                                                                 \
 294             *p++ = v;                                           \
 295             *p++ = v >> 32;                                     \
 296          } else {                                               \
 297             *p++ = regs[i].value;                               \
 298             if (regs[i].is_address)                             \
 299                *p++ = regs[i].value >> 32;                      \
 300          }                                                      \
 301       }                                                         \
 302    } while (0)
 303
 304 /* Emits a sequence of register writes in order using a pkt4.  This will check
 305  * (at runtime on a !NDEBUG build) that the registers were actually set up in
 306  * order in the code.
 307  *
 308  * Note that references to buffers aren't automatically added to the CS,
 309  * unlike in freedreno.  We are clever in various places to avoid duplicating
 310  * the reference add work.
 311  *
 312  * Also, 64-bit address registers don't have a way (currently) to set a 64-bit
 313  * address without having a reference to a BO, since the .dword field in the
 314  * register's struct is only 32-bit wide.  We should fix this in the pack
 315  * codegen later.
 316  */
 317 #define tu_cs_emit_regs(cs, ...) do {                   \
 318    const struct fd_reg_pair regs[] = { __VA_ARGS__ };   \
 319    unsigned count = ARRAY_SIZE(regs);                   \
 320                                                         \
 321    STATIC_ASSERT(count > 0);                            \
 322    STATIC_ASSERT(count <= 16);                          \
 323                                                         \
 324    tu_cs_emit_pkt4(cs, regs[0].reg, count);             \
 325    uint32_t *p = cs->cur;                               \
 326    __ONE_REG( 0, regs);                                 \
 327    __ONE_REG( 1, regs);                                 \
 328    __ONE_REG( 2, regs);                                 \
 329    __ONE_REG( 3, regs);                                 \
 330    __ONE_REG( 4, regs);                                 \
 331    __ONE_REG( 5, regs);                                 \
 332    __ONE_REG( 6, regs);                                 \
 333    __ONE_REG( 7, regs);                                 \
 334    __ONE_REG( 8, regs);                                 \
 335    __ONE_REG( 9, regs);                                 \
 336    __ONE_REG(10, regs);                                 \
 337    __ONE_REG(11, regs);                                 \
 338    __ONE_REG(12, regs);                                 \
 339    __ONE_REG(13, regs);                                 \
 340    __ONE_REG(14, regs);                                 \
 341    __ONE_REG(15, regs);                                 \
 342    cs->cur = p;                                         \
 343    } while (0)
 344
 345 #endif /* TU_CS_H */