/* * Copyright © 2017 Broadcom * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ /** @file v3d_cpu_tiling.h * * Contains load/store functions common to both v3d and vc4. The utile layout * stayed the same, though the way utiles get laid out has changed. */ static inline void v3d_load_utile(void *cpu, uint32_t cpu_stride, void *gpu, uint32_t gpu_stride) { #if defined(V3D_BUILD_NEON) && defined(PIPE_ARCH_ARM) if (gpu_stride == 8) { __asm__ volatile ( /* Load from the GPU in one shot, no interleave, to * d0-d7. */ "vldm %[gpu], {q0, q1, q2, q3}\n" /* Store each 8-byte line to cpu-side destination, * incrementing it by the stride each time. */ "vst1.8 d0, [%[cpu]], %[cpu_stride]\n" "vst1.8 d1, [%[cpu]], %[cpu_stride]\n" "vst1.8 d2, [%[cpu]], %[cpu_stride]\n" "vst1.8 d3, [%[cpu]], %[cpu_stride]\n" "vst1.8 d4, [%[cpu]], %[cpu_stride]\n" "vst1.8 d5, [%[cpu]], %[cpu_stride]\n" "vst1.8 d6, [%[cpu]], %[cpu_stride]\n" "vst1.8 d7, [%[cpu]]\n" : [cpu] "+r"(cpu) : [gpu] "r"(gpu), [cpu_stride] "r"(cpu_stride) : "q0", "q1", "q2", "q3"); return; } else if (gpu_stride == 16) { void *cpu2 = cpu + 8; __asm__ volatile ( /* Load from the GPU in one shot, no interleave, to * d0-d7. */ "vldm %[gpu], {q0, q1, q2, q3};\n" /* Store each 16-byte line in 2 parts to the cpu-side * destination. (vld1 can only store one d-register * at a time). */ "vst1.8 d0, [%[cpu]], %[cpu_stride]\n" "vst1.8 d1, [%[cpu2]],%[cpu_stride]\n" "vst1.8 d2, [%[cpu]], %[cpu_stride]\n" "vst1.8 d3, [%[cpu2]],%[cpu_stride]\n" "vst1.8 d4, [%[cpu]], %[cpu_stride]\n" "vst1.8 d5, [%[cpu2]],%[cpu_stride]\n" "vst1.8 d6, [%[cpu]]\n" "vst1.8 d7, [%[cpu2]]\n" : [cpu] "+r"(cpu), [cpu2] "+r"(cpu2) : [gpu] "r"(gpu), [cpu_stride] "r"(cpu_stride) : "q0", "q1", "q2", "q3"); return; } #elif defined (PIPE_ARCH_AARCH64) if (gpu_stride == 8) { __asm__ volatile ( /* Load from the GPU in one shot, no interleave, to * d0-d7. */ "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%[gpu]]\n" /* Store each 8-byte line to cpu-side destination, * incrementing it by the stride each time. */ "st1 {v0.D}[0], [%[cpu]], %[cpu_stride]\n" "st1 {v0.D}[1], [%[cpu]], %[cpu_stride]\n" "st1 {v1.D}[0], [%[cpu]], %[cpu_stride]\n" "st1 {v1.D}[1], [%[cpu]], %[cpu_stride]\n" "st1 {v2.D}[0], [%[cpu]], %[cpu_stride]\n" "st1 {v2.D}[1], [%[cpu]], %[cpu_stride]\n" "st1 {v3.D}[0], [%[cpu]], %[cpu_stride]\n" "st1 {v3.D}[1], [%[cpu]]\n" : [cpu] "+r"(cpu) : [gpu] "r"(gpu), [cpu_stride] "r"(cpu_stride) : "v0", "v1", "v2", "v3"); return; } else if (gpu_stride == 16) { void *cpu2 = cpu + 8; __asm__ volatile ( /* Load from the GPU in one shot, no interleave, to * d0-d7. */ "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%[gpu]]\n" /* Store each 16-byte line in 2 parts to the cpu-side * destination. (vld1 can only store one d-register * at a time). */ "st1 {v0.D}[0], [%[cpu]], %[cpu_stride]\n" "st1 {v0.D}[1], [%[cpu2]],%[cpu_stride]\n" "st1 {v1.D}[0], [%[cpu]], %[cpu_stride]\n" "st1 {v1.D}[1], [%[cpu2]],%[cpu_stride]\n" "st1 {v2.D}[0], [%[cpu]], %[cpu_stride]\n" "st1 {v2.D}[1], [%[cpu2]],%[cpu_stride]\n" "st1 {v3.D}[0], [%[cpu]]\n" "st1 {v3.D}[1], [%[cpu2]]\n" : [cpu] "+r"(cpu), [cpu2] "+r"(cpu2) : [gpu] "r"(gpu), [cpu_stride] "r"(cpu_stride) : "v0", "v1", "v2", "v3"); return; } #endif for (uint32_t gpu_offset = 0; gpu_offset < 64; gpu_offset += gpu_stride) { memcpy(cpu, gpu + gpu_offset, gpu_stride); cpu += cpu_stride; } } static inline void v3d_store_utile(void *gpu, uint32_t gpu_stride, void *cpu, uint32_t cpu_stride) { #if defined(V3D_BUILD_NEON) && defined(PIPE_ARCH_ARM) if (gpu_stride == 8) { __asm__ volatile ( /* Load each 8-byte line from cpu-side source, * incrementing it by the stride each time. */ "vld1.8 d0, [%[cpu]], %[cpu_stride]\n" "vld1.8 d1, [%[cpu]], %[cpu_stride]\n" "vld1.8 d2, [%[cpu]], %[cpu_stride]\n" "vld1.8 d3, [%[cpu]], %[cpu_stride]\n" "vld1.8 d4, [%[cpu]], %[cpu_stride]\n" "vld1.8 d5, [%[cpu]], %[cpu_stride]\n" "vld1.8 d6, [%[cpu]], %[cpu_stride]\n" "vld1.8 d7, [%[cpu]]\n" /* Load from the GPU in one shot, no interleave, to * d0-d7. */ "vstm %[gpu], {q0, q1, q2, q3}\n" : [cpu] "+r"(cpu) : [gpu] "r"(gpu), [cpu_stride] "r"(cpu_stride) : "q0", "q1", "q2", "q3"); return; } else if (gpu_stride == 16) { void *cpu2 = cpu + 8; __asm__ volatile ( /* Load each 16-byte line in 2 parts from the cpu-side * destination. (vld1 can only store one d-register * at a time). */ "vld1.8 d0, [%[cpu]], %[cpu_stride]\n" "vld1.8 d1, [%[cpu2]],%[cpu_stride]\n" "vld1.8 d2, [%[cpu]], %[cpu_stride]\n" "vld1.8 d3, [%[cpu2]],%[cpu_stride]\n" "vld1.8 d4, [%[cpu]], %[cpu_stride]\n" "vld1.8 d5, [%[cpu2]],%[cpu_stride]\n" "vld1.8 d6, [%[cpu]]\n" "vld1.8 d7, [%[cpu2]]\n" /* Store to the GPU in one shot, no interleave. */ "vstm %[gpu], {q0, q1, q2, q3}\n" : [cpu] "+r"(cpu), [cpu2] "+r"(cpu2) : [gpu] "r"(gpu), [cpu_stride] "r"(cpu_stride) : "q0", "q1", "q2", "q3"); return; } #elif defined (PIPE_ARCH_AARCH64) if (gpu_stride == 8) { __asm__ volatile ( /* Load each 8-byte line from cpu-side source, * incrementing it by the stride each time. */ "ld1 {v0.D}[0], [%[cpu]], %[cpu_stride]\n" "ld1 {v0.D}[1], [%[cpu]], %[cpu_stride]\n" "ld1 {v1.D}[0], [%[cpu]], %[cpu_stride]\n" "ld1 {v1.D}[1], [%[cpu]], %[cpu_stride]\n" "ld1 {v2.D}[0], [%[cpu]], %[cpu_stride]\n" "ld1 {v2.D}[1], [%[cpu]], %[cpu_stride]\n" "ld1 {v3.D}[0], [%[cpu]], %[cpu_stride]\n" "ld1 {v3.D}[1], [%[cpu]]\n" /* Store to the GPU in one shot, no interleave. */ "st1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%[gpu]]\n" : [cpu] "+r"(cpu) : [gpu] "r"(gpu), [cpu_stride] "r"(cpu_stride) : "v0", "v1", "v2", "v3"); return; } else if (gpu_stride == 16) { void *cpu2 = cpu + 8; __asm__ volatile ( /* Load each 16-byte line in 2 parts from the cpu-side * destination. (vld1 can only store one d-register * at a time). */ "ld1 {v0.D}[0], [%[cpu]], %[cpu_stride]\n" "ld1 {v0.D}[1], [%[cpu2]],%[cpu_stride]\n" "ld1 {v1.D}[0], [%[cpu]], %[cpu_stride]\n" "ld1 {v1.D}[1], [%[cpu2]],%[cpu_stride]\n" "ld1 {v2.D}[0], [%[cpu]], %[cpu_stride]\n" "ld1 {v2.D}[1], [%[cpu2]],%[cpu_stride]\n" "ld1 {v3.D}[0], [%[cpu]]\n" "ld1 {v3.D}[1], [%[cpu2]]\n" /* Store to the GPU in one shot, no interleave. */ "st1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%[gpu]]\n" : [cpu] "+r"(cpu), [cpu2] "+r"(cpu2) : [gpu] "r"(gpu), [cpu_stride] "r"(cpu_stride) : "v0", "v1", "v2", "v3"); return; } #endif for (uint32_t gpu_offset = 0; gpu_offset < 64; gpu_offset += gpu_stride) { memcpy(gpu + gpu_offset, cpu, gpu_stride); cpu += cpu_stride; } }