2 * Copyright (C) 2018-2019 Alyssa Rosenzweig
3 * Copyright (C) 2019 Collabora, Ltd.
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26 #include "pan_context.h"
28 /* See mali_job for notes on how this works. But basically, for small vertex
29 * counts, we have a lookup table, and for large vertex counts, we look at the
30 * high bits as a heuristic. This has to match exactly how the hardware
31 * calculates this (which is why the algorithm is so weird) or else instancing
34 /* Given an odd number (of the form 2k + 1), compute k */
35 #define ODD(odd) ((odd - 1) >> 1)
37 /* Given the shift/odd pair, recover the original padded integer */
40 pan_expand_shift_odd(struct pan_shift_odd o
)
42 unsigned odd
= 2*o
.odd
+ 1;
43 unsigned shift
= 1 << o
.shift
;
47 static inline struct pan_shift_odd
48 pan_factored(unsigned pot
, unsigned odd
)
50 struct pan_shift_odd out
;
52 assert(util_is_power_of_two_or_zero(pot
));
55 /* Odd is of the form (2k + 1) = (k << 1) + 1 = (k << 1) | 1.
57 * So (odd >> 1) = ((k << 1) | 1) >> 1 = ((k << 1) >> 1) | (1 >> 1)
62 /* POT is the form (1 << shift) */
63 out
.shift
= __builtin_ctz(pot
);
69 /* For small vertices. Second argument is whether the primitive takes a
70 * power-of-two argument, which determines how rounding works. True for POINTS
71 * and LINES, false for TRIANGLES. Presumably true for QUADS but you'd be crazy
72 * to try instanced quads on ES class hardware <3 */
100 static struct pan_shift_odd
101 panfrost_small_padded_vertex_count(unsigned idx
)
108 static struct pan_shift_odd
109 panfrost_large_padded_vertex_count(uint32_t vertex_count
)
111 struct pan_shift_odd out
= { 0 };
113 /* First, we have to find the highest set one */
114 unsigned highest
= 32 - __builtin_clz(vertex_count
);
116 /* Using that, we mask out the highest 4-bits */
117 unsigned n
= highest
- 4;
118 unsigned nibble
= (vertex_count
>> n
) & 0xF;
120 /* Great, we have the nibble. Now we can just try possibilities. Note
121 * that we don't care about the bottom most bit in most cases, and we
122 * know the top bit must be 1 */
124 unsigned middle_two
= (nibble
>> 1) & 0x3;
126 switch (middle_two
) {
129 return pan_factored(1 << n
, 9);
131 return pan_factored(1 << (n
+ 1), 5);
133 return pan_factored(1 << (n
+ 2), 3);
135 return pan_factored(1 << (n
+ 1), 7);
137 return pan_factored(1 << (n
+ 4), 1);
139 unreachable("Invalid two bits");
146 panfrost_padded_vertex_count(
147 unsigned vertex_count
,
150 assert(vertex_count
> 0);
152 if (vertex_count
< 20) {
153 /* Add an off-by-one if it won't align naturally (quirk of the hardware) */
157 return panfrost_small_padded_vertex_count(vertex_count
);
159 return panfrost_large_padded_vertex_count(vertex_count
);
162 /* The much, much more irritating case -- instancing is enabled. See
163 * panfrost_job.h for notes on how this works */
166 panfrost_vertex_instanced(
167 struct panfrost_job
*batch
,
168 struct panfrost_resource
*rsrc
,
170 union mali_attr
*attrs
,
172 unsigned vertex_count
,
173 unsigned instance_count
)
175 /* First, grab the padded vertex count */
177 struct pan_shift_odd o
= {
178 .shift
= batch
->ctx
->payloads
[PIPE_SHADER_FRAGMENT
].instance_shift
,
179 .odd
= batch
->ctx
->payloads
[PIPE_SHADER_FRAGMENT
].instance_odd
,
182 unsigned padded_count
= batch
->ctx
->padded_count
;
184 /* Depending if there is an instance divisor or not, packing varies.
185 * When there is a divisor, the hardware-level divisor is actually the
186 * product of the instance divisor and the padded count */
188 unsigned hw_divisor
= padded_count
* divisor
;
191 /* Per-vertex attributes use the MODULO mode. First, compute
194 attrs
->elements
|= MALI_ATTR_MODULO
;
195 attrs
->shift
= o
.shift
;
196 attrs
->extra_flags
= o
.odd
;
199 } else if (util_is_power_of_two_or_zero(hw_divisor
)) {
200 /* If there is a divisor but the hardware divisor works out to
201 * a power of two (not terribly exceptional), we can use an
202 * easy path (just shifting) */
204 attrs
->elements
|= MALI_ATTR_POT_DIVIDE
;
205 attrs
->shift
= __builtin_ctz(hw_divisor
);
209 /* We have a NPOT divisor. Here's the fun one (multipling by
210 * the inverse and shifting) */
213 unsigned shift
= util_logbase2(hw_divisor
);
215 /* m = ceil(2^(32 + shift) / d) */
216 uint64_t shift_hi
= 32 + shift
;
217 uint64_t t
= 1ll << shift_hi
;
219 double hw_divisor_d
= hw_divisor
;
220 double m_f
= ceil(t_f
/ hw_divisor_d
);
224 uint32_t magic_divisor
= m
, extra_flags
= 0;
226 /* e = 2^(shift + 32) % d */
227 uint64_t e
= t
% hw_divisor
;
229 /* Apply round-down algorithm? e <= 2^shift?. XXX: The blob
230 * seems to use a different condition */
231 if (e
<= (1ll << shift
)) {
232 magic_divisor
= m
- 1;
236 /* Top flag implicitly set */
237 assert(magic_divisor
& (1u << 31));
238 magic_divisor
&= ~(1u << 31);
240 /* Upload to two different slots */
242 attrs
[0].elements
|= MALI_ATTR_NPOT_DIVIDE
;
243 attrs
[0].shift
= shift
;
244 attrs
[0].extra_flags
= extra_flags
;
247 attrs
[1].magic_divisor
= magic_divisor
;
249 attrs
[1].divisor
= divisor
;
256 panfrost_emit_vertex_data(struct panfrost_job
*batch
)
258 struct panfrost_context
*ctx
= batch
->ctx
;
259 struct panfrost_vertex_state
*so
= ctx
->vertex
;
261 /* Staged mali_attr, and index into them. i =/= k, depending on the
262 * vertex buffer mask and instancing. Twice as much room is allocated,
263 * for a worst case of NPOT_DIVIDEs which take up extra slot */
264 union mali_attr attrs
[PIPE_MAX_ATTRIBS
* 2];
267 unsigned vertex_count
= ctx
->vertex_count
;
268 unsigned instanced_count
= ctx
->instance_count
;
270 for (unsigned i
= 0; i
< so
->num_elements
; ++i
) {
271 /* We map a mali_attr to be 1:1 with the mali_attr_meta, which
272 * means duplicating some vertex buffers (who cares? aside from
273 * maybe some caching implications but I somehow doubt that
276 struct pipe_vertex_element
*elem
= &so
->pipe
[i
];
277 unsigned vbi
= elem
->vertex_buffer_index
;
279 /* The exception to 1:1 mapping is that we can have multiple
280 * entries (NPOT divisors), so we fixup anyways */
284 if (!(ctx
->vb_mask
& (1 << vbi
))) continue;
286 struct pipe_vertex_buffer
*buf
= &ctx
->vertex_buffers
[vbi
];
287 struct panfrost_resource
*rsrc
= (struct panfrost_resource
*) (buf
->buffer
.resource
);
291 /* Align to 64 bytes by masking off the lower bits. This
292 * will be adjusted back when we fixup the src_offset in
295 mali_ptr raw_addr
= rsrc
->bo
->gpu
+ buf
->buffer_offset
;
296 mali_ptr addr
= raw_addr
& ~63;
297 unsigned chopped_addr
= raw_addr
- addr
;
299 /* Add a dependency of the batch on the vertex buffer */
300 panfrost_job_add_bo(batch
, rsrc
->bo
);
302 /* Set common fields */
303 attrs
[k
].elements
= addr
;
304 attrs
[k
].stride
= buf
->stride
;
306 /* Since we advanced the base pointer, we shrink the buffer
308 attrs
[k
].size
= rsrc
->base
.width0
- buf
->buffer_offset
;
310 /* We need to add the extra size we masked off (for
311 * correctness) so the data doesn't get clamped away */
312 attrs
[k
].size
+= chopped_addr
;
314 /* For non-instancing make sure we initialize */
315 attrs
[k
].shift
= attrs
[k
].extra_flags
= 0;
317 /* Instancing uses a dramatically different code path than
318 * linear, so dispatch for the actual emission now that the
319 * common code is finished */
321 unsigned divisor
= elem
->instance_divisor
;
323 if (divisor
&& instanced_count
== 1) {
324 /* Silly corner case where there's a divisor(=1) but
325 * there's no legitimate instancing. So we want *every*
326 * attribute to be the same. So set stride to zero so
327 * we don't go anywhere. */
329 attrs
[k
].size
= attrs
[k
].stride
+ chopped_addr
;
331 attrs
[k
++].elements
|= MALI_ATTR_LINEAR
;
332 } else if (instanced_count
<= 1) {
333 /* Normal, non-instanced attributes */
334 attrs
[k
++].elements
|= MALI_ATTR_LINEAR
;
336 k
+= panfrost_vertex_instanced(
337 batch
, rsrc
, divisor
, &attrs
[k
], addr
, vertex_count
, instanced_count
);
341 /* Upload whatever we emitted and go */
343 ctx
->payloads
[PIPE_SHADER_VERTEX
].postfix
.attributes
=
344 panfrost_upload_transient(ctx
, attrs
, k
* sizeof(union mali_attr
));