2 * Copyright (C) 2014-2015 Etnaviv Project
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
24 * Christian Gmeiner <christian.gmeiner@gmail.com>
30 #include "etnaviv_drmif.h"
31 #include "etnaviv_priv.h"
33 static pthread_mutex_t idx_lock
= PTHREAD_MUTEX_INITIALIZER
;
35 static void *grow(void *ptr
, uint32_t nr
, uint32_t *max
, uint32_t sz
)
37 if ((nr
+ 1) > *max
) {
38 if ((*max
* 2) < (nr
+ 1))
42 ptr
= realloc(ptr
, *max
* sz
);
48 #define APPEND(x, name) ({ \
49 (x)->name = grow((x)->name, (x)->nr_ ## name, &(x)->max_ ## name, sizeof((x)->name[0])); \
50 (x)->nr_ ## name ++; \
53 void etna_cmd_stream_realloc(struct etna_cmd_stream
*stream
, size_t n
)
59 * Increase the command buffer size by 1 kiB. Here we pick 1 kiB
60 * increment to prevent it from growing too much too quickly.
62 size
= ALIGN(stream
->size
+ n
, 1024);
64 /* Command buffer is too big for older kernel versions */
68 buffer
= realloc(stream
->buffer
, size
* 4);
72 stream
->buffer
= buffer
;
78 WARN_MSG("command buffer too long, forcing flush.");
79 etna_cmd_stream_force_flush(stream
);
82 static inline struct etna_cmd_stream_priv
*
83 etna_cmd_stream_priv(struct etna_cmd_stream
*stream
)
85 return (struct etna_cmd_stream_priv
*)stream
;
88 struct etna_cmd_stream
*etna_cmd_stream_new(struct etna_pipe
*pipe
,
90 void (*force_flush
)(struct etna_cmd_stream
*stream
, void *priv
),
93 struct etna_cmd_stream_priv
*stream
= NULL
;
96 ERROR_MSG("invalid size of 0");
100 stream
= calloc(1, sizeof(*stream
));
102 ERROR_MSG("allocation failed");
106 /* allocate even number of 32-bit words */
107 size
= ALIGN(size
, 2);
109 stream
->base
.buffer
= malloc(size
* sizeof(uint32_t));
110 if (!stream
->base
.buffer
) {
111 ERROR_MSG("allocation failed");
115 stream
->base
.size
= size
;
117 stream
->force_flush
= force_flush
;
118 stream
->force_flush_priv
= priv
;
120 return &stream
->base
;
124 etna_cmd_stream_del(&stream
->base
);
129 void etna_cmd_stream_del(struct etna_cmd_stream
*stream
)
131 struct etna_cmd_stream_priv
*priv
= etna_cmd_stream_priv(stream
);
133 free(stream
->buffer
);
134 free(priv
->submit
.relocs
);
135 free(priv
->submit
.pmrs
);
139 void etna_cmd_stream_force_flush(struct etna_cmd_stream
*stream
)
141 struct etna_cmd_stream_priv
*priv
= etna_cmd_stream_priv(stream
);
143 if (priv
->force_flush
)
144 priv
->force_flush(stream
, priv
->force_flush_priv
);
147 uint32_t etna_cmd_stream_timestamp(struct etna_cmd_stream
*stream
)
149 return etna_cmd_stream_priv(stream
)->last_timestamp
;
152 static uint32_t append_bo(struct etna_cmd_stream
*stream
, struct etna_bo
*bo
)
154 struct etna_cmd_stream_priv
*priv
= etna_cmd_stream_priv(stream
);
157 idx
= APPEND(&priv
->submit
, bos
);
158 idx
= APPEND(priv
, bos
);
160 priv
->submit
.bos
[idx
].flags
= 0;
161 priv
->submit
.bos
[idx
].handle
= bo
->handle
;
162 priv
->submit
.bos
[idx
].presumed
= bo
->va
;
164 priv
->bos
[idx
] = etna_bo_ref(bo
);
169 /* add (if needed) bo, return idx: */
170 static uint32_t bo2idx(struct etna_cmd_stream
*stream
, struct etna_bo
*bo
,
173 struct etna_cmd_stream_priv
*priv
= etna_cmd_stream_priv(stream
);
176 pthread_mutex_lock(&idx_lock
);
178 if (bo
->current_stream
== stream
) {
184 priv
->bo_table
= drmHashCreate();
186 if (!drmHashLookup(priv
->bo_table
, bo
->handle
, &val
)) {
188 idx
= (uint32_t)(uintptr_t)val
;
190 idx
= append_bo(stream
, bo
);
191 val
= (void *)(uintptr_t)idx
;
192 drmHashInsert(priv
->bo_table
, bo
->handle
, val
);
195 bo
->current_stream
= stream
;
198 pthread_mutex_unlock(&idx_lock
);
200 if (flags
& ETNA_RELOC_READ
)
201 priv
->submit
.bos
[idx
].flags
|= ETNA_SUBMIT_BO_READ
;
202 if (flags
& ETNA_RELOC_WRITE
)
203 priv
->submit
.bos
[idx
].flags
|= ETNA_SUBMIT_BO_WRITE
;
208 void etna_cmd_stream_flush(struct etna_cmd_stream
*stream
, int in_fence_fd
,
211 struct etna_cmd_stream_priv
*priv
= etna_cmd_stream_priv(stream
);
212 int ret
, id
= priv
->pipe
->id
;
213 struct etna_gpu
*gpu
= priv
->pipe
->gpu
;
215 struct drm_etnaviv_gem_submit req
= {
218 .bos
= VOID2U64(priv
->submit
.bos
),
219 .nr_bos
= priv
->submit
.nr_bos
,
220 .relocs
= VOID2U64(priv
->submit
.relocs
),
221 .nr_relocs
= priv
->submit
.nr_relocs
,
222 .pmrs
= VOID2U64(priv
->submit
.pmrs
),
223 .nr_pmrs
= priv
->submit
.nr_pmrs
,
224 .stream
= VOID2U64(stream
->buffer
),
225 .stream_size
= stream
->offset
* 4, /* in bytes */
228 if (in_fence_fd
!= -1) {
229 req
.flags
|= ETNA_SUBMIT_FENCE_FD_IN
| ETNA_SUBMIT_NO_IMPLICIT
;
230 req
.fence_fd
= in_fence_fd
;
234 req
.flags
|= ETNA_SUBMIT_FENCE_FD_OUT
;
236 if (gpu
->dev
->use_softpin
)
237 req
.flags
|= ETNA_SUBMIT_SOFTPIN
;
239 ret
= drmCommandWriteRead(gpu
->dev
->fd
, DRM_ETNAVIV_GEM_SUBMIT
,
243 ERROR_MSG("submit failed: %d (%s)", ret
, strerror(errno
));
245 priv
->last_timestamp
= req
.fence
;
247 for (uint32_t i
= 0; i
< priv
->nr_bos
; i
++) {
248 struct etna_bo
*bo
= priv
->bos
[i
];
250 bo
->current_stream
= NULL
;
254 if (priv
->bo_table
) {
255 drmHashDestroy(priv
->bo_table
);
256 priv
->bo_table
= NULL
;
260 *out_fence_fd
= req
.fence_fd
;
263 priv
->submit
.nr_bos
= 0;
264 priv
->submit
.nr_relocs
= 0;
265 priv
->submit
.nr_pmrs
= 0;
269 void etna_cmd_stream_reloc(struct etna_cmd_stream
*stream
,
270 const struct etna_reloc
*r
)
272 struct etna_cmd_stream_priv
*priv
= etna_cmd_stream_priv(stream
);
273 struct drm_etnaviv_gem_submit_reloc
*reloc
;
274 uint32_t addr
= r
->bo
->va
+ r
->offset
;
275 uint32_t bo_idx
= bo2idx(stream
, r
->bo
, r
->flags
);
278 if (!priv
->pipe
->gpu
->dev
->use_softpin
) {
279 idx
= APPEND(&priv
->submit
, relocs
);
280 reloc
= &priv
->submit
.relocs
[idx
];
282 reloc
->reloc_idx
= bo_idx
;
283 reloc
->reloc_offset
= r
->offset
;
284 reloc
->submit_offset
= stream
->offset
* 4; /* in bytes */
288 etna_cmd_stream_emit(stream
, addr
);
291 void etna_cmd_stream_ref_bo(struct etna_cmd_stream
*stream
, struct etna_bo
*bo
,
294 bo2idx(stream
, bo
, flags
);
297 void etna_cmd_stream_perf(struct etna_cmd_stream
*stream
, const struct etna_perf
*p
)
299 struct etna_cmd_stream_priv
*priv
= etna_cmd_stream_priv(stream
);
300 struct drm_etnaviv_gem_submit_pmr
*pmr
;
301 uint32_t idx
= APPEND(&priv
->submit
, pmrs
);
303 pmr
= &priv
->submit
.pmrs
[idx
];
305 pmr
->flags
= p
->flags
;
306 pmr
->sequence
= p
->sequence
;
307 pmr
->read_offset
= p
->offset
;
308 pmr
->read_idx
= bo2idx(stream
, p
->bo
, ETNA_SUBMIT_BO_READ
| ETNA_SUBMIT_BO_WRITE
);
309 pmr
->domain
= p
->signal
->domain
->id
;
310 pmr
->signal
= p
->signal
->signal
;