2 * Copyright © 2016-2018 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
26 #include <sys/types.h>
31 #ifndef HAVE_MEMFD_CREATE
32 #include <sys/syscall.h>
35 memfd_create(const char *name
, unsigned int flags
)
37 return syscall(SYS_memfd_create
, name
, flags
);
42 struct list_head link
;
43 struct gen_batch_decode_bo bo
;
59 const uint8_t *aub_data
;
63 add_gtt_bo_map(struct aub_mem
*mem
, struct gen_batch_decode_bo bo
, bool ppgtt
, bool unmap_after_use
)
65 struct bo_map
*m
= calloc(1, sizeof(*m
));
69 m
->unmap_after_use
= unmap_after_use
;
70 list_add(&m
->link
, &mem
->maps
);
74 aub_mem_clear_bo_maps(struct aub_mem
*mem
)
76 list_for_each_entry_safe(struct bo_map
, i
, &mem
->maps
, link
) {
77 if (i
->unmap_after_use
)
78 munmap((void *)i
->bo
.map
, i
->bo
.size
);
84 static inline struct ggtt_entry
*
85 ggtt_entry_next(struct ggtt_entry
*entry
)
89 struct rb_node
*node
= rb_node_next(&entry
->node
);
92 return rb_node_data(struct ggtt_entry
, node
, node
);
96 cmp_uint64(uint64_t a
, uint64_t b
)
106 cmp_ggtt_entry(const struct rb_node
*node
, const void *addr
)
108 struct ggtt_entry
*entry
= rb_node_data(struct ggtt_entry
, node
, node
);
109 return cmp_uint64(entry
->virt_addr
, *(const uint64_t *)addr
);
112 static struct ggtt_entry
*
113 ensure_ggtt_entry(struct aub_mem
*mem
, uint64_t virt_addr
)
115 struct rb_node
*node
= rb_tree_search_sloppy(&mem
->ggtt
, &virt_addr
,
118 if (!node
|| (cmp
= cmp_ggtt_entry(node
, &virt_addr
))) {
119 struct ggtt_entry
*new_entry
= calloc(1, sizeof(*new_entry
));
120 new_entry
->virt_addr
= virt_addr
;
121 rb_tree_insert_at(&mem
->ggtt
, node
, &new_entry
->node
, cmp
> 0);
122 node
= &new_entry
->node
;
125 return rb_node_data(struct ggtt_entry
, node
, node
);
128 static struct ggtt_entry
*
129 search_ggtt_entry(struct aub_mem
*mem
, uint64_t virt_addr
)
133 struct rb_node
*node
= rb_tree_search(&mem
->ggtt
, &virt_addr
, cmp_ggtt_entry
);
138 return rb_node_data(struct ggtt_entry
, node
, node
);
142 cmp_phys_mem(const struct rb_node
*node
, const void *addr
)
144 struct phys_mem
*mem
= rb_node_data(struct phys_mem
, node
, node
);
145 return cmp_uint64(mem
->phys_addr
, *(uint64_t *)addr
);
148 static struct phys_mem
*
149 ensure_phys_mem(struct aub_mem
*mem
, uint64_t phys_addr
)
151 struct rb_node
*node
= rb_tree_search_sloppy(&mem
->mem
, &phys_addr
, cmp_phys_mem
);
153 if (!node
|| (cmp
= cmp_phys_mem(node
, &phys_addr
))) {
154 struct phys_mem
*new_mem
= calloc(1, sizeof(*new_mem
));
155 new_mem
->phys_addr
= phys_addr
;
156 new_mem
->fd_offset
= mem
->mem_fd_len
;
158 MAYBE_UNUSED
int ftruncate_res
= ftruncate(mem
->mem_fd
, mem
->mem_fd_len
+= 4096);
159 assert(ftruncate_res
== 0);
161 new_mem
->data
= mmap(NULL
, 4096, PROT_READ
| PROT_WRITE
, MAP_SHARED
,
162 mem
->mem_fd
, new_mem
->fd_offset
);
163 assert(new_mem
->data
!= MAP_FAILED
);
165 rb_tree_insert_at(&mem
->mem
, node
, &new_mem
->node
, cmp
> 0);
166 node
= &new_mem
->node
;
169 return rb_node_data(struct phys_mem
, node
, node
);
172 static struct phys_mem
*
173 search_phys_mem(struct aub_mem
*mem
, uint64_t phys_addr
)
177 struct rb_node
*node
= rb_tree_search(&mem
->mem
, &phys_addr
, cmp_phys_mem
);
182 return rb_node_data(struct phys_mem
, node
, node
);
186 aub_mem_local_write(void *_mem
, uint64_t address
,
187 const void *data
, uint32_t size
)
189 struct aub_mem
*mem
= _mem
;
190 struct gen_batch_decode_bo bo
= {
195 add_gtt_bo_map(mem
, bo
, false, false);
199 aub_mem_ggtt_entry_write(void *_mem
, uint64_t address
,
200 const void *_data
, uint32_t _size
)
202 struct aub_mem
*mem
= _mem
;
203 uint64_t virt_addr
= (address
/ sizeof(uint64_t)) << 12;
204 const uint64_t *data
= _data
;
205 size_t size
= _size
/ sizeof(*data
);
206 for (const uint64_t *entry
= data
;
208 entry
++, virt_addr
+= 4096) {
209 struct ggtt_entry
*pt
= ensure_ggtt_entry(mem
, virt_addr
);
210 pt
->phys_addr
= *entry
;
215 aub_mem_phys_write(void *_mem
, uint64_t phys_address
,
216 const void *data
, uint32_t size
)
218 struct aub_mem
*mem
= _mem
;
219 uint32_t to_write
= size
;
220 for (uint64_t page
= phys_address
& ~0xfff; page
< phys_address
+ size
; page
+= 4096) {
221 struct phys_mem
*pmem
= ensure_phys_mem(mem
, page
);
222 uint64_t offset
= MAX2(page
, phys_address
) - page
;
223 uint32_t size_this_page
= MIN2(to_write
, 4096 - offset
);
224 to_write
-= size_this_page
;
225 memcpy(pmem
->data
+ offset
, data
, size_this_page
);
226 pmem
->aub_data
= data
- offset
;
227 data
= (const uint8_t *)data
+ size_this_page
;
232 aub_mem_ggtt_write(void *_mem
, uint64_t virt_address
,
233 const void *data
, uint32_t size
)
235 struct aub_mem
*mem
= _mem
;
236 uint32_t to_write
= size
;
237 for (uint64_t page
= virt_address
& ~0xfff; page
< virt_address
+ size
; page
+= 4096) {
238 struct ggtt_entry
*entry
= search_ggtt_entry(mem
, page
);
239 assert(entry
&& entry
->phys_addr
& 0x1);
241 uint64_t offset
= MAX2(page
, virt_address
) - page
;
242 uint32_t size_this_page
= MIN2(to_write
, 4096 - offset
);
243 to_write
-= size_this_page
;
245 uint64_t phys_page
= entry
->phys_addr
& ~0xfff; /* Clear the validity bits. */
246 aub_mem_phys_write(mem
, phys_page
+ offset
, data
, size_this_page
);
247 data
= (const uint8_t *)data
+ size_this_page
;
251 struct gen_batch_decode_bo
252 aub_mem_get_ggtt_bo(void *_mem
, uint64_t address
)
254 struct aub_mem
*mem
= _mem
;
255 struct gen_batch_decode_bo bo
= {0};
257 list_for_each_entry(struct bo_map
, i
, &mem
->maps
, link
)
258 if (!i
->ppgtt
&& i
->bo
.addr
<= address
&& i
->bo
.addr
+ i
->bo
.size
> address
)
263 struct ggtt_entry
*start
=
264 (struct ggtt_entry
*)rb_tree_search_sloppy(&mem
->ggtt
, &address
,
266 if (start
&& start
->virt_addr
< address
)
267 start
= ggtt_entry_next(start
);
271 struct ggtt_entry
*last
= start
;
272 for (struct ggtt_entry
*i
= ggtt_entry_next(last
);
273 i
&& last
->virt_addr
+ 4096 == i
->virt_addr
;
274 last
= i
, i
= ggtt_entry_next(last
))
277 bo
.addr
= MIN2(address
, start
->virt_addr
);
278 bo
.size
= last
->virt_addr
- bo
.addr
+ 4096;
279 bo
.map
= mmap(NULL
, bo
.size
, PROT_READ
, MAP_SHARED
| MAP_ANONYMOUS
, -1, 0);
280 assert(bo
.map
!= MAP_FAILED
);
282 for (struct ggtt_entry
*i
= start
;
284 i
= i
== last
? NULL
: ggtt_entry_next(i
)) {
285 uint64_t phys_addr
= i
->phys_addr
& ~0xfff;
286 struct phys_mem
*phys_mem
= search_phys_mem(mem
, phys_addr
);
291 uint32_t map_offset
= i
->virt_addr
- address
;
292 void *res
= mmap((uint8_t *)bo
.map
+ map_offset
, 4096, PROT_READ
,
293 MAP_SHARED
| MAP_FIXED
, mem
->mem_fd
, phys_mem
->fd_offset
);
294 assert(res
!= MAP_FAILED
);
297 add_gtt_bo_map(mem
, bo
, false, true);
302 static struct phys_mem
*
303 ppgtt_walk(struct aub_mem
*mem
, uint64_t pml4
, uint64_t address
)
306 uint64_t addr
= pml4
;
307 for (int level
= 4; level
> 0; level
--) {
308 struct phys_mem
*table
= search_phys_mem(mem
, addr
);
311 int index
= (address
>> shift
) & 0x1ff;
312 uint64_t entry
= ((uint64_t *)table
->data
)[index
];
315 addr
= entry
& ~0xfff;
318 return search_phys_mem(mem
, addr
);
322 ppgtt_mapped(struct aub_mem
*mem
, uint64_t pml4
, uint64_t address
)
324 return ppgtt_walk(mem
, pml4
, address
) != NULL
;
327 struct gen_batch_decode_bo
328 aub_mem_get_ppgtt_bo(void *_mem
, uint64_t address
)
330 struct aub_mem
*mem
= _mem
;
331 struct gen_batch_decode_bo bo
= {0};
333 list_for_each_entry(struct bo_map
, i
, &mem
->maps
, link
)
334 if (i
->ppgtt
&& i
->bo
.addr
<= address
&& i
->bo
.addr
+ i
->bo
.size
> address
)
339 if (!ppgtt_mapped(mem
, mem
->pml4
, address
))
342 /* Map everything until the first gap since we don't know how much the
343 * decoder actually needs.
345 uint64_t end
= address
;
346 while (ppgtt_mapped(mem
, mem
->pml4
, end
))
350 bo
.size
= end
- address
;
351 bo
.map
= mmap(NULL
, bo
.size
, PROT_READ
, MAP_SHARED
| MAP_ANONYMOUS
, -1, 0);
352 assert(bo
.map
!= MAP_FAILED
);
354 for (uint64_t page
= address
; page
< end
; page
+= 4096) {
355 struct phys_mem
*phys_mem
= ppgtt_walk(mem
, mem
->pml4
, page
);
357 void *res
= mmap((uint8_t *)bo
.map
+ (page
- bo
.addr
), 4096, PROT_READ
,
358 MAP_SHARED
| MAP_FIXED
, mem
->mem_fd
, phys_mem
->fd_offset
);
359 assert(res
!= MAP_FAILED
);
362 add_gtt_bo_map(mem
, bo
, true, true);
368 aub_mem_init(struct aub_mem
*mem
)
370 memset(mem
, 0, sizeof(*mem
));
372 list_inithead(&mem
->maps
);
374 mem
->mem_fd
= memfd_create("phys memory", 0);
376 return mem
->mem_fd
!= -1;
380 aub_mem_fini(struct aub_mem
*mem
)
382 if (mem
->mem_fd
== -1)
385 aub_mem_clear_bo_maps(mem
);
388 rb_tree_foreach_safe(struct ggtt_entry
, entry
, &mem
->ggtt
, node
) {
389 rb_tree_remove(&mem
->ggtt
, &entry
->node
);
392 rb_tree_foreach_safe(struct phys_mem
, entry
, &mem
->mem
, node
) {
393 rb_tree_remove(&mem
->mem
, &entry
->node
);
401 struct gen_batch_decode_bo
402 aub_mem_get_phys_addr_data(struct aub_mem
*mem
, uint64_t phys_addr
)
404 struct phys_mem
*page
= search_phys_mem(mem
, phys_addr
);
406 (struct gen_batch_decode_bo
) { .map
= page
->data
, .addr
= page
->phys_addr
, .size
= 4096 } :
407 (struct gen_batch_decode_bo
) {};
410 struct gen_batch_decode_bo
411 aub_mem_get_ppgtt_addr_data(struct aub_mem
*mem
, uint64_t virt_addr
)
413 struct phys_mem
*page
= ppgtt_walk(mem
, mem
->pml4
, virt_addr
);
415 (struct gen_batch_decode_bo
) { .map
= page
->data
, .addr
= virt_addr
& ~((1ULL << 12) - 1), .size
= 4096 } :
416 (struct gen_batch_decode_bo
) {};
419 struct gen_batch_decode_bo
420 aub_mem_get_ppgtt_addr_aub_data(struct aub_mem
*mem
, uint64_t virt_addr
)
422 struct phys_mem
*page
= ppgtt_walk(mem
, mem
->pml4
, virt_addr
);
424 (struct gen_batch_decode_bo
) { .map
= page
->aub_data
, .addr
= virt_addr
& ~((1ULL << 12) - 1), .size
= 4096 } :
425 (struct gen_batch_decode_bo
) {};