2 * Copyright © 2016-2018 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
26 #include <sys/types.h>
30 #include "util/anon_file.h"
33 struct list_head link
;
34 struct gen_batch_decode_bo bo
;
50 const uint8_t *aub_data
;
54 add_gtt_bo_map(struct aub_mem
*mem
, struct gen_batch_decode_bo bo
, bool ppgtt
, bool unmap_after_use
)
56 struct bo_map
*m
= calloc(1, sizeof(*m
));
60 m
->unmap_after_use
= unmap_after_use
;
61 list_add(&m
->link
, &mem
->maps
);
65 aub_mem_clear_bo_maps(struct aub_mem
*mem
)
67 list_for_each_entry_safe(struct bo_map
, i
, &mem
->maps
, link
) {
68 if (i
->unmap_after_use
)
69 munmap((void *)i
->bo
.map
, i
->bo
.size
);
75 static inline struct ggtt_entry
*
76 ggtt_entry_next(struct ggtt_entry
*entry
)
80 struct rb_node
*node
= rb_node_next(&entry
->node
);
83 return rb_node_data(struct ggtt_entry
, node
, node
);
87 cmp_uint64(uint64_t a
, uint64_t b
)
97 cmp_ggtt_entry(const struct rb_node
*node
, const void *addr
)
99 struct ggtt_entry
*entry
= rb_node_data(struct ggtt_entry
, node
, node
);
100 return cmp_uint64(entry
->virt_addr
, *(const uint64_t *)addr
);
103 static struct ggtt_entry
*
104 ensure_ggtt_entry(struct aub_mem
*mem
, uint64_t virt_addr
)
106 struct rb_node
*node
= rb_tree_search_sloppy(&mem
->ggtt
, &virt_addr
,
109 if (!node
|| (cmp
= cmp_ggtt_entry(node
, &virt_addr
))) {
110 struct ggtt_entry
*new_entry
= calloc(1, sizeof(*new_entry
));
111 new_entry
->virt_addr
= virt_addr
;
112 rb_tree_insert_at(&mem
->ggtt
, node
, &new_entry
->node
, cmp
< 0);
113 node
= &new_entry
->node
;
116 return rb_node_data(struct ggtt_entry
, node
, node
);
119 static struct ggtt_entry
*
120 search_ggtt_entry(struct aub_mem
*mem
, uint64_t virt_addr
)
124 struct rb_node
*node
= rb_tree_search(&mem
->ggtt
, &virt_addr
, cmp_ggtt_entry
);
129 return rb_node_data(struct ggtt_entry
, node
, node
);
133 cmp_phys_mem(const struct rb_node
*node
, const void *addr
)
135 struct phys_mem
*mem
= rb_node_data(struct phys_mem
, node
, node
);
136 return cmp_uint64(mem
->phys_addr
, *(uint64_t *)addr
);
139 static struct phys_mem
*
140 ensure_phys_mem(struct aub_mem
*mem
, uint64_t phys_addr
)
142 struct rb_node
*node
= rb_tree_search_sloppy(&mem
->mem
, &phys_addr
, cmp_phys_mem
);
144 if (!node
|| (cmp
= cmp_phys_mem(node
, &phys_addr
))) {
145 struct phys_mem
*new_mem
= calloc(1, sizeof(*new_mem
));
146 new_mem
->phys_addr
= phys_addr
;
147 new_mem
->fd_offset
= mem
->mem_fd_len
;
149 ASSERTED
int ftruncate_res
= ftruncate(mem
->mem_fd
, mem
->mem_fd_len
+= 4096);
150 assert(ftruncate_res
== 0);
152 new_mem
->data
= mmap(NULL
, 4096, PROT_READ
| PROT_WRITE
, MAP_SHARED
,
153 mem
->mem_fd
, new_mem
->fd_offset
);
154 assert(new_mem
->data
!= MAP_FAILED
);
156 rb_tree_insert_at(&mem
->mem
, node
, &new_mem
->node
, cmp
< 0);
157 node
= &new_mem
->node
;
160 return rb_node_data(struct phys_mem
, node
, node
);
163 static struct phys_mem
*
164 search_phys_mem(struct aub_mem
*mem
, uint64_t phys_addr
)
168 struct rb_node
*node
= rb_tree_search(&mem
->mem
, &phys_addr
, cmp_phys_mem
);
173 return rb_node_data(struct phys_mem
, node
, node
);
177 aub_mem_local_write(void *_mem
, uint64_t address
,
178 const void *data
, uint32_t size
)
180 struct aub_mem
*mem
= _mem
;
181 struct gen_batch_decode_bo bo
= {
186 add_gtt_bo_map(mem
, bo
, false, false);
190 aub_mem_ggtt_entry_write(void *_mem
, uint64_t address
,
191 const void *_data
, uint32_t _size
)
193 struct aub_mem
*mem
= _mem
;
194 uint64_t virt_addr
= (address
/ sizeof(uint64_t)) << 12;
195 const uint64_t *data
= _data
;
196 size_t size
= _size
/ sizeof(*data
);
197 for (const uint64_t *entry
= data
;
199 entry
++, virt_addr
+= 4096) {
200 struct ggtt_entry
*pt
= ensure_ggtt_entry(mem
, virt_addr
);
201 pt
->phys_addr
= *entry
;
206 aub_mem_phys_write(void *_mem
, uint64_t phys_address
,
207 const void *data
, uint32_t size
)
209 struct aub_mem
*mem
= _mem
;
210 uint32_t to_write
= size
;
211 for (uint64_t page
= phys_address
& ~0xfff; page
< phys_address
+ size
; page
+= 4096) {
212 struct phys_mem
*pmem
= ensure_phys_mem(mem
, page
);
213 uint64_t offset
= MAX2(page
, phys_address
) - page
;
214 uint32_t size_this_page
= MIN2(to_write
, 4096 - offset
);
215 to_write
-= size_this_page
;
216 memcpy(pmem
->data
+ offset
, data
, size_this_page
);
217 pmem
->aub_data
= data
- offset
;
218 data
= (const uint8_t *)data
+ size_this_page
;
223 aub_mem_ggtt_write(void *_mem
, uint64_t virt_address
,
224 const void *data
, uint32_t size
)
226 struct aub_mem
*mem
= _mem
;
227 uint32_t to_write
= size
;
228 for (uint64_t page
= virt_address
& ~0xfff; page
< virt_address
+ size
; page
+= 4096) {
229 struct ggtt_entry
*entry
= search_ggtt_entry(mem
, page
);
230 assert(entry
&& entry
->phys_addr
& 0x1);
232 uint64_t offset
= MAX2(page
, virt_address
) - page
;
233 uint32_t size_this_page
= MIN2(to_write
, 4096 - offset
);
234 to_write
-= size_this_page
;
236 uint64_t phys_page
= entry
->phys_addr
& ~0xfff; /* Clear the validity bits. */
237 aub_mem_phys_write(mem
, phys_page
+ offset
, data
, size_this_page
);
238 data
= (const uint8_t *)data
+ size_this_page
;
242 struct gen_batch_decode_bo
243 aub_mem_get_ggtt_bo(void *_mem
, uint64_t address
)
245 struct aub_mem
*mem
= _mem
;
246 struct gen_batch_decode_bo bo
= {0};
248 list_for_each_entry(struct bo_map
, i
, &mem
->maps
, link
)
249 if (!i
->ppgtt
&& i
->bo
.addr
<= address
&& i
->bo
.addr
+ i
->bo
.size
> address
)
254 struct ggtt_entry
*start
=
255 (struct ggtt_entry
*)rb_tree_search_sloppy(&mem
->ggtt
, &address
,
257 if (start
&& start
->virt_addr
< address
)
258 start
= ggtt_entry_next(start
);
262 struct ggtt_entry
*last
= start
;
263 for (struct ggtt_entry
*i
= ggtt_entry_next(last
);
264 i
&& last
->virt_addr
+ 4096 == i
->virt_addr
;
265 last
= i
, i
= ggtt_entry_next(last
))
268 bo
.addr
= MIN2(address
, start
->virt_addr
);
269 bo
.size
= last
->virt_addr
- bo
.addr
+ 4096;
270 bo
.map
= mmap(NULL
, bo
.size
, PROT_READ
, MAP_SHARED
| MAP_ANONYMOUS
, -1, 0);
271 assert(bo
.map
!= MAP_FAILED
);
273 for (struct ggtt_entry
*i
= start
;
275 i
= i
== last
? NULL
: ggtt_entry_next(i
)) {
276 uint64_t phys_addr
= i
->phys_addr
& ~0xfff;
277 struct phys_mem
*phys_mem
= search_phys_mem(mem
, phys_addr
);
282 uint32_t map_offset
= i
->virt_addr
- address
;
284 mmap((uint8_t *)bo
.map
+ map_offset
, 4096, PROT_READ
,
285 MAP_SHARED
| MAP_FIXED
, mem
->mem_fd
, phys_mem
->fd_offset
);
286 assert(res
!= MAP_FAILED
);
289 add_gtt_bo_map(mem
, bo
, false, true);
294 static struct phys_mem
*
295 ppgtt_walk(struct aub_mem
*mem
, uint64_t pml4
, uint64_t address
)
298 uint64_t addr
= pml4
;
299 for (int level
= 4; level
> 0; level
--) {
300 struct phys_mem
*table
= search_phys_mem(mem
, addr
);
303 int index
= (address
>> shift
) & 0x1ff;
304 uint64_t entry
= ((uint64_t *)table
->data
)[index
];
307 addr
= entry
& ~0xfff;
310 return search_phys_mem(mem
, addr
);
314 ppgtt_mapped(struct aub_mem
*mem
, uint64_t pml4
, uint64_t address
)
316 return ppgtt_walk(mem
, pml4
, address
) != NULL
;
319 struct gen_batch_decode_bo
320 aub_mem_get_ppgtt_bo(void *_mem
, uint64_t address
)
322 struct aub_mem
*mem
= _mem
;
323 struct gen_batch_decode_bo bo
= {0};
325 list_for_each_entry(struct bo_map
, i
, &mem
->maps
, link
)
326 if (i
->ppgtt
&& i
->bo
.addr
<= address
&& i
->bo
.addr
+ i
->bo
.size
> address
)
331 if (!ppgtt_mapped(mem
, mem
->pml4
, address
))
334 /* Map everything until the first gap since we don't know how much the
335 * decoder actually needs.
337 uint64_t end
= address
;
338 while (ppgtt_mapped(mem
, mem
->pml4
, end
))
342 bo
.size
= end
- address
;
343 bo
.map
= mmap(NULL
, bo
.size
, PROT_READ
, MAP_SHARED
| MAP_ANONYMOUS
, -1, 0);
344 assert(bo
.map
!= MAP_FAILED
);
346 for (uint64_t page
= address
; page
< end
; page
+= 4096) {
347 struct phys_mem
*phys_mem
= ppgtt_walk(mem
, mem
->pml4
, page
);
350 mmap((uint8_t *)bo
.map
+ (page
- bo
.addr
), 4096, PROT_READ
,
351 MAP_SHARED
| MAP_FIXED
, mem
->mem_fd
, phys_mem
->fd_offset
);
352 assert(res
!= MAP_FAILED
);
355 add_gtt_bo_map(mem
, bo
, true, true);
361 aub_mem_init(struct aub_mem
*mem
)
363 memset(mem
, 0, sizeof(*mem
));
365 list_inithead(&mem
->maps
);
367 mem
->mem_fd
= os_create_anonymous_file(0, "phys memory");
369 return mem
->mem_fd
!= -1;
373 aub_mem_fini(struct aub_mem
*mem
)
375 if (mem
->mem_fd
== -1)
378 aub_mem_clear_bo_maps(mem
);
381 rb_tree_foreach_safe(struct ggtt_entry
, entry
, &mem
->ggtt
, node
) {
382 rb_tree_remove(&mem
->ggtt
, &entry
->node
);
385 rb_tree_foreach_safe(struct phys_mem
, entry
, &mem
->mem
, node
) {
386 rb_tree_remove(&mem
->mem
, &entry
->node
);
394 struct gen_batch_decode_bo
395 aub_mem_get_phys_addr_data(struct aub_mem
*mem
, uint64_t phys_addr
)
397 struct phys_mem
*page
= search_phys_mem(mem
, phys_addr
);
399 (struct gen_batch_decode_bo
) { .map
= page
->data
, .addr
= page
->phys_addr
, .size
= 4096 } :
400 (struct gen_batch_decode_bo
) {};
403 struct gen_batch_decode_bo
404 aub_mem_get_ppgtt_addr_data(struct aub_mem
*mem
, uint64_t virt_addr
)
406 struct phys_mem
*page
= ppgtt_walk(mem
, mem
->pml4
, virt_addr
);
408 (struct gen_batch_decode_bo
) { .map
= page
->data
, .addr
= virt_addr
& ~((1ULL << 12) - 1), .size
= 4096 } :
409 (struct gen_batch_decode_bo
) {};
412 struct gen_batch_decode_bo
413 aub_mem_get_ppgtt_addr_aub_data(struct aub_mem
*mem
, uint64_t virt_addr
)
415 struct phys_mem
*page
= ppgtt_walk(mem
, mem
->pml4
, virt_addr
);
417 (struct gen_batch_decode_bo
) { .map
= page
->aub_data
, .addr
= virt_addr
& ~((1ULL << 12) - 1), .size
= 4096 } :
418 (struct gen_batch_decode_bo
) {};