2 * Copyright 2012 Nouveau Project
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Christoph Bumiller
25 #include "nvc0/nvc0_context.h"
26 #include "nvc0/nve4_compute.h"
28 #include "codegen/nv50_ir_driver.h"
31 static void nve4_compute_dump_launch_desc(const struct nve4_cp_launch_desc
*);
32 static void gp100_compute_dump_launch_desc(const struct gp100_cp_launch_desc
*);
37 nve4_screen_compute_setup(struct nvc0_screen
*screen
,
38 struct nouveau_pushbuf
*push
)
40 struct nouveau_device
*dev
= screen
->base
.device
;
41 struct nouveau_object
*chan
= screen
->base
.channel
;
47 switch (dev
->chipset
& ~0xf) {
50 obj_class
= NVF0_COMPUTE_CLASS
; /* GK110 */
53 obj_class
= NVE4_COMPUTE_CLASS
; /* GK104 */
56 obj_class
= GM107_COMPUTE_CLASS
;
59 obj_class
= GM200_COMPUTE_CLASS
;
62 obj_class
= (dev
->chipset
== 0x130 || dev
->chipset
== 0x13b) ?
63 GP100_COMPUTE_CLASS
: GP104_COMPUTE_CLASS
;
66 NOUVEAU_ERR("unsupported chipset: NV%02x\n", dev
->chipset
);
70 ret
= nouveau_object_new(chan
, 0xbeef00c0, obj_class
, NULL
, 0,
73 NOUVEAU_ERR("Failed to allocate compute object: %d\n", ret
);
77 BEGIN_NVC0(push
, SUBC_CP(NV01_SUBCHAN_OBJECT
), 1);
78 PUSH_DATA (push
, screen
->compute
->oclass
);
80 BEGIN_NVC0(push
, NVE4_CP(TEMP_ADDRESS_HIGH
), 2);
81 PUSH_DATAh(push
, screen
->tls
->offset
);
82 PUSH_DATA (push
, screen
->tls
->offset
);
83 /* No idea why there are 2. Divide size by 2 to be safe.
84 * Actually this might be per-MP TEMP size and looks like I'm only using
85 * 2 MPs instead of all 8.
87 BEGIN_NVC0(push
, NVE4_CP(MP_TEMP_SIZE_HIGH(0)), 3);
88 PUSH_DATAh(push
, screen
->tls
->size
/ screen
->mp_count
);
89 PUSH_DATA (push
, (screen
->tls
->size
/ screen
->mp_count
) & ~0x7fff);
90 PUSH_DATA (push
, 0xff);
91 BEGIN_NVC0(push
, NVE4_CP(MP_TEMP_SIZE_HIGH(1)), 3);
92 PUSH_DATAh(push
, screen
->tls
->size
/ screen
->mp_count
);
93 PUSH_DATA (push
, (screen
->tls
->size
/ screen
->mp_count
) & ~0x7fff);
94 PUSH_DATA (push
, 0xff);
96 /* Unified address space ? Who needs that ? Certainly not OpenCL.
98 * FATAL: Buffers with addresses inside [0x1000000, 0x3000000] will NOT be
99 * accessible. We cannot prevent that at the moment, so expect failure.
101 BEGIN_NVC0(push
, NVE4_CP(LOCAL_BASE
), 1);
102 PUSH_DATA (push
, 0xff << 24);
103 BEGIN_NVC0(push
, NVE4_CP(SHARED_BASE
), 1);
104 PUSH_DATA (push
, 0xfe << 24);
106 BEGIN_NVC0(push
, NVE4_CP(CODE_ADDRESS_HIGH
), 2);
107 PUSH_DATAh(push
, screen
->text
->offset
);
108 PUSH_DATA (push
, screen
->text
->offset
);
110 BEGIN_NVC0(push
, SUBC_CP(0x0310), 1);
111 PUSH_DATA (push
, (obj_class
>= NVF0_COMPUTE_CLASS
) ? 0x400 : 0x300);
113 /* NOTE: these do not affect the state used by the 3D object */
114 BEGIN_NVC0(push
, NVE4_CP(TIC_ADDRESS_HIGH
), 3);
115 PUSH_DATAh(push
, screen
->txc
->offset
);
116 PUSH_DATA (push
, screen
->txc
->offset
);
117 PUSH_DATA (push
, NVC0_TIC_MAX_ENTRIES
- 1);
118 BEGIN_NVC0(push
, NVE4_CP(TSC_ADDRESS_HIGH
), 3);
119 PUSH_DATAh(push
, screen
->txc
->offset
+ 65536);
120 PUSH_DATA (push
, screen
->txc
->offset
+ 65536);
121 PUSH_DATA (push
, NVC0_TSC_MAX_ENTRIES
- 1);
123 if (obj_class
>= NVF0_COMPUTE_CLASS
) {
124 /* The blob calls GK110_COMPUTE.FIRMWARE[0x6], along with the args (0x1)
125 * passed with GK110_COMPUTE.GRAPH.SCRATCH[0x2]. This is currently
126 * disabled because our firmware doesn't support these commands and the
127 * GPU hangs if they are used. */
128 BEGIN_NIC0(push
, SUBC_CP(0x0248), 64);
129 for (i
= 63; i
>= 0; i
--)
130 PUSH_DATA(push
, 0x38000 | i
);
131 IMMED_NVC0(push
, SUBC_CP(NV50_GRAPH_SERIALIZE
), 0);
134 BEGIN_NVC0(push
, NVE4_CP(TEX_CB_INDEX
), 1);
135 PUSH_DATA (push
, 7); /* does not interfere with 3D */
137 /* Disabling this UNK command avoid a read fault when using texelFetch()
138 * from a compute shader for weird reasons.
139 if (obj_class == NVF0_COMPUTE_CLASS)
140 IMMED_NVC0(push, SUBC_CP(0x02c4), 1);
143 address
= screen
->uniform_bo
->offset
+ NVC0_CB_AUX_INFO(5);
145 /* MS sample coordinate offsets: these do not work with _ALT modes ! */
146 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH
), 2);
147 PUSH_DATAh(push
, address
+ NVC0_CB_AUX_MS_INFO
);
148 PUSH_DATA (push
, address
+ NVC0_CB_AUX_MS_INFO
);
149 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_LINE_LENGTH_IN
), 2);
150 PUSH_DATA (push
, 64);
152 BEGIN_1IC0(push
, NVE4_CP(UPLOAD_EXEC
), 17);
153 PUSH_DATA (push
, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR
| (0x20 << 1));
154 PUSH_DATA (push
, 0); /* 0 */
156 PUSH_DATA (push
, 1); /* 1 */
158 PUSH_DATA (push
, 0); /* 2 */
160 PUSH_DATA (push
, 1); /* 3 */
162 PUSH_DATA (push
, 2); /* 4 */
164 PUSH_DATA (push
, 3); /* 5 */
166 PUSH_DATA (push
, 2); /* 6 */
168 PUSH_DATA (push
, 3); /* 7 */
171 #ifdef NOUVEAU_NVE4_MP_TRAP_HANDLER
172 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH
), 2);
173 PUSH_DATAh(push
, screen
->parm
->offset
+ NVE4_CP_INPUT_TRAP_INFO_PTR
);
174 PUSH_DATA (push
, screen
->parm
->offset
+ NVE4_CP_INPUT_TRAP_INFO_PTR
);
175 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_LINE_LENGTH_IN
), 2);
176 PUSH_DATA (push
, 28);
178 BEGIN_1IC0(push
, NVE4_CP(UPLOAD_EXEC
), 8);
180 PUSH_DATA (push
, screen
->parm
->offset
+ NVE4_CP_PARAM_TRAP_INFO
);
181 PUSH_DATAh(push
, screen
->parm
->offset
+ NVE4_CP_PARAM_TRAP_INFO
);
182 PUSH_DATA (push
, screen
->tls
->offset
);
183 PUSH_DATAh(push
, screen
->tls
->offset
);
184 PUSH_DATA (push
, screen
->tls
->size
/ 2); /* MP TEMP block size */
185 PUSH_DATA (push
, screen
->tls
->size
/ 2 / 64); /* warp TEMP block size */
186 PUSH_DATA (push
, 0); /* warp cfstack size */
189 BEGIN_NVC0(push
, NVE4_CP(FLUSH
), 1);
190 PUSH_DATA (push
, NVE4_COMPUTE_FLUSH_CB
);
196 gm107_compute_validate_surfaces(struct nvc0_context
*nvc0
,
197 struct pipe_image_view
*view
, int slot
)
199 struct nv04_resource
*res
= nv04_resource(view
->resource
);
200 struct nouveau_pushbuf
*push
= nvc0
->base
.pushbuf
;
201 struct nvc0_screen
*screen
= nvc0
->screen
;
202 struct nouveau_bo
*txc
= nvc0
->screen
->txc
;
203 struct nv50_tic_entry
*tic
;
207 tic
= nv50_tic_entry(nvc0
->images_tic
[s
][slot
]);
209 res
= nv04_resource(tic
->pipe
.texture
);
210 nvc0_update_tic(nvc0
, tic
, res
);
213 tic
->id
= nvc0_screen_tic_alloc(nvc0
->screen
, tic
);
215 /* upload the texture view */
216 PUSH_SPACE(push
, 16);
217 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH
), 2);
218 PUSH_DATAh(push
, txc
->offset
+ (tic
->id
* 32));
219 PUSH_DATA (push
, txc
->offset
+ (tic
->id
* 32));
220 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_LINE_LENGTH_IN
), 2);
221 PUSH_DATA (push
, 32);
223 BEGIN_1IC0(push
, NVE4_CP(UPLOAD_EXEC
), 9);
224 PUSH_DATA (push
, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR
| (0x20 << 1));
225 PUSH_DATAp(push
, &tic
->tic
[0], 8);
227 BEGIN_NIC0(push
, NVE4_CP(TIC_FLUSH
), 1);
228 PUSH_DATA (push
, (tic
->id
<< 4) | 1);
230 if (res
->status
& NOUVEAU_BUFFER_STATUS_GPU_WRITING
) {
231 BEGIN_NIC0(push
, NVE4_CP(TEX_CACHE_CTL
), 1);
232 PUSH_DATA (push
, (tic
->id
<< 4) | 1);
234 nvc0
->screen
->tic
.lock
[tic
->id
/ 32] |= 1 << (tic
->id
% 32);
236 res
->status
&= ~NOUVEAU_BUFFER_STATUS_GPU_WRITING
;
237 res
->status
|= NOUVEAU_BUFFER_STATUS_GPU_READING
;
239 BCTX_REFN(nvc0
->bufctx_cp
, CP_SUF
, res
, RD
);
241 address
= screen
->uniform_bo
->offset
+ NVC0_CB_AUX_INFO(s
);
243 /* upload the texture handle */
244 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH
), 2);
245 PUSH_DATAh(push
, address
+ NVC0_CB_AUX_TEX_INFO(slot
+ 32));
246 PUSH_DATA (push
, address
+ NVC0_CB_AUX_TEX_INFO(slot
+ 32));
247 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_LINE_LENGTH_IN
), 2);
249 PUSH_DATA (push
, 0x1);
250 BEGIN_1IC0(push
, NVE4_CP(UPLOAD_EXEC
), 2);
251 PUSH_DATA (push
, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR
| (0x20 << 1));
252 PUSH_DATA (push
, tic
->id
);
254 BEGIN_NVC0(push
, NVE4_CP(FLUSH
), 1);
255 PUSH_DATA (push
, NVE4_COMPUTE_FLUSH_CB
);
259 nve4_compute_validate_surfaces(struct nvc0_context
*nvc0
)
261 struct nouveau_pushbuf
*push
= nvc0
->base
.pushbuf
;
266 if (!nvc0
->images_dirty
[s
])
269 address
= nvc0
->screen
->uniform_bo
->offset
+ NVC0_CB_AUX_INFO(s
);
271 for (i
= 0; i
< NVC0_MAX_IMAGES
; ++i
) {
272 struct pipe_image_view
*view
= &nvc0
->images
[s
][i
];
274 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH
), 2);
275 PUSH_DATAh(push
, address
+ NVC0_CB_AUX_SU_INFO(i
));
276 PUSH_DATA (push
, address
+ NVC0_CB_AUX_SU_INFO(i
));
277 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_LINE_LENGTH_IN
), 2);
278 PUSH_DATA (push
, 16 * 4);
279 PUSH_DATA (push
, 0x1);
280 BEGIN_1IC0(push
, NVE4_CP(UPLOAD_EXEC
), 1 + 16);
281 PUSH_DATA (push
, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR
| (0x20 << 1));
283 if (view
->resource
) {
284 struct nv04_resource
*res
= nv04_resource(view
->resource
);
286 if (res
->base
.target
== PIPE_BUFFER
) {
287 if (view
->access
& PIPE_IMAGE_ACCESS_WRITE
)
288 nvc0_mark_image_range_valid(view
);
291 nve4_set_surface_info(push
, view
, nvc0
);
292 BCTX_REFN(nvc0
->bufctx_cp
, CP_SUF
, res
, RDWR
);
294 if (nvc0
->screen
->base
.class_3d
>= GM107_3D_CLASS
)
295 gm107_compute_validate_surfaces(nvc0
, view
, i
);
297 for (j
= 0; j
< 16; j
++)
303 /* Thankfully, textures with samplers follow the normal rules. */
305 nve4_compute_validate_samplers(struct nvc0_context
*nvc0
)
307 bool need_flush
= nve4_validate_tsc(nvc0
, 5);
309 BEGIN_NVC0(nvc0
->base
.pushbuf
, NVE4_CP(TSC_FLUSH
), 1);
310 PUSH_DATA (nvc0
->base
.pushbuf
, 0);
313 /* Invalidate all 3D samplers because they are aliased. */
314 for (int s
= 0; s
< 5; s
++)
315 nvc0
->samplers_dirty
[s
] = ~0;
316 nvc0
->dirty_3d
|= NVC0_NEW_3D_SAMPLERS
;
319 /* (Code duplicated at bottom for various non-convincing reasons.
320 * E.g. we might want to use the COMPUTE subchannel to upload TIC/TSC
321 * entries to avoid a subchannel switch.
322 * Same for texture cache flushes.
323 * Also, the bufctx differs, and more IFs in the 3D version looks ugly.)
325 static void nve4_compute_validate_textures(struct nvc0_context
*);
328 nve4_compute_set_tex_handles(struct nvc0_context
*nvc0
)
330 struct nouveau_pushbuf
*push
= nvc0
->base
.pushbuf
;
331 struct nvc0_screen
*screen
= nvc0
->screen
;
333 const unsigned s
= nvc0_shader_stage(PIPE_SHADER_COMPUTE
);
335 uint32_t dirty
= nvc0
->textures_dirty
[s
] | nvc0
->samplers_dirty
[s
];
340 n
= util_logbase2(dirty
) + 1 - i
;
343 address
= screen
->uniform_bo
->offset
+ NVC0_CB_AUX_INFO(s
);
345 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH
), 2);
346 PUSH_DATAh(push
, address
+ NVC0_CB_AUX_TEX_INFO(i
));
347 PUSH_DATA (push
, address
+ NVC0_CB_AUX_TEX_INFO(i
));
348 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_LINE_LENGTH_IN
), 2);
349 PUSH_DATA (push
, n
* 4);
350 PUSH_DATA (push
, 0x1);
351 BEGIN_1IC0(push
, NVE4_CP(UPLOAD_EXEC
), 1 + n
);
352 PUSH_DATA (push
, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR
| (0x20 << 1));
353 PUSH_DATAp(push
, &nvc0
->tex_handles
[s
][i
], n
);
355 BEGIN_NVC0(push
, NVE4_CP(FLUSH
), 1);
356 PUSH_DATA (push
, NVE4_COMPUTE_FLUSH_CB
);
358 nvc0
->textures_dirty
[s
] = 0;
359 nvc0
->samplers_dirty
[s
] = 0;
363 nve4_compute_validate_constbufs(struct nvc0_context
*nvc0
)
365 struct nouveau_pushbuf
*push
= nvc0
->base
.pushbuf
;
368 while (nvc0
->constbuf_dirty
[s
]) {
369 int i
= ffs(nvc0
->constbuf_dirty
[s
]) - 1;
370 nvc0
->constbuf_dirty
[s
] &= ~(1 << i
);
372 if (nvc0
->constbuf
[s
][i
].user
) {
373 struct nouveau_bo
*bo
= nvc0
->screen
->uniform_bo
;
374 const unsigned base
= NVC0_CB_USR_INFO(s
);
375 const unsigned size
= nvc0
->constbuf
[s
][0].size
;
376 assert(i
== 0); /* we really only want OpenGL uniforms here */
377 assert(nvc0
->constbuf
[s
][0].u
.data
);
379 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH
), 2);
380 PUSH_DATAh(push
, bo
->offset
+ base
);
381 PUSH_DATA (push
, bo
->offset
+ base
);
382 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_LINE_LENGTH_IN
), 2);
383 PUSH_DATA (push
, size
);
384 PUSH_DATA (push
, 0x1);
385 BEGIN_1IC0(push
, NVE4_CP(UPLOAD_EXEC
), 1 + (size
/ 4));
386 PUSH_DATA (push
, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR
| (0x20 << 1));
387 PUSH_DATAp(push
, nvc0
->constbuf
[s
][0].u
.data
, size
/ 4);
390 struct nv04_resource
*res
=
391 nv04_resource(nvc0
->constbuf
[s
][i
].u
.buf
);
394 = nvc0
->screen
->uniform_bo
->offset
+ NVC0_CB_AUX_INFO(s
);
396 assert(i
> 0); /* we really only want uniform buffer objects */
398 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH
), 2);
399 PUSH_DATAh(push
, address
+ NVC0_CB_AUX_UBO_INFO(i
- 1));
400 PUSH_DATA (push
, address
+ NVC0_CB_AUX_UBO_INFO(i
- 1));
401 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_LINE_LENGTH_IN
), 2);
402 PUSH_DATA (push
, 4 * 4);
403 PUSH_DATA (push
, 0x1);
404 BEGIN_1IC0(push
, NVE4_CP(UPLOAD_EXEC
), 1 + 4);
405 PUSH_DATA (push
, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR
| (0x20 << 1));
407 PUSH_DATA (push
, res
->address
+ nvc0
->constbuf
[s
][i
].offset
);
408 PUSH_DATAh(push
, res
->address
+ nvc0
->constbuf
[s
][i
].offset
);
409 PUSH_DATA (push
, nvc0
->constbuf
[5][i
].size
);
411 BCTX_REFN(nvc0
->bufctx_cp
, CP_CB(i
), res
, RD
);
413 res
->cb_bindings
[s
] |= 1 << i
;
418 BEGIN_NVC0(push
, NVE4_CP(FLUSH
), 1);
419 PUSH_DATA (push
, NVE4_COMPUTE_FLUSH_CB
);
423 nve4_compute_validate_buffers(struct nvc0_context
*nvc0
)
425 struct nouveau_pushbuf
*push
= nvc0
->base
.pushbuf
;
430 address
= nvc0
->screen
->uniform_bo
->offset
+ NVC0_CB_AUX_INFO(s
);
432 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH
), 2);
433 PUSH_DATAh(push
, address
+ NVC0_CB_AUX_BUF_INFO(0));
434 PUSH_DATA (push
, address
+ NVC0_CB_AUX_BUF_INFO(0));
435 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_LINE_LENGTH_IN
), 2);
436 PUSH_DATA (push
, 4 * NVC0_MAX_BUFFERS
* 4);
437 PUSH_DATA (push
, 0x1);
438 BEGIN_1IC0(push
, NVE4_CP(UPLOAD_EXEC
), 1 + 4 * NVC0_MAX_BUFFERS
);
439 PUSH_DATA (push
, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR
| (0x20 << 1));
441 for (i
= 0; i
< NVC0_MAX_BUFFERS
; i
++) {
442 if (nvc0
->buffers
[s
][i
].buffer
) {
443 struct nv04_resource
*res
=
444 nv04_resource(nvc0
->buffers
[s
][i
].buffer
);
445 PUSH_DATA (push
, res
->address
+ nvc0
->buffers
[s
][i
].buffer_offset
);
446 PUSH_DATAh(push
, res
->address
+ nvc0
->buffers
[s
][i
].buffer_offset
);
447 PUSH_DATA (push
, nvc0
->buffers
[s
][i
].buffer_size
);
449 BCTX_REFN(nvc0
->bufctx_cp
, CP_BUF
, res
, RDWR
);
450 util_range_add(&res
->valid_buffer_range
,
451 nvc0
->buffers
[s
][i
].buffer_offset
,
452 nvc0
->buffers
[s
][i
].buffer_offset
+
453 nvc0
->buffers
[s
][i
].buffer_size
);
463 static struct nvc0_state_validate
464 validate_list_cp
[] = {
465 { nvc0_compprog_validate
, NVC0_NEW_CP_PROGRAM
},
466 { nve4_compute_validate_textures
, NVC0_NEW_CP_TEXTURES
},
467 { nve4_compute_validate_samplers
, NVC0_NEW_CP_SAMPLERS
},
468 { nve4_compute_set_tex_handles
, NVC0_NEW_CP_TEXTURES
|
469 NVC0_NEW_CP_SAMPLERS
},
470 { nve4_compute_validate_surfaces
, NVC0_NEW_CP_SURFACES
},
471 { nvc0_compute_validate_globals
, NVC0_NEW_CP_GLOBALS
},
472 { nve4_compute_validate_buffers
, NVC0_NEW_CP_BUFFERS
},
473 { nve4_compute_validate_constbufs
, NVC0_NEW_CP_CONSTBUF
},
477 nve4_state_validate_cp(struct nvc0_context
*nvc0
, uint32_t mask
)
481 ret
= nvc0_state_validate(nvc0
, mask
, validate_list_cp
,
482 ARRAY_SIZE(validate_list_cp
), &nvc0
->dirty_cp
,
485 if (unlikely(nvc0
->state
.flushed
))
486 nvc0_bufctx_fence(nvc0
, nvc0
->bufctx_cp
, true);
491 nve4_compute_upload_input(struct nvc0_context
*nvc0
,
492 const struct pipe_grid_info
*info
)
494 struct nvc0_screen
*screen
= nvc0
->screen
;
495 struct nouveau_pushbuf
*push
= nvc0
->base
.pushbuf
;
496 struct nvc0_program
*cp
= nvc0
->compprog
;
499 address
= screen
->uniform_bo
->offset
+ NVC0_CB_AUX_INFO(5);
502 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH
), 2);
503 PUSH_DATAh(push
, screen
->uniform_bo
->offset
+ NVC0_CB_USR_INFO(5));
504 PUSH_DATA (push
, screen
->uniform_bo
->offset
+ NVC0_CB_USR_INFO(5));
505 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_LINE_LENGTH_IN
), 2);
506 PUSH_DATA (push
, cp
->parm_size
);
507 PUSH_DATA (push
, 0x1);
508 BEGIN_1IC0(push
, NVE4_CP(UPLOAD_EXEC
), 1 + (cp
->parm_size
/ 4));
509 PUSH_DATA (push
, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR
| (0x20 << 1));
510 PUSH_DATAp(push
, info
->input
, cp
->parm_size
/ 4);
512 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH
), 2);
513 PUSH_DATAh(push
, address
+ NVC0_CB_AUX_GRID_INFO(0));
514 PUSH_DATA (push
, address
+ NVC0_CB_AUX_GRID_INFO(0));
515 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_LINE_LENGTH_IN
), 2);
516 PUSH_DATA (push
, 8 * 4);
517 PUSH_DATA (push
, 0x1);
519 if (unlikely(info
->indirect
)) {
520 struct nv04_resource
*res
= nv04_resource(info
->indirect
);
521 uint32_t offset
= res
->offset
+ info
->indirect_offset
;
523 nouveau_pushbuf_space(push
, 32, 0, 1);
524 PUSH_REFN(push
, res
->bo
, NOUVEAU_BO_RD
| res
->domain
);
526 BEGIN_1IC0(push
, NVE4_CP(UPLOAD_EXEC
), 1 + 8);
527 PUSH_DATA (push
, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR
| (0x20 << 1));
528 PUSH_DATAp(push
, info
->block
, 3);
529 nouveau_pushbuf_data(push
, res
->bo
, offset
,
530 NVC0_IB_ENTRY_1_NO_PREFETCH
| 3 * 4);
532 BEGIN_1IC0(push
, NVE4_CP(UPLOAD_EXEC
), 1 + 8);
533 PUSH_DATA (push
, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR
| (0x20 << 1));
534 PUSH_DATAp(push
, info
->block
, 3);
535 PUSH_DATAp(push
, info
->grid
, 3);
538 PUSH_DATA (push
, info
->work_dim
);
540 BEGIN_NVC0(push
, NVE4_CP(FLUSH
), 1);
541 PUSH_DATA (push
, NVE4_COMPUTE_FLUSH_CB
);
544 static inline uint8_t
545 nve4_compute_derive_cache_split(struct nvc0_context
*nvc0
, uint32_t shared_size
)
547 if (shared_size
> (32 << 10))
548 return NVC0_3D_CACHE_SPLIT_48K_SHARED_16K_L1
;
549 if (shared_size
> (16 << 10))
550 return NVE4_3D_CACHE_SPLIT_32K_SHARED_32K_L1
;
551 return NVC1_3D_CACHE_SPLIT_16K_SHARED_48K_L1
;
555 nve4_compute_setup_launch_desc(struct nvc0_context
*nvc0
,
556 struct nve4_cp_launch_desc
*desc
,
557 const struct pipe_grid_info
*info
)
559 const struct nvc0_screen
*screen
= nvc0
->screen
;
560 const struct nvc0_program
*cp
= nvc0
->compprog
;
562 nve4_cp_launch_desc_init_default(desc
);
564 desc
->entry
= nvc0_program_symbol_offset(cp
, info
->pc
);
566 desc
->griddim_x
= info
->grid
[0];
567 desc
->griddim_y
= info
->grid
[1];
568 desc
->griddim_z
= info
->grid
[2];
569 desc
->blockdim_x
= info
->block
[0];
570 desc
->blockdim_y
= info
->block
[1];
571 desc
->blockdim_z
= info
->block
[2];
573 desc
->shared_size
= align(cp
->cp
.smem_size
, 0x100);
574 desc
->local_size_p
= (cp
->hdr
[1] & 0xfffff0) + align(cp
->cp
.lmem_size
, 0x10);
575 desc
->local_size_n
= 0;
576 desc
->cstack_size
= 0x800;
577 desc
->cache_split
= nve4_compute_derive_cache_split(nvc0
, cp
->cp
.smem_size
);
579 desc
->gpr_alloc
= cp
->num_gprs
;
580 desc
->bar_alloc
= cp
->num_barriers
;
582 // Only bind user uniforms and the driver constant buffer through the
583 // launch descriptor because UBOs are sticked to the driver cb to avoid the
584 // limitation of 8 CBs.
585 if (nvc0
->constbuf
[5][0].user
|| cp
->parm_size
) {
586 nve4_cp_launch_desc_set_cb(desc
, 0, screen
->uniform_bo
,
587 NVC0_CB_USR_INFO(5), 1 << 16);
589 nve4_cp_launch_desc_set_cb(desc
, 7, screen
->uniform_bo
,
590 NVC0_CB_AUX_INFO(5), 1 << 11);
594 gp100_compute_setup_launch_desc(struct nvc0_context
*nvc0
,
595 struct gp100_cp_launch_desc
*desc
,
596 const struct pipe_grid_info
*info
)
598 const struct nvc0_screen
*screen
= nvc0
->screen
;
599 const struct nvc0_program
*cp
= nvc0
->compprog
;
601 gp100_cp_launch_desc_init_default(desc
);
603 desc
->entry
= nvc0_program_symbol_offset(cp
, info
->pc
);
605 desc
->griddim_x
= info
->grid
[0];
606 desc
->griddim_y
= info
->grid
[1];
607 desc
->griddim_z
= info
->grid
[2];
608 desc
->blockdim_x
= info
->block
[0];
609 desc
->blockdim_y
= info
->block
[1];
610 desc
->blockdim_z
= info
->block
[2];
612 desc
->shared_size
= align(cp
->cp
.smem_size
, 0x100);
613 desc
->local_size_p
= (cp
->hdr
[1] & 0xfffff0) + align(cp
->cp
.lmem_size
, 0x10);
614 desc
->local_size_n
= 0;
615 desc
->cstack_size
= 0x800;
617 desc
->gpr_alloc
= cp
->num_gprs
;
618 desc
->bar_alloc
= cp
->num_barriers
;
620 // Only bind user uniforms and the driver constant buffer through the
621 // launch descriptor because UBOs are sticked to the driver cb to avoid the
622 // limitation of 8 CBs.
623 if (nvc0
->constbuf
[5][0].user
|| cp
->parm_size
) {
624 gp100_cp_launch_desc_set_cb(desc
, 0, screen
->uniform_bo
,
625 NVC0_CB_USR_INFO(5), 1 << 16);
627 gp100_cp_launch_desc_set_cb(desc
, 7, screen
->uniform_bo
,
628 NVC0_CB_AUX_INFO(5), 1 << 11);
632 nve4_compute_alloc_launch_desc(struct nouveau_context
*nv
,
633 struct nouveau_bo
**pbo
, uint64_t *pgpuaddr
)
635 uint8_t *ptr
= nouveau_scratch_get(nv
, 512, pgpuaddr
, pbo
);
638 if (*pgpuaddr
& 255) {
639 unsigned adj
= 256 - (*pgpuaddr
& 255);
647 nve4_upload_indirect_desc(struct nouveau_pushbuf
*push
,
648 struct nv04_resource
*res
, uint64_t gpuaddr
,
649 uint32_t length
, uint32_t bo_offset
)
651 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH
), 2);
652 PUSH_DATAh(push
, gpuaddr
);
653 PUSH_DATA (push
, gpuaddr
);
654 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_LINE_LENGTH_IN
), 2);
655 PUSH_DATA (push
, length
);
658 nouveau_pushbuf_space(push
, 32, 0, 1);
659 PUSH_REFN(push
, res
->bo
, NOUVEAU_BO_RD
| res
->domain
);
661 BEGIN_1IC0(push
, NVE4_CP(UPLOAD_EXEC
), 1 + (length
/ 4));
662 PUSH_DATA (push
, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR
| (0x08 << 1));
663 nouveau_pushbuf_data(push
, res
->bo
, bo_offset
,
664 NVC0_IB_ENTRY_1_NO_PREFETCH
| length
);
668 nve4_launch_grid(struct pipe_context
*pipe
, const struct pipe_grid_info
*info
)
670 struct nvc0_context
*nvc0
= nvc0_context(pipe
);
671 struct nouveau_pushbuf
*push
= nvc0
->base
.pushbuf
;
673 uint64_t desc_gpuaddr
;
674 struct nouveau_bo
*desc_bo
;
677 desc
= nve4_compute_alloc_launch_desc(&nvc0
->base
, &desc_bo
, &desc_gpuaddr
);
682 BCTX_REFN_bo(nvc0
->bufctx_cp
, CP_DESC
, NOUVEAU_BO_GART
| NOUVEAU_BO_RD
,
685 list_for_each_entry(struct nvc0_resident
, resident
, &nvc0
->tex_head
, list
) {
686 nvc0_add_resident(nvc0
->bufctx_cp
, NVC0_BIND_CP_BINDLESS
, resident
->buf
,
690 list_for_each_entry(struct nvc0_resident
, resident
, &nvc0
->img_head
, list
) {
691 nvc0_add_resident(nvc0
->bufctx_cp
, NVC0_BIND_CP_BINDLESS
, resident
->buf
,
695 ret
= !nve4_state_validate_cp(nvc0
, ~0);
699 if (nvc0
->screen
->compute
->oclass
>= GP100_COMPUTE_CLASS
)
700 gp100_compute_setup_launch_desc(nvc0
, desc
, info
);
702 nve4_compute_setup_launch_desc(nvc0
, desc
, info
);
704 nve4_compute_upload_input(nvc0
, info
);
707 if (debug_get_num_option("NV50_PROG_DEBUG", 0)) {
708 if (nvc0
->screen
->compute
->oclass
>= GP100_COMPUTE_CLASS
)
709 gp100_compute_dump_launch_desc(desc
);
711 nve4_compute_dump_launch_desc(desc
);
715 if (unlikely(info
->indirect
)) {
716 struct nv04_resource
*res
= nv04_resource(info
->indirect
);
717 uint32_t offset
= res
->offset
+ info
->indirect_offset
;
719 /* upload the descriptor */
720 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH
), 2);
721 PUSH_DATAh(push
, desc_gpuaddr
);
722 PUSH_DATA (push
, desc_gpuaddr
);
723 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_LINE_LENGTH_IN
), 2);
724 PUSH_DATA (push
, 256);
726 BEGIN_1IC0(push
, NVE4_CP(UPLOAD_EXEC
), 1 + (256 / 4));
727 PUSH_DATA (push
, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR
| (0x08 << 1));
728 PUSH_DATAp(push
, (const uint32_t *)desc
, 256 / 4);
730 if (nvc0
->screen
->compute
->oclass
>= GP100_COMPUTE_CLASS
) {
731 nve4_upload_indirect_desc(push
, res
, desc_gpuaddr
+ 48, 12, offset
);
733 /* overwrite griddim_x and griddim_y as two 32-bits integers even
734 * if griddim_y must be a 16-bits integer */
735 nve4_upload_indirect_desc(push
, res
, desc_gpuaddr
+ 48, 8, offset
);
737 /* overwrite the 16 high bits of griddim_y with griddim_z because
738 * we need (z << 16) | x */
739 nve4_upload_indirect_desc(push
, res
, desc_gpuaddr
+ 54, 4, offset
+ 8);
743 /* upload descriptor and flush */
744 BEGIN_NVC0(push
, NVE4_CP(LAUNCH_DESC_ADDRESS
), 1);
745 PUSH_DATA (push
, desc_gpuaddr
>> 8);
746 BEGIN_NVC0(push
, NVE4_CP(LAUNCH
), 1);
747 PUSH_DATA (push
, 0x3);
748 BEGIN_NVC0(push
, SUBC_CP(NV50_GRAPH_SERIALIZE
), 1);
753 NOUVEAU_ERR("Failed to launch grid !\n");
754 nouveau_scratch_done(&nvc0
->base
);
755 nouveau_bufctx_reset(nvc0
->bufctx_cp
, NVC0_BIND_CP_DESC
);
756 nouveau_bufctx_reset(nvc0
->bufctx_cp
, NVC0_BIND_CP_BINDLESS
);
760 #define NVE4_TIC_ENTRY_INVALID 0x000fffff
763 nve4_compute_validate_textures(struct nvc0_context
*nvc0
)
765 struct nouveau_bo
*txc
= nvc0
->screen
->txc
;
766 struct nouveau_pushbuf
*push
= nvc0
->base
.pushbuf
;
767 const unsigned s
= 5;
769 uint32_t commands
[2][32];
770 unsigned n
[2] = { 0, 0 };
772 for (i
= 0; i
< nvc0
->num_textures
[s
]; ++i
) {
773 struct nv50_tic_entry
*tic
= nv50_tic_entry(nvc0
->textures
[s
][i
]);
774 struct nv04_resource
*res
;
775 const bool dirty
= !!(nvc0
->textures_dirty
[s
] & (1 << i
));
778 nvc0
->tex_handles
[s
][i
] |= NVE4_TIC_ENTRY_INVALID
;
781 res
= nv04_resource(tic
->pipe
.texture
);
782 nvc0_update_tic(nvc0
, tic
, res
);
785 tic
->id
= nvc0_screen_tic_alloc(nvc0
->screen
, tic
);
787 PUSH_SPACE(push
, 16);
788 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH
), 2);
789 PUSH_DATAh(push
, txc
->offset
+ (tic
->id
* 32));
790 PUSH_DATA (push
, txc
->offset
+ (tic
->id
* 32));
791 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_LINE_LENGTH_IN
), 2);
792 PUSH_DATA (push
, 32);
794 BEGIN_1IC0(push
, NVE4_CP(UPLOAD_EXEC
), 9);
795 PUSH_DATA (push
, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR
| (0x20 << 1));
796 PUSH_DATAp(push
, &tic
->tic
[0], 8);
798 commands
[0][n
[0]++] = (tic
->id
<< 4) | 1;
800 if (res
->status
& NOUVEAU_BUFFER_STATUS_GPU_WRITING
) {
801 commands
[1][n
[1]++] = (tic
->id
<< 4) | 1;
803 nvc0
->screen
->tic
.lock
[tic
->id
/ 32] |= 1 << (tic
->id
% 32);
805 res
->status
&= ~NOUVEAU_BUFFER_STATUS_GPU_WRITING
;
806 res
->status
|= NOUVEAU_BUFFER_STATUS_GPU_READING
;
808 nvc0
->tex_handles
[s
][i
] &= ~NVE4_TIC_ENTRY_INVALID
;
809 nvc0
->tex_handles
[s
][i
] |= tic
->id
;
811 BCTX_REFN(nvc0
->bufctx_cp
, CP_TEX(i
), res
, RD
);
813 for (; i
< nvc0
->state
.num_textures
[s
]; ++i
) {
814 nvc0
->tex_handles
[s
][i
] |= NVE4_TIC_ENTRY_INVALID
;
815 nvc0
->textures_dirty
[s
] |= 1 << i
;
819 BEGIN_NIC0(push
, NVE4_CP(TIC_FLUSH
), n
[0]);
820 PUSH_DATAp(push
, commands
[0], n
[0]);
823 BEGIN_NIC0(push
, NVE4_CP(TEX_CACHE_CTL
), n
[1]);
824 PUSH_DATAp(push
, commands
[1], n
[1]);
827 nvc0
->state
.num_textures
[s
] = nvc0
->num_textures
[s
];
829 /* Invalidate all 3D textures because they are aliased. */
830 for (int s
= 0; s
< 5; s
++) {
831 for (int i
= 0; i
< nvc0
->num_textures
[s
]; i
++)
832 nouveau_bufctx_reset(nvc0
->bufctx_3d
, NVC0_BIND_3D_TEX(s
, i
));
833 nvc0
->textures_dirty
[s
] = ~0;
835 nvc0
->dirty_3d
|= NVC0_NEW_3D_TEXTURES
;
840 static const char *nve4_cache_split_name(unsigned value
)
843 case NVC1_3D_CACHE_SPLIT_16K_SHARED_48K_L1
: return "16K_SHARED_48K_L1";
844 case NVE4_3D_CACHE_SPLIT_32K_SHARED_32K_L1
: return "32K_SHARED_32K_L1";
845 case NVC0_3D_CACHE_SPLIT_48K_SHARED_16K_L1
: return "48K_SHARED_16K_L1";
852 nve4_compute_dump_launch_desc(const struct nve4_cp_launch_desc
*desc
)
854 const uint32_t *data
= (const uint32_t *)desc
;
858 debug_printf("COMPUTE LAUNCH DESCRIPTOR:\n");
860 for (i
= 0; i
< sizeof(*desc
); i
+= 4) {
862 debug_printf("[%x]: 0x%08x\n", i
, data
[i
/ 4]);
866 debug_printf("...\n");
871 debug_printf("entry = 0x%x\n", desc
->entry
);
872 debug_printf("grid dimensions = %ux%ux%u\n",
873 desc
->griddim_x
, desc
->griddim_y
, desc
->griddim_z
);
874 debug_printf("block dimensions = %ux%ux%u\n",
875 desc
->blockdim_x
, desc
->blockdim_y
, desc
->blockdim_z
);
876 debug_printf("s[] size: 0x%x\n", desc
->shared_size
);
877 debug_printf("l[] size: -0x%x / +0x%x\n",
878 desc
->local_size_n
, desc
->local_size_p
);
879 debug_printf("stack size: 0x%x\n", desc
->cstack_size
);
880 debug_printf("barrier count: %u\n", desc
->bar_alloc
);
881 debug_printf("$r count: %u\n", desc
->gpr_alloc
);
882 debug_printf("cache split: %s\n", nve4_cache_split_name(desc
->cache_split
));
883 debug_printf("linked tsc: %d\n", desc
->linked_tsc
);
885 for (i
= 0; i
< 8; ++i
) {
887 uint32_t size
= desc
->cb
[i
].size
;
888 bool valid
= !!(desc
->cb_mask
& (1 << i
));
890 address
= ((uint64_t)desc
->cb
[i
].address_h
<< 32) | desc
->cb
[i
].address_l
;
892 if (!valid
&& !address
&& !size
)
894 debug_printf("CB[%u]: address = 0x%"PRIx64
", size 0x%x%s\n",
895 i
, address
, size
, valid
? "" : " (invalid)");
900 gp100_compute_dump_launch_desc(const struct gp100_cp_launch_desc
*desc
)
902 const uint32_t *data
= (const uint32_t *)desc
;
906 debug_printf("COMPUTE LAUNCH DESCRIPTOR:\n");
908 for (i
= 0; i
< sizeof(*desc
); i
+= 4) {
910 debug_printf("[%x]: 0x%08x\n", i
, data
[i
/ 4]);
914 debug_printf("...\n");
919 debug_printf("entry = 0x%x\n", desc
->entry
);
920 debug_printf("grid dimensions = %ux%ux%u\n",
921 desc
->griddim_x
, desc
->griddim_y
, desc
->griddim_z
);
922 debug_printf("block dimensions = %ux%ux%u\n",
923 desc
->blockdim_x
, desc
->blockdim_y
, desc
->blockdim_z
);
924 debug_printf("s[] size: 0x%x\n", desc
->shared_size
);
925 debug_printf("l[] size: -0x%x / +0x%x\n",
926 desc
->local_size_n
, desc
->local_size_p
);
927 debug_printf("stack size: 0x%x\n", desc
->cstack_size
);
928 debug_printf("barrier count: %u\n", desc
->bar_alloc
);
929 debug_printf("$r count: %u\n", desc
->gpr_alloc
);
930 debug_printf("linked tsc: %d\n", desc
->linked_tsc
);
932 for (i
= 0; i
< 8; ++i
) {
934 uint32_t size
= desc
->cb
[i
].size_sh4
<< 4;
935 bool valid
= !!(desc
->cb_mask
& (1 << i
));
937 address
= ((uint64_t)desc
->cb
[i
].address_h
<< 32) | desc
->cb
[i
].address_l
;
939 if (!valid
&& !address
&& !size
)
941 debug_printf("CB[%u]: address = 0x%"PRIx64
", size 0x%x%s\n",
942 i
, address
, size
, valid
? "" : " (invalid)");
947 #ifdef NOUVEAU_NVE4_MP_TRAP_HANDLER
949 nve4_compute_trap_info(struct nvc0_context
*nvc0
)
951 struct nvc0_screen
*screen
= nvc0
->screen
;
952 struct nouveau_bo
*bo
= screen
->parm
;
954 volatile struct nve4_mp_trap_info
*info
;
957 ret
= nouveau_bo_map(bo
, NOUVEAU_BO_RDWR
, nvc0
->base
.client
);
960 map
= (uint8_t *)bo
->map
;
961 info
= (volatile struct nve4_mp_trap_info
*)(map
+ NVE4_CP_PARAM_TRAP_INFO
);
964 debug_printf("trapstat = %08x\n", info
->trapstat
);
965 debug_printf("warperr = %08x\n", info
->warperr
);
966 debug_printf("PC = %x\n", info
->pc
);
967 debug_printf("tid = %u %u %u\n",
968 info
->tid
[0], info
->tid
[1], info
->tid
[2]);
969 debug_printf("ctaid = %u %u %u\n",
970 info
->ctaid
[0], info
->ctaid
[1], info
->ctaid
[2]);
971 for (i
= 0; i
<= 63; ++i
)
972 debug_printf("$r%i = %08x\n", i
, info
->r
[i
]);
973 for (i
= 0; i
<= 6; ++i
)
974 debug_printf("$p%i = %i\n", i
, (info
->flags
>> i
) & 1);
975 debug_printf("$c = %x\n", info
->flags
>> 12);