2 * Copyright (c) 2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 #include "anv_private.h"
26 #include "genxml/gen_macros.h"
27 #include "genxml/genX_pack.h"
30 * Chunk of L3 cache reserved for some specific purpose.
32 enum anv_l3_partition
{
33 /** Shared local memory. */
35 /** Unified return buffer. */
37 /** Union of DC and RO. */
39 /** Data cluster RW partition. */
41 /** Union of IS, C and T. */
43 /** Instruction and state cache. */
45 /** Constant cache. */
49 /** Number of supported L3 partitions. */
54 * L3 configuration represented as the number of ways allocated for each
55 * partition. \sa get_l3_way_size().
57 struct anv_l3_config
{
64 * IVB/HSW validated L3 configurations. The first entry will be used as
65 * default by gen7_restore_default_l3_config(), otherwise the ordering is
68 static const struct anv_l3_config ivb_l3_configs
[] = {
69 /* SLM URB ALL DC RO IS C T */
70 {{ 0, 32, 0, 0, 32, 0, 0, 0 }},
71 {{ 0, 32, 0, 16, 16, 0, 0, 0 }},
72 {{ 0, 32, 0, 4, 0, 8, 4, 16 }},
73 {{ 0, 28, 0, 8, 0, 8, 4, 16 }},
74 {{ 0, 28, 0, 16, 0, 8, 4, 8 }},
75 {{ 0, 28, 0, 8, 0, 16, 4, 8 }},
76 {{ 0, 28, 0, 0, 0, 16, 4, 16 }},
77 {{ 0, 32, 0, 0, 0, 16, 0, 16 }},
78 {{ 0, 28, 0, 4, 32, 0, 0, 0 }},
79 {{ 16, 16, 0, 16, 16, 0, 0, 0 }},
80 {{ 16, 16, 0, 8, 0, 8, 8, 8 }},
81 {{ 16, 16, 0, 4, 0, 8, 4, 16 }},
82 {{ 16, 16, 0, 4, 0, 16, 4, 8 }},
83 {{ 16, 16, 0, 0, 32, 0, 0, 0 }},
89 #if GEN_GEN == 7 && !GEN_IS_HASWELL
92 * VLV validated L3 configurations. \sa ivb_l3_configs.
94 static const struct anv_l3_config vlv_l3_configs
[] = {
95 /* SLM URB ALL DC RO IS C T */
96 {{ 0, 64, 0, 0, 32, 0, 0, 0 }},
97 {{ 0, 80, 0, 0, 16, 0, 0, 0 }},
98 {{ 0, 80, 0, 8, 8, 0, 0, 0 }},
99 {{ 0, 64, 0, 16, 16, 0, 0, 0 }},
100 {{ 0, 60, 0, 4, 32, 0, 0, 0 }},
101 {{ 32, 32, 0, 16, 16, 0, 0, 0 }},
102 {{ 32, 40, 0, 8, 16, 0, 0, 0 }},
103 {{ 32, 40, 0, 16, 8, 0, 0, 0 }},
112 * BDW validated L3 configurations. \sa ivb_l3_configs.
114 static const struct anv_l3_config bdw_l3_configs
[] = {
115 /* SLM URB ALL DC RO IS C T */
116 {{ 0, 48, 48, 0, 0, 0, 0, 0 }},
117 {{ 0, 48, 0, 16, 32, 0, 0, 0 }},
118 {{ 0, 32, 0, 16, 48, 0, 0, 0 }},
119 {{ 0, 32, 0, 0, 64, 0, 0, 0 }},
120 {{ 0, 32, 64, 0, 0, 0, 0, 0 }},
121 {{ 24, 16, 48, 0, 0, 0, 0, 0 }},
122 {{ 24, 16, 0, 16, 32, 0, 0, 0 }},
123 {{ 24, 16, 0, 32, 16, 0, 0, 0 }},
129 #if GEN_GEN == 8 || GEN_GEN == 9
132 * CHV/SKL validated L3 configurations. \sa ivb_l3_configs.
134 static const struct anv_l3_config chv_l3_configs
[] = {
135 /* SLM URB ALL DC RO IS C T */
136 {{ 0, 48, 48, 0, 0, 0, 0, 0 }},
137 {{ 0, 48, 0, 16, 32, 0, 0, 0 }},
138 {{ 0, 32, 0, 16, 48, 0, 0, 0 }},
139 {{ 0, 32, 0, 0, 64, 0, 0, 0 }},
140 {{ 0, 32, 64, 0, 0, 0, 0, 0 }},
141 {{ 32, 16, 48, 0, 0, 0, 0, 0 }},
142 {{ 32, 16, 0, 16, 32, 0, 0, 0 }},
143 {{ 32, 16, 0, 32, 16, 0, 0, 0 }},
150 * Return a zero-terminated array of validated L3 configurations for the
153 static inline const struct anv_l3_config
*
154 get_l3_configs(const struct brw_device_info
*devinfo
)
156 assert(devinfo
->gen
== GEN_GEN
);
158 return ivb_l3_configs
;
160 return (devinfo
->is_baytrail
? vlv_l3_configs
: ivb_l3_configs
);
162 return (devinfo
->is_cherryview
? chv_l3_configs
: bdw_l3_configs
);
164 return chv_l3_configs
;
166 #error GEN not supported
171 * Return the size of an L3 way in KB.
174 get_l3_way_size(const struct brw_device_info
*devinfo
)
176 if (devinfo
->is_baytrail
)
179 else if (devinfo
->is_cherryview
|| devinfo
->gt
== 1)
183 return 8 * devinfo
->num_slices
;
187 * L3 configuration represented as a vector of weights giving the desired
188 * relative size of each partition. The scale is arbitrary, only the ratios
189 * between weights will have an influence on the selection of the closest L3
192 struct anv_l3_weights
{
197 * L1-normalize a vector of L3 partition weights.
199 static struct anv_l3_weights
200 norm_l3_weights(struct anv_l3_weights w
)
204 for (unsigned i
= 0; i
< NUM_L3P
; i
++)
207 for (unsigned i
= 0; i
< NUM_L3P
; i
++)
214 * Get the relative partition weights of the specified L3 configuration.
216 static struct anv_l3_weights
217 get_config_l3_weights(const struct anv_l3_config
*cfg
)
220 struct anv_l3_weights w
;
222 for (unsigned i
= 0; i
< NUM_L3P
; i
++)
225 return norm_l3_weights(w
);
227 const struct anv_l3_weights w
= { { 0 } };
233 * Distance between two L3 configurations represented as vectors of weights.
234 * Usually just the L1 metric except when the two configurations are
235 * considered incompatible in which case the distance will be infinite. Note
236 * that the compatibility condition is asymmetric -- They will be considered
237 * incompatible whenever the reference configuration \p w0 requires SLM, DC,
238 * or URB but \p w1 doesn't provide it.
241 diff_l3_weights(struct anv_l3_weights w0
, struct anv_l3_weights w1
)
243 if ((w0
.w
[L3P_SLM
] && !w1
.w
[L3P_SLM
]) ||
244 (w0
.w
[L3P_DC
] && !w1
.w
[L3P_DC
] && !w1
.w
[L3P_ALL
]) ||
245 (w0
.w
[L3P_URB
] && !w1
.w
[L3P_URB
])) {
251 for (unsigned i
= 0; i
< NUM_L3P
; i
++)
252 dw
+= fabs(w0
.w
[i
] - w1
.w
[i
]);
259 * Return the closest validated L3 configuration for the specified device and
262 static const struct anv_l3_config
*
263 get_l3_config(const struct brw_device_info
*devinfo
, struct anv_l3_weights w0
)
265 const struct anv_l3_config
*const cfgs
= get_l3_configs(devinfo
);
266 const struct anv_l3_config
*cfg_best
= NULL
;
267 float dw_best
= HUGE_VALF
;
269 for (const struct anv_l3_config
*cfg
= cfgs
; cfg
->n
[L3P_URB
]; cfg
++) {
270 const float dw
= diff_l3_weights(w0
, get_config_l3_weights(cfg
));
282 * Return a reasonable default L3 configuration for the specified device based
283 * on whether SLM and DC are required. In the non-SLM non-DC case the result
284 * is intended to approximately resemble the hardware defaults.
286 static struct anv_l3_weights
287 get_default_l3_weights(const struct brw_device_info
*devinfo
,
288 bool needs_dc
, bool needs_slm
)
290 struct anv_l3_weights w
= {{ 0 }};
292 w
.w
[L3P_SLM
] = needs_slm
;
295 if (devinfo
->gen
>= 8) {
298 w
.w
[L3P_DC
] = needs_dc
? 0.1 : 0;
299 w
.w
[L3P_RO
] = devinfo
->is_baytrail
? 0.5 : 1.0;
302 return norm_l3_weights(w
);
306 * Calculate the desired L3 partitioning based on the current state of the
307 * pipeline. For now this simply returns the conservative defaults calculated
308 * by get_default_l3_weights(), but we could probably do better by gathering
309 * more statistics from the pipeline state (e.g. guess of expected URB usage
310 * and bound surfaces), or by using feed-back from performance counters.
312 static struct anv_l3_weights
313 get_pipeline_state_l3_weights(const struct anv_pipeline
*pipeline
)
315 bool needs_dc
= false, needs_slm
= false;
317 for (unsigned i
= 0; i
< MESA_SHADER_STAGES
; i
++) {
318 const struct brw_stage_prog_data
*prog_data
= pipeline
->prog_data
[i
];
320 needs_dc
|= pipeline
->needs_data_cache
;
321 needs_slm
|= prog_data
&& prog_data
->total_shared
;
324 return get_default_l3_weights(&pipeline
->device
->info
,
325 needs_dc
, needs_slm
);
328 #define emit_lri(batch, reg, imm) \
329 anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_IMM), lri) { \
330 lri.RegisterOffset = __anv_reg_num(reg); \
331 lri.DataDWord = imm; \
334 #define IVB_L3SQCREG1_SQGHPCI_DEFAULT 0x00730000
335 #define VLV_L3SQCREG1_SQGHPCI_DEFAULT 0x00d30000
336 #define HSW_L3SQCREG1_SQGHPCI_DEFAULT 0x00610000
339 * Program the hardware to use the specified L3 configuration.
342 setup_l3_config(struct anv_cmd_buffer
*cmd_buffer
/*, struct brw_context *brw*/,
343 const struct anv_l3_config
*cfg
)
345 const bool has_slm
= cfg
->n
[L3P_SLM
];
347 /* According to the hardware docs, the L3 partitioning can only be changed
348 * while the pipeline is completely drained and the caches are flushed,
349 * which involves a first PIPE_CONTROL flush which stalls the pipeline...
351 anv_batch_emit(&cmd_buffer
->batch
, GENX(PIPE_CONTROL
), pc
) {
352 pc
.DCFlushEnable
= true;
353 pc
.PostSyncOperation
= NoWrite
;
354 pc
.CommandStreamerStallEnable
= true;
357 /* ...followed by a second pipelined PIPE_CONTROL that initiates
358 * invalidation of the relevant caches. Note that because RO invalidation
359 * happens at the top of the pipeline (i.e. right away as the PIPE_CONTROL
360 * command is processed by the CS) we cannot combine it with the previous
361 * stalling flush as the hardware documentation suggests, because that
362 * would cause the CS to stall on previous rendering *after* RO
363 * invalidation and wouldn't prevent the RO caches from being polluted by
364 * concurrent rendering before the stall completes. This intentionally
365 * doesn't implement the SKL+ hardware workaround suggesting to enable CS
366 * stall on PIPE_CONTROLs with the texture cache invalidation bit set for
367 * GPGPU workloads because the previous and subsequent PIPE_CONTROLs
368 * already guarantee that there is no concurrent GPGPU kernel execution
369 * (see SKL HSD 2132585).
371 anv_batch_emit(&cmd_buffer
->batch
, GENX(PIPE_CONTROL
), pc
) {
372 pc
.TextureCacheInvalidationEnable
= true;
373 pc
.ConstantCacheInvalidationEnable
= true;
374 pc
.InstructionCacheInvalidateEnable
= true;
375 pc
.StateCacheInvalidationEnable
= true;
376 pc
.PostSyncOperation
= NoWrite
;
379 /* Now send a third stalling flush to make sure that invalidation is
380 * complete when the L3 configuration registers are modified.
382 anv_batch_emit(&cmd_buffer
->batch
, GENX(PIPE_CONTROL
), pc
) {
383 pc
.DCFlushEnable
= true;
384 pc
.PostSyncOperation
= NoWrite
;
385 pc
.CommandStreamerStallEnable
= true;
390 assert(!cfg
->n
[L3P_IS
] && !cfg
->n
[L3P_C
] && !cfg
->n
[L3P_T
]);
393 anv_pack_struct(&l3cr
, GENX(L3CNTLREG
),
394 .SLMEnable
= has_slm
,
395 .URBAllocation
= cfg
->n
[L3P_URB
],
396 .ROAllocation
= cfg
->n
[L3P_RO
],
397 .DCAllocation
= cfg
->n
[L3P_DC
],
398 .AllAllocation
= cfg
->n
[L3P_ALL
]);
400 /* Set up the L3 partitioning. */
401 emit_lri(&cmd_buffer
->batch
, GENX(L3CNTLREG
), l3cr
);
405 const bool has_dc
= cfg
->n
[L3P_DC
] || cfg
->n
[L3P_ALL
];
406 const bool has_is
= cfg
->n
[L3P_IS
] || cfg
->n
[L3P_RO
] || cfg
->n
[L3P_ALL
];
407 const bool has_c
= cfg
->n
[L3P_C
] || cfg
->n
[L3P_RO
] || cfg
->n
[L3P_ALL
];
408 const bool has_t
= cfg
->n
[L3P_T
] || cfg
->n
[L3P_RO
] || cfg
->n
[L3P_ALL
];
410 assert(!cfg
->n
[L3P_ALL
]);
412 /* When enabled SLM only uses a portion of the L3 on half of the banks,
413 * the matching space on the remaining banks has to be allocated to a
414 * client (URB for all validated configurations) set to the
415 * lower-bandwidth 2-bank address hashing mode.
417 const struct brw_device_info
*devinfo
= &cmd_buffer
->device
->info
;
418 const bool urb_low_bw
= has_slm
&& !devinfo
->is_baytrail
;
419 assert(!urb_low_bw
|| cfg
->n
[L3P_URB
] == cfg
->n
[L3P_SLM
]);
421 /* Minimum number of ways that can be allocated to the URB. */
422 const unsigned n0_urb
= (devinfo
->is_baytrail
? 32 : 0);
423 assert(cfg
->n
[L3P_URB
] >= n0_urb
);
425 uint32_t l3sqcr1
, l3cr2
, l3cr3
;
426 anv_pack_struct(&l3sqcr1
, GENX(L3SQCREG1
),
427 .ConvertDC_UC
= !has_dc
,
428 .ConvertIS_UC
= !has_is
,
429 .ConvertC_UC
= !has_c
,
430 .ConvertT_UC
= !has_t
);
432 GEN_IS_HASWELL
? HSW_L3SQCREG1_SQGHPCI_DEFAULT
:
433 devinfo
->is_baytrail
? VLV_L3SQCREG1_SQGHPCI_DEFAULT
:
434 IVB_L3SQCREG1_SQGHPCI_DEFAULT
;
436 anv_pack_struct(&l3cr2
, GENX(L3CNTLREG2
),
437 .SLMEnable
= has_slm
,
438 .URBLowBandwidth
= urb_low_bw
,
439 .URBAllocation
= cfg
->n
[L3P_URB
],
441 .ALLAllocation
= cfg
->n
[L3P_ALL
],
443 .ROAllocation
= cfg
->n
[L3P_RO
],
444 .DCAllocation
= cfg
->n
[L3P_DC
]);
446 anv_pack_struct(&l3cr3
, GENX(L3CNTLREG3
),
447 .ISAllocation
= cfg
->n
[L3P_IS
],
449 .CAllocation
= cfg
->n
[L3P_C
],
451 .TAllocation
= cfg
->n
[L3P_T
],
454 /* Set up the L3 partitioning. */
455 emit_lri(&cmd_buffer
->batch
, GENX(L3SQCREG1
), l3sqcr1
);
456 emit_lri(&cmd_buffer
->batch
, GENX(L3CNTLREG2
), l3cr2
);
457 emit_lri(&cmd_buffer
->batch
, GENX(L3CNTLREG3
), l3cr3
);
460 if (cmd_buffer
->device
->instance
->physicalDevice
.cmd_parser_version
>= 4) {
461 /* Enable L3 atomics on HSW if we have a DC partition, otherwise keep
462 * them disabled to avoid crashing the system hard.
464 uint32_t scratch1
, chicken3
;
465 anv_pack_struct(&scratch1
, GENX(SCRATCH1
),
466 .L3AtomicDisable
= !has_dc
);
467 anv_pack_struct(&chicken3
, GENX(CHICKEN3
),
468 .L3AtomicDisable
= !has_dc
);
469 emit_lri(&cmd_buffer
->batch
, GENX(SCRATCH1
), scratch1
);
470 emit_lri(&cmd_buffer
->batch
, GENX(CHICKEN3
), chicken3
);
479 * Return the unit brw_context::urb::size is expressed in, in KB. \sa
480 * brw_device_info::urb::size.
483 get_urb_size_scale(const struct brw_device_info
*devinfo
)
485 return (devinfo
->gen
>= 8 ? devinfo
->num_slices
: 1);
489 genX(setup_pipeline_l3_config
)(struct anv_pipeline
*pipeline
)
491 const struct anv_l3_weights w
= get_pipeline_state_l3_weights(pipeline
);
492 const struct brw_device_info
*devinfo
= &pipeline
->device
->info
;
493 const struct anv_l3_config
*const cfg
= get_l3_config(devinfo
, w
);
494 pipeline
->urb
.l3_config
= cfg
;
496 unsigned sz
= cfg
->n
[L3P_URB
] * get_l3_way_size(devinfo
);
499 /* From the SKL "L3 Allocation and Programming" documentation:
501 * "URB is limited to 1008KB due to programming restrictions. This is not
502 * a restriction of the L3 implementation, but of the FF and other clients.
503 * Therefore, in a GT4 implementation it is possible for the programmed
504 * allocation of the L3 data array to provide 3*384KB=1152KB for URB, but
505 * only 1008KB of this will be used."
510 pipeline
->urb
.total_size
= sz
/ get_urb_size_scale(devinfo
);
514 * Print out the specified L3 configuration.
517 dump_l3_config(const struct anv_l3_config
*cfg
)
519 fprintf(stderr
, "SLM=%d URB=%d ALL=%d DC=%d RO=%d IS=%d C=%d T=%d\n",
520 cfg
->n
[L3P_SLM
], cfg
->n
[L3P_URB
], cfg
->n
[L3P_ALL
],
521 cfg
->n
[L3P_DC
], cfg
->n
[L3P_RO
],
522 cfg
->n
[L3P_IS
], cfg
->n
[L3P_C
], cfg
->n
[L3P_T
]);
526 genX(cmd_buffer_config_l3
)(struct anv_cmd_buffer
*cmd_buffer
,
527 const struct anv_pipeline
*pipeline
)
529 struct anv_cmd_state
*state
= &cmd_buffer
->state
;
530 const struct anv_l3_config
*const cfg
= pipeline
->urb
.l3_config
;
532 if (cfg
!= state
->current_l3_config
) {
533 setup_l3_config(cmd_buffer
, cfg
);
534 state
->current_l3_config
= cfg
;
536 if (unlikely(INTEL_DEBUG
& DEBUG_L3
)) {
537 fprintf(stderr
, "L3 config transition: ");