#include "util/u_blitter.h"
#include "util/u_slab.h"
#include "util/u_suballoc.h"
-#include "r600.h"
+#include "util/u_double_list.h"
+#include "util/u_transfer.h"
#include "r600_llvm.h"
#include "r600_public.h"
#include "r600_resource.h"
-#define R600_NUM_ATOMS 40
-
-#define R600_TRACE_CS 0
+#define R600_NUM_ATOMS 41
/* the number of CS dwords for flushing and drawing */
#define R600_MAX_FLUSH_CS_DWORDS 16
#define R600_MAP_BUFFER_ALIGNMENT 64
+#define R600_ERR(fmt, args...) \
+ fprintf(stderr, "EE %s:%d %s - "fmt, __FILE__, __LINE__, __func__, ##args)
+
+#define R600_CONTEXT_INVAL_READ_CACHES (1 << 0)
+#define R600_CONTEXT_STREAMOUT_FLUSH (1 << 1)
+#define R600_CONTEXT_WAIT_3D_IDLE (1 << 2)
+#define R600_CONTEXT_WAIT_CP_DMA_IDLE (1 << 3)
+#define R600_CONTEXT_FLUSH_AND_INV (1 << 4)
+#define R600_CONTEXT_FLUSH_AND_INV_CB_META (1 << 5)
+#define R600_CONTEXT_PS_PARTIAL_FLUSH (1 << 6)
+#define R600_CONTEXT_FLUSH_AND_INV_DB_META (1 << 7)
+
+#define R600_QUERY_DRAW_CALLS (PIPE_QUERY_DRIVER_SPECIFIC + 0)
+#define R600_QUERY_REQUESTED_VRAM (PIPE_QUERY_DRIVER_SPECIFIC + 1)
+#define R600_QUERY_REQUESTED_GTT (PIPE_QUERY_DRIVER_SPECIFIC + 2)
+#define R600_QUERY_BUFFER_WAIT_TIME (PIPE_QUERY_DRIVER_SPECIFIC + 3)
+
+struct r600_context;
struct r600_bytecode;
struct r600_shader_key;
struct pipe_viewport_state state;
};
-struct compute_memory_pool;
-void compute_memory_pool_delete(struct compute_memory_pool* pool);
-struct compute_memory_pool* compute_memory_pool_new(
- struct r600_screen *rscreen);
-
struct r600_pipe_fences {
struct r600_resource *bo;
unsigned *data;
pipe_mutex mutex;
};
-enum r600_msaa_texture_mode {
- /* If the hw can fetch the first sample only (no decompression available).
- * This means MSAA texturing is not fully implemented. */
- MSAA_TEXTURE_SAMPLE_ZERO,
-
- /* If the hw can fetch decompressed MSAA textures.
- * Supported families: R600, R700, Evergreen.
- * Cayman cannot use this, because it cannot do the decompression. */
- MSAA_TEXTURE_DECOMPRESSED,
-
- /* If the hw can fetch compressed MSAA textures, which means shaders can
- * read resolved FMASK. This yields the best performance.
- * Supported families: Evergreen, Cayman. */
- MSAA_TEXTURE_COMPRESSED
-};
-
typedef boolean (*r600g_dma_blit_t)(struct pipe_context *ctx,
struct pipe_resource *dst,
unsigned dst_level,
/* logging */
#define DBG_TEX_DEPTH (1 << 0)
#define DBG_COMPUTE (1 << 1)
+#define DBG_VM (1 << 2)
+#define DBG_TRACE_CS (1 << 3)
/* shaders */
#define DBG_FS (1 << 8)
#define DBG_VS (1 << 9)
/* features */
#define DBG_NO_HYPERZ (1 << 16)
#define DBG_NO_LLVM (1 << 17)
+#define DBG_NO_CP_DMA (1 << 18)
+#define DBG_NO_ASYNC_DMA (1 << 19)
+#define DBG_NO_DISCARD_RANGE (1 << 20)
+/* shader backend */
+#define DBG_SB (1 << 21)
+#define DBG_SB_CS (1 << 22)
+#define DBG_SB_DRY_RUN (1 << 23)
+#define DBG_SB_STAT (1 << 24)
+#define DBG_SB_DUMP (1 << 25)
+#define DBG_SB_NO_FALLBACK (1 << 26)
+#define DBG_SB_DISASM (1 << 27)
+
+struct r600_tiling_info {
+ unsigned num_channels;
+ unsigned num_banks;
+ unsigned group_bytes;
+};
struct r600_screen {
struct pipe_screen screen;
bool has_streamout;
bool has_msaa;
bool has_cp_dma;
- enum r600_msaa_texture_mode msaa_texture_support;
+ bool has_compressed_msaa_texturing;
struct r600_tiling_info tiling_info;
struct r600_pipe_fences fences;
* XXX: Not sure if this is the best place for global_pool. Also,
* it's not thread safe, so it won't work with multiple contexts. */
struct compute_memory_pool *global_pool;
-#if R600_TRACE_CS
struct r600_resource *trace_bo;
uint32_t *trace_ptr;
unsigned cs_count;
-#endif
r600g_dma_blit_t dma_blit;
+
+ /* Auxiliary context. Mainly used to initialize resources.
+ * It must be locked prior to using and flushed before unlocking. */
+ struct pipe_context *aux_context;
+ pipe_mutex aux_context_lock;
};
struct r600_pipe_sampler_view {
struct r600_pipe_shader_selector *shader;
};
+struct r600_query_buffer {
+ /* The buffer where query results are stored. */
+ struct r600_resource *buf;
+ /* Offset of the next free result after current query data */
+ unsigned results_end;
+ /* If a query buffer is full, a new buffer is created and the old one
+ * is put in here. When we calculate the result, we sum up the samples
+ * from all buffers. */
+ struct r600_query_buffer *previous;
+};
+
+struct r600_query {
+ /* The query buffer and how many results are in it. */
+ struct r600_query_buffer buffer;
+ /* The type of query */
+ unsigned type;
+ /* Size of the result in memory for both begin_query and end_query,
+ * this can be one or two numbers, or it could even be a size of a structure. */
+ unsigned result_size;
+ /* The number of dwords for begin_query or end_query. */
+ unsigned num_cs_dw;
+ /* linked list of queries */
+ struct list_head list;
+ /* for custom non-GPU queries */
+ uint64_t begin_result;
+ uint64_t end_result;
+};
+
+struct r600_so_target {
+ struct pipe_stream_output_target b;
+
+ /* The buffer where BUFFER_FILLED_SIZE is stored. */
+ struct r600_resource *buf_filled_size;
+ unsigned buf_filled_size_offset;
+
+ unsigned stride_in_dw;
+ unsigned so_index;
+};
+
struct r600_streamout {
struct r600_atom begin_atom;
bool begin_emitted;
void *custom_dsa_flush;
void *custom_blend_resolve;
void *custom_blend_decompress;
- void *custom_blend_fmask_decompress;
/* With rasterizer discard, there doesn't have to be a pixel shader.
* In that case, we bind this one: */
void *dummy_pixel_shader;
/* Queries. */
/* The list of active queries. Only one query of each type can be active. */
int num_occlusion_queries;
+ int num_pipelinestat_queries;
/* Keep track of non-timer queries, because they should be suspended
* during context flushing.
* The timer queries (TIME_ELAPSED) shouldn't be suspended. */
unsigned num_cs_dw_nontimer_queries_suspend;
/* If queries have been suspended. */
bool nontimer_queries_suspended;
+ unsigned num_draw_calls;
/* Render condition. */
struct pipe_query *current_render_cond;
unsigned current_render_cond_mode;
boolean predicate_drawing;
+ void *sb_context;
struct r600_isa *isa;
};
cs->cdw += cb->num_dw;
}
-#if R600_TRACE_CS
void r600_trace_emit(struct r600_context *rctx);
-#endif
static INLINE void r600_emit_atom(struct r600_context *rctx, struct r600_atom *atom)
{
atom->emit(rctx, atom);
atom->dirty = false;
-#if R600_TRACE_CS
if (rctx->screen->trace_bo) {
r600_trace_emit(rctx);
}
-#endif
}
static INLINE void r600_set_cso_state(struct r600_cso_state *state, void *cso)
r600_set_cso_state(state, cso);
}
+/* compute_memory_pool.c */
+struct compute_memory_pool;
+void compute_memory_pool_delete(struct compute_memory_pool* pool);
+struct compute_memory_pool* compute_memory_pool_new(
+ struct r600_screen *rscreen);
+
/* evergreen_state.c */
struct pipe_sampler_view *
evergreen_create_sampler_view_custom(struct pipe_context *ctx,
void *evergreen_create_db_flush_dsa(struct r600_context *rctx);
void *evergreen_create_resolve_blend(struct r600_context *rctx);
void *evergreen_create_decompress_blend(struct r600_context *rctx);
-void *evergreen_create_fmask_decompress_blend(struct r600_context *rctx);
boolean evergreen_is_format_supported(struct pipe_screen *screen,
enum pipe_format format,
enum pipe_texture_target target,
/* r600_blit.c */
void r600_copy_buffer(struct pipe_context *ctx, struct pipe_resource *dst, unsigned dstx,
struct pipe_resource *src, const struct pipe_box *src_box);
+void r600_screen_clear_buffer(struct r600_screen *rscreen, struct pipe_resource *dst,
+ unsigned offset, unsigned size, unsigned char value);
void r600_init_blit_functions(struct r600_context *rctx);
void r600_blit_decompress_depth(struct pipe_context *ctx,
struct r600_texture *texture,
void *r600_buffer_mmap_sync_with_rings(struct r600_context *ctx,
struct r600_resource *resource,
unsigned usage);
+const char * r600_llvm_gpu_string(enum radeon_family family);
+
/* r600_query.c */
void r600_init_query_functions(struct r600_context *rctx);
boolean vtx);
/* r600_hw_context.c */
+void r600_get_backend_mask(struct r600_context *ctx);
+void r600_context_flush(struct r600_context *ctx, unsigned flags);
+void r600_begin_new_cs(struct r600_context *ctx);
+void r600_context_emit_fence(struct r600_context *ctx, struct r600_resource *fence,
+ unsigned offset, unsigned value);
+void r600_flush_emit(struct r600_context *ctx);
+void r600_need_cs_space(struct r600_context *ctx, unsigned num_dw, boolean count_draw_in);
+void r600_need_dma_space(struct r600_context *ctx, unsigned num_dw);
+void r600_cp_dma_copy_buffer(struct r600_context *rctx,
+ struct pipe_resource *dst, uint64_t dst_offset,
+ struct pipe_resource *src, uint64_t src_offset,
+ unsigned size);
+void evergreen_cp_dma_clear_buffer(struct r600_context *rctx,
+ struct pipe_resource *dst, uint64_t offset,
+ unsigned size, uint32_t clear_value);
+void r600_dma_copy(struct r600_context *rctx,
+ struct pipe_resource *dst,
+ struct pipe_resource *src,
+ uint64_t dst_offset,
+ uint64_t src_offset,
+ uint64_t size);
+boolean r600_dma_blit(struct pipe_context *ctx,
+ struct pipe_resource *dst,
+ unsigned dst_level,
+ unsigned dst_x, unsigned dst_y, unsigned dst_z,
+ struct pipe_resource *src,
+ unsigned src_level,
+ const struct pipe_box *src_box);
void r600_emit_streamout_begin(struct r600_context *ctx, struct r600_atom *atom);
void r600_emit_streamout_end(struct r600_context *ctx);
*/
void evergreen_flush_vgt_streamout(struct r600_context *ctx);
void evergreen_set_streamout_enable(struct r600_context *ctx, unsigned buffer_enable_bit);
+void evergreen_dma_copy(struct r600_context *rctx,
+ struct pipe_resource *dst,
+ struct pipe_resource *src,
+ uint64_t dst_offset,
+ uint64_t src_offset,
+ uint64_t size);
+boolean evergreen_dma_blit(struct pipe_context *ctx,
+ struct pipe_resource *dst,
+ unsigned dst_level,
+ unsigned dst_x, unsigned dst_y, unsigned dst_z,
+ struct pipe_resource *src,
+ unsigned src_level,
+ const struct pipe_box *src_box);
/* r600_state_common.c */
void r600_init_common_state_functions(struct r600_context *rctx);
unsigned r600_tex_compare(unsigned compare);
bool sampler_state_needs_border_color(const struct pipe_sampler_state *state);
+/* r600_uvd.c */
+struct pipe_video_decoder *r600_uvd_create_decoder(struct pipe_context *context,
+ enum pipe_video_profile profile,
+ enum pipe_video_entrypoint entrypoint,
+ enum pipe_video_chroma_format chroma_format,
+ unsigned width, unsigned height,
+ unsigned max_references, bool expect_chunked_decode);
+
+struct pipe_video_buffer *r600_video_buffer_create(struct pipe_context *pipe,
+ const struct pipe_video_buffer *tmpl);
+
+int r600_uvd_get_video_param(struct pipe_screen *screen,
+ enum pipe_video_profile profile,
+ enum pipe_video_cap param);
+
/*
* Helpers for building command buffers
*/
r600_write_value(cs, value);
}
+static INLINE void r600_write_context_reg_flag(struct radeon_winsys_cs *cs, unsigned reg, unsigned value, unsigned flag)
+{
+ if (flag & RADEON_CP_PACKET3_COMPUTE_MODE) {
+ r600_write_compute_context_reg(cs, reg, value);
+ } else {
+ r600_write_context_reg(cs, reg, value);
+ }
+
+}
static INLINE void r600_write_ctl_const(struct radeon_winsys_cs *cs, unsigned reg, unsigned value)
{
r600_write_ctl_const_seq(cs, reg, 1);