st/vdpau: only send buffers with B8G8R8A8 format to X

PresentPixmap only works if the pixmap depth matches with the
window depth, otherwise it returns a BadMatch protocol error.
Even if the depths match, the result won't look correctly
if the VDPAU RGB component order doesn't match the X11 one so
we only allow the X11 format.
For other buffers we copy them to a buffer which is send to X.

v2: only send buffers with format VDP_RGBA_FORMAT_B8G8R8A8
v3: reword commit message
v4: add comment explaining the code

Signed-off-by: Nayan Deshmukh <nayan26deshmukh@gmail.com>
Reviewed-by: Christian König <christian.koenig@amd.com>

diff --git a/src/gallium/state_trackers/vdpau/output.c b/src/gallium/state_trackers/vdpau/output.c
index 98a8011f0a603de01d26a8645927e3d5d27e5cdf..8b26f7a81ab491f968c522f45a93cae352992541 100644 (file)
--- a/src/gallium/state_trackers/vdpau/output.c
+++ b/src/gallium/state_trackers/vdpau/output.c
@@ -75,6 +75,13 @@ vlVdpOutputSurfaceCreate(VdpDevice device,
 
    memset(&res_tmpl, 0, sizeof(res_tmpl));
 
+   /*
+    * The output won't look correctly when this buffer is send to X,
+    * if the VDPAU RGB component order doesn't match the X11 one so
+    * we only allow the X11 format
+    */
+   vlsurface->send_to_X = rgba_format == VDP_RGBA_FORMAT_B8G8R8A8;
+
    res_tmpl.target = PIPE_TEXTURE_2D;
    res_tmpl.format = VdpFormatRGBAToPipe(rgba_format);
    res_tmpl.width0 = width;

diff --git a/src/gallium/state_trackers/vdpau/presentation.c b/src/gallium/state_trackers/vdpau/presentation.c
index d4793692f902305e457308fc9e949532b9c5d59e..78cafc81afe9922e33536e9c627ec5f2f25f182b 100644 (file)
--- a/src/gallium/state_trackers/vdpau/presentation.c
+++ b/src/gallium/state_trackers/vdpau/presentation.c
@@ -231,7 +231,7 @@ vlVdpPresentationQueueDisplay(VdpPresentationQueue presentation_queue,
    vscreen = pq->device->vscreen;
 
    pipe_mutex_lock(pq->device->mutex);
-   if (vscreen->set_back_texture_from_output)
+   if (vscreen->set_back_texture_from_output && surf->send_to_X)
       vscreen->set_back_texture_from_output(vscreen, surf->surface->texture, clip_width, clip_height);
    tex = vscreen->texture_from_drawable(vscreen, (void *)pq->drawable);
    if (!tex) {
@@ -239,7 +239,7 @@ vlVdpPresentationQueueDisplay(VdpPresentationQueue presentation_queue,
       return VDP_STATUS_INVALID_HANDLE;
    }
 
-   if (!vscreen->set_back_texture_from_output) {
+   if (!vscreen->set_back_texture_from_output || !surf->send_to_X) {
       dirty_area = vscreen->get_dirty_area(vscreen);
 
       memset(&surf_templ, 0, sizeof(surf_templ));
@@ -289,7 +289,7 @@ vlVdpPresentationQueueDisplay(VdpPresentationQueue presentation_queue,
       framenum++;
    }
 
-   if (!vscreen->set_back_texture_from_output) {
+   if (!vscreen->set_back_texture_from_output || !surf->send_to_X) {
       pipe_resource_reference(&tex, NULL);
       pipe_surface_reference(&surf_draw, NULL);
    }

diff --git a/src/gallium/state_trackers/vdpau/vdpau_private.h b/src/gallium/state_trackers/vdpau/vdpau_private.h
index 490a0bd308a1d845d8527deb1080d33e0bcd4dda..8356608f6d7b4065ce49a811de06beaadd18da81 100644 (file)
--- a/src/gallium/state_trackers/vdpau/vdpau_private.h
+++ b/src/gallium/state_trackers/vdpau/vdpau_private.h
@@ -415,6 +415,7 @@ typedef struct
    struct pipe_fence_handle *fence;
    struct vl_compositor_state cstate;
    struct u_rect dirty_area;
+   bool send_to_X;
 } vlVdpOutputSurface;
 
 typedef struct