integer overflow in XF86DRIOpenConnection() [CVE-2013-1993 1/2]
[mesa.git] / src / glx / XF86dri.c
1 /**************************************************************************
2
3 Copyright 1998-1999 Precision Insight, Inc., Cedar Park, Texas.
4 Copyright 2000 VA Linux Systems, Inc.
5 All Rights Reserved.
6
7 Permission is hereby granted, free of charge, to any person obtaining a
8 copy of this software and associated documentation files (the
9 "Software"), to deal in the Software without restriction, including
10 without limitation the rights to use, copy, modify, merge, publish,
11 distribute, sub license, and/or sell copies of the Software, and to
12 permit persons to whom the Software is furnished to do so, subject to
13 the following conditions:
14
15 The above copyright notice and this permission notice (including the
16 next paragraph) shall be included in all copies or substantial portions
17 of the Software.
18
19 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20 OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22 IN NO EVENT SHALL PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR
23 ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24 TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25 SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26
27 **************************************************************************/
28
29 /*
30 * Authors:
31 * Kevin E. Martin <martin@valinux.com>
32 * Jens Owen <jens@tungstengraphics.com>
33 * Rickard E. (Rik) Faith <faith@valinux.com>
34 *
35 */
36
37 /* THIS IS NOT AN X CONSORTIUM STANDARD */
38
39 #if defined(GLX_DIRECT_RENDERING) && !defined(GLX_USE_APPLEGL)
40
41 #include <X11/Xlibint.h>
42 #include <X11/Xfuncproto.h>
43 #include <X11/extensions/Xext.h>
44 #include <X11/extensions/extutil.h>
45 #include "xf86dristr.h"
46 #include <limits.h>
47
48 static XExtensionInfo _xf86dri_info_data;
49 static XExtensionInfo *xf86dri_info = &_xf86dri_info_data;
50 static char xf86dri_extension_name[] = XF86DRINAME;
51
52 #define XF86DRICheckExtension(dpy,i,val) \
53 XextCheckExtension (dpy, i, xf86dri_extension_name, val)
54
55 /*****************************************************************************
56 * *
57 * private utility routines *
58 * *
59 *****************************************************************************/
60
61 static int close_display(Display * dpy, XExtCodes * extCodes);
62 static /* const */ XExtensionHooks xf86dri_extension_hooks = {
63 NULL, /* create_gc */
64 NULL, /* copy_gc */
65 NULL, /* flush_gc */
66 NULL, /* free_gc */
67 NULL, /* create_font */
68 NULL, /* free_font */
69 close_display, /* close_display */
70 NULL, /* wire_to_event */
71 NULL, /* event_to_wire */
72 NULL, /* error */
73 NULL, /* error_string */
74 };
75
76 static
77 XEXT_GENERATE_FIND_DISPLAY(find_display, xf86dri_info,
78 xf86dri_extension_name,
79 &xf86dri_extension_hooks, 0, NULL)
80
81 static
82 XEXT_GENERATE_CLOSE_DISPLAY(close_display, xf86dri_info)
83
84
85 /*****************************************************************************
86 * *
87 * public XFree86-DRI Extension routines *
88 * *
89 *****************************************************************************/
90 #if 0
91 #include <stdio.h>
92 #define TRACE(msg) fprintf(stderr,"XF86DRI%s\n", msg);
93 #else
94 #define TRACE(msg)
95 #endif
96
97 Bool
98 XF86DRIQueryExtension(Display * dpy, int *event_basep,
99 int *error_basep)
100 {
101 XExtDisplayInfo *info = find_display(dpy);
102
103 TRACE("QueryExtension...");
104 if (XextHasExtension(info)) {
105 *event_basep = info->codes->first_event;
106 *error_basep = info->codes->first_error;
107 TRACE("QueryExtension... return True");
108 return True;
109 }
110 else {
111 TRACE("QueryExtension... return False");
112 return False;
113 }
114 }
115
116 Bool
117 XF86DRIQueryVersion(Display * dpy, int *majorVersion, int *minorVersion,
118 int *patchVersion)
119 {
120 XExtDisplayInfo *info = find_display(dpy);
121 xXF86DRIQueryVersionReply rep;
122 xXF86DRIQueryVersionReq *req;
123
124 TRACE("QueryVersion...");
125 XF86DRICheckExtension(dpy, info, False);
126
127 LockDisplay(dpy);
128 GetReq(XF86DRIQueryVersion, req);
129 req->reqType = info->codes->major_opcode;
130 req->driReqType = X_XF86DRIQueryVersion;
131 if (!_XReply(dpy, (xReply *) & rep, 0, xFalse)) {
132 UnlockDisplay(dpy);
133 SyncHandle();
134 TRACE("QueryVersion... return False");
135 return False;
136 }
137 *majorVersion = rep.majorVersion;
138 *minorVersion = rep.minorVersion;
139 *patchVersion = rep.patchVersion;
140 UnlockDisplay(dpy);
141 SyncHandle();
142 TRACE("QueryVersion... return True");
143 return True;
144 }
145
146 Bool
147 XF86DRIQueryDirectRenderingCapable(Display * dpy, int screen,
148 Bool * isCapable)
149 {
150 XExtDisplayInfo *info = find_display(dpy);
151 xXF86DRIQueryDirectRenderingCapableReply rep;
152 xXF86DRIQueryDirectRenderingCapableReq *req;
153
154 TRACE("QueryDirectRenderingCapable...");
155 XF86DRICheckExtension(dpy, info, False);
156
157 LockDisplay(dpy);
158 GetReq(XF86DRIQueryDirectRenderingCapable, req);
159 req->reqType = info->codes->major_opcode;
160 req->driReqType = X_XF86DRIQueryDirectRenderingCapable;
161 req->screen = screen;
162 if (!_XReply(dpy, (xReply *) & rep, 0, xFalse)) {
163 UnlockDisplay(dpy);
164 SyncHandle();
165 TRACE("QueryDirectRenderingCapable... return False");
166 return False;
167 }
168 *isCapable = rep.isCapable;
169 UnlockDisplay(dpy);
170 SyncHandle();
171 TRACE("QueryDirectRenderingCapable... return True");
172 return True;
173 }
174
175 Bool
176 XF86DRIOpenConnection(Display * dpy, int screen, drm_handle_t * hSAREA,
177 char **busIdString)
178 {
179 XExtDisplayInfo *info = find_display(dpy);
180 xXF86DRIOpenConnectionReply rep;
181 xXF86DRIOpenConnectionReq *req;
182
183 TRACE("OpenConnection...");
184 XF86DRICheckExtension(dpy, info, False);
185
186 LockDisplay(dpy);
187 GetReq(XF86DRIOpenConnection, req);
188 req->reqType = info->codes->major_opcode;
189 req->driReqType = X_XF86DRIOpenConnection;
190 req->screen = screen;
191 if (!_XReply(dpy, (xReply *) & rep, 0, xFalse)) {
192 UnlockDisplay(dpy);
193 SyncHandle();
194 TRACE("OpenConnection... return False");
195 return False;
196 }
197
198 *hSAREA = rep.hSAREALow;
199 if (sizeof(drm_handle_t) == 8) {
200 int shift = 32; /* var to prevent warning on next line */
201 *hSAREA |= ((drm_handle_t) rep.hSAREAHigh) << shift;
202 }
203
204 if (rep.length) {
205 if (rep.busIdStringLength < INT_MAX)
206 *busIdString = calloc(rep.busIdStringLength + 1, 1);
207 else
208 *busIdString = NULL;
209 if (*busIdString == NULL) {
210 _XEatData(dpy, ((rep.busIdStringLength + 3) & ~3));
211 UnlockDisplay(dpy);
212 SyncHandle();
213 TRACE("OpenConnection... return False");
214 return False;
215 }
216 _XReadPad(dpy, *busIdString, rep.busIdStringLength);
217 }
218 else {
219 *busIdString = NULL;
220 }
221 UnlockDisplay(dpy);
222 SyncHandle();
223 TRACE("OpenConnection... return True");
224 return True;
225 }
226
227 Bool
228 XF86DRIAuthConnection(Display * dpy, int screen, drm_magic_t magic)
229 {
230 XExtDisplayInfo *info = find_display(dpy);
231 xXF86DRIAuthConnectionReq *req;
232 xXF86DRIAuthConnectionReply rep;
233
234 TRACE("AuthConnection...");
235 XF86DRICheckExtension(dpy, info, False);
236
237 LockDisplay(dpy);
238 GetReq(XF86DRIAuthConnection, req);
239 req->reqType = info->codes->major_opcode;
240 req->driReqType = X_XF86DRIAuthConnection;
241 req->screen = screen;
242 req->magic = magic;
243 rep.authenticated = 0;
244 if (!_XReply(dpy, (xReply *) & rep, 0, xFalse) || !rep.authenticated) {
245 UnlockDisplay(dpy);
246 SyncHandle();
247 TRACE("AuthConnection... return False");
248 return False;
249 }
250 UnlockDisplay(dpy);
251 SyncHandle();
252 TRACE("AuthConnection... return True");
253 return True;
254 }
255
256 Bool
257 XF86DRICloseConnection(Display * dpy, int screen)
258 {
259 XExtDisplayInfo *info = find_display(dpy);
260 xXF86DRICloseConnectionReq *req;
261
262 TRACE("CloseConnection...");
263
264 XF86DRICheckExtension(dpy, info, False);
265
266 LockDisplay(dpy);
267 GetReq(XF86DRICloseConnection, req);
268 req->reqType = info->codes->major_opcode;
269 req->driReqType = X_XF86DRICloseConnection;
270 req->screen = screen;
271 UnlockDisplay(dpy);
272 SyncHandle();
273 TRACE("CloseConnection... return True");
274 return True;
275 }
276
277 Bool
278 XF86DRIGetClientDriverName(Display * dpy, int screen,
279 int *ddxDriverMajorVersion,
280 int *ddxDriverMinorVersion,
281 int *ddxDriverPatchVersion,
282 char **clientDriverName)
283 {
284 XExtDisplayInfo *info = find_display(dpy);
285 xXF86DRIGetClientDriverNameReply rep;
286 xXF86DRIGetClientDriverNameReq *req;
287
288 TRACE("GetClientDriverName...");
289 XF86DRICheckExtension(dpy, info, False);
290
291 LockDisplay(dpy);
292 GetReq(XF86DRIGetClientDriverName, req);
293 req->reqType = info->codes->major_opcode;
294 req->driReqType = X_XF86DRIGetClientDriverName;
295 req->screen = screen;
296 if (!_XReply(dpy, (xReply *) & rep, 0, xFalse)) {
297 UnlockDisplay(dpy);
298 SyncHandle();
299 TRACE("GetClientDriverName... return False");
300 return False;
301 }
302
303 *ddxDriverMajorVersion = rep.ddxDriverMajorVersion;
304 *ddxDriverMinorVersion = rep.ddxDriverMinorVersion;
305 *ddxDriverPatchVersion = rep.ddxDriverPatchVersion;
306
307 if (rep.length) {
308 if (!
309 (*clientDriverName =
310 calloc(rep.clientDriverNameLength + 1, 1))) {
311 _XEatData(dpy, ((rep.clientDriverNameLength + 3) & ~3));
312 UnlockDisplay(dpy);
313 SyncHandle();
314 TRACE("GetClientDriverName... return False");
315 return False;
316 }
317 _XReadPad(dpy, *clientDriverName, rep.clientDriverNameLength);
318 }
319 else {
320 *clientDriverName = NULL;
321 }
322 UnlockDisplay(dpy);
323 SyncHandle();
324 TRACE("GetClientDriverName... return True");
325 return True;
326 }
327
328 Bool
329 XF86DRICreateContextWithConfig(Display * dpy, int screen, int configID,
330 XID * context, drm_context_t * hHWContext)
331 {
332 XExtDisplayInfo *info = find_display(dpy);
333 xXF86DRICreateContextReply rep;
334 xXF86DRICreateContextReq *req;
335
336 TRACE("CreateContext...");
337 XF86DRICheckExtension(dpy, info, False);
338
339 LockDisplay(dpy);
340 GetReq(XF86DRICreateContext, req);
341 req->reqType = info->codes->major_opcode;
342 req->driReqType = X_XF86DRICreateContext;
343 req->visual = configID;
344 req->screen = screen;
345 *context = XAllocID(dpy);
346 req->context = *context;
347 if (!_XReply(dpy, (xReply *) & rep, 0, xFalse)) {
348 UnlockDisplay(dpy);
349 SyncHandle();
350 TRACE("CreateContext... return False");
351 return False;
352 }
353 *hHWContext = rep.hHWContext;
354 UnlockDisplay(dpy);
355 SyncHandle();
356 TRACE("CreateContext... return True");
357 return True;
358 }
359
360 Bool
361 XF86DRICreateContext(Display * dpy, int screen, Visual * visual,
362 XID * context, drm_context_t * hHWContext)
363 {
364 return XF86DRICreateContextWithConfig(dpy, screen, visual->visualid,
365 context, hHWContext);
366 }
367
368 Bool
369 XF86DRIDestroyContext(Display * dpy, int screen, XID context)
370 {
371 XExtDisplayInfo *info = find_display(dpy);
372 xXF86DRIDestroyContextReq *req;
373
374 TRACE("DestroyContext...");
375 XF86DRICheckExtension(dpy, info, False);
376
377 LockDisplay(dpy);
378 GetReq(XF86DRIDestroyContext, req);
379 req->reqType = info->codes->major_opcode;
380 req->driReqType = X_XF86DRIDestroyContext;
381 req->screen = screen;
382 req->context = context;
383 UnlockDisplay(dpy);
384 SyncHandle();
385 TRACE("DestroyContext... return True");
386 return True;
387 }
388
389 Bool
390 XF86DRICreateDrawable(Display * dpy, int screen,
391 XID drawable, drm_drawable_t * hHWDrawable)
392 {
393 XExtDisplayInfo *info = find_display(dpy);
394 xXF86DRICreateDrawableReply rep;
395 xXF86DRICreateDrawableReq *req;
396
397 TRACE("CreateDrawable...");
398 XF86DRICheckExtension(dpy, info, False);
399
400 LockDisplay(dpy);
401 GetReq(XF86DRICreateDrawable, req);
402 req->reqType = info->codes->major_opcode;
403 req->driReqType = X_XF86DRICreateDrawable;
404 req->screen = screen;
405 req->drawable = drawable;
406 if (!_XReply(dpy, (xReply *) & rep, 0, xFalse)) {
407 UnlockDisplay(dpy);
408 SyncHandle();
409 TRACE("CreateDrawable... return False");
410 return False;
411 }
412 *hHWDrawable = rep.hHWDrawable;
413 UnlockDisplay(dpy);
414 SyncHandle();
415 TRACE("CreateDrawable... return True");
416 return True;
417 }
418
419 static int
420 noopErrorHandler(Display * dpy, XErrorEvent * xerr)
421 {
422 return 0;
423 }
424
425 Bool
426 XF86DRIDestroyDrawable(Display * dpy, int screen, XID drawable)
427 {
428 XExtDisplayInfo *info = find_display(dpy);
429 xXF86DRIDestroyDrawableReq *req;
430 int (*oldXErrorHandler) (Display *, XErrorEvent *);
431
432 TRACE("DestroyDrawable...");
433 XF86DRICheckExtension(dpy, info, False);
434
435 /* This is called from the DRI driver, which used call it like this
436 *
437 * if (windowExists(drawable))
438 * destroyDrawable(drawable);
439 *
440 * which is a textbook race condition - the window may disappear
441 * from the server between checking for its existance and
442 * destroying it. Instead we change the semantics of
443 * __DRIinterfaceMethodsRec::destroyDrawable() to succeed even if
444 * the windows is gone, by wrapping the destroy call in an error
445 * handler. */
446
447 XSync(dpy, False);
448 oldXErrorHandler = XSetErrorHandler(noopErrorHandler);
449
450 LockDisplay(dpy);
451 GetReq(XF86DRIDestroyDrawable, req);
452 req->reqType = info->codes->major_opcode;
453 req->driReqType = X_XF86DRIDestroyDrawable;
454 req->screen = screen;
455 req->drawable = drawable;
456 UnlockDisplay(dpy);
457 SyncHandle();
458
459 XSetErrorHandler(oldXErrorHandler);
460
461 TRACE("DestroyDrawable... return True");
462 return True;
463 }
464
465 Bool
466 XF86DRIGetDrawableInfo(Display * dpy, int screen, Drawable drawable,
467 unsigned int *index, unsigned int *stamp,
468 int *X, int *Y, int *W, int *H,
469 int *numClipRects, drm_clip_rect_t ** pClipRects,
470 int *backX, int *backY,
471 int *numBackClipRects,
472 drm_clip_rect_t ** pBackClipRects)
473 {
474 XExtDisplayInfo *info = find_display(dpy);
475 xXF86DRIGetDrawableInfoReply rep;
476 xXF86DRIGetDrawableInfoReq *req;
477 int total_rects;
478
479 TRACE("GetDrawableInfo...");
480 XF86DRICheckExtension(dpy, info, False);
481
482 LockDisplay(dpy);
483 GetReq(XF86DRIGetDrawableInfo, req);
484 req->reqType = info->codes->major_opcode;
485 req->driReqType = X_XF86DRIGetDrawableInfo;
486 req->screen = screen;
487 req->drawable = drawable;
488
489 if (!_XReply(dpy, (xReply *) & rep, 1, xFalse)) {
490 UnlockDisplay(dpy);
491 SyncHandle();
492 TRACE("GetDrawableInfo... return False");
493 return False;
494 }
495 *index = rep.drawableTableIndex;
496 *stamp = rep.drawableTableStamp;
497 *X = (int) rep.drawableX;
498 *Y = (int) rep.drawableY;
499 *W = (int) rep.drawableWidth;
500 *H = (int) rep.drawableHeight;
501 *numClipRects = rep.numClipRects;
502 total_rects = *numClipRects;
503
504 *backX = rep.backX;
505 *backY = rep.backY;
506 *numBackClipRects = rep.numBackClipRects;
507 total_rects += *numBackClipRects;
508
509 #if 0
510 /* Because of the fix in Xserver/GL/dri/xf86dri.c, this check breaks
511 * backwards compatibility (Because of the >> 2 shift) but the fix
512 * enables multi-threaded apps to work.
513 */
514 if (rep.length != ((((SIZEOF(xXF86DRIGetDrawableInfoReply) -
515 SIZEOF(xGenericReply) +
516 total_rects * sizeof(drm_clip_rect_t)) +
517 3) & ~3) >> 2)) {
518 _XEatData(dpy, rep.length);
519 UnlockDisplay(dpy);
520 SyncHandle();
521 TRACE("GetDrawableInfo... return False");
522 return False;
523 }
524 #endif
525
526 if (*numClipRects) {
527 int len = sizeof(drm_clip_rect_t) * (*numClipRects);
528
529 *pClipRects = calloc(len, 1);
530 if (*pClipRects)
531 _XRead(dpy, (char *) *pClipRects, len);
532 }
533 else {
534 *pClipRects = NULL;
535 }
536
537 if (*numBackClipRects) {
538 int len = sizeof(drm_clip_rect_t) * (*numBackClipRects);
539
540 *pBackClipRects = calloc(len, 1);
541 if (*pBackClipRects)
542 _XRead(dpy, (char *) *pBackClipRects, len);
543 }
544 else {
545 *pBackClipRects = NULL;
546 }
547
548 UnlockDisplay(dpy);
549 SyncHandle();
550 TRACE("GetDrawableInfo... return True");
551 return True;
552 }
553
554 Bool
555 XF86DRIGetDeviceInfo(Display * dpy, int screen, drm_handle_t * hFrameBuffer,
556 int *fbOrigin, int *fbSize, int *fbStride,
557 int *devPrivateSize, void **pDevPrivate)
558 {
559 XExtDisplayInfo *info = find_display(dpy);
560 xXF86DRIGetDeviceInfoReply rep;
561 xXF86DRIGetDeviceInfoReq *req;
562
563 TRACE("GetDeviceInfo...");
564 XF86DRICheckExtension(dpy, info, False);
565
566 LockDisplay(dpy);
567 GetReq(XF86DRIGetDeviceInfo, req);
568 req->reqType = info->codes->major_opcode;
569 req->driReqType = X_XF86DRIGetDeviceInfo;
570 req->screen = screen;
571 if (!_XReply(dpy, (xReply *) & rep, 0, xFalse)) {
572 UnlockDisplay(dpy);
573 SyncHandle();
574 TRACE("GetDeviceInfo... return False");
575 return False;
576 }
577
578 *hFrameBuffer = rep.hFrameBufferLow;
579 if (sizeof(drm_handle_t) == 8) {
580 int shift = 32; /* var to prevent warning on next line */
581 *hFrameBuffer |= ((drm_handle_t) rep.hFrameBufferHigh) << shift;
582 }
583
584 *fbOrigin = rep.framebufferOrigin;
585 *fbSize = rep.framebufferSize;
586 *fbStride = rep.framebufferStride;
587 *devPrivateSize = rep.devPrivateSize;
588
589 if (rep.length) {
590 if (!(*pDevPrivate = calloc(rep.devPrivateSize, 1))) {
591 _XEatData(dpy, ((rep.devPrivateSize + 3) & ~3));
592 UnlockDisplay(dpy);
593 SyncHandle();
594 TRACE("GetDeviceInfo... return False");
595 return False;
596 }
597 _XRead(dpy, (char *) *pDevPrivate, rep.devPrivateSize);
598 }
599 else {
600 *pDevPrivate = NULL;
601 }
602
603 UnlockDisplay(dpy);
604 SyncHandle();
605 TRACE("GetDeviceInfo... return True");
606 return True;
607 }
608
609 Bool
610 XF86DRIOpenFullScreen(Display * dpy, int screen, Drawable drawable)
611 {
612 /* This function and the underlying X protocol are deprecated.
613 */
614 (void) dpy;
615 (void) screen;
616 (void) drawable;
617 return False;
618 }
619
620 Bool
621 XF86DRICloseFullScreen(Display * dpy, int screen, Drawable drawable)
622 {
623 /* This function and the underlying X protocol are deprecated.
624 */
625 (void) dpy;
626 (void) screen;
627 (void) drawable;
628 return True;
629 }
630
631 #endif /* GLX_DIRECT_RENDERING */