src/glut/glx/glut_overlay.c

   1
   2 /* Copyright (c) Mark J. Kilgard, 1996, 1997.  */
   3
   4 /* This program is freely distributable without licensing fees
   5    and is provided without guarantee or warrantee expressed or
   6    implied. This program is -not- in the public domain. */
   7
   8 #ifdef __VMS
   9 #include <GL/vms_x_fix.h>
  10 #endif
  11
  12 #include <stdlib.h>
  13 #include <stdio.h>
  14 #include <string.h>
  15 #include <assert.h>
  16
  17 #if !defined(_WIN32)
  18 #include <X11/Xlib.h>
  19 #include <X11/Xutil.h>
  20 #if defined (__vms)
  21 #include <Xmu/StdCmap.h>  /* for XmuLookupStandardColormap */
  22 #else
  23 #include <X11/Xmu/StdCmap.h>  /* for XmuLookupStandardColormap */
  24 #endif
  25 #endif /* !_WIN32 */
  26
  27 #include "glutint.h"
  28 #include "layerutil.h"
  29
  30 static Criterion requiredOverlayCriteria[] =
  31 {
  32   {LEVEL, EQ, 1},       /* This entry gets poked in
  33                            determineOverlayVisual. */
  34   {TRANSPARENT, EQ, 1},
  35   {XPSEUDOCOLOR, EQ, 1},
  36   {RGBA, EQ, 0},
  37   {BUFFER_SIZE, GTE, 1}
  38 };
  39 static int numRequiredOverlayCriteria = sizeof(requiredOverlayCriteria) / sizeof(Criterion);
  40 static int requiredOverlayCriteriaMask =
  41 (1 << LEVEL) | (1 << TRANSPARENT) | (1 << XSTATICGRAY) | (1 << RGBA) | (1 << CI_MODE);
  42
  43 #if !defined(_WIN32)
  44 static int
  45 checkOverlayAcceptability(XVisualInfo * vi, unsigned int mode)
  46 {
  47   int value;
  48
  49   /* Must support OpenGL. */
  50   glXGetConfig(__glutDisplay, vi, GLX_USE_GL, &value);
  51   if (!value)
  52     return 1;
  53
  54   /* Must be color index. */
  55   glXGetConfig(__glutDisplay, vi, GLX_RGBA, &value);
  56   if (value)
  57     return 1;
  58
  59   /* Must match single/double buffering request. */
  60   glXGetConfig(__glutDisplay, vi, GLX_DOUBLEBUFFER, &value);
  61   if (GLUT_WIND_IS_DOUBLE(mode) != (value != 0))
  62     return 1;
  63
  64   /* Must match mono/stereo request. */
  65   glXGetConfig(__glutDisplay, vi, GLX_STEREO, &value);
  66   if (GLUT_WIND_IS_STEREO(mode) != (value != 0))
  67     return 1;
  68
  69   /* Alpha and accumulation buffers incompatible with color
  70      index. */
  71   if (GLUT_WIND_HAS_ALPHA(mode) || GLUT_WIND_HAS_ACCUM(mode))
  72     return 1;
  73
  74   /* Look for depth buffer if requested. */
  75   glXGetConfig(__glutDisplay, vi, GLX_DEPTH_SIZE, &value);
  76   if (GLUT_WIND_HAS_DEPTH(mode) && (value <= 0))
  77     return 1;
  78
  79   /* Look for stencil buffer if requested. */
  80   glXGetConfig(__glutDisplay, vi, GLX_STENCIL_SIZE, &value);
  81   if (GLUT_WIND_HAS_STENCIL(mode) && (value <= 0))
  82     return 1;
  83
  84 #if defined(GLX_VERSION_1_1) && (defined(GLX_SGIS_multisample) || defined(GLX_ARB_multisample))
  85   /* XXX Multisampled overlay color index??  Pretty unlikely. */
  86   /* Look for multisampling if requested. */
  87   if (__glutIsSupportedByGLX("GLX_SGIS_multisample") ||
  88       __glutIsSupportedByGLX("GLX_ARB_multisample"))
  89     glXGetConfig(__glutDisplay, vi, GLX_SAMPLES_SGIS, &value);
  90   else
  91     value = 0;
  92   if (GLUT_WIND_IS_MULTISAMPLE(mode) && (value <= 0))
  93     return 1;
  94 #endif
  95
  96   return 0;
  97 }
  98 #endif
  99
 100 static XVisualInfo *
 101 getOverlayVisualInfoCI(unsigned int mode)
 102 {
 103 #if !defined(_WIN32)
 104   XLayerVisualInfo *vi;
 105   XLayerVisualInfo template;
 106   XVisualInfo *goodVisual, *returnVisual;
 107   int nitems, i, j, bad;
 108
 109   /* The GLX 1.0 glXChooseVisual is does not permit queries
 110      based on pixel transparency (and GLX_BUFFER_SIZE uses
 111      "smallest that meets" its requirement instead of "largest
 112      that meets" that GLUT wants. So, GLUT implements its own
 113      visual selection routine for color index overlays. */
 114
 115   /* Try three overlay layers. */
 116   for (i = 1; i <= 3; i++) {
 117     template.vinfo.screen = __glutScreen;
 118     template.vinfo.class = PseudoColor;
 119     template.layer = i;
 120     template.type = TransparentPixel;
 121     vi = __glutXGetLayerVisualInfo(__glutDisplay,
 122       VisualTransparentType | VisualScreenMask | VisualClassMask | VisualLayerMask,
 123       &template, &nitems);
 124     if (vi) {
 125       /* Check list for acceptable visual meeting requirements
 126          of requested display mode. */
 127       for (j = 0; j < nitems; j++) {
 128         bad = checkOverlayAcceptability(&vi[j].vinfo, mode);
 129         if (bad) {
 130           /* Set vi[j].vinfo.visual to mark it unacceptable. */
 131           vi[j].vinfo.visual = NULL;
 132         }
 133       }
 134
 135       /* Look through list to find deepest acceptable visual. */
 136       goodVisual = NULL;
 137       for (j = 0; j < nitems; j++) {
 138         if (vi[j].vinfo.visual) {
 139           if (goodVisual == NULL) {
 140             goodVisual = &vi[j].vinfo;
 141           } else {
 142             if (goodVisual->depth < vi[j].vinfo.depth) {
 143               goodVisual = &vi[j].vinfo;
 144             }
 145           }
 146         }
 147       }
 148
 149       /* If a visual is found, clean up and return the visual. */
 150       if (goodVisual) {
 151         returnVisual = (XVisualInfo *) malloc(sizeof(XVisualInfo));
 152         if (returnVisual) {
 153           *returnVisual = *goodVisual;
 154         }
 155         XFree(vi);
 156         return returnVisual;
 157       }
 158       XFree(vi);
 159     }
 160   }
 161 #endif /* !_WIN32 */
 162   return NULL;
 163 }
 164
 165 /* ARGSUSED */
 166 static XVisualInfo *
 167 getOverlayVisualInfoRGB(unsigned int mode)
 168 {
 169
 170   /* XXX For now, transparent RGBA overlays are not supported
 171      by GLUT.  RGBA overlays raise difficult questions about
 172      what the transparent pixel (really color) value should be.
 173
 174      Color index overlay transparency is "easy" because the
 175      transparent pixel value does not affect displayable colors
 176      (except for stealing one color cell) since colors are
 177      determined by indirection through a colormap, and because
 178      it is uncommon for arbitrary pixel values in color index to
 179      be "calculated" (as can occur with a host of RGBA operations
 180      like lighting, blending, etc) so it is easy to avoid the
 181      transparent pixel value.
 182
 183      Since it is typically easy to avoid the transparent pixel
 184      value in color index mode, if GLUT tells the programmer what
 185      pixel is transparent, then most program can easily avoid
 186      generating that pixel value except when they intend
 187      transparency.  GLUT returns whatever transparent pixel value
 188      is provided by the system through glutGet(
 189      GLUT_TRANSPARENT_INDEX).
 190
 191      Theory versus practice for RGBA overlay transparency: In
 192      theory, the reasonable thing is enabling overlay transparency
 193      when an overlay pixel's destination alpha is 0 because this
 194      allows overlay transparency to be controlled via alpha and all
 195      visibile colors are permited, but no hardware I am aware of
 196      supports this practice (and it requires destination alpha which
 197      is typically optional and quite uncommon for overlay windows!).
 198
 199      In practice, the choice of  transparent pixel value is typically
 200      "hardwired" into most graphics hardware to a single pixel value.
 201      SGI hardware uses true black (0,0,0) without regard for the
 202      destination alpha.  This is far from ideal because true black (a
 203      common color that is easy to accidently generate) can not be
 204      generated in an RGBA overlay. I am not sure what other vendors
 205      do.
 206
 207      Pragmatically, most of the typical things you want to do in the
 208      overlays can be done in color index (rubber banding, pop-up
 209      menus, etc.).  One solution for GLUT would be to simply
 210      "advertise" what RGB triple (or possibly RGBA quadruple or simply
 211      A alone) generates transparency.  The problem with this approach
 212      is that it forces programmers to avoid whatever arbitrary color
 213      various systems decide is transparent.  This is a difficult
 214      burden to place on programmers that want to portably make use of
 215      overlays.
 216
 217      To actually support transparent RGBA overlays, there are really
 218      two reaonsable options.  ONE: Simply mandate that true black is
 219      the RGBA overlay transparent color (what IRIS GL did).  This is
 220      nice for programmers since only one option, nice for existing SGI
 221      hardware, bad for anyone (including SGI) who wants to improve
 222      upon "true black" RGB transparency.
 223
 224      Or TWO: Provide a set of queriable "transparency types" (like
 225      "true black" or "alpha == 0" or "true white" or even a queriable
 226      transparent color).  This is harder for programmers, OK for
 227      existing SGI hardware, and it leaves open the issue of what other
 228      modes are reasonable.
 229
 230      Option TWO seems the more general approach, but since hardware
 231      designers will likely only implement a single mode (this is a
 232      scan out issue where bandwidth is pressing issue), codifying
 233      multiple speculative approaches nobody may ever implement seems
 234      silly.  And option ONE fiats a suboptimal solution.
 235
 236      Therefore, I defer any decision of how GLUT should support RGBA
 237      overlay transparency and leave support for it unimplemented.
 238      Nobody has been pressing me for RGBA overlay transparency (though
 239      people have requested color index overlay transparency
 240      repeatedly).  Geez, if you read this far you are either really
 241      bored or maybe actually  interested in this topic.  Anyway, if
 242      you have ideas (particularly if you plan on implementing a
 243      hardware scheme for RGBA overlay transparency), I'd be
 244      interested.
 245
 246      For the record, SGI's expiremental Framebufer Configuration
 247      experimental GLX extension uses option TWO.  Transparency modes
 248      for "none" and "RGB" are defined (others could be defined later).
 249      What RGB value is the transparent one must be queried.
 250
 251      I was hoping GLUT could have something that required less work
 252      from the programmer to use portably. -mjk */
 253
 254   __glutWarning("RGBA overlays are not supported by GLUT (for now).");
 255   return NULL;
 256 }
 257
 258 static XVisualInfo *
 259 getOverlayVisualInfo(unsigned int mode)
 260 {
 261   /* XXX GLUT_LUMINANCE not implemented for GLUT 3.0. */
 262   if (GLUT_WIND_IS_LUMINANCE(mode))
 263     return NULL;
 264
 265   if (GLUT_WIND_IS_RGB(mode))
 266     return getOverlayVisualInfoRGB(mode);
 267   else
 268     return getOverlayVisualInfoCI(mode);
 269 }
 270
 271 #if !defined(_WIN32)
 272
 273 /* The GLUT overlay can come and go, and the overlay window has
 274    a distinct X window ID.  Logically though, GLUT treats the
 275    normal and overlay windows as a unified window.  In
 276    particular, X input events typically go to the overlay window
 277    since it is "on top of" the normal window.  When an overlay
 278    window ID is destroyed (due to glutRemoveOverlay or a call to
 279    glutEstablishOverlay when an overlay already exists), we
 280    still keep track of the overlay window ID until we get back a
 281    DestroyNotify event for the overlay window. Otherwise, we
 282    could lose track of X input events sent to a destroyed
 283    overlay.  To avoid this, we keep the destroyed overlay window
 284    ID on a "stale window" list.  This lets us properly route X
 285    input events generated on destroyed overlay windows to the
 286    proper GLUT window. */
 287 static void
 288 addStaleWindow(GLUTwindow * window, Window win)
 289 {
 290   GLUTstale *entry;
 291
 292   entry = (GLUTstale *) malloc(sizeof(GLUTstale));
 293   if (!entry)
 294     __glutFatalError("out of memory");
 295   entry->window = window;
 296   entry->win = win;
 297   entry->next = __glutStaleWindowList;
 298   __glutStaleWindowList = entry;
 299 }
 300
 301 #endif
 302
 303 void
 304 __glutFreeOverlay(GLUToverlay * overlay)
 305 {
 306   if (overlay->visAlloced)
 307     XFree(overlay->vis);
 308   XDestroyWindow(__glutDisplay, overlay->win);
 309   glXDestroyContext(__glutDisplay, overlay->ctx);
 310   if (overlay->colormap) {
 311     /* Only color index overlays have colormap data structure. */
 312     __glutFreeColormap(overlay->colormap);
 313   }
 314   free(overlay);
 315 }
 316
 317 static XVisualInfo *
 318 determineOverlayVisual(int *treatAsSingle, Bool * visAlloced, void **fbc)
 319 {
 320   if (__glutDisplayString) {
 321     XVisualInfo *vi;
 322     int i;
 323
 324     /* __glutDisplayString should be NULL except if
 325        glutInitDisplayString has been called to register a
 326        different display string.  Calling glutInitDisplayString
 327        means using a string instead of an integer mask determine
 328
 329        the visual to use. Using the function pointer variable
 330        __glutDetermineVisualFromString below avoids linking in
 331        the code for implementing glutInitDisplayString (ie,
 332        glut_dstr.o) unless glutInitDisplayString gets called by
 333        the application. */
 334
 335     assert(__glutDetermineVisualFromString);
 336
 337     /* Try three overlay layers. */
 338     *visAlloced = False;
 339     *fbc = NULL;
 340     for (i = 1; i <= 3; i++) {
 341       requiredOverlayCriteria[0].value = i;
 342       vi = __glutDetermineVisualFromString(__glutDisplayString, treatAsSingle,
 343         requiredOverlayCriteria, numRequiredOverlayCriteria,
 344         requiredOverlayCriteriaMask, fbc);
 345       if (vi) {
 346         return vi;
 347       }
 348     }
 349     return NULL;
 350   } else {
 351     *visAlloced = True;
 352     *fbc = NULL;
 353     return __glutDetermineVisual(__glutDisplayMode,
 354       treatAsSingle, getOverlayVisualInfo);
 355   }
 356 }
 357
 358 /* CENTRY */
 359 void GLUTAPIENTRY
 360 glutEstablishOverlay(void)
 361 {
 362   GLUToverlay *overlay;
 363   GLUTwindow *window;
 364   XSetWindowAttributes wa;
 365   void *fbc;
 366
 367   /* Register a routine to free an overlay with glut_win.c;
 368      this keeps glut_win.c from pulling in all of
 369      glut_overlay.c when no overlay functionality is used by
 370      the application. */
 371   __glutFreeOverlayFunc = __glutFreeOverlay;
 372
 373   window = __glutCurrentWindow;
 374
 375   /* Allow for an existant overlay to be re-established perhaps
 376      if you wanted a different display mode. */
 377   if (window->overlay) {
 378 #if !defined(_WIN32)
 379     addStaleWindow(window, window->overlay->win);
 380 #endif
 381     __glutFreeOverlay(window->overlay);
 382   }
 383   overlay = (GLUToverlay *) malloc(sizeof(GLUToverlay));
 384   if (!overlay)
 385     __glutFatalError("out of memory.");
 386
 387   overlay->vis = determineOverlayVisual(&overlay->treatAsSingle,
 388     &overlay->visAlloced, &fbc);
 389   if (!overlay->vis) {
 390     __glutFatalError("lacks overlay support.");
 391   }
 392   overlay->ctx = NULL;
 393 #if defined(GLX_VERSION_1_1) && defined(GLX_SGIX_fbconfig)
 394   if (fbc) {
 395     window->ctx = __glut_glXCreateContextWithConfigSGIX(__glutDisplay, fbc,
 396       GLX_RGBA_TYPE_SGIX, None, __glutTryDirect);
 397   } else
 398 #endif
 399   {
 400     overlay->ctx = glXCreateContext(__glutDisplay, overlay->vis,
 401       None, __glutTryDirect);
 402   }
 403   if (!overlay->ctx) {
 404     __glutFatalError(
 405       "failed to create overlay OpenGL rendering context.");
 406   }
 407 #if !defined(_WIN32)
 408   overlay->isDirect = glXIsDirect(__glutDisplay, overlay->ctx);
 409   if (__glutForceDirect) {
 410     if (!overlay->isDirect) {
 411       __glutFatalError("direct rendering not possible.");
 412     }
 413   }
 414 #endif
 415   __glutSetupColormap(overlay->vis, &overlay->colormap, &overlay->cmap);
 416   overlay->transparentPixel = __glutGetTransparentPixel(__glutDisplay,
 417     overlay->vis);
 418   wa.colormap = overlay->cmap;
 419   wa.background_pixel = overlay->transparentPixel;
 420   wa.event_mask = window->eventMask & GLUT_OVERLAY_EVENT_FILTER_MASK;
 421   wa.border_pixel = 0;
 422 #if defined(_WIN32)
 423   /* XXX Overlays not supported in Win32 yet. */
 424 #else
 425   overlay->win = XCreateWindow(__glutDisplay,
 426     window->win,
 427     0, 0, window->width, window->height, 0,
 428     overlay->vis->depth, InputOutput, overlay->vis->visual,
 429     CWBackPixel | CWBorderPixel | CWEventMask | CWColormap,
 430     &wa);
 431 #endif
 432   if (window->children) {
 433     /* Overlay window must be lowered below any GLUT
 434        subwindows. */
 435     XLowerWindow(__glutDisplay, overlay->win);
 436   }
 437   XMapWindow(__glutDisplay, overlay->win);
 438   overlay->shownState = 1;
 439
 440   overlay->display = NULL;
 441
 442   /* Make sure a reshape gets delivered. */
 443   window->forceReshape = True;
 444
 445 #if !defined(_WIN32)
 446   __glutPutOnWorkList(__glutToplevelOf(window), GLUT_COLORMAP_WORK);
 447 #endif
 448
 449   window->overlay = overlay;
 450   glutUseLayer(GLUT_OVERLAY);
 451
 452   if (overlay->treatAsSingle) {
 453     glDrawBuffer(GL_FRONT);
 454     glReadBuffer(GL_FRONT);
 455   }
 456 }
 457
 458 void GLUTAPIENTRY
 459 glutRemoveOverlay(void)
 460 {
 461   GLUTwindow *window = __glutCurrentWindow;
 462   GLUToverlay *overlay = __glutCurrentWindow->overlay;
 463
 464   if (!window->overlay)
 465     return;
 466
 467   /* If using overlay, switch to the normal layer. */
 468   if (window->renderWin == overlay->win) {
 469     glutUseLayer(GLUT_NORMAL);
 470   }
 471 #if !defined(_WIN32)
 472   addStaleWindow(window, overlay->win);
 473 #endif
 474   __glutFreeOverlay(overlay);
 475   window->overlay = NULL;
 476 #if !defined(_WIN32)
 477   __glutPutOnWorkList(__glutToplevelOf(window), GLUT_COLORMAP_WORK);
 478 #endif
 479 }
 480
 481 void GLUTAPIENTRY
 482 glutUseLayer(GLenum layer)
 483 {
 484   GLUTwindow *window = __glutCurrentWindow;
 485
 486   switch (layer) {
 487   case GLUT_NORMAL:
 488 #ifdef _WIN32
 489     window->renderDc = window->hdc;
 490 #endif
 491     window->renderWin = window->win;
 492     window->renderCtx = window->ctx;
 493     break;
 494   case GLUT_OVERLAY:
 495     /* Did you crash here?  Calling glutUseLayer(GLUT_OVERLAY)
 496        without an overlay established is erroneous.  Fix your
 497        code. */
 498 #ifdef _WIN32
 499     window->renderDc = window->overlay->hdc;
 500 #endif
 501     window->renderWin = window->overlay->win;
 502     window->renderCtx = window->overlay->ctx;
 503     break;
 504   default:
 505     __glutWarning("glutUseLayer: unknown layer, %d.", layer);
 506     break;
 507   }
 508   __glutSetWindow(window);
 509 }
 510
 511 void GLUTAPIENTRY
 512 glutPostOverlayRedisplay(void)
 513 {
 514   __glutPostRedisplay(__glutCurrentWindow, GLUT_OVERLAY_REDISPLAY_WORK);
 515 }
 516
 517 /* The advantage of this routine is that it saves the cost of a
 518    glutSetWindow call (entailing an expensive OpenGL context
 519    switch), particularly useful when multiple windows need
 520    redisplays posted at the same times. */
 521 void GLUTAPIENTRY
 522 glutPostWindowOverlayRedisplay(int win)
 523 {
 524   __glutPostRedisplay(__glutWindowList[win - 1], GLUT_OVERLAY_REDISPLAY_WORK);
 525 }
 526
 527 void GLUTAPIENTRY
 528 glutOverlayDisplayFunc(GLUTdisplayCB displayFunc)
 529 {
 530   if (!__glutCurrentWindow->overlay) {
 531     __glutWarning("glutOverlayDisplayFunc: window has no overlay established");
 532     return;
 533   }
 534   __glutCurrentWindow->overlay->display = displayFunc;
 535 }
 536
 537 void GLUTAPIENTRY
 538 glutHideOverlay(void)
 539 {
 540   if (!__glutCurrentWindow->overlay) {
 541     __glutWarning("glutHideOverlay: window has no overlay established");
 542     return;
 543   }
 544   XUnmapWindow(__glutDisplay, __glutCurrentWindow->overlay->win);
 545   __glutCurrentWindow->overlay->shownState = 0;
 546 }
 547
 548 void GLUTAPIENTRY
 549 glutShowOverlay(void)
 550 {
 551   if (!__glutCurrentWindow->overlay) {
 552     __glutWarning("glutShowOverlay: window has no overlay established");
 553     return;
 554   }
 555   XMapWindow(__glutDisplay, __glutCurrentWindow->overlay->win);
 556   __glutCurrentWindow->overlay->shownState = 1;
 557 }
 558
 559 int GLUTAPIENTRY
 560 glutLayerGet(GLenum param)
 561 {
 562   switch (param) {
 563   case GLUT_OVERLAY_POSSIBLE:
 564     {
 565       XVisualInfo *vi;
 566       Bool dummy, visAlloced;
 567       void *fbc;
 568
 569       vi = determineOverlayVisual(&dummy, &visAlloced, &fbc);
 570       if (vi) {
 571         if (visAlloced)
 572           XFree(vi);
 573         return 1;
 574       }
 575       return 0;
 576     }
 577   case GLUT_LAYER_IN_USE:
 578     return __glutCurrentWindow->renderWin != __glutCurrentWindow->win;
 579   case GLUT_HAS_OVERLAY:
 580     return __glutCurrentWindow->overlay != NULL;
 581   case GLUT_TRANSPARENT_INDEX:
 582     if (__glutCurrentWindow->overlay) {
 583       return __glutCurrentWindow->overlay->transparentPixel;
 584     } else {
 585       return -1;
 586     }
 587   case GLUT_NORMAL_DAMAGED:
 588     /* __glutWindowDamaged is used so the damage state within
 589        the window (or overlay belwo) can be cleared before
 590        calling a display callback so on return, the state does
 591        not have to be cleared (since upon return from the
 592        callback the window could be destroyed (or layer
 593        removed). */
 594     return (__glutCurrentWindow->workMask & GLUT_REPAIR_WORK)
 595       || __glutWindowDamaged;
 596   case GLUT_OVERLAY_DAMAGED:
 597     if (__glutCurrentWindow->overlay) {
 598       return (__glutCurrentWindow->workMask & GLUT_OVERLAY_REPAIR_WORK)
 599         || __glutWindowDamaged;
 600     } else {
 601       return -1;
 602     }
 603   default:
 604     __glutWarning("invalid glutLayerGet param: %d", param);
 605     return -1;
 606   }
 607 }
 608 /* ENDCENTRY */