src/glut/glx/glut_overlay.c

   1
   2 /* Copyright (c) Mark J. Kilgard, 1996, 1997.  */
   3
   4 /* This program is freely distributable without licensing fees
   5    and is provided without guarantee or warrantee expressed or
   6    implied. This program is -not- in the public domain. */
   7
   8 #ifdef __VMS
   9 #include <GL/vms_x_fix.h>
  10 #endif
  11
  12 #include <stdlib.h>
  13 #include <stdio.h>
  14 #include <string.h>
  15 #include <assert.h>
  16
  17 #if !defined(_WIN32)
  18 #include <X11/Xlib.h>
  19 #include <X11/Xutil.h>
  20 #include <X11/Xatom.h>  /* for XA_RGB_DEFAULT_MAP atom */
  21 #if defined (__vms)
  22 #include <Xmu/StdCmap.h>  /* for XmuLookupStandardColormap */
  23 #else
  24 #include <X11/Xmu/StdCmap.h>  /* for XmuLookupStandardColormap */
  25 #endif
  26 #endif /* !_WIN32 */
  27
  28 #include "glutint.h"
  29 #include "layerutil.h"
  30
  31 static Criterion requiredOverlayCriteria[] =
  32 {
  33   {LEVEL, EQ, 1},       /* This entry gets poked in
  34                            determineOverlayVisual. */
  35   {TRANSPARENT, EQ, 1},
  36   {XPSEUDOCOLOR, EQ, 1},
  37   {RGBA, EQ, 0},
  38   {BUFFER_SIZE, GTE, 1}
  39 };
  40 static int numRequiredOverlayCriteria = sizeof(requiredOverlayCriteria) / sizeof(Criterion);
  41 static int requiredOverlayCriteriaMask =
  42 (1 << LEVEL) | (1 << TRANSPARENT) | (1 << XSTATICGRAY) | (1 << RGBA) | (1 << CI_MODE);
  43
  44 #if !defined(_WIN32)
  45 static int
  46 checkOverlayAcceptability(XVisualInfo * vi, unsigned int mode)
  47 {
  48   int value;
  49
  50   /* Must support OpenGL. */
  51   glXGetConfig(__glutDisplay, vi, GLX_USE_GL, &value);
  52   if (!value)
  53     return 1;
  54
  55   /* Must be color index. */
  56   glXGetConfig(__glutDisplay, vi, GLX_RGBA, &value);
  57   if (value)
  58     return 1;
  59
  60   /* Must match single/double buffering request. */
  61   glXGetConfig(__glutDisplay, vi, GLX_DOUBLEBUFFER, &value);
  62   if (GLUT_WIND_IS_DOUBLE(mode) != (value != 0))
  63     return 1;
  64
  65   /* Must match mono/stereo request. */
  66   glXGetConfig(__glutDisplay, vi, GLX_STEREO, &value);
  67   if (GLUT_WIND_IS_STEREO(mode) != (value != 0))
  68     return 1;
  69
  70   /* Alpha and accumulation buffers incompatible with color
  71      index. */
  72   if (GLUT_WIND_HAS_ALPHA(mode) || GLUT_WIND_HAS_ACCUM(mode))
  73     return 1;
  74
  75   /* Look for depth buffer if requested. */
  76   glXGetConfig(__glutDisplay, vi, GLX_DEPTH_SIZE, &value);
  77   if (GLUT_WIND_HAS_DEPTH(mode) && (value <= 0))
  78     return 1;
  79
  80   /* Look for stencil buffer if requested. */
  81   glXGetConfig(__glutDisplay, vi, GLX_STENCIL_SIZE, &value);
  82   if (GLUT_WIND_HAS_STENCIL(mode) && (value <= 0))
  83     return 1;
  84
  85 #if defined(GLX_VERSION_1_1) && defined(GLX_SGIS_multisample)
  86   /* XXX Multisampled overlay color index??  Pretty unlikely. */
  87   /* Look for multisampling if requested. */
  88   if (__glutIsSupportedByGLX("GLX_SGIS_multisample"))
  89     glXGetConfig(__glutDisplay, vi, GLX_SAMPLES_SGIS, &value);
  90   else
  91     value = 0;
  92   if (GLUT_WIND_IS_MULTISAMPLE(mode) && (value <= 0))
  93     return 1;
  94 #endif
  95
  96   return 0;
  97 }
  98 #endif
  99
 100 static XVisualInfo *
 101 getOverlayVisualInfoCI(unsigned int mode)
 102 {
 103 #if !defined(_WIN32)
 104   XLayerVisualInfo *vi;
 105   XLayerVisualInfo template;
 106   XVisualInfo *goodVisual, *returnVisual;
 107   int nitems, i, j, bad;
 108
 109   /* The GLX 1.0 glXChooseVisual is does not permit queries
 110      based on pixel transparency (and GLX_BUFFER_SIZE uses
 111      "smallest that meets" its requirement instead of "largest
 112      that meets" that GLUT wants. So, GLUT implements its own
 113      visual selection routine for color index overlays. */
 114
 115   /* Try three overlay layers. */
 116   for (i = 1; i <= 3; i++) {
 117     template.vinfo.screen = __glutScreen;
 118     template.vinfo.class = PseudoColor;
 119     template.layer = i;
 120     template.type = TransparentPixel;
 121     vi = __glutXGetLayerVisualInfo(__glutDisplay,
 122       VisualTransparentType | VisualScreenMask | VisualClassMask | VisualLayerMask,
 123       &template, &nitems);
 124     if (vi) {
 125       /* Check list for acceptable visual meeting requirements
 126          of requested display mode. */
 127       for (j = 0; j < nitems; j++) {
 128         bad = checkOverlayAcceptability(&vi[j].vinfo, mode);
 129         if (bad) {
 130           /* Set vi[j].vinfo.visual to mark it unacceptable. */
 131           vi[j].vinfo.visual = NULL;
 132         }
 133       }
 134
 135       /* Look through list to find deepest acceptable visual. */
 136       goodVisual = NULL;
 137       for (j = 0; j < nitems; j++) {
 138         if (vi[j].vinfo.visual) {
 139           if (goodVisual == NULL) {
 140             goodVisual = &vi[j].vinfo;
 141           } else {
 142             if (goodVisual->depth < vi[j].vinfo.depth) {
 143               goodVisual = &vi[j].vinfo;
 144             }
 145           }
 146         }
 147       }
 148
 149       /* If a visual is found, clean up and return the visual. */
 150       if (goodVisual) {
 151         returnVisual = (XVisualInfo *) malloc(sizeof(XVisualInfo));
 152         if (returnVisual) {
 153           *returnVisual = *goodVisual;
 154         }
 155         XFree(vi);
 156         return returnVisual;
 157       }
 158       XFree(vi);
 159     }
 160   }
 161 #endif /* !_WIN32 */
 162   return NULL;
 163 }
 164
 165 /* ARGSUSED */
 166 static XVisualInfo *
 167 getOverlayVisualInfoRGB(unsigned int mode)
 168 {
 169
 170   /* XXX For now, transparent RGBA overlays are not supported
 171      by GLUT.  RGBA overlays raise difficult questions about
 172      what the transparent pixel (really color) value should be.
 173
 174      Color index overlay transparency is "easy" because the
 175      transparent pixel value does not affect displayable colors
 176      (except for stealing one color cell) since colors are
 177      determined by indirection through a colormap, and because
 178      it is uncommon for arbitrary pixel values in color index to
 179      be "calculated" (as can occur with a host of RGBA operations
 180      like lighting, blending, etc) so it is easy to avoid the
 181      transparent pixel value.
 182
 183      Since it is typically easy to avoid the transparent pixel
 184      value in color index mode, if GLUT tells the programmer what
 185      pixel is transparent, then most program can easily avoid
 186      generating that pixel value except when they intend
 187      transparency.  GLUT returns whatever transparent pixel value
 188      is provided by the system through glutGet(
 189      GLUT_TRANSPARENT_INDEX).
 190
 191      Theory versus practice for RGBA overlay transparency: In
 192      theory, the reasonable thing is enabling overlay transparency
 193      when an overlay pixel's destination alpha is 0 because this
 194      allows overlay transparency to be controlled via alpha and all
 195      visibile colors are permited, but no hardware I am aware of
 196      supports this practice (and it requires destination alpha which
 197      is typically optional and quite uncommon for overlay windows!).
 198
 199      In practice, the choice of  transparent pixel value is typically
 200      "hardwired" into most graphics hardware to a single pixel value.
 201      SGI hardware uses true black (0,0,0) without regard for the
 202      destination alpha.  This is far from ideal because true black (a
 203      common color that is easy to accidently generate) can not be
 204      generated in an RGBA overlay. I am not sure what other vendors
 205      do.
 206
 207      Pragmatically, most of the typical things you want to do in the
 208      overlays can be done in color index (rubber banding, pop-up
 209      menus, etc.).  One solution for GLUT would be to simply
 210      "advertise" what RGB triple (or possibly RGBA quadruple or simply
 211      A alone) generates transparency.  The problem with this approach
 212      is that it forces programmers to avoid whatever arbitrary color
 213      various systems decide is transparent.  This is a difficult
 214      burden to place on programmers that want to portably make use of
 215      overlays.
 216
 217      To actually support transparent RGBA overlays, there are really
 218      two reaonsable options.  ONE: Simply mandate that true black is
 219      the RGBA overlay transparent color (what IRIS GL did).  This is
 220      nice for programmers since only one option, nice for existing SGI
 221      hardware, bad for anyone (including SGI) who wants to improve
 222      upon "true black" RGB transparency.
 223
 224      Or TWO: Provide a set of queriable "transparency types" (like
 225      "true black" or "alpha == 0" or "true white" or even a queriable
 226      transparent color).  This is harder for programmers, OK for
 227      existing SGI hardware, and it leaves open the issue of what other
 228      modes are reasonable.
 229
 230      Option TWO seems the more general approach, but since hardware
 231      designers will likely only implement a single mode (this is a
 232      scan out issue where bandwidth is pressing issue), codifying
 233      multiple speculative approaches nobody may ever implement seems
 234      silly.  And option ONE fiats a suboptimal solution.
 235
 236      Therefore, I defer any decision of how GLUT should support RGBA
 237      overlay transparency and leave support for it unimplemented.
 238      Nobody has been pressing me for RGBA overlay transparency (though
 239      people have requested color index overlay transparency
 240      repeatedly).  Geez, if you read this far you are either really
 241      bored or maybe actually  interested in this topic.  Anyway, if
 242      you have ideas (particularly if you plan on implementing a
 243      hardware scheme for RGBA overlay transparency), I'd be
 244      interested.
 245
 246      For the record, SGI's expiremental Framebufer Configuration
 247      experimental GLX extension uses option TWO.  Transparency modes
 248      for "none" and "RGB" are defined (others could be defined later).
 249      What RGB value is the transparent one must be queried.
 250
 251      I was hoping GLUT could have something that required less work
 252      from the programmer to use portably. -mjk */
 253
 254   __glutWarning("RGBA overlays are not supported by GLUT (for now).");
 255   return NULL;
 256 }
 257
 258 static XVisualInfo *
 259 getOverlayVisualInfo(unsigned int mode)
 260 {
 261   /* XXX GLUT_LUMINANCE not implemented for GLUT 3.0. */
 262   if (GLUT_WIND_IS_LUMINANCE(mode))
 263     return NULL;
 264
 265   if (GLUT_WIND_IS_RGB(mode))
 266     return getOverlayVisualInfoRGB(mode);
 267   else
 268     return getOverlayVisualInfoCI(mode);
 269 }
 270
 271 #if !defined(_WIN32)
 272
 273 /* The GLUT overlay can come and go, and the overlay window has
 274    a distinct X window ID.  Logically though, GLUT treats the
 275    normal and overlay windows as a unified window.  In
 276    particular, X input events typically go to the overlay window
 277    since it is "on top of" the normal window.  When an overlay
 278    window ID is destroyed (due to glutRemoveOverlay or a call to
 279    glutEstablishOverlay when an overlay already exists), we
 280    still keep track of the overlay window ID until we get back a
 281    DestroyNotify event for the overlay window. Otherwise, we
 282    could lose track of X input events sent to a destroyed
 283    overlay.  To avoid this, we keep the destroyed overlay window
 284    ID on a "stale window" list.  This lets us properly route X
 285    input events generated on destroyed overlay windows to the
 286    proper GLUT window. */
 287 static void
 288 addStaleWindow(GLUTwindow * window, Window win)
 289 {
 290   GLUTstale *entry;
 291
 292   entry = (GLUTstale *) malloc(sizeof(GLUTstale));
 293   if (!entry)
 294     __glutFatalError("out of memory");
 295   entry->window = window;
 296   entry->win = win;
 297   entry->next = __glutStaleWindowList;
 298   __glutStaleWindowList = entry;
 299 }
 300
 301 #endif
 302
 303 void
 304 __glutFreeOverlay(GLUToverlay * overlay)
 305 {
 306   if (overlay->visAlloced)
 307     XFree(overlay->vis);
 308   XDestroyWindow(__glutDisplay, overlay->win);
 309   glXDestroyContext(__glutDisplay, overlay->ctx);
 310   if (overlay->colormap) {
 311     /* Only color index overlays have colormap data structure. */
 312     __glutFreeColormap(overlay->colormap);
 313   }
 314   free(overlay);
 315 }
 316
 317 static XVisualInfo *
 318 determineOverlayVisual(int *treatAsSingle, Bool * visAlloced, void **fbc)
 319 {
 320   if (__glutDisplayString) {
 321     XVisualInfo *vi;
 322     int i;
 323
 324     /* __glutDisplayString should be NULL except if
 325        glutInitDisplayString has been called to register a
 326        different display string.  Calling glutInitDisplayString
 327        means using a string instead of an integer mask determine
 328
 329        the visual to use. Using the function pointer variable
 330        __glutDetermineVisualFromString below avoids linking in
 331        the code for implementing glutInitDisplayString (ie,
 332        glut_dstr.o) unless glutInitDisplayString gets called by
 333        the application. */
 334
 335     assert(__glutDetermineVisualFromString);
 336
 337     /* Try three overlay layers. */
 338     *visAlloced = False;
 339     *fbc = NULL;
 340     for (i = 1; i <= 3; i++) {
 341       requiredOverlayCriteria[0].value = i;
 342       vi = __glutDetermineVisualFromString(__glutDisplayString, treatAsSingle,
 343         requiredOverlayCriteria, numRequiredOverlayCriteria,
 344         requiredOverlayCriteriaMask, fbc);
 345       if (vi) {
 346         return vi;
 347       }
 348     }
 349     return NULL;
 350   } else {
 351     *visAlloced = True;
 352     *fbc = NULL;
 353     return __glutDetermineVisual(__glutDisplayMode,
 354       treatAsSingle, getOverlayVisualInfo);
 355   }
 356 }
 357
 358 /* CENTRY */
 359 void GLUTAPIENTRY
 360 glutEstablishOverlay(void)
 361 {
 362   GLUToverlay *overlay;
 363   GLUTwindow *window;
 364   XSetWindowAttributes wa;
 365 #if defined(GLX_VERSION_1_1) && defined(GLX_SGIX_fbconfig)
 366   GLXFBConfigSGIX fbc;
 367 #else
 368   void *fbc;
 369 #endif
 370
 371   /* Register a routine to free an overlay with glut_win.c;
 372      this keeps glut_win.c from pulling in all of
 373      glut_overlay.c when no overlay functionality is used by
 374      the application. */
 375   __glutFreeOverlayFunc = __glutFreeOverlay;
 376
 377   window = __glutCurrentWindow;
 378
 379   /* Allow for an existant overlay to be re-established perhaps
 380      if you wanted a different display mode. */
 381   if (window->overlay) {
 382 #if !defined(_WIN32)
 383     addStaleWindow(window, window->overlay->win);
 384 #endif
 385     __glutFreeOverlay(window->overlay);
 386   }
 387   overlay = (GLUToverlay *) malloc(sizeof(GLUToverlay));
 388   if (!overlay)
 389     __glutFatalError("out of memory.");
 390
 391   overlay->vis = determineOverlayVisual(&overlay->treatAsSingle,
 392     &overlay->visAlloced, (void **) &fbc);
 393   if (!overlay->vis) {
 394     __glutFatalError("lacks overlay support.");
 395   }
 396 #if defined(GLX_VERSION_1_1) && defined(GLX_SGIX_fbconfig)
 397   if (fbc) {
 398     window->ctx = __glut_glXCreateContextWithConfigSGIX(__glutDisplay, fbc,
 399       GLX_RGBA_TYPE_SGIX, None, __glutTryDirect);
 400   } else
 401 #endif
 402   {
 403     overlay->ctx = glXCreateContext(__glutDisplay, overlay->vis,
 404       None, __glutTryDirect);
 405   }
 406   if (!overlay->ctx) {
 407     __glutFatalError(
 408       "failed to create overlay OpenGL rendering context.");
 409   }
 410 #if !defined(_WIN32)
 411   overlay->isDirect = glXIsDirect(__glutDisplay, overlay->ctx);
 412   if (__glutForceDirect) {
 413     if (!overlay->isDirect) {
 414       __glutFatalError("direct rendering not possible.");
 415     }
 416   }
 417 #endif
 418   __glutSetupColormap(overlay->vis, &overlay->colormap, &overlay->cmap);
 419   overlay->transparentPixel = __glutGetTransparentPixel(__glutDisplay,
 420     overlay->vis);
 421   wa.colormap = overlay->cmap;
 422   wa.background_pixel = overlay->transparentPixel;
 423   wa.event_mask = window->eventMask & GLUT_OVERLAY_EVENT_FILTER_MASK;
 424   wa.border_pixel = 0;
 425 #if defined(_WIN32)
 426   /* XXX Overlays not supported in Win32 yet. */
 427 #else
 428   overlay->win = XCreateWindow(__glutDisplay,
 429     window->win,
 430     0, 0, window->width, window->height, 0,
 431     overlay->vis->depth, InputOutput, overlay->vis->visual,
 432     CWBackPixel | CWBorderPixel | CWEventMask | CWColormap,
 433     &wa);
 434 #endif
 435   if (window->children) {
 436     /* Overlay window must be lowered below any GLUT
 437        subwindows. */
 438     XLowerWindow(__glutDisplay, overlay->win);
 439   }
 440   XMapWindow(__glutDisplay, overlay->win);
 441   overlay->shownState = 1;
 442
 443   overlay->display = NULL;
 444
 445   /* Make sure a reshape gets delivered. */
 446   window->forceReshape = True;
 447
 448 #if !defined(_WIN32)
 449   __glutPutOnWorkList(__glutToplevelOf(window), GLUT_COLORMAP_WORK);
 450 #endif
 451
 452   window->overlay = overlay;
 453   glutUseLayer(GLUT_OVERLAY);
 454
 455   if (overlay->treatAsSingle) {
 456     glDrawBuffer(GL_FRONT);
 457     glReadBuffer(GL_FRONT);
 458   }
 459 }
 460
 461 void GLUTAPIENTRY
 462 glutRemoveOverlay(void)
 463 {
 464   GLUTwindow *window = __glutCurrentWindow;
 465   GLUToverlay *overlay = __glutCurrentWindow->overlay;
 466
 467   if (!window->overlay)
 468     return;
 469
 470   /* If using overlay, switch to the normal layer. */
 471   if (window->renderWin == overlay->win) {
 472     glutUseLayer(GLUT_NORMAL);
 473   }
 474 #if !defined(_WIN32)
 475   addStaleWindow(window, overlay->win);
 476 #endif
 477   __glutFreeOverlay(overlay);
 478   window->overlay = NULL;
 479 #if !defined(_WIN32)
 480   __glutPutOnWorkList(__glutToplevelOf(window), GLUT_COLORMAP_WORK);
 481 #endif
 482 }
 483
 484 void GLUTAPIENTRY
 485 glutUseLayer(GLenum layer)
 486 {
 487   GLUTwindow *window = __glutCurrentWindow;
 488
 489   switch (layer) {
 490   case GLUT_NORMAL:
 491 #ifdef _WIN32
 492     window->renderDc = window->hdc;
 493 #endif
 494     window->renderWin = window->win;
 495     window->renderCtx = window->ctx;
 496     break;
 497   case GLUT_OVERLAY:
 498     /* Did you crash here?  Calling glutUseLayer(GLUT_OVERLAY)
 499        without an overlay established is erroneous.  Fix your
 500        code. */
 501 #ifdef _WIN32
 502     window->renderDc = window->overlay->hdc;
 503 #endif
 504     window->renderWin = window->overlay->win;
 505     window->renderCtx = window->overlay->ctx;
 506     break;
 507   default:
 508     __glutWarning("glutUseLayer: unknown layer, %d.", layer);
 509     break;
 510   }
 511   __glutSetWindow(window);
 512 }
 513
 514 void GLUTAPIENTRY
 515 glutPostOverlayRedisplay(void)
 516 {
 517   __glutPostRedisplay(__glutCurrentWindow, GLUT_OVERLAY_REDISPLAY_WORK);
 518 }
 519
 520 /* The advantage of this routine is that it saves the cost of a
 521    glutSetWindow call (entailing an expensive OpenGL context
 522    switch), particularly useful when multiple windows need
 523    redisplays posted at the same times. */
 524 void GLUTAPIENTRY
 525 glutPostWindowOverlayRedisplay(int win)
 526 {
 527   __glutPostRedisplay(__glutWindowList[win - 1], GLUT_OVERLAY_REDISPLAY_WORK);
 528 }
 529
 530 void GLUTAPIENTRY
 531 glutOverlayDisplayFunc(GLUTdisplayCB displayFunc)
 532 {
 533   if (!__glutCurrentWindow->overlay) {
 534     __glutWarning("glutOverlayDisplayFunc: window has no overlay established");
 535     return;
 536   }
 537   __glutCurrentWindow->overlay->display = displayFunc;
 538 }
 539
 540 void GLUTAPIENTRY
 541 glutHideOverlay(void)
 542 {
 543   if (!__glutCurrentWindow->overlay) {
 544     __glutWarning("glutHideOverlay: window has no overlay established");
 545     return;
 546   }
 547   XUnmapWindow(__glutDisplay, __glutCurrentWindow->overlay->win);
 548   __glutCurrentWindow->overlay->shownState = 0;
 549 }
 550
 551 void GLUTAPIENTRY
 552 glutShowOverlay(void)
 553 {
 554   if (!__glutCurrentWindow->overlay) {
 555     __glutWarning("glutShowOverlay: window has no overlay established");
 556     return;
 557   }
 558   XMapWindow(__glutDisplay, __glutCurrentWindow->overlay->win);
 559   __glutCurrentWindow->overlay->shownState = 1;
 560 }
 561
 562 int GLUTAPIENTRY
 563 glutLayerGet(GLenum param)
 564 {
 565   switch (param) {
 566   case GLUT_OVERLAY_POSSIBLE:
 567     {
 568       XVisualInfo *vi;
 569       Bool dummy, visAlloced;
 570       void *fbc;
 571
 572       vi = determineOverlayVisual(&dummy, &visAlloced, &fbc);
 573       if (vi) {
 574         if (visAlloced)
 575           XFree(vi);
 576         return 1;
 577       }
 578       return 0;
 579     }
 580   case GLUT_LAYER_IN_USE:
 581     return __glutCurrentWindow->renderWin != __glutCurrentWindow->win;
 582   case GLUT_HAS_OVERLAY:
 583     return __glutCurrentWindow->overlay != NULL;
 584   case GLUT_TRANSPARENT_INDEX:
 585     if (__glutCurrentWindow->overlay) {
 586       return __glutCurrentWindow->overlay->transparentPixel;
 587     } else {
 588       return -1;
 589     }
 590   case GLUT_NORMAL_DAMAGED:
 591     /* __glutWindowDamaged is used so the damage state within
 592        the window (or overlay belwo) can be cleared before
 593        calling a display callback so on return, the state does
 594        not have to be cleared (since upon return from the
 595        callback the window could be destroyed (or layer
 596        removed). */
 597     return (__glutCurrentWindow->workMask & GLUT_REPAIR_WORK)
 598       || __glutWindowDamaged;
 599   case GLUT_OVERLAY_DAMAGED:
 600     if (__glutCurrentWindow->overlay) {
 601       return (__glutCurrentWindow->workMask & GLUT_OVERLAY_REPAIR_WORK)
 602         || __glutWindowDamaged;
 603     } else {
 604       return -1;
 605     }
 606   default:
 607     __glutWarning("invalid glutLayerGet param: %d", param);
 608     return -1;
 609   }
 610 }
 611 /* ENDCENTRY */