src/glut/glx/glut_overlay.c

   1
   2 /* Copyright (c) Mark J. Kilgard, 1996, 1997.  */
   3
   4 /* This program is freely distributable without licensing fees
   5    and is provided without guarantee or warrantee expressed or
   6    implied. This program is -not- in the public domain. */
   7
   8 #ifdef __VMS
   9 #include <GL/vms_x_fix.h>
  10 #endif
  11
  12 #include <stdlib.h>
  13 #include <stdio.h>
  14 #include <string.h>
  15 #include <assert.h>
  16
  17 #if !defined(_WIN32)
  18 #include <X11/Xlib.h>
  19 #include <X11/Xutil.h>
  20 #include <X11/Xatom.h>  /* for XA_RGB_DEFAULT_MAP atom */
  21 #if defined (__vms)
  22 #include <Xmu/StdCmap.h>  /* for XmuLookupStandardColormap */
  23 #else
  24 #include <X11/Xmu/StdCmap.h>  /* for XmuLookupStandardColormap */
  25 #endif
  26 #endif /* !_WIN32 */
  27
  28 #include "glutint.h"
  29 #include "layerutil.h"
  30
  31 static Criterion requiredOverlayCriteria[] =
  32 {
  33   {LEVEL, EQ, 1},       /* This entry gets poked in
  34                            determineOverlayVisual. */
  35   {TRANSPARENT, EQ, 1},
  36   {XPSEUDOCOLOR, EQ, 1},
  37   {RGBA, EQ, 0},
  38   {BUFFER_SIZE, GTE, 1}
  39 };
  40 static int numRequiredOverlayCriteria = sizeof(requiredOverlayCriteria) / sizeof(Criterion);
  41 static int requiredOverlayCriteriaMask =
  42 (1 << LEVEL) | (1 << TRANSPARENT) | (1 << XSTATICGRAY) | (1 << RGBA) | (1 << CI_MODE);
  43
  44 #if !defined(_WIN32)
  45 static int
  46 checkOverlayAcceptability(XVisualInfo * vi, unsigned int mode)
  47 {
  48   int value;
  49
  50   /* Must support OpenGL. */
  51   glXGetConfig(__glutDisplay, vi, GLX_USE_GL, &value);
  52   if (!value)
  53     return 1;
  54
  55   /* Must be color index. */
  56   glXGetConfig(__glutDisplay, vi, GLX_RGBA, &value);
  57   if (value)
  58     return 1;
  59
  60   /* Must match single/double buffering request. */
  61   glXGetConfig(__glutDisplay, vi, GLX_DOUBLEBUFFER, &value);
  62   if (GLUT_WIND_IS_DOUBLE(mode) != (value != 0))
  63     return 1;
  64
  65   /* Must match mono/stereo request. */
  66   glXGetConfig(__glutDisplay, vi, GLX_STEREO, &value);
  67   if (GLUT_WIND_IS_STEREO(mode) != (value != 0))
  68     return 1;
  69
  70   /* Alpha and accumulation buffers incompatible with color
  71      index. */
  72   if (GLUT_WIND_HAS_ALPHA(mode) || GLUT_WIND_HAS_ACCUM(mode))
  73     return 1;
  74
  75   /* Look for depth buffer if requested. */
  76   glXGetConfig(__glutDisplay, vi, GLX_DEPTH_SIZE, &value);
  77   if (GLUT_WIND_HAS_DEPTH(mode) && (value <= 0))
  78     return 1;
  79
  80   /* Look for stencil buffer if requested. */
  81   glXGetConfig(__glutDisplay, vi, GLX_STENCIL_SIZE, &value);
  82   if (GLUT_WIND_HAS_STENCIL(mode) && (value <= 0))
  83     return 1;
  84
  85 #if defined(GLX_VERSION_1_1) && defined(GLX_SGIS_multisample)
  86   /* XXX Multisampled overlay color index??  Pretty unlikely. */
  87   /* Look for multisampling if requested. */
  88   if (__glutIsSupportedByGLX("GLX_SGIS_multisample"))
  89     glXGetConfig(__glutDisplay, vi, GLX_SAMPLES_SGIS, &value);
  90   else
  91     value = 0;
  92   if (GLUT_WIND_IS_MULTISAMPLE(mode) && (value <= 0))
  93     return 1;
  94 #endif
  95
  96   return 0;
  97 }
  98 #endif
  99
 100 static XVisualInfo *
 101 getOverlayVisualInfoCI(unsigned int mode)
 102 {
 103 #if !defined(_WIN32)
 104   XLayerVisualInfo *vi;
 105   XLayerVisualInfo template;
 106   XVisualInfo *goodVisual, *returnVisual;
 107   int nitems, i, j, bad;
 108
 109   /* The GLX 1.0 glXChooseVisual is does not permit queries
 110      based on pixel transparency (and GLX_BUFFER_SIZE uses
 111      "smallest that meets" its requirement instead of "largest
 112      that meets" that GLUT wants. So, GLUT implements its own
 113      visual selection routine for color index overlays. */
 114
 115   /* Try three overlay layers. */
 116   for (i = 1; i <= 3; i++) {
 117     template.vinfo.screen = __glutScreen;
 118     template.vinfo.class = PseudoColor;
 119     template.layer = i;
 120     template.type = TransparentPixel;
 121     vi = __glutXGetLayerVisualInfo(__glutDisplay,
 122       VisualTransparentType | VisualScreenMask | VisualClassMask | VisualLayerMask,
 123       &template, &nitems);
 124     if (vi) {
 125       /* Check list for acceptable visual meeting requirements
 126          of requested display mode. */
 127       for (j = 0; j < nitems; j++) {
 128         bad = checkOverlayAcceptability(&vi[j].vinfo, mode);
 129         if (bad) {
 130           /* Set vi[j].vinfo.visual to mark it unacceptable. */
 131           vi[j].vinfo.visual = NULL;
 132         }
 133       }
 134
 135       /* Look through list to find deepest acceptable visual. */
 136       goodVisual = NULL;
 137       for (j = 0; j < nitems; j++) {
 138         if (vi[j].vinfo.visual) {
 139           if (goodVisual == NULL) {
 140             goodVisual = &vi[j].vinfo;
 141           } else {
 142             if (goodVisual->depth < vi[j].vinfo.depth) {
 143               goodVisual = &vi[j].vinfo;
 144             }
 145           }
 146         }
 147       }
 148
 149       /* If a visual is found, clean up and return the visual. */
 150       if (goodVisual) {
 151         returnVisual = (XVisualInfo *) malloc(sizeof(XVisualInfo));
 152         if (returnVisual) {
 153           *returnVisual = *goodVisual;
 154         }
 155         XFree(vi);
 156         return returnVisual;
 157       }
 158       XFree(vi);
 159     }
 160   }
 161 #endif /* !_WIN32 */
 162   return NULL;
 163 }
 164
 165 /* ARGSUSED */
 166 static XVisualInfo *
 167 getOverlayVisualInfoRGB(unsigned int mode)
 168 {
 169
 170   /* XXX For now, transparent RGBA overlays are not supported
 171      by GLUT.  RGBA overlays raise difficult questions about
 172      what the transparent pixel (really color) value should be.
 173
 174      Color index overlay transparency is "easy" because the
 175      transparent pixel value does not affect displayable colors
 176      (except for stealing one color cell) since colors are
 177      determined by indirection through a colormap, and because
 178      it is uncommon for arbitrary pixel values in color index to
 179      be "calculated" (as can occur with a host of RGBA operations
 180      like lighting, blending, etc) so it is easy to avoid the
 181      transparent pixel value.
 182
 183      Since it is typically easy to avoid the transparent pixel
 184      value in color index mode, if GLUT tells the programmer what
 185      pixel is transparent, then most program can easily avoid
 186      generating that pixel value except when they intend
 187      transparency.  GLUT returns whatever transparent pixel value
 188      is provided by the system through glutGet(
 189      GLUT_TRANSPARENT_INDEX).
 190
 191      Theory versus practice for RGBA overlay transparency: In
 192      theory, the reasonable thing is enabling overlay transparency
 193      when an overlay pixel's destination alpha is 0 because this
 194      allows overlay transparency to be controlled via alpha and all
 195      visibile colors are permited, but no hardware I am aware of
 196      supports this practice (and it requires destination alpha which
 197      is typically optional and quite uncommon for overlay windows!).
 198
 199      In practice, the choice of  transparent pixel value is typically
 200      "hardwired" into most graphics hardware to a single pixel value.
 201      SGI hardware uses true black (0,0,0) without regard for the
 202      destination alpha.  This is far from ideal because true black (a
 203      common color that is easy to accidently generate) can not be
 204      generated in an RGBA overlay. I am not sure what other vendors
 205      do.
 206
 207      Pragmatically, most of the typical things you want to do in the
 208      overlays can be done in color index (rubber banding, pop-up
 209      menus, etc.).  One solution for GLUT would be to simply
 210      "advertise" what RGB triple (or possibly RGBA quadruple or simply
 211      A alone) generates transparency.  The problem with this approach
 212      is that it forces programmers to avoid whatever arbitrary color
 213      various systems decide is transparent.  This is a difficult
 214      burden to place on programmers that want to portably make use of
 215      overlays.
 216
 217      To actually support transparent RGBA overlays, there are really
 218      two reaonsable options.  ONE: Simply mandate that true black is
 219      the RGBA overlay transparent color (what IRIS GL did).  This is
 220      nice for programmers since only one option, nice for existing SGI
 221      hardware, bad for anyone (including SGI) who wants to improve
 222      upon "true black" RGB transparency.
 223
 224      Or TWO: Provide a set of queriable "transparency types" (like
 225      "true black" or "alpha == 0" or "true white" or even a queriable
 226      transparent color).  This is harder for programmers, OK for
 227      existing SGI hardware, and it leaves open the issue of what other
 228      modes are reasonable.
 229
 230      Option TWO seems the more general approach, but since hardware
 231      designers will likely only implement a single mode (this is a
 232      scan out issue where bandwidth is pressing issue), codifying
 233      multiple speculative approaches nobody may ever implement seems
 234      silly.  And option ONE fiats a suboptimal solution.
 235
 236      Therefore, I defer any decision of how GLUT should support RGBA
 237      overlay transparency and leave support for it unimplemented.
 238      Nobody has been pressing me for RGBA overlay transparency (though
 239      people have requested color index overlay transparency
 240      repeatedly).  Geez, if you read this far you are either really
 241      bored or maybe actually  interested in this topic.  Anyway, if
 242      you have ideas (particularly if you plan on implementing a
 243      hardware scheme for RGBA overlay transparency), I'd be
 244      interested.
 245
 246      For the record, SGI's expiremental Framebufer Configuration
 247      experimental GLX extension uses option TWO.  Transparency modes
 248      for "none" and "RGB" are defined (others could be defined later).
 249      What RGB value is the transparent one must be queried.
 250
 251      I was hoping GLUT could have something that required less work
 252      from the programmer to use portably. -mjk */
 253
 254   __glutWarning("RGBA overlays are not supported by GLUT (for now).");
 255   return NULL;
 256 }
 257
 258 static XVisualInfo *
 259 getOverlayVisualInfo(unsigned int mode)
 260 {
 261   /* XXX GLUT_LUMINANCE not implemented for GLUT 3.0. */
 262   if (GLUT_WIND_IS_LUMINANCE(mode))
 263     return NULL;
 264
 265   if (GLUT_WIND_IS_RGB(mode))
 266     return getOverlayVisualInfoRGB(mode);
 267   else
 268     return getOverlayVisualInfoCI(mode);
 269 }
 270
 271 #if !defined(_WIN32)
 272
 273 /* The GLUT overlay can come and go, and the overlay window has
 274    a distinct X window ID.  Logically though, GLUT treats the
 275    normal and overlay windows as a unified window.  In
 276    particular, X input events typically go to the overlay window
 277    since it is "on top of" the normal window.  When an overlay
 278    window ID is destroyed (due to glutRemoveOverlay or a call to
 279    glutEstablishOverlay when an overlay already exists), we
 280    still keep track of the overlay window ID until we get back a
 281    DestroyNotify event for the overlay window. Otherwise, we
 282    could lose track of X input events sent to a destroyed
 283    overlay.  To avoid this, we keep the destroyed overlay window
 284    ID on a "stale window" list.  This lets us properly route X
 285    input events generated on destroyed overlay windows to the
 286    proper GLUT window. */
 287 static void
 288 addStaleWindow(GLUTwindow * window, Window win)
 289 {
 290   GLUTstale *entry;
 291
 292   entry = (GLUTstale *) malloc(sizeof(GLUTstale));
 293   if (!entry)
 294     __glutFatalError("out of memory");
 295   entry->window = window;
 296   entry->win = win;
 297   entry->next = __glutStaleWindowList;
 298   __glutStaleWindowList = entry;
 299 }
 300
 301 #endif
 302
 303 void
 304 __glutFreeOverlay(GLUToverlay * overlay)
 305 {
 306   if (overlay->visAlloced)
 307     XFree(overlay->vis);
 308   XDestroyWindow(__glutDisplay, overlay->win);
 309   glXDestroyContext(__glutDisplay, overlay->ctx);
 310   if (overlay->colormap) {
 311     /* Only color index overlays have colormap data structure. */
 312     __glutFreeColormap(overlay->colormap);
 313   }
 314   free(overlay);
 315 }
 316
 317 static XVisualInfo *
 318 determineOverlayVisual(int *treatAsSingle, Bool * visAlloced, void **fbc)
 319 {
 320   if (__glutDisplayString) {
 321     XVisualInfo *vi;
 322     int i;
 323
 324     /* __glutDisplayString should be NULL except if
 325        glutInitDisplayString has been called to register a
 326        different display string.  Calling glutInitDisplayString
 327        means using a string instead of an integer mask determine
 328
 329        the visual to use. Using the function pointer variable
 330        __glutDetermineVisualFromString below avoids linking in
 331        the code for implementing glutInitDisplayString (ie,
 332        glut_dstr.o) unless glutInitDisplayString gets called by
 333        the application. */
 334
 335     assert(__glutDetermineVisualFromString);
 336
 337     /* Try three overlay layers. */
 338     *visAlloced = False;
 339     *fbc = NULL;
 340     for (i = 1; i <= 3; i++) {
 341       requiredOverlayCriteria[0].value = i;
 342       vi = __glutDetermineVisualFromString(__glutDisplayString, treatAsSingle,
 343         requiredOverlayCriteria, numRequiredOverlayCriteria,
 344         requiredOverlayCriteriaMask, fbc);
 345       if (vi) {
 346         return vi;
 347       }
 348     }
 349     return NULL;
 350   } else {
 351     *visAlloced = True;
 352     *fbc = NULL;
 353     return __glutDetermineVisual(__glutDisplayMode,
 354       treatAsSingle, getOverlayVisualInfo);
 355   }
 356 }
 357
 358 /* CENTRY */
 359 void GLUTAPIENTRY
 360 glutEstablishOverlay(void)
 361 {
 362   GLUToverlay *overlay;
 363   GLUTwindow *window;
 364   XSetWindowAttributes wa;
 365   void *fbc;
 366
 367   /* Register a routine to free an overlay with glut_win.c;
 368      this keeps glut_win.c from pulling in all of
 369      glut_overlay.c when no overlay functionality is used by
 370      the application. */
 371   __glutFreeOverlayFunc = __glutFreeOverlay;
 372
 373   window = __glutCurrentWindow;
 374
 375   /* Allow for an existant overlay to be re-established perhaps
 376      if you wanted a different display mode. */
 377   if (window->overlay) {
 378 #if !defined(_WIN32)
 379     addStaleWindow(window, window->overlay->win);
 380 #endif
 381     __glutFreeOverlay(window->overlay);
 382   }
 383   overlay = (GLUToverlay *) malloc(sizeof(GLUToverlay));
 384   if (!overlay)
 385     __glutFatalError("out of memory.");
 386
 387   overlay->vis = determineOverlayVisual(&overlay->treatAsSingle,
 388     &overlay->visAlloced, &fbc);
 389   if (!overlay->vis) {
 390     __glutFatalError("lacks overlay support.");
 391   }
 392 #if defined(GLX_VERSION_1_1) && defined(GLX_SGIX_fbconfig)
 393   if (fbc) {
 394     window->ctx = __glut_glXCreateContextWithConfigSGIX(__glutDisplay, fbc,
 395       GLX_RGBA_TYPE_SGIX, None, __glutTryDirect);
 396   } else
 397 #endif
 398   {
 399     overlay->ctx = glXCreateContext(__glutDisplay, overlay->vis,
 400       None, __glutTryDirect);
 401   }
 402   if (!overlay->ctx) {
 403     __glutFatalError(
 404       "failed to create overlay OpenGL rendering context.");
 405   }
 406 #if !defined(_WIN32)
 407   overlay->isDirect = glXIsDirect(__glutDisplay, overlay->ctx);
 408   if (__glutForceDirect) {
 409     if (!overlay->isDirect) {
 410       __glutFatalError("direct rendering not possible.");
 411     }
 412   }
 413 #endif
 414   __glutSetupColormap(overlay->vis, &overlay->colormap, &overlay->cmap);
 415   overlay->transparentPixel = __glutGetTransparentPixel(__glutDisplay,
 416     overlay->vis);
 417   wa.colormap = overlay->cmap;
 418   wa.background_pixel = overlay->transparentPixel;
 419   wa.event_mask = window->eventMask & GLUT_OVERLAY_EVENT_FILTER_MASK;
 420   wa.border_pixel = 0;
 421 #if defined(_WIN32)
 422   /* XXX Overlays not supported in Win32 yet. */
 423 #else
 424   overlay->win = XCreateWindow(__glutDisplay,
 425     window->win,
 426     0, 0, window->width, window->height, 0,
 427     overlay->vis->depth, InputOutput, overlay->vis->visual,
 428     CWBackPixel | CWBorderPixel | CWEventMask | CWColormap,
 429     &wa);
 430 #endif
 431   if (window->children) {
 432     /* Overlay window must be lowered below any GLUT
 433        subwindows. */
 434     XLowerWindow(__glutDisplay, overlay->win);
 435   }
 436   XMapWindow(__glutDisplay, overlay->win);
 437   overlay->shownState = 1;
 438
 439   overlay->display = NULL;
 440
 441   /* Make sure a reshape gets delivered. */
 442   window->forceReshape = True;
 443
 444 #if !defined(_WIN32)
 445   __glutPutOnWorkList(__glutToplevelOf(window), GLUT_COLORMAP_WORK);
 446 #endif
 447
 448   window->overlay = overlay;
 449   glutUseLayer(GLUT_OVERLAY);
 450
 451   if (overlay->treatAsSingle) {
 452     glDrawBuffer(GL_FRONT);
 453     glReadBuffer(GL_FRONT);
 454   }
 455 }
 456
 457 void GLUTAPIENTRY
 458 glutRemoveOverlay(void)
 459 {
 460   GLUTwindow *window = __glutCurrentWindow;
 461   GLUToverlay *overlay = __glutCurrentWindow->overlay;
 462
 463   if (!window->overlay)
 464     return;
 465
 466   /* If using overlay, switch to the normal layer. */
 467   if (window->renderWin == overlay->win) {
 468     glutUseLayer(GLUT_NORMAL);
 469   }
 470 #if !defined(_WIN32)
 471   addStaleWindow(window, overlay->win);
 472 #endif
 473   __glutFreeOverlay(overlay);
 474   window->overlay = NULL;
 475 #if !defined(_WIN32)
 476   __glutPutOnWorkList(__glutToplevelOf(window), GLUT_COLORMAP_WORK);
 477 #endif
 478 }
 479
 480 void GLUTAPIENTRY
 481 glutUseLayer(GLenum layer)
 482 {
 483   GLUTwindow *window = __glutCurrentWindow;
 484
 485   switch (layer) {
 486   case GLUT_NORMAL:
 487 #ifdef _WIN32
 488     window->renderDc = window->hdc;
 489 #endif
 490     window->renderWin = window->win;
 491     window->renderCtx = window->ctx;
 492     break;
 493   case GLUT_OVERLAY:
 494     /* Did you crash here?  Calling glutUseLayer(GLUT_OVERLAY)
 495        without an overlay established is erroneous.  Fix your
 496        code. */
 497 #ifdef _WIN32
 498     window->renderDc = window->overlay->hdc;
 499 #endif
 500     window->renderWin = window->overlay->win;
 501     window->renderCtx = window->overlay->ctx;
 502     break;
 503   default:
 504     __glutWarning("glutUseLayer: unknown layer, %d.", layer);
 505     break;
 506   }
 507   __glutSetWindow(window);
 508 }
 509
 510 void GLUTAPIENTRY
 511 glutPostOverlayRedisplay(void)
 512 {
 513   __glutPostRedisplay(__glutCurrentWindow, GLUT_OVERLAY_REDISPLAY_WORK);
 514 }
 515
 516 /* The advantage of this routine is that it saves the cost of a
 517    glutSetWindow call (entailing an expensive OpenGL context
 518    switch), particularly useful when multiple windows need
 519    redisplays posted at the same times. */
 520 void GLUTAPIENTRY
 521 glutPostWindowOverlayRedisplay(int win)
 522 {
 523   __glutPostRedisplay(__glutWindowList[win - 1], GLUT_OVERLAY_REDISPLAY_WORK);
 524 }
 525
 526 void GLUTAPIENTRY
 527 glutOverlayDisplayFunc(GLUTdisplayCB displayFunc)
 528 {
 529   if (!__glutCurrentWindow->overlay) {
 530     __glutWarning("glutOverlayDisplayFunc: window has no overlay established");
 531     return;
 532   }
 533   __glutCurrentWindow->overlay->display = displayFunc;
 534 }
 535
 536 void GLUTAPIENTRY
 537 glutHideOverlay(void)
 538 {
 539   if (!__glutCurrentWindow->overlay) {
 540     __glutWarning("glutHideOverlay: window has no overlay established");
 541     return;
 542   }
 543   XUnmapWindow(__glutDisplay, __glutCurrentWindow->overlay->win);
 544   __glutCurrentWindow->overlay->shownState = 0;
 545 }
 546
 547 void GLUTAPIENTRY
 548 glutShowOverlay(void)
 549 {
 550   if (!__glutCurrentWindow->overlay) {
 551     __glutWarning("glutShowOverlay: window has no overlay established");
 552     return;
 553   }
 554   XMapWindow(__glutDisplay, __glutCurrentWindow->overlay->win);
 555   __glutCurrentWindow->overlay->shownState = 1;
 556 }
 557
 558 int GLUTAPIENTRY
 559 glutLayerGet(GLenum param)
 560 {
 561   switch (param) {
 562   case GLUT_OVERLAY_POSSIBLE:
 563     {
 564       XVisualInfo *vi;
 565       Bool dummy, visAlloced;
 566       void *fbc;
 567
 568       vi = determineOverlayVisual(&dummy, &visAlloced, &fbc);
 569       if (vi) {
 570         if (visAlloced)
 571           XFree(vi);
 572         return 1;
 573       }
 574       return 0;
 575     }
 576   case GLUT_LAYER_IN_USE:
 577     return __glutCurrentWindow->renderWin != __glutCurrentWindow->win;
 578   case GLUT_HAS_OVERLAY:
 579     return __glutCurrentWindow->overlay != NULL;
 580   case GLUT_TRANSPARENT_INDEX:
 581     if (__glutCurrentWindow->overlay) {
 582       return __glutCurrentWindow->overlay->transparentPixel;
 583     } else {
 584       return -1;
 585     }
 586   case GLUT_NORMAL_DAMAGED:
 587     /* __glutWindowDamaged is used so the damage state within
 588        the window (or overlay belwo) can be cleared before
 589        calling a display callback so on return, the state does
 590        not have to be cleared (since upon return from the
 591        callback the window could be destroyed (or layer
 592        removed). */
 593     return (__glutCurrentWindow->workMask & GLUT_REPAIR_WORK)
 594       || __glutWindowDamaged;
 595   case GLUT_OVERLAY_DAMAGED:
 596     if (__glutCurrentWindow->overlay) {
 597       return (__glutCurrentWindow->workMask & GLUT_OVERLAY_REPAIR_WORK)
 598         || __glutWindowDamaged;
 599     } else {
 600       return -1;
 601     }
 602   default:
 603     __glutWarning("invalid glutLayerGet param: %d", param);
 604     return -1;
 605   }
 606 }
 607 /* ENDCENTRY */