src/mesa/drivers/dri/intel/intel_mipmap_tree.h

   1 /**************************************************************************
   2  *
   3  * Copyright 2006 Tungsten Graphics, Inc., Cedar Park, Texas.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 #ifndef INTEL_MIPMAP_TREE_H
  29 #define INTEL_MIPMAP_TREE_H
  30
  31 #include <assert.h>
  32
  33 #include "intel_regions.h"
  34 #include "intel_resolve_map.h"
  35
  36 #ifdef __cplusplus
  37 extern "C" {
  38 #endif
  39
  40 /* A layer on top of the intel_regions code which adds:
  41  *
  42  * - Code to size and layout a region to hold a set of mipmaps.
  43  * - Query to determine if a new image fits in an existing tree.
  44  * - More refcounting
  45  *     - maybe able to remove refcounting from intel_region?
  46  * - ?
  47  *
  48  * The fixed mipmap layout of intel hardware where one offset
  49  * specifies the position of all images in a mipmap hierachy
  50  * complicates the implementation of GL texture image commands,
  51  * compared to hardware where each image is specified with an
  52  * independent offset.
  53  *
  54  * In an ideal world, each texture object would be associated with a
  55  * single bufmgr buffer or 2d intel_region, and all the images within
  56  * the texture object would slot into the tree as they arrive.  The
  57  * reality can be a little messier, as images can arrive from the user
  58  * with sizes that don't fit in the existing tree, or in an order
  59  * where the tree layout cannot be guessed immediately.
  60  *
  61  * This structure encodes an idealized mipmap tree.  The GL image
  62  * commands build these where possible, otherwise store the images in
  63  * temporary system buffers.
  64  */
  65
  66 struct intel_resolve_map;
  67 struct intel_texture_image;
  68
  69 /**
  70  * When calling intel_miptree_map() on an ETC-transcoded-to-RGB miptree or a
  71  * depthstencil-split-to-separate-stencil miptree, we'll normally make a
  72  * tmeporary and recreate the kind of data requested by Mesa core, since we're
  73  * satisfying some glGetTexImage() request or something.
  74  *
  75  * However, occasionally you want to actually map the miptree's current data
  76  * without transcoding back.  This flag to intel_miptree_map() gets you that.
  77  */
  78 #define BRW_MAP_DIRECT_BIT      0x80000000
  79
  80 struct intel_miptree_map {
  81    /** Bitfield of GL_MAP_READ_BIT, GL_MAP_WRITE_BIT, GL_MAP_INVALIDATE_BIT */
  82    GLbitfield mode;
  83    /** Region of interest for the map. */
  84    int x, y, w, h;
  85    /** Possibly malloced temporary buffer for the mapping. */
  86    void *buffer;
  87    /** Possible pointer to a BO temporary for the mapping. */
  88    drm_intel_bo *bo;
  89    /** Pointer to the start of (map_x, map_y) returned by the mapping. */
  90    void *ptr;
  91    /** Stride of the mapping. */
  92    int stride;
  93
  94    /**
  95     * intel_mipmap_tree::singlesample_mt is temporary storage that persists
  96     * only for the duration of the map.
  97     */
  98    bool singlesample_mt_is_tmp;
  99 };
 100
 101 /**
 102  * Describes the location of each texture image within a texture region.
 103  */
 104 struct intel_mipmap_level
 105 {
 106    /** Offset to this miptree level, used in computing x_offset. */
 107    GLuint level_x;
 108    /** Offset to this miptree level, used in computing y_offset. */
 109    GLuint level_y;
 110    GLuint width;
 111    GLuint height;
 112
 113    /**
 114     * \brief Number of 2D slices in this miplevel.
 115     *
 116     * The exact semantics of depth varies according to the texture target:
 117     *    - For GL_TEXTURE_CUBE_MAP, depth is 6.
 118     *    - For GL_TEXTURE_2D_ARRAY, depth is the number of array slices. It is
 119     *      identical for all miplevels in the texture.
 120     *    - For GL_TEXTURE_3D, it is the texture's depth at this miplevel. Its
 121     *      value, like width and height, varies with miplevel.
 122     *    - For other texture types, depth is 1.
 123     */
 124    GLuint depth;
 125
 126    /**
 127     * \brief List of 2D images in this mipmap level.
 128     *
 129     * This may be a list of cube faces, array slices in 2D array texture, or
 130     * layers in a 3D texture. The list's length is \c depth.
 131     */
 132    struct intel_mipmap_slice {
 133       /**
 134        * \name Offset to slice
 135        * \{
 136        *
 137        * Hardware formats are so diverse that that there is no unified way to
 138        * compute the slice offsets, so we store them in this table.
 139        *
 140        * The (x, y) offset to slice \c s at level \c l relative the miptrees
 141        * base address is
 142        * \code
 143        *     x = mt->level[l].slice[s].x_offset
 144        *     y = mt->level[l].slice[s].y_offset
 145        */
 146       GLuint x_offset;
 147       GLuint y_offset;
 148       /** \} */
 149
 150       /**
 151        * Mapping information. Persistent for the duration of
 152        * intel_miptree_map/unmap on this slice.
 153        */
 154       struct intel_miptree_map *map;
 155
 156       /**
 157        * \brief Is HiZ enabled for this slice?
 158        *
 159        * If \c mt->level[l].slice[s].has_hiz is set, then (1) \c mt->hiz_mt
 160        * has been allocated and (2) the HiZ memory corresponding to this slice
 161        * resides at \c mt->hiz_mt->level[l].slice[s].
 162        */
 163       bool has_hiz;
 164    } *slice;
 165 };
 166
 167 /**
 168  * Enum for keeping track of the different MSAA layouts supported by Gen7.
 169  */
 170 enum intel_msaa_layout
 171 {
 172    /**
 173     * Ordinary surface with no MSAA.
 174     */
 175    INTEL_MSAA_LAYOUT_NONE,
 176
 177    /**
 178     * Interleaved Multisample Surface.  The additional samples are
 179     * accommodated by scaling up the width and the height of the surface so
 180     * that all the samples corresponding to a pixel are located at nearby
 181     * memory locations.
 182     */
 183    INTEL_MSAA_LAYOUT_IMS,
 184
 185    /**
 186     * Uncompressed Multisample Surface.  The surface is stored as a 2D array,
 187     * with array slice n containing all pixel data for sample n.
 188     */
 189    INTEL_MSAA_LAYOUT_UMS,
 190
 191    /**
 192     * Compressed Multisample Surface.  The surface is stored as in
 193     * INTEL_MSAA_LAYOUT_UMS, but there is an additional buffer called the MCS
 194     * (Multisample Control Surface) buffer.  Each pixel in the MCS buffer
 195     * indicates the mapping from sample number to array slice.  This allows
 196     * the common case (where all samples constituting a pixel have the same
 197     * color value) to be stored efficiently by just using a single array
 198     * slice.
 199     */
 200    INTEL_MSAA_LAYOUT_CMS,
 201 };
 202
 203 struct intel_mipmap_tree
 204 {
 205    /* Effectively the key:
 206     */
 207    GLenum target;
 208
 209    /**
 210     * Generally, this is just the same as the gl_texture_image->TexFormat or
 211     * gl_renderbuffer->Format.
 212     *
 213     * However, for textures and renderbuffers with packed depth/stencil formats
 214     * on hardware where we want or need to use separate stencil, there will be
 215     * two miptrees for storing the data.  If the depthstencil texture or rb is
 216     * MESA_FORMAT_Z32_FLOAT_X24S8, then mt->format will be
 217     * MESA_FORMAT_Z32_FLOAT, otherwise for MESA_FORMAT_S8_Z24 objects it will be
 218     * MESA_FORMAT_X8_Z24.
 219     *
 220     * For ETC1/ETC2 textures, this is one of the uncompressed mesa texture
 221     * formats if the hardware lacks support for ETC1/ETC2. See @ref wraps_etc.
 222     */
 223    gl_format format;
 224
 225    /** This variable stores the value of ETC compressed texture format */
 226    gl_format etc_format;
 227
 228    /**
 229     * The X offset of each image in the miptree must be aligned to this. See
 230     * the "Alignment Unit Size" section of the BSpec.
 231     */
 232    unsigned int align_w;
 233    unsigned int align_h; /**< \see align_w */
 234
 235    GLuint first_level;
 236    GLuint last_level;
 237
 238    /**
 239     * Level zero image dimensions.  These dimensions correspond to the
 240     * physical layout of data in memory.  Accordingly, they account for the
 241     * extra width, height, and or depth that must be allocated in order to
 242     * accommodate multisample formats, and they account for the extra factor
 243     * of 6 in depth that must be allocated in order to accommodate cubemap
 244     * textures.
 245     */
 246    GLuint physical_width0, physical_height0, physical_depth0;
 247
 248    GLuint cpp;
 249    GLuint num_samples;
 250    bool compressed;
 251
 252    /**
 253     * Level zero image dimensions.  These dimensions correspond to the
 254     * logical width, height, and depth of the region as seen by client code.
 255     * Accordingly, they do not account for the extra width, height, and/or
 256     * depth that must be allocated in order to accommodate multisample
 257     * formats, nor do they account for the extra factor of 6 in depth that
 258     * must be allocated in order to accommodate cubemap textures.
 259     */
 260    uint32_t logical_width0, logical_height0, logical_depth0;
 261
 262    /**
 263     * For 1D array, 2D array, cube, and 2D multisampled surfaces on Gen7: true
 264     * if the surface only contains LOD 0, and hence no space is for LOD's
 265     * other than 0 in between array slices.
 266     *
 267     * Corresponds to the surface_array_spacing bit in gen7_surface_state.
 268     */
 269    bool array_spacing_lod0;
 270
 271    /**
 272     * MSAA layout used by this buffer.
 273     */
 274    enum intel_msaa_layout msaa_layout;
 275
 276    /* Derived from the above:
 277     */
 278    GLuint total_width;
 279    GLuint total_height;
 280
 281    /* The 3DSTATE_CLEAR_PARAMS value associated with the last depth clear to
 282     * this depth mipmap tree, if any.
 283     */
 284    uint32_t depth_clear_value;
 285
 286    /* Includes image offset tables:
 287     */
 288    struct intel_mipmap_level level[MAX_TEXTURE_LEVELS];
 289
 290    /* The data is held here:
 291     */
 292    struct intel_region *region;
 293
 294    /* Offset into region bo where miptree starts:
 295     */
 296    uint32_t offset;
 297
 298    /**
 299     * \brief Singlesample miptree.
 300     *
 301     * This is used under two cases.
 302     *
 303     * --- Case 1: As persistent singlesample storage for multisample window
 304     *  system front and back buffers ---
 305     *
 306     * Suppose that the window system FBO was created with a multisample
 307     * config.  Let `back_irb` be the `intel_renderbuffer` for the FBO's back
 308     * buffer. Then `back_irb` contains two miptrees: a parent multisample
 309     * miptree (back_irb->mt) and a child singlesample miptree
 310     * (back_irb->mt->singlesample_mt).  The DRM buffer shared with DRI2
 311     * belongs to `back_irb->mt->singlesample_mt` and contains singlesample
 312     * data.  The singlesample miptree is created at the same time as and
 313     * persists for the lifetime of its parent multisample miptree.
 314     *
 315     * When access to the singlesample data is needed, such as at
 316     * eglSwapBuffers and glReadPixels, an automatic downsample occurs from
 317     * `back_rb->mt` to `back_rb->mt->singlesample_mt` when necessary.
 318     *
 319     * This description of the back buffer applies analogously to the front
 320     * buffer.
 321     *
 322     *
 323     * --- Case 2: As temporary singlesample storage for mapping multisample
 324     *  miptrees ---
 325     *
 326     * Suppose the intel_miptree_map is called on a multisample miptree, `mt`,
 327     * for which case 1 does not apply (that is, `mt` does not belong to
 328     * a front or back buffer).  Then `mt->singlesample_mt` is null at the
 329     * start of the call. intel_miptree_map will create a temporary
 330     * singlesample miptree, store it at `mt->singlesample_mt`, downsample from
 331     * `mt` to `mt->singlesample_mt` if necessary, then map
 332     * `mt->singlesample_mt`. The temporary miptree is later deleted during
 333     * intel_miptree_unmap.
 334     */
 335    struct intel_mipmap_tree *singlesample_mt;
 336
 337    /**
 338     * \brief A downsample is needed from this miptree to singlesample_mt.
 339     */
 340    bool need_downsample;
 341
 342    /**
 343     * \brief HiZ miptree
 344     *
 345     * The hiz miptree contains the miptree's hiz buffer. To allocate the hiz
 346     * miptree, use intel_miptree_alloc_hiz().
 347     *
 348     * To determine if hiz is enabled, do not check this pointer. Instead, use
 349     * intel_miptree_slice_has_hiz().
 350     */
 351    struct intel_mipmap_tree *hiz_mt;
 352
 353    /**
 354     * \brief Map of miptree slices to needed resolves.
 355     *
 356     * This is used only when the miptree has a child HiZ miptree.
 357     *
 358     * Let \c mt be a depth miptree with HiZ enabled. Then the resolve map is
 359     * \c mt->hiz_map. The resolve map of the child HiZ miptree, \c
 360     * mt->hiz_mt->hiz_map, is unused.
 361     */
 362    struct intel_resolve_map hiz_map;
 363
 364    /**
 365     * \brief Stencil miptree for depthstencil textures.
 366     *
 367     * This miptree is used for depthstencil textures and renderbuffers that
 368     * require separate stencil.  It always has the true copy of the stencil
 369     * bits, regardless of mt->format.
 370     *
 371     * \see intel_miptree_map_depthstencil()
 372     * \see intel_miptree_unmap_depthstencil()
 373     */
 374    struct intel_mipmap_tree *stencil_mt;
 375
 376    /**
 377     * \brief MCS miptree for multisampled textures.
 378     *
 379     * This miptree contains the "multisample control surface", which stores
 380     * the necessary information to implement compressed MSAA on Gen7+
 381     * (INTEL_MSAA_FORMAT_CMS).
 382     */
 383    struct intel_mipmap_tree *mcs_mt;
 384
 385    /* These are also refcounted:
 386     */
 387    GLuint refcount;
 388 };
 389
 390
 391
 392 struct intel_mipmap_tree *intel_miptree_create(struct intel_context *intel,
 393                                                GLenum target,
 394                                                gl_format format,
 395                                                GLuint first_level,
 396                                                GLuint last_level,
 397                                                GLuint width0,
 398                                                GLuint height0,
 399                                                GLuint depth0,
 400                                                bool expect_accelerated_upload,
 401                                                GLuint num_samples,
 402                                                bool force_y_tiling);
 403
 404 struct intel_mipmap_tree *
 405 intel_miptree_create_layout(struct intel_context *intel,
 406                             GLenum target,
 407                             gl_format format,
 408                             GLuint first_level,
 409                             GLuint last_level,
 410                             GLuint width0,
 411                             GLuint height0,
 412                             GLuint depth0,
 413                             bool for_region,
 414                             GLuint num_samples);
 415
 416 struct intel_mipmap_tree *
 417 intel_miptree_create_for_region(struct intel_context *intel,
 418                                 GLenum target,
 419                                 gl_format format,
 420                                 struct intel_region *region);
 421
 422 struct intel_mipmap_tree*
 423 intel_miptree_create_for_dri2_buffer(struct intel_context *intel,
 424                                      unsigned dri_attachment,
 425                                      gl_format format,
 426                                      uint32_t num_samples,
 427                                      struct intel_region *region);
 428
 429 /**
 430  * Create a miptree appropriate as the storage for a non-texture renderbuffer.
 431  * The miptree has the following properties:
 432  *     - The target is GL_TEXTURE_2D.
 433  *     - There are no levels other than the base level 0.
 434  *     - Depth is 1.
 435  */
 436 struct intel_mipmap_tree*
 437 intel_miptree_create_for_renderbuffer(struct intel_context *intel,
 438                                       gl_format format,
 439                                       uint32_t width,
 440                                       uint32_t height,
 441                                       uint32_t num_samples);
 442
 443 /** \brief Assert that the level and layer are valid for the miptree. */
 444 static inline void
 445 intel_miptree_check_level_layer(struct intel_mipmap_tree *mt,
 446                                 uint32_t level,
 447                                 uint32_t layer)
 448 {
 449    assert(level >= mt->first_level);
 450    assert(level <= mt->last_level);
 451    assert(layer < mt->level[level].depth);
 452 }
 453
 454 int intel_miptree_pitch_align (struct intel_context *intel,
 455                                struct intel_mipmap_tree *mt,
 456                                uint32_t tiling,
 457                                int pitch);
 458
 459 void intel_miptree_reference(struct intel_mipmap_tree **dst,
 460                              struct intel_mipmap_tree *src);
 461
 462 void intel_miptree_release(struct intel_mipmap_tree **mt);
 463
 464 /* Check if an image fits an existing mipmap tree layout
 465  */
 466 bool intel_miptree_match_image(struct intel_mipmap_tree *mt,
 467                                     struct gl_texture_image *image);
 468
 469 void
 470 intel_miptree_get_image_offset(struct intel_mipmap_tree *mt,
 471                                GLuint level, GLuint slice,
 472                                GLuint *x, GLuint *y);
 473
 474 void
 475 intel_miptree_get_dimensions_for_image(struct gl_texture_image *image,
 476                                        int *width, int *height, int *depth);
 477
 478 void
 479 intel_miptree_get_tile_offsets(struct intel_mipmap_tree *mt,
 480                                GLuint level, GLuint slice,
 481                                uint32_t *tile_x,
 482                                uint32_t *tile_y);
 483
 484 void intel_miptree_set_level_info(struct intel_mipmap_tree *mt,
 485                                   GLuint level,
 486                                   GLuint x, GLuint y,
 487                                   GLuint w, GLuint h, GLuint d);
 488
 489 void intel_miptree_set_image_offset(struct intel_mipmap_tree *mt,
 490                                     GLuint level,
 491                                     GLuint img, GLuint x, GLuint y);
 492
 493 void
 494 intel_miptree_copy_teximage(struct intel_context *intel,
 495                             struct intel_texture_image *intelImage,
 496                             struct intel_mipmap_tree *dst_mt, bool invalidate);
 497
 498 /**
 499  * Copy the stencil data from \c mt->stencil_mt->region to \c mt->region for
 500  * the given miptree slice.
 501  *
 502  * \see intel_mipmap_tree::stencil_mt
 503  */
 504 void
 505 intel_miptree_s8z24_scatter(struct intel_context *intel,
 506                             struct intel_mipmap_tree *mt,
 507                             uint32_t level,
 508                             uint32_t slice);
 509
 510 /**
 511  * Copy the stencil data in \c mt->stencil_mt->region to \c mt->region for the
 512  * given miptree slice.
 513  *
 514  * \see intel_mipmap_tree::stencil_mt
 515  */
 516 void
 517 intel_miptree_s8z24_gather(struct intel_context *intel,
 518                            struct intel_mipmap_tree *mt,
 519                            uint32_t level,
 520                            uint32_t layer);
 521
 522 bool
 523 intel_miptree_alloc_mcs(struct intel_context *intel,
 524                         struct intel_mipmap_tree *mt,
 525                         GLuint num_samples);
 526
 527 /**
 528  * \name Miptree HiZ functions
 529  * \{
 530  *
 531  * It is safe to call the "slice_set_need_resolve" and "slice_resolve"
 532  * functions on a miptree without HiZ. In that case, each function is a no-op.
 533  */
 534
 535 /**
 536  * \brief Allocate the miptree's embedded HiZ miptree.
 537  * \see intel_mipmap_tree:hiz_mt
 538  * \return false if allocation failed
 539  */
 540
 541 bool
 542 intel_miptree_alloc_hiz(struct intel_context *intel,
 543                         struct intel_mipmap_tree *mt);
 544
 545 bool
 546 intel_miptree_slice_has_hiz(struct intel_mipmap_tree *mt,
 547                             uint32_t level,
 548                             uint32_t layer);
 549
 550 void
 551 intel_miptree_slice_set_needs_hiz_resolve(struct intel_mipmap_tree *mt,
 552                                           uint32_t level,
 553                                           uint32_t depth);
 554 void
 555 intel_miptree_slice_set_needs_depth_resolve(struct intel_mipmap_tree *mt,
 556                                             uint32_t level,
 557                                             uint32_t depth);
 558
 559 /**
 560  * \return false if no resolve was needed
 561  */
 562 bool
 563 intel_miptree_slice_resolve_hiz(struct intel_context *intel,
 564                                 struct intel_mipmap_tree *mt,
 565                                 unsigned int level,
 566                                 unsigned int depth);
 567
 568 /**
 569  * \return false if no resolve was needed
 570  */
 571 bool
 572 intel_miptree_slice_resolve_depth(struct intel_context *intel,
 573                                   struct intel_mipmap_tree *mt,
 574                                   unsigned int level,
 575                                   unsigned int depth);
 576
 577 /**
 578  * \return false if no resolve was needed
 579  */
 580 bool
 581 intel_miptree_all_slices_resolve_hiz(struct intel_context *intel,
 582                                      struct intel_mipmap_tree *mt);
 583
 584 /**
 585  * \return false if no resolve was needed
 586  */
 587 bool
 588 intel_miptree_all_slices_resolve_depth(struct intel_context *intel,
 589                                        struct intel_mipmap_tree *mt);
 590
 591 /**\}*/
 592
 593 void
 594 intel_miptree_downsample(struct intel_context *intel,
 595                          struct intel_mipmap_tree *mt);
 596
 597 void
 598 intel_miptree_upsample(struct intel_context *intel,
 599                        struct intel_mipmap_tree *mt);
 600
 601 /* i915_mipmap_tree.c:
 602  */
 603 void i915_miptree_layout(struct intel_mipmap_tree *mt);
 604 void i945_miptree_layout(struct intel_mipmap_tree *mt);
 605 void brw_miptree_layout(struct intel_context *intel,
 606                         struct intel_mipmap_tree *mt);
 607
 608 void *intel_miptree_map_raw(struct intel_context *intel,
 609                             struct intel_mipmap_tree *mt);
 610
 611 void intel_miptree_unmap_raw(struct intel_context *intel,
 612                              struct intel_mipmap_tree *mt);
 613
 614 void
 615 intel_miptree_map(struct intel_context *intel,
 616                   struct intel_mipmap_tree *mt,
 617                   unsigned int level,
 618                   unsigned int slice,
 619                   unsigned int x,
 620                   unsigned int y,
 621                   unsigned int w,
 622                   unsigned int h,
 623                   GLbitfield mode,
 624                   void **out_ptr,
 625                   int *out_stride);
 626
 627 void
 628 intel_miptree_unmap(struct intel_context *intel,
 629                     struct intel_mipmap_tree *mt,
 630                     unsigned int level,
 631                     unsigned int slice);
 632
 633 #ifdef I915
 634 static inline void
 635 intel_hiz_exec(struct intel_context *intel, struct intel_mipmap_tree *mt,
 636                unsigned int level, unsigned int layer, enum gen6_hiz_op op)
 637 {
 638    /* Stub on i915.  It would be nice if we didn't execute resolve code at all
 639     * there.
 640     */
 641 }
 642 #else
 643 void
 644 intel_hiz_exec(struct intel_context *intel, struct intel_mipmap_tree *mt,
 645                unsigned int level, unsigned int layer, enum gen6_hiz_op op);
 646 #endif
 647
 648 #ifdef __cplusplus
 649 }
 650 #endif
 651
 652 #endif