Revert "Revert "i965/fs: Use align1 mode on ternary instructions on Gen10+""

[mesa.git] / src / intel / compiler / brw_fs_surface_builder.cpp

/*
 * Copyright © 2013-2015 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include "isl/isl.h"
#include "brw_fs_surface_builder.h"
#include "brw_fs.h"

using namespace brw;

namespace brw {
   namespace surface_access {
      namespace {
         /**
          * Generate a logical send opcode for a surface message and return
          * the result.
          */
         fs_reg
         emit_send(const fs_builder &bld, enum opcode opcode,
                   const fs_reg &addr, const fs_reg &src, const fs_reg &surface,
                   unsigned dims, unsigned arg, unsigned rsize,
                   brw_predicate pred = BRW_PREDICATE_NONE)
         {
            /* Reduce the dynamically uniform surface index to a single
             * scalar.
             */
            const fs_reg usurface = bld.emit_uniformize(surface);
            const fs_reg srcs[] = {
               addr, src, usurface, brw_imm_ud(dims), brw_imm_ud(arg)
            };
            const fs_reg dst = bld.vgrf(BRW_REGISTER_TYPE_UD, rsize);
            fs_inst *inst = bld.emit(opcode, dst, srcs, ARRAY_SIZE(srcs));

            inst->size_written = rsize * dst.component_size(inst->exec_size);
            inst->predicate = pred;
            return dst;
         }
      }

      /**
       * Emit an untyped surface read opcode.  \p dims determines the number
       * of components of the address and \p size the number of components of
       * the returned value.
       */
      fs_reg
      emit_untyped_read(const fs_builder &bld,
                        const fs_reg &surface, const fs_reg &addr,
                        unsigned dims, unsigned size,
                        brw_predicate pred)
      {
         return emit_send(bld, SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL,
                          addr, fs_reg(), surface, dims, size, size, pred);
      }

      /**
       * Emit an untyped surface write opcode.  \p dims determines the number
       * of components of the address and \p size the number of components of
       * the argument.
       */
      void
      emit_untyped_write(const fs_builder &bld, const fs_reg &surface,
                         const fs_reg &addr, const fs_reg &src,
                         unsigned dims, unsigned size,
                         brw_predicate pred)
      {
         emit_send(bld, SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL,
                   addr, src, surface, dims, size, 0, pred);
      }

      /**
       * Emit an untyped surface atomic opcode.  \p dims determines the number
       * of components of the address and \p rsize the number of components of
       * the returned value (either zero or one).
       */
      fs_reg
      emit_untyped_atomic(const fs_builder &bld,
                          const fs_reg &surface, const fs_reg &addr,
                          const fs_reg &src0, const fs_reg &src1,
                          unsigned dims, unsigned rsize, unsigned op,
                          brw_predicate pred)
      {
         /* FINISHME: Factor out this frequently recurring pattern into a
          * helper function.
          */
         const unsigned n = (src0.file != BAD_FILE) + (src1.file != BAD_FILE);
         const fs_reg srcs[] = { src0, src1 };
         const fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD, n);
         bld.LOAD_PAYLOAD(tmp, srcs, n, 0);

         return emit_send(bld, SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL,
                          addr, tmp, surface, dims, op, rsize, pred);
      }

      /**
       * Emit a typed surface read opcode.  \p dims determines the number of
       * components of the address and \p size the number of components of the
       * returned value.
       */
      fs_reg
      emit_typed_read(const fs_builder &bld, const fs_reg &surface,
                      const fs_reg &addr, unsigned dims, unsigned size)
      {
         return emit_send(bld, SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL,
                          addr, fs_reg(), surface, dims, size, size);
      }

      /**
       * Emit a typed surface write opcode.  \p dims determines the number of
       * components of the address and \p size the number of components of the
       * argument.
       */
      void
      emit_typed_write(const fs_builder &bld, const fs_reg &surface,
                       const fs_reg &addr, const fs_reg &src,
                       unsigned dims, unsigned size)
      {
         emit_send(bld, SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL,
                   addr, src, surface, dims, size, 0);
      }

      /**
       * Emit a typed surface atomic opcode.  \p dims determines the number of
       * components of the address and \p rsize the number of components of
       * the returned value (either zero or one).
       */
      fs_reg
      emit_typed_atomic(const fs_builder &bld, const fs_reg &surface,
                        const fs_reg &addr,
                        const fs_reg &src0, const fs_reg &src1,
                        unsigned dims, unsigned rsize, unsigned op,
                        brw_predicate pred)
      {
         /* FINISHME: Factor out this frequently recurring pattern into a
          * helper function.
          */
         const unsigned n = (src0.file != BAD_FILE) + (src1.file != BAD_FILE);
         const fs_reg srcs[] = { src0, src1 };
         const fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD, n);
         bld.LOAD_PAYLOAD(tmp, srcs, n, 0);

         return emit_send(bld, SHADER_OPCODE_TYPED_ATOMIC_LOGICAL,
                          addr, tmp, surface, dims, op, rsize);
      }

      fs_reg
      emit_byte_scattered_read(const fs_builder &bld,
                               const fs_reg &surface, const fs_reg &addr,
                               unsigned dims, unsigned size,
                               unsigned bit_size, brw_predicate pred)
      {
         return emit_send(bld, SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL,
                          addr, fs_reg(), surface, dims, bit_size, size, pred);
      }

      void
      emit_byte_scattered_write(const fs_builder &bld, const fs_reg &surface,
                                const fs_reg &addr, const fs_reg &src,
                                unsigned dims, unsigned size,
                                unsigned bit_size, brw_predicate pred)
      {
         emit_send(bld, SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL,
                   addr, src, surface, dims, bit_size, 0, pred);
      }
   }
}

namespace {
   namespace image_format_info {
      /* The higher compiler layers use the GL enums for image formats even if
       * they come in from SPIR-V or Vulkan.  We need to turn them into an ISL
       * enum before we can use them.
       */
      static enum isl_format
      isl_format_for_gl_format(uint32_t gl_format)
      {
         switch (gl_format) {
         case GL_R8:             return ISL_FORMAT_R8_UNORM;
         case GL_R8_SNORM:       return ISL_FORMAT_R8_SNORM;
         case GL_R8UI:           return ISL_FORMAT_R8_UINT;
         case GL_R8I:            return ISL_FORMAT_R8_SINT;
         case GL_RG8:            return ISL_FORMAT_R8G8_UNORM;
         case GL_RG8_SNORM:      return ISL_FORMAT_R8G8_SNORM;
         case GL_RG8UI:          return ISL_FORMAT_R8G8_UINT;
         case GL_RG8I:           return ISL_FORMAT_R8G8_SINT;
         case GL_RGBA8:          return ISL_FORMAT_R8G8B8A8_UNORM;
         case GL_RGBA8_SNORM:    return ISL_FORMAT_R8G8B8A8_SNORM;
         case GL_RGBA8UI:        return ISL_FORMAT_R8G8B8A8_UINT;
         case GL_RGBA8I:         return ISL_FORMAT_R8G8B8A8_SINT;
         case GL_R11F_G11F_B10F: return ISL_FORMAT_R11G11B10_FLOAT;
         case GL_RGB10_A2:       return ISL_FORMAT_R10G10B10A2_UNORM;
         case GL_RGB10_A2UI:     return ISL_FORMAT_R10G10B10A2_UINT;
         case GL_R16:            return ISL_FORMAT_R16_UNORM;
         case GL_R16_SNORM:      return ISL_FORMAT_R16_SNORM;
         case GL_R16F:           return ISL_FORMAT_R16_FLOAT;
         case GL_R16UI:          return ISL_FORMAT_R16_UINT;
         case GL_R16I:           return ISL_FORMAT_R16_SINT;
         case GL_RG16:           return ISL_FORMAT_R16G16_UNORM;
         case GL_RG16_SNORM:     return ISL_FORMAT_R16G16_SNORM;
         case GL_RG16F:          return ISL_FORMAT_R16G16_FLOAT;
         case GL_RG16UI:         return ISL_FORMAT_R16G16_UINT;
         case GL_RG16I:          return ISL_FORMAT_R16G16_SINT;
         case GL_RGBA16:         return ISL_FORMAT_R16G16B16A16_UNORM;
         case GL_RGBA16_SNORM:   return ISL_FORMAT_R16G16B16A16_SNORM;
         case GL_RGBA16F:        return ISL_FORMAT_R16G16B16A16_FLOAT;
         case GL_RGBA16UI:       return ISL_FORMAT_R16G16B16A16_UINT;
         case GL_RGBA16I:        return ISL_FORMAT_R16G16B16A16_SINT;
         case GL_R32F:           return ISL_FORMAT_R32_FLOAT;
         case GL_R32UI:          return ISL_FORMAT_R32_UINT;
         case GL_R32I:           return ISL_FORMAT_R32_SINT;
         case GL_RG32F:          return ISL_FORMAT_R32G32_FLOAT;
         case GL_RG32UI:         return ISL_FORMAT_R32G32_UINT;
         case GL_RG32I:          return ISL_FORMAT_R32G32_SINT;
         case GL_RGBA32F:        return ISL_FORMAT_R32G32B32A32_FLOAT;
         case GL_RGBA32UI:       return ISL_FORMAT_R32G32B32A32_UINT;
         case GL_RGBA32I:        return ISL_FORMAT_R32G32B32A32_SINT;
         case GL_NONE:           return ISL_FORMAT_UNSUPPORTED;
         default:
            assert(!"Invalid image format");
            return ISL_FORMAT_UNSUPPORTED;
         }
      }

      /**
       * Simple 4-tuple of scalars used to pass around per-color component
       * values.
       */
      struct color_u {
         color_u(unsigned x = 0) : r(x), g(x), b(x), a(x)
         {
         }

         color_u(unsigned r, unsigned g, unsigned b, unsigned a) :
            r(r), g(g), b(b), a(a)
         {
         }

         unsigned
         operator[](unsigned i) const
         {
            const unsigned xs[] = { r, g, b, a };
            return xs[i];
         }

         unsigned r, g, b, a;
      };

      /**
       * Return the per-channel bitfield widths for a given image format.
       */
      inline color_u
      get_bit_widths(isl_format format)
      {
         const isl_format_layout *fmtl = isl_format_get_layout(format);

         return color_u(fmtl->channels.r.bits,
                        fmtl->channels.g.bits,
                        fmtl->channels.b.bits,
                        fmtl->channels.a.bits);
      }

      /**
       * Return the per-channel bitfield shifts for a given image format.
       */
      inline color_u
      get_bit_shifts(isl_format format)
      {
         const color_u widths = get_bit_widths(format);
         return color_u(0, widths.r, widths.r + widths.g,
                        widths.r + widths.g + widths.b);
      }

      /**
       * Return true if all present components have the same bit width.
       */
      inline bool
      is_homogeneous(isl_format format)
      {
         const color_u widths = get_bit_widths(format);
         return ((widths.g == 0 || widths.g == widths.r) &&
                 (widths.b == 0 || widths.b == widths.r) &&
                 (widths.a == 0 || widths.a == widths.r));
      }

      /**
       * Return true if the format conversion boils down to a trivial copy.
       */
      inline bool
      is_conversion_trivial(const gen_device_info *devinfo, isl_format format)
      {
         return (get_bit_widths(format).r == 32 && is_homogeneous(format)) ||
                 format == isl_lower_storage_image_format(devinfo, format);
      }

      /**
       * Return true if the hardware natively supports some format with
       * compatible bitfield layout, but possibly different data types.
       */
      inline bool
      has_supported_bit_layout(const gen_device_info *devinfo,
                               isl_format format)
      {
         const color_u widths = get_bit_widths(format);
         const color_u lower_widths = get_bit_widths(
            isl_lower_storage_image_format(devinfo, format));

         return (widths.r == lower_widths.r &&
                 widths.g == lower_widths.g &&
                 widths.b == lower_widths.b &&
                 widths.a == lower_widths.a);
      }

      /**
       * Return true if we are required to spread individual components over
       * several components of the format used by the hardware (RG32 and
       * friends implemented as RGBA16UI).
       */
      inline bool
      has_split_bit_layout(const gen_device_info *devinfo, isl_format format)
      {
         const isl_format lower_format =
            isl_lower_storage_image_format(devinfo, format);

         return (isl_format_get_num_channels(format) <
                 isl_format_get_num_channels(lower_format));
      }

      /**
       * Return true if the hardware returns garbage in the unused high bits
       * of each component.  This may happen on IVB because we rely on the
       * undocumented behavior that typed reads from surfaces of the
       * unsupported R8 and R16 formats return useful data in their least
       * significant bits.
       */
      inline bool
      has_undefined_high_bits(const gen_device_info *devinfo,
                              isl_format format)
      {
         const isl_format lower_format =
            isl_lower_storage_image_format(devinfo, format);

         return (devinfo->gen == 7 && !devinfo->is_haswell &&
                 (lower_format == ISL_FORMAT_R16_UINT ||
                  lower_format == ISL_FORMAT_R8_UINT));
      }

      /**
       * Return true if the format represents values as signed integers
       * requiring sign extension when unpacking.
       */
      inline bool
      needs_sign_extension(isl_format format)
      {
         return isl_format_has_snorm_channel(format) ||
                isl_format_has_sint_channel(format);
      }
   }

   namespace image_validity {
      /**
       * Check whether the bound image is suitable for untyped access.
       */
      static brw_predicate
      emit_untyped_image_check(const fs_builder &bld, const fs_reg &image,
                               brw_predicate pred)
      {
         const gen_device_info *devinfo = bld.shader->devinfo;
         const fs_reg stride = offset(image, bld, BRW_IMAGE_PARAM_STRIDE_OFFSET);

         if (devinfo->gen == 7 && !devinfo->is_haswell) {
            /* Check whether the first stride component (i.e. the Bpp value)
             * is greater than four, what on Gen7 indicates that a surface of
             * type RAW has been bound for untyped access.  Reading or writing
             * to a surface of type other than RAW using untyped surface
             * messages causes a hang on IVB and VLV.
             */
            set_predicate(pred,
                          bld.CMP(bld.null_reg_ud(), stride, brw_imm_d(4),
                                  BRW_CONDITIONAL_G));

            return BRW_PREDICATE_NORMAL;
         } else {
            /* More recent generations handle the format mismatch
             * gracefully.
             */
            return pred;
         }
      }

      /**
       * Check whether there is an image bound at the given index and write
       * the comparison result to f0.0.  Returns an appropriate predication
       * mode to use on subsequent image operations.
       */
      static brw_predicate
      emit_typed_atomic_check(const fs_builder &bld, const fs_reg &image)
      {
         const gen_device_info *devinfo = bld.shader->devinfo;
         const fs_reg size = offset(image, bld, BRW_IMAGE_PARAM_SIZE_OFFSET);

         if (devinfo->gen == 7 && !devinfo->is_haswell) {
            /* Check the first component of the size field to find out if the
             * image is bound.  Necessary on IVB for typed atomics because
             * they don't seem to respect null surfaces and will happily
             * corrupt or read random memory when no image is bound.
             */
            bld.CMP(bld.null_reg_ud(),
                    retype(size, BRW_REGISTER_TYPE_UD),
                    brw_imm_d(0), BRW_CONDITIONAL_NZ);

            return BRW_PREDICATE_NORMAL;
         } else {
            /* More recent platforms implement compliant behavior when a null
             * surface is bound.
             */
            return BRW_PREDICATE_NONE;
         }
      }

      /**
       * Check whether the provided coordinates are within the image bounds
       * and write the comparison result to f0.0.  Returns an appropriate
       * predication mode to use on subsequent image operations.
       */
      static brw_predicate
      emit_bounds_check(const fs_builder &bld, const fs_reg &image,
                        const fs_reg &addr, unsigned dims)
      {
         const fs_reg size = offset(image, bld, BRW_IMAGE_PARAM_SIZE_OFFSET);

         for (unsigned c = 0; c < dims; ++c)
            set_predicate(c == 0 ? BRW_PREDICATE_NONE : BRW_PREDICATE_NORMAL,
                          bld.CMP(bld.null_reg_ud(),
                                  offset(retype(addr, BRW_REGISTER_TYPE_UD), bld, c),
                                  offset(size, bld, c),
                                  BRW_CONDITIONAL_L));

         return BRW_PREDICATE_NORMAL;
      }
   }

   namespace image_coordinates {
      /**
       * Return the total number of coordinates needed to address a texel of
       * the surface, which may be more than the sum of \p surf_dims and \p
       * arr_dims if padding is required.
       */
      static unsigned
      num_image_coordinates(const fs_builder &bld,
                            unsigned surf_dims, unsigned arr_dims,
                            isl_format format)
      {
         /* HSW in vec4 mode and our software coordinate handling for untyped
          * reads want the array index to be at the Z component.
          */
         const bool array_index_at_z =
            format != ISL_FORMAT_UNSUPPORTED &&
            !isl_has_matching_typed_storage_image_format(
               bld.shader->devinfo, format);
         const unsigned zero_dims =
            ((surf_dims == 1 && arr_dims == 1 && array_index_at_z) ? 1 : 0);

         return surf_dims + zero_dims + arr_dims;
      }

      /**
       * Transform image coordinates into the form expected by the
       * implementation.
       */
      static fs_reg
      emit_image_coordinates(const fs_builder &bld, const fs_reg &addr,
                             unsigned surf_dims, unsigned arr_dims,
                             isl_format format)
      {
         const unsigned dims =
            num_image_coordinates(bld, surf_dims, arr_dims, format);

         if (dims > surf_dims + arr_dims) {
            assert(surf_dims == 1 && arr_dims == 1 && dims == 3);
            /* The array index is required to be passed in as the Z component,
             * insert a zero at the Y component to shift it to the right
             * position.
             *
             * FINISHME: Factor out this frequently recurring pattern into a
             * helper function.
             */
            const fs_reg srcs[] = { addr, brw_imm_d(0), offset(addr, bld, 1) };
            const fs_reg dst = bld.vgrf(addr.type, dims);
            bld.LOAD_PAYLOAD(dst, srcs, dims, 0);
            return dst;
         } else {
            return addr;
         }
      }

      /**
       * Calculate the offset in memory of the texel given by \p coord.
       *
       * This is meant to be used with untyped surface messages to access a
       * tiled surface, what involves taking into account the tiling and
       * swizzling modes of the surface manually so it will hopefully not
       * happen very often.
       *
       * The tiling algorithm implemented here matches either the X or Y
       * tiling layouts supported by the hardware depending on the tiling
       * coefficients passed to the program as uniforms.  See Volume 1 Part 2
       * Section 4.5 "Address Tiling Function" of the IVB PRM for an in-depth
       * explanation of the hardware tiling format.
       */
      static fs_reg
      emit_address_calculation(const fs_builder &bld, const fs_reg &image,
                               const fs_reg &coord, unsigned dims)
      {
         const gen_device_info *devinfo = bld.shader->devinfo;
         const fs_reg off = offset(image, bld, BRW_IMAGE_PARAM_OFFSET_OFFSET);
         const fs_reg stride = offset(image, bld, BRW_IMAGE_PARAM_STRIDE_OFFSET);
         const fs_reg tile = offset(image, bld, BRW_IMAGE_PARAM_TILING_OFFSET);
         const fs_reg swz = offset(image, bld, BRW_IMAGE_PARAM_SWIZZLING_OFFSET);
         const fs_reg addr = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
         const fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
         const fs_reg minor = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
         const fs_reg major = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
         const fs_reg dst = bld.vgrf(BRW_REGISTER_TYPE_UD);

         /* Shift the coordinates by the fixed surface offset.  It may be
          * non-zero if the image is a single slice of a higher-dimensional
          * surface, or if a non-zero mipmap level of the surface is bound to
          * the pipeline.  The offset needs to be applied here rather than at
          * surface state set-up time because the desired slice-level may
          * start mid-tile, so simply shifting the surface base address
          * wouldn't give a well-formed tiled surface in the general case.
          */
         for (unsigned c = 0; c < 2; ++c)
            bld.ADD(offset(addr, bld, c), offset(off, bld, c),
                    (c < dims ?
                     offset(retype(coord, BRW_REGISTER_TYPE_UD), bld, c) :
                     fs_reg(brw_imm_d(0))));

         /* The layout of 3-D textures in memory is sort-of like a tiling
          * format.  At each miplevel, the slices are arranged in rows of
          * 2^level slices per row.  The slice row is stored in tmp.y and
          * the slice within the row is stored in tmp.x.
          *
          * The layout of 2-D array textures and cubemaps is much simpler:
          * Depending on whether the ARYSPC_LOD0 layout is in use it will be
          * stored in memory as an array of slices, each one being a 2-D
          * arrangement of miplevels, or as a 2D arrangement of miplevels,
          * each one being an array of slices.  In either case the separation
          * between slices of the same LOD is equal to the qpitch value
          * provided as stride.w.
          *
          * This code can be made to handle either 2D arrays and 3D textures
          * by passing in the miplevel as tile.z for 3-D textures and 0 in
          * tile.z for 2-D array textures.
          *
          * See Volume 1 Part 1 of the Gen7 PRM, sections 6.18.4.7 "Surface
          * Arrays" and 6.18.6 "3D Surfaces" for a more extensive discussion
          * of the hardware 3D texture and 2D array layouts.
          */
         if (dims > 2) {
            /* Decompose z into a major (tmp.y) and a minor (tmp.x)
             * index.
             */
            bld.BFE(offset(tmp, bld, 0), offset(tile, bld, 2), brw_imm_d(0),
                    offset(retype(coord, BRW_REGISTER_TYPE_UD), bld, 2));
            bld.SHR(offset(tmp, bld, 1),
                    offset(retype(coord, BRW_REGISTER_TYPE_UD), bld, 2),
                    offset(tile, bld, 2));

            /* Take into account the horizontal (tmp.x) and vertical (tmp.y)
             * slice offset.
             */
            for (unsigned c = 0; c < 2; ++c) {
               bld.MUL(offset(tmp, bld, c),
                       offset(stride, bld, 2 + c), offset(tmp, bld, c));
               bld.ADD(offset(addr, bld, c),
                       offset(addr, bld, c), offset(tmp, bld, c));
            }
         }

         if (dims > 1) {
            /* Calculate the major/minor x and y indices.  In order to
             * accommodate both X and Y tiling, the Y-major tiling format is
             * treated as being a bunch of narrow X-tiles placed next to each
             * other.  This means that the tile width for Y-tiling is actually
             * the width of one sub-column of the Y-major tile where each 4K
             * tile has 8 512B sub-columns.
             *
             * The major Y value is the row of tiles in which the pixel lives.
             * The major X value is the tile sub-column in which the pixel
             * lives; for X tiling, this is the same as the tile column, for Y
             * tiling, each tile has 8 sub-columns.  The minor X and Y indices
             * are the position within the sub-column.
             */
            for (unsigned c = 0; c < 2; ++c) {
               /* Calculate the minor x and y indices. */
               bld.BFE(offset(minor, bld, c), offset(tile, bld, c),
                       brw_imm_d(0), offset(addr, bld, c));

               /* Calculate the major x and y indices. */
               bld.SHR(offset(major, bld, c),
                       offset(addr, bld, c), offset(tile, bld, c));
            }

            /* Calculate the texel index from the start of the tile row and
             * the vertical coordinate of the row.
             * Equivalent to:
             *   tmp.x = (major.x << tile.y << tile.x) +
             *           (minor.y << tile.x) + minor.x
             *   tmp.y = major.y << tile.y
             */
            bld.SHL(tmp, major, offset(tile, bld, 1));
            bld.ADD(tmp, tmp, offset(minor, bld, 1));
            bld.SHL(tmp, tmp, offset(tile, bld, 0));
            bld.ADD(tmp, tmp, minor);
            bld.SHL(offset(tmp, bld, 1),
                    offset(major, bld, 1), offset(tile, bld, 1));

            /* Add it to the start of the tile row. */
            bld.MUL(offset(tmp, bld, 1),
                    offset(tmp, bld, 1), offset(stride, bld, 1));
            bld.ADD(tmp, tmp, offset(tmp, bld, 1));

            /* Multiply by the Bpp value. */
            bld.MUL(dst, tmp, stride);

            if (devinfo->gen < 8 && !devinfo->is_baytrail) {
               /* Take into account the two dynamically specified shifts.
                * Both need are used to implement swizzling of X-tiled
                * surfaces.  For Y-tiled surfaces only one bit needs to be
                * XOR-ed with bit 6 of the memory address, so a swz value of
                * 0xff (actually interpreted as 31 by the hardware) will be
                * provided to cause the relevant bit of tmp.y to be zero and
                * turn the first XOR into the identity.  For linear surfaces
                * or platforms lacking address swizzling both shifts will be
                * 0xff causing the relevant bits of both tmp.x and .y to be
                * zero, what effectively disables swizzling.
                */
               for (unsigned c = 0; c < 2; ++c)
                  bld.SHR(offset(tmp, bld, c), dst, offset(swz, bld, c));

               /* XOR tmp.x and tmp.y with bit 6 of the memory address. */
               bld.XOR(tmp, tmp, offset(tmp, bld, 1));
               bld.AND(tmp, tmp, brw_imm_d(1 << 6));
               bld.XOR(dst, dst, tmp);
            }

         } else {
            /* Multiply by the Bpp/stride value.  Note that the addr.y may be
             * non-zero even if the image is one-dimensional because a
             * vertical offset may have been applied above to select a
             * non-zero slice or level of a higher-dimensional texture.
             */
            bld.MUL(offset(addr, bld, 1),
                    offset(addr, bld, 1), offset(stride, bld, 1));
            bld.ADD(addr, addr, offset(addr, bld, 1));
            bld.MUL(dst, addr, stride);
         }

         return dst;
      }
   }

   namespace image_format_conversion {
      using image_format_info::color_u;

      namespace {
         /**
          * Maximum representable value in an unsigned integer with the given
          * number of bits.
          */
         inline unsigned
         scale(unsigned n)
         {
            return (1 << n) - 1;
         }
      }

      /**
       * Pack the vector \p src in a bitfield given the per-component bit
       * shifts and widths.  Note that bitfield components are not allowed to
       * cross 32-bit boundaries.
       */
      static fs_reg
      emit_pack(const fs_builder &bld, const fs_reg &src,
                const color_u &shifts, const color_u &widths)
      {
         const fs_reg dst = bld.vgrf(BRW_REGISTER_TYPE_UD, 4);
         bool seen[4] = {};

         for (unsigned c = 0; c < 4; ++c) {
            if (widths[c]) {
               const fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD);

               /* Shift each component left to the correct bitfield position. */
               bld.SHL(tmp, offset(src, bld, c), brw_imm_ud(shifts[c] % 32));

               /* Add everything up. */
               if (seen[shifts[c] / 32]) {
                  bld.OR(offset(dst, bld, shifts[c] / 32),
                         offset(dst, bld, shifts[c] / 32), tmp);
               } else {
                  bld.MOV(offset(dst, bld, shifts[c] / 32), tmp);
                  seen[shifts[c] / 32] = true;
               }
            }
         }

         return dst;
      }

      /**
       * Unpack a vector from the bitfield \p src given the per-component bit
       * shifts and widths.  Note that bitfield components are not allowed to
       * cross 32-bit boundaries.
       */
      static fs_reg
      emit_unpack(const fs_builder &bld, const fs_reg &src,
                  const color_u &shifts, const color_u &widths)
      {
         const fs_reg dst = bld.vgrf(src.type, 4);

         for (unsigned c = 0; c < 4; ++c) {
            if (widths[c]) {
               /* Shift left to discard the most significant bits. */
               bld.SHL(offset(dst, bld, c),
                       offset(src, bld, shifts[c] / 32),
                       brw_imm_ud(32 - shifts[c] % 32 - widths[c]));

               /* Shift back to the least significant bits using an arithmetic
                * shift to get sign extension on signed types.
                */
               bld.ASR(offset(dst, bld, c),
                       offset(dst, bld, c), brw_imm_ud(32 - widths[c]));
            }
         }

         return dst;
      }

      /**
       * Convert an integer vector into another integer vector of the
       * specified bit widths, properly handling overflow.
       */
      static fs_reg
      emit_convert_to_integer(const fs_builder &bld, const fs_reg &src,
                              const color_u &widths, bool is_signed)
      {
         const unsigned s = (is_signed ? 1 : 0);
         const fs_reg dst = bld.vgrf(
            is_signed ? BRW_REGISTER_TYPE_D : BRW_REGISTER_TYPE_UD, 4);
         assert(src.type == dst.type);

         for (unsigned c = 0; c < 4; ++c) {
            if (widths[c]) {
               /* Clamp to the maximum value. */
               bld.emit_minmax(offset(dst, bld, c), offset(src, bld, c),
                               brw_imm_d((int)scale(widths[c] - s)),
                               BRW_CONDITIONAL_L);

               /* Clamp to the minimum value. */
               if (is_signed)
                  bld.emit_minmax(offset(dst, bld, c), offset(dst, bld, c),
                                  brw_imm_d(-(int)scale(widths[c] - s) - 1),
                                  BRW_CONDITIONAL_GE);

               /* Mask off all but the bits we actually want.  Otherwise, if
                * we pass a negative number into the hardware when it's
                * expecting something like UINT8, it will happily clamp it to
                * +255 for us.
                */
               if (is_signed && widths[c] < 32)
                  bld.AND(offset(dst, bld, c), offset(dst, bld, c),
                          brw_imm_d(scale(widths[c])));
            }
         }

         return dst;
      }

      /**
       * Convert a normalized fixed-point vector of the specified signedness
       * and bit widths into a floating point vector.
       */
      static fs_reg
      emit_convert_from_scaled(const fs_builder &bld, const fs_reg &src,
                               const color_u &widths, bool is_signed)
      {
         const unsigned s = (is_signed ? 1 : 0);
         const fs_reg dst = bld.vgrf(BRW_REGISTER_TYPE_F, 4);

         for (unsigned c = 0; c < 4; ++c) {
            if (widths[c]) {
               /* Convert to float. */
               bld.MOV(offset(dst, bld, c), offset(src, bld, c));

               /* Divide by the normalization constants. */
               bld.MUL(offset(dst, bld, c), offset(dst, bld, c),
                       brw_imm_f(1.0f / scale(widths[c] - s)));

               /* Clamp to the minimum value. */
               if (is_signed)
                  bld.emit_minmax(offset(dst, bld, c),
                                  offset(dst, bld, c), brw_imm_f(-1.0f),
                                  BRW_CONDITIONAL_GE);
            }
         }
         return dst;
      }

      /**
       * Convert a floating-point vector into a normalized fixed-point vector
       * of the specified signedness and bit widths.
       */
      static fs_reg
      emit_convert_to_scaled(const fs_builder &bld, const fs_reg &src,
                             const color_u &widths, bool is_signed)
      {
         const unsigned s = (is_signed ? 1 : 0);
         const fs_reg dst = bld.vgrf(
            is_signed ? BRW_REGISTER_TYPE_D : BRW_REGISTER_TYPE_UD, 4);
         const fs_reg fdst = retype(dst, BRW_REGISTER_TYPE_F);

         for (unsigned c = 0; c < 4; ++c) {
            if (widths[c]) {
               /* Clamp the normalized floating-point argument. */
               if (is_signed) {
                  bld.emit_minmax(offset(fdst, bld, c), offset(src, bld, c),
                                  brw_imm_f(-1.0f), BRW_CONDITIONAL_GE);

                  bld.emit_minmax(offset(fdst, bld, c), offset(fdst, bld, c),
                                  brw_imm_f(1.0f), BRW_CONDITIONAL_L);
               } else {
                  set_saturate(true, bld.MOV(offset(fdst, bld, c),
                                             offset(src, bld, c)));
               }

               /* Multiply by the normalization constants. */
               bld.MUL(offset(fdst, bld, c), offset(fdst, bld, c),
                       brw_imm_f((float)scale(widths[c] - s)));

               /* Convert to integer. */
               bld.RNDE(offset(fdst, bld, c), offset(fdst, bld, c));
               bld.MOV(offset(dst, bld, c), offset(fdst, bld, c));

               /* Mask off all but the bits we actually want.  Otherwise, if
                * we pass a negative number into the hardware when it's
                * expecting something like UINT8, it will happily clamp it to
                * +255 for us.
                */
               if (is_signed && widths[c] < 32)
                  bld.AND(offset(dst, bld, c), offset(dst, bld, c),
                          brw_imm_d(scale(widths[c])));
            }
         }

         return dst;
      }

      /**
       * Convert a floating point vector of the specified bit widths into a
       * 32-bit floating point vector.
       */
      static fs_reg
      emit_convert_from_float(const fs_builder &bld, const fs_reg &src,
                              const color_u &widths)
      {
         const fs_reg dst = bld.vgrf(BRW_REGISTER_TYPE_UD, 4);
         const fs_reg fdst = retype(dst, BRW_REGISTER_TYPE_F);

         for (unsigned c = 0; c < 4; ++c) {
            if (widths[c]) {
               bld.MOV(offset(dst, bld, c), offset(src, bld, c));

               /* Extend 10-bit and 11-bit floating point numbers to 15 bits.
                * This works because they have a 5-bit exponent just like the
                * 16-bit floating point format, and they have no sign bit.
                */
               if (widths[c] < 16)
                  bld.SHL(offset(dst, bld, c),
                          offset(dst, bld, c), brw_imm_ud(15 - widths[c]));

               /* Convert to 32-bit floating point. */
               bld.F16TO32(offset(fdst, bld, c), offset(dst, bld, c));
            }
         }

         return fdst;
      }

      /**
       * Convert a vector into a floating point vector of the specified bit
       * widths.
       */
      static fs_reg
      emit_convert_to_float(const fs_builder &bld, const fs_reg &src,
                            const color_u &widths)
      {
         const fs_reg dst = bld.vgrf(BRW_REGISTER_TYPE_UD, 4);
         const fs_reg fdst = retype(dst, BRW_REGISTER_TYPE_F);

         for (unsigned c = 0; c < 4; ++c) {
            if (widths[c]) {
               bld.MOV(offset(fdst, bld, c), offset(src, bld, c));

               /* Clamp to the minimum value. */
               if (widths[c] < 16)
                  bld.emit_minmax(offset(fdst, bld, c), offset(fdst, bld, c),
                                  brw_imm_f(0.0f), BRW_CONDITIONAL_GE);

               /* Convert to 16-bit floating-point. */
               bld.F32TO16(offset(dst, bld, c), offset(fdst, bld, c));

               /* Discard the least significant bits to get floating point
                * numbers of the requested width.  This works because the
                * 10-bit and 11-bit floating point formats have a 5-bit
                * exponent just like the 16-bit format, and they have no sign
                * bit.
                */
               if (widths[c] < 16)
                  bld.SHR(offset(dst, bld, c), offset(dst, bld, c),
                          brw_imm_ud(15 - widths[c]));
            }
         }

         return dst;
      }

      /**
       * Fill missing components of a vector with 0, 0, 0, 1.
       */
      static fs_reg
      emit_pad(const fs_builder &bld, const fs_reg &src,
               const color_u &widths)
      {
         const fs_reg dst = bld.vgrf(src.type, 4);
         const unsigned pad[] = { 0, 0, 0, 1 };

         for (unsigned c = 0; c < 4; ++c)
            bld.MOV(offset(dst, bld, c),
                    widths[c] ? offset(src, bld, c)
                              : fs_reg(brw_imm_ud(pad[c])));

         return dst;
      }
   }
}

namespace brw {
   namespace image_access {
      /**
       * Load a vector from a surface of the given format and dimensionality
       * at the given coordinates.  \p surf_dims and \p arr_dims give the
       * number of non-array and array coordinates of the image respectively.
       */
      fs_reg
      emit_image_load(const fs_builder &bld,
                      const fs_reg &image, const fs_reg &addr,
                      unsigned surf_dims, unsigned arr_dims,
                      unsigned gl_format)
      {
         using namespace image_format_info;
         using namespace image_format_conversion;
         using namespace image_validity;
         using namespace image_coordinates;
         using namespace surface_access;
         const gen_device_info *devinfo = bld.shader->devinfo;
         const isl_format format = isl_format_for_gl_format(gl_format);
         const isl_format lower_format =
            isl_lower_storage_image_format(devinfo, format);
         fs_reg tmp;

         /* Transform the image coordinates into actual surface coordinates. */
         const fs_reg saddr =
            emit_image_coordinates(bld, addr, surf_dims, arr_dims, format);
         const unsigned dims =
            num_image_coordinates(bld, surf_dims, arr_dims, format);

         if (isl_has_matching_typed_storage_image_format(devinfo, format)) {
            /* Hopefully we get here most of the time... */
            tmp = emit_typed_read(bld, image, saddr, dims,
                                  isl_format_get_num_channels(lower_format));
         } else {
            /* Untyped surface reads return 32 bits of the surface per
             * component, without any sort of unpacking or type conversion,
             */
            const unsigned size = isl_format_get_layout(format)->bpb / 32;
            /* they don't properly handle out of bounds access, so we have to
             * check manually if the coordinates are valid and predicate the
             * surface read on the result,
             */
            const brw_predicate pred =
               emit_untyped_image_check(bld, image,
                                        emit_bounds_check(bld, image,
                                                          saddr, dims));

            /* and they don't know about surface coordinates, we need to
             * convert them to a raw memory offset.
             */
            const fs_reg laddr = emit_address_calculation(bld, image, saddr, dims);

            tmp = emit_untyped_read(bld, image, laddr, 1, size, pred);

            /* An out of bounds surface access should give zero as result. */
            for (unsigned c = 0; c < size; ++c)
               set_predicate(pred, bld.SEL(offset(tmp, bld, c),
                                           offset(tmp, bld, c), brw_imm_d(0)));
         }

         /* Set the register type to D instead of UD if the data type is
          * represented as a signed integer in memory so that sign extension
          * is handled correctly by unpack.
          */
         if (needs_sign_extension(format))
            tmp = retype(tmp, BRW_REGISTER_TYPE_D);

         if (!has_supported_bit_layout(devinfo, format)) {
            /* Unpack individual vector components from the bitfield if the
             * hardware is unable to do it for us.
             */
            if (has_split_bit_layout(devinfo, format))
               tmp = emit_pack(bld, tmp, get_bit_shifts(lower_format),
                               get_bit_widths(lower_format));
            else
               tmp = emit_unpack(bld, tmp, get_bit_shifts(format),
                                 get_bit_widths(format));

         } else if ((needs_sign_extension(format) &&
                     !is_conversion_trivial(devinfo, format)) ||
                    has_undefined_high_bits(devinfo, format)) {
            /* Perform a trivial unpack even though the bit layout matches in
             * order to get the most significant bits of each component
             * initialized properly.
             */
            tmp = emit_unpack(bld, tmp, color_u(0, 32, 64, 96),
                              get_bit_widths(format));
         }

         if (!isl_format_has_int_channel(format)) {
            if (is_conversion_trivial(devinfo, format)) {
               /* Just need to cast the vector to the target type. */
               tmp = retype(tmp, BRW_REGISTER_TYPE_F);
            } else {
               /* Do the right sort of type conversion to float. */
               if (isl_format_has_float_channel(format))
                  tmp = emit_convert_from_float(
                     bld, tmp, get_bit_widths(format));
               else
                  tmp = emit_convert_from_scaled(
                     bld, tmp, get_bit_widths(format),
                     isl_format_has_snorm_channel(format));
            }
         }

         /* Initialize missing components of the result. */
         return emit_pad(bld, tmp, get_bit_widths(format));
      }

      /**
       * Store a vector in a surface of the given format and dimensionality at
       * the given coordinates.  \p surf_dims and \p arr_dims give the number
       * of non-array and array coordinates of the image respectively.
       */
      void
      emit_image_store(const fs_builder &bld, const fs_reg &image,
                       const fs_reg &addr, const fs_reg &src,
                       unsigned surf_dims, unsigned arr_dims,
                       unsigned gl_format)
      {
         using namespace image_format_info;
         using namespace image_format_conversion;
         using namespace image_validity;
         using namespace image_coordinates;
         using namespace surface_access;
         const isl_format format = isl_format_for_gl_format(gl_format);
         const gen_device_info *devinfo = bld.shader->devinfo;

         /* Transform the image coordinates into actual surface coordinates. */
         const fs_reg saddr =
            emit_image_coordinates(bld, addr, surf_dims, arr_dims, format);
         const unsigned dims =
            num_image_coordinates(bld, surf_dims, arr_dims, format);

         if (gl_format == GL_NONE) {
            /* We don't know what the format is, but that's fine because it
             * implies write-only access, and typed surface writes are always
             * able to take care of type conversion and packing for us.
             */
            emit_typed_write(bld, image, saddr, src, dims, 4);

         } else {
            const isl_format lower_format =
               isl_lower_storage_image_format(devinfo, format);
            fs_reg tmp = src;

            if (!is_conversion_trivial(devinfo, format)) {
               /* Do the right sort of type conversion. */
               if (isl_format_has_float_channel(format))
                  tmp = emit_convert_to_float(bld, tmp, get_bit_widths(format));

               else if (isl_format_has_int_channel(format))
                  tmp = emit_convert_to_integer(bld, tmp, get_bit_widths(format),
                                                isl_format_has_sint_channel(format));

               else
                  tmp = emit_convert_to_scaled(bld, tmp, get_bit_widths(format),
                                               isl_format_has_snorm_channel(format));
            }

            /* We're down to bit manipulation at this point. */
            tmp = retype(tmp, BRW_REGISTER_TYPE_UD);

            if (!has_supported_bit_layout(devinfo, format)) {
               /* Pack the vector components into a bitfield if the hardware
                * is unable to do it for us.
                */
               if (has_split_bit_layout(devinfo, format))
                  tmp = emit_unpack(bld, tmp, get_bit_shifts(lower_format),
                                    get_bit_widths(lower_format));

               else
                  tmp = emit_pack(bld, tmp, get_bit_shifts(format),
                                  get_bit_widths(format));
            }

            if (isl_has_matching_typed_storage_image_format(devinfo, format)) {
               /* Hopefully we get here most of the time... */
               emit_typed_write(bld, image, saddr, tmp, dims,
                                isl_format_get_num_channels(lower_format));

            } else {
               /* Untyped surface writes store 32 bits of the surface per
                * component, without any sort of packing or type conversion,
                */
               const unsigned size = isl_format_get_layout(format)->bpb / 32;

               /* they don't properly handle out of bounds access, so we have
                * to check manually if the coordinates are valid and predicate
                * the surface write on the result,
                */
               const brw_predicate pred =
                  emit_untyped_image_check(bld, image,
                                           emit_bounds_check(bld, image,
                                                             saddr, dims));

               /* and, phew, they don't know about surface coordinates, we
                * need to convert them to a raw memory offset.
                */
               const fs_reg laddr = emit_address_calculation(
                  bld, image, saddr, dims);

               emit_untyped_write(bld, image, laddr, tmp, 1, size, pred);
            }
         }
      }

      /**
       * Perform an atomic read-modify-write operation in a surface of the
       * given dimensionality at the given coordinates.  \p surf_dims and \p
       * arr_dims give the number of non-array and array coordinates of the
       * image respectively.  Main building block of the imageAtomic GLSL
       * built-ins.
       */
      fs_reg
      emit_image_atomic(const fs_builder &bld,
                        const fs_reg &image, const fs_reg &addr,
                        const fs_reg &src0, const fs_reg &src1,
                        unsigned surf_dims, unsigned arr_dims,
                        unsigned rsize, unsigned op)
      {
         using namespace image_validity;
         using namespace image_coordinates;
         using namespace surface_access;
         /* Avoid performing an atomic operation on an unbound surface. */
         const brw_predicate pred = emit_typed_atomic_check(bld, image);

         /* Transform the image coordinates into actual surface coordinates. */
         const fs_reg saddr =
            emit_image_coordinates(bld, addr, surf_dims, arr_dims,
                                   ISL_FORMAT_R32_UINT);
         const unsigned dims =
            num_image_coordinates(bld, surf_dims, arr_dims,
                                  ISL_FORMAT_R32_UINT);

         /* Thankfully we can do without untyped atomics here. */
         const fs_reg tmp = emit_typed_atomic(bld, image, saddr, src0, src1,
                                              dims, rsize, op, pred);

         /* An unbound surface access should give zero as result. */
         if (rsize && pred)
            set_predicate(pred, bld.SEL(tmp, tmp, brw_imm_d(0)));

         return retype(tmp, src0.type);
      }
   }
}