src/intel/common/gen_clflush.h

   1 /*
   2  * Copyright © 2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #ifndef GEN_CLFLUSH_H
  25 #define GEN_CLFLUSH_H
  26
  27 #define CACHELINE_SIZE 64
  28 #define CACHELINE_MASK 63
  29
  30 static inline void
  31 gen_clflush_range(void *start, size_t size)
  32 {
  33    void *p = (void *) (((uintptr_t) start) & ~CACHELINE_MASK);
  34    void *end = start + size;
  35
  36    while (p < end) {
  37       __builtin_ia32_clflush(p);
  38       p += CACHELINE_SIZE;
  39    }
  40 }
  41
  42 static inline void
  43 gen_flush_range(void *start, size_t size)
  44 {
  45    __builtin_ia32_mfence();
  46    gen_clflush_range(start, size);
  47 }
  48
  49 static inline void
  50 gen_invalidate_range(void *start, size_t size)
  51 {
  52    gen_clflush_range(start, size);
  53
  54    /* Modern Atom CPUs (Baytrail+) have issues with clflush serialization,
  55     * where mfence is not a sufficient synchronization barrier.  We must
  56     * double clflush the last cacheline.  This guarantees it will be ordered
  57     * after the preceding clflushes, and then the mfence guards against
  58     * prefetches crossing the clflush boundary.
  59     *
  60     * See kernel commit 396f5d62d1a5fd99421855a08ffdef8edb43c76e
  61     * ("drm: Restore double clflush on the last partial cacheline")
  62     * and https://bugs.freedesktop.org/show_bug.cgi?id=92845.
  63     */
  64    __builtin_ia32_clflush(start + size - 1);
  65    __builtin_ia32_mfence();
  66 }
  67
  68 #endif