src/util/u_cpu_detect.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2008 Dennis Smit
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * on the rights to use, copy, modify, merge, publish, distribute, sub
  10  * license, and/or sell copies of the Software, and to permit persons to whom
  11  * the Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.  IN NO EVENT SHALL
  20  * AUTHORS, COPYRIGHT HOLDERS, AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  21  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  22  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  23  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  24  *
  25  **************************************************************************/
  26
  27 /**
  28  * @file
  29  * CPU feature detection.
  30  *
  31  * @author Dennis Smit
  32  * @author Based on the work of Eric Anholt <anholt@FreeBSD.org>
  33  */
  34
  35 #include "pipe/p_config.h"
  36
  37 #include "util/u_debug.h"
  38 #include "u_cpu_detect.h"
  39 #include "c11/threads.h"
  40
  41 #if defined(PIPE_ARCH_PPC)
  42 #if defined(PIPE_OS_APPLE)
  43 #include <sys/sysctl.h>
  44 #else
  45 #include <signal.h>
  46 #include <setjmp.h>
  47 #endif
  48 #endif
  49
  50 #if defined(PIPE_OS_NETBSD) || defined(PIPE_OS_OPENBSD)
  51 #include <sys/param.h>
  52 #include <sys/sysctl.h>
  53 #include <machine/cpu.h>
  54 #endif
  55
  56 #if defined(PIPE_OS_FREEBSD) || defined(PIPE_OS_DRAGONFLY)
  57 #include <sys/types.h>
  58 #include <sys/sysctl.h>
  59 #if __has_include(<sys/auxv.h>)
  60 #include <sys/auxv.h>
  61 #define HAVE_ELF_AUX_INFO
  62 #endif
  63 #if defined(PIPE_ARCH_PPC)
  64 #include <machine/cpu.h>
  65 #endif
  66 #endif
  67
  68 #if defined(PIPE_OS_LINUX)
  69 #include <signal.h>
  70 #include <fcntl.h>
  71 #include <elf.h>
  72 #endif
  73
  74 #ifdef PIPE_OS_UNIX
  75 #include <unistd.h>
  76 #endif
  77
  78 #if defined(HAS_ANDROID_CPUFEATURES)
  79 #include <cpu-features.h>
  80 #endif
  81
  82 #if defined(PIPE_OS_WINDOWS)
  83 #include <windows.h>
  84 #if defined(PIPE_CC_MSVC)
  85 #include <intrin.h>
  86 #endif
  87 #endif
  88
  89
  90 #ifdef DEBUG
  91 DEBUG_GET_ONCE_BOOL_OPTION(dump_cpu, "GALLIUM_DUMP_CPU", FALSE)
  92 #endif
  93
  94
  95 struct util_cpu_caps util_cpu_caps;
  96
  97 #if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
  98 static int has_cpuid(void);
  99 #endif
 100
 101
 102 #if defined(PIPE_ARCH_PPC) && !defined(PIPE_OS_APPLE) && !defined(PIPE_OS_BSD) && !defined(PIPE_OS_LINUX)
 103 static jmp_buf  __lv_powerpc_jmpbuf;
 104 static volatile sig_atomic_t __lv_powerpc_canjump = 0;
 105
 106 static void
 107 sigill_handler(int sig)
 108 {
 109    if (!__lv_powerpc_canjump) {
 110       signal (sig, SIG_DFL);
 111       raise (sig);
 112    }
 113
 114    __lv_powerpc_canjump = 0;
 115    longjmp(__lv_powerpc_jmpbuf, 1);
 116 }
 117 #endif
 118
 119 #if defined(PIPE_ARCH_PPC)
 120 static void
 121 check_os_altivec_support(void)
 122 {
 123 #if defined(__ALTIVEC__)
 124    util_cpu_caps.has_altivec = 1;
 125 #endif
 126 #if defined(__VSX__)
 127    util_cpu_caps.has_vsx = 1;
 128 #endif
 129 #if defined(__ALTIVEC__) && defined(__VSX__)
 130 /* Do nothing */
 131 #elif defined(PIPE_OS_APPLE) || defined(PIPE_OS_NETBSD) || defined(PIPE_OS_OPENBSD)
 132 #ifdef HW_VECTORUNIT
 133    int sels[2] = {CTL_HW, HW_VECTORUNIT};
 134 #else
 135    int sels[2] = {CTL_MACHDEP, CPU_ALTIVEC};
 136 #endif
 137    int has_vu = 0;
 138    int len = sizeof (has_vu);
 139    int err;
 140
 141    err = sysctl(sels, 2, &has_vu, &len, NULL, 0);
 142
 143    if (err == 0) {
 144       if (has_vu != 0) {
 145          util_cpu_caps.has_altivec = 1;
 146       }
 147    }
 148 #elif defined(PIPE_OS_FREEBSD) /* !PIPE_OS_APPLE && !PIPE_OS_NETBSD && !PIPE_OS_OPENBSD */
 149    unsigned long hwcap = 0;
 150 #ifdef HAVE_ELF_AUX_INFO
 151    elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap));
 152 #else
 153    size_t len = sizeof(hwcap);
 154    sysctlbyname("hw.cpu_features", &hwcap, &len, NULL, 0);
 155 #endif
 156    if (hwcap & PPC_FEATURE_HAS_ALTIVEC)
 157       util_cpu_caps.has_altivec = 1;
 158    if (hwcap & PPC_FEATURE_HAS_VSX)
 159       util_cpu_caps.has_vsx = 1;
 160 #elif defined(PIPE_OS_LINUX) /* !PIPE_OS_FREEBSD */
 161 #if defined(PIPE_ARCH_PPC_64)
 162     Elf64_auxv_t aux;
 163 #else
 164     Elf32_auxv_t aux;
 165 #endif
 166     int fd = open("/proc/self/auxv", O_RDONLY | O_CLOEXEC);
 167     if (fd >= 0) {
 168        while (read(fd, &aux, sizeof(aux)) == sizeof(aux)) {
 169           if (aux.a_type == AT_HWCAP) {
 170              char *env_vsx = getenv("GALLIVM_VSX");
 171              uint64_t hwcap = aux.a_un.a_val;
 172              util_cpu_caps.has_altivec = (hwcap >> 28) & 1;
 173              if (!env_vsx || env_vsx[0] != '0') {
 174                 util_cpu_caps.has_vsx  = (hwcap >>  7) & 1;
 175              }
 176              break;
 177           }
 178        }
 179        close(fd);
 180     }
 181 #else /* !PIPE_OS_APPLE && !PIPE_OS_BSD && !PIPE_OS_LINUX */
 182    /* not on Apple/Darwin or Linux, do it the brute-force way */
 183    /* this is borrowed from the libmpeg2 library */
 184    signal(SIGILL, sigill_handler);
 185    if (setjmp(__lv_powerpc_jmpbuf)) {
 186       signal(SIGILL, SIG_DFL);
 187    } else {
 188       boolean enable_altivec = TRUE;    /* Default: enable  if available, and if not overridden */
 189       boolean enable_vsx = TRUE;
 190 #ifdef DEBUG
 191       /* Disabling Altivec code generation is not the same as disabling VSX code generation,
 192        * which can be done simply by passing -mattr=-vsx to the LLVM compiler; cf.
 193        * lp_build_create_jit_compiler_for_module().
 194        * If you want to disable Altivec code generation, the best place to do it is here.
 195        */
 196       char *env_control = getenv("GALLIVM_ALTIVEC");    /* 1=enable (default); 0=disable */
 197       if (env_control && env_control[0] == '0') {
 198          enable_altivec = FALSE;
 199       }
 200 #endif
 201       /* VSX instructions can be explicitly enabled/disabled via GALLIVM_VSX=1 or 0 */
 202       char *env_vsx = getenv("GALLIVM_VSX");
 203       if (env_vsx && env_vsx[0] == '0') {
 204          enable_vsx = FALSE;
 205       }
 206       if (enable_altivec) {
 207          __lv_powerpc_canjump = 1;
 208
 209          __asm __volatile
 210             ("mtspr 256, %0\n\t"
 211              "vand %%v0, %%v0, %%v0"
 212              :
 213              : "r" (-1));
 214
 215          util_cpu_caps.has_altivec = 1;
 216
 217          if (enable_vsx) {
 218             __asm __volatile("xxland %vs0, %vs0, %vs0");
 219             util_cpu_caps.has_vsx = 1;
 220          }
 221          signal(SIGILL, SIG_DFL);
 222       } else {
 223          util_cpu_caps.has_altivec = 0;
 224       }
 225    }
 226 #endif /* !PIPE_OS_APPLE && !PIPE_OS_LINUX */
 227 }
 228 #endif /* PIPE_ARCH_PPC */
 229
 230
 231 #if defined(PIPE_ARCH_X86) || defined (PIPE_ARCH_X86_64)
 232 static int has_cpuid(void)
 233 {
 234 #if defined(PIPE_ARCH_X86)
 235 #if defined(PIPE_OS_GCC)
 236    int a, c;
 237
 238    __asm __volatile
 239       ("pushf\n"
 240        "popl %0\n"
 241        "movl %0, %1\n"
 242        "xorl $0x200000, %0\n"
 243        "push %0\n"
 244        "popf\n"
 245        "pushf\n"
 246        "popl %0\n"
 247        : "=a" (a), "=c" (c)
 248        :
 249        : "cc");
 250
 251    return a != c;
 252 #else
 253    /* FIXME */
 254    return 1;
 255 #endif
 256 #elif defined(PIPE_ARCH_X86_64)
 257    return 1;
 258 #else
 259    return 0;
 260 #endif
 261 }
 262
 263
 264 /**
 265  * @sa cpuid.h included in gcc-4.3 onwards.
 266  * @sa http://msdn.microsoft.com/en-us/library/hskdteyh.aspx
 267  */
 268 static inline void
 269 cpuid(uint32_t ax, uint32_t *p)
 270 {
 271 #if defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86)
 272    __asm __volatile (
 273      "xchgl %%ebx, %1\n\t"
 274      "cpuid\n\t"
 275      "xchgl %%ebx, %1"
 276      : "=a" (p[0]),
 277        "=S" (p[1]),
 278        "=c" (p[2]),
 279        "=d" (p[3])
 280      : "0" (ax)
 281    );
 282 #elif defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86_64)
 283    __asm __volatile (
 284      "cpuid\n\t"
 285      : "=a" (p[0]),
 286        "=b" (p[1]),
 287        "=c" (p[2]),
 288        "=d" (p[3])
 289      : "0" (ax)
 290    );
 291 #elif defined(PIPE_CC_MSVC)
 292    __cpuid(p, ax);
 293 #else
 294    p[0] = 0;
 295    p[1] = 0;
 296    p[2] = 0;
 297    p[3] = 0;
 298 #endif
 299 }
 300
 301 /**
 302  * @sa cpuid.h included in gcc-4.4 onwards.
 303  * @sa http://msdn.microsoft.com/en-us/library/hskdteyh%28v=vs.90%29.aspx
 304  */
 305 static inline void
 306 cpuid_count(uint32_t ax, uint32_t cx, uint32_t *p)
 307 {
 308 #if defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86)
 309    __asm __volatile (
 310      "xchgl %%ebx, %1\n\t"
 311      "cpuid\n\t"
 312      "xchgl %%ebx, %1"
 313      : "=a" (p[0]),
 314        "=S" (p[1]),
 315        "=c" (p[2]),
 316        "=d" (p[3])
 317      : "0" (ax), "2" (cx)
 318    );
 319 #elif defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86_64)
 320    __asm __volatile (
 321      "cpuid\n\t"
 322      : "=a" (p[0]),
 323        "=b" (p[1]),
 324        "=c" (p[2]),
 325        "=d" (p[3])
 326      : "0" (ax), "2" (cx)
 327    );
 328 #elif defined(PIPE_CC_MSVC)
 329    __cpuidex(p, ax, cx);
 330 #else
 331    p[0] = 0;
 332    p[1] = 0;
 333    p[2] = 0;
 334    p[3] = 0;
 335 #endif
 336 }
 337
 338
 339 static inline uint64_t xgetbv(void)
 340 {
 341 #if defined(PIPE_CC_GCC)
 342    uint32_t eax, edx;
 343
 344    __asm __volatile (
 345      ".byte 0x0f, 0x01, 0xd0" // xgetbv isn't supported on gcc < 4.4
 346      : "=a"(eax),
 347        "=d"(edx)
 348      : "c"(0)
 349    );
 350
 351    return ((uint64_t)edx << 32) | eax;
 352 #elif defined(PIPE_CC_MSVC) && defined(_MSC_FULL_VER) && defined(_XCR_XFEATURE_ENABLED_MASK)
 353    return _xgetbv(_XCR_XFEATURE_ENABLED_MASK);
 354 #else
 355    return 0;
 356 #endif
 357 }
 358
 359
 360 #if defined(PIPE_ARCH_X86)
 361 PIPE_ALIGN_STACK static inline boolean sse2_has_daz(void)
 362 {
 363    struct {
 364       uint32_t pad1[7];
 365       uint32_t mxcsr_mask;
 366       uint32_t pad2[128-8];
 367    } PIPE_ALIGN_VAR(16) fxarea;
 368
 369    fxarea.mxcsr_mask = 0;
 370 #if defined(PIPE_CC_GCC)
 371    __asm __volatile ("fxsave %0" : "+m" (fxarea));
 372 #elif defined(PIPE_CC_MSVC) || defined(PIPE_CC_ICL)
 373    _fxsave(&fxarea);
 374 #else
 375    fxarea.mxcsr_mask = 0;
 376 #endif
 377    return !!(fxarea.mxcsr_mask & (1 << 6));
 378 }
 379 #endif
 380
 381 #endif /* X86 or X86_64 */
 382
 383 #if defined(PIPE_ARCH_ARM)
 384 static void
 385 check_os_arm_support(void)
 386 {
 387    /*
 388     * On Android, the cpufeatures library is preferred way of checking
 389     * CPU capabilities. However, it is not available for standalone Mesa
 390     * builds, i.e. when Android build system (Android.mk-based) is not
 391     * used. Because of this we cannot use PIPE_OS_ANDROID here, but rather
 392     * have a separate macro that only gets enabled from respective Android.mk.
 393     */
 394 #if defined(__ARM_NEON) || defined(__ARM_NEON__)
 395    util_cpu_caps.has_neon = 1;
 396 #elif defined(PIPE_OS_FREEBSD) && defined(HAVE_ELF_AUX_INFO)
 397    unsigned long hwcap = 0;
 398    elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap));
 399    if (hwcap & HWCAP_NEON)
 400       util_cpu_caps.has_neon = 1;
 401 #elif defined(HAS_ANDROID_CPUFEATURES)
 402    AndroidCpuFamily cpu_family = android_getCpuFamily();
 403    uint64_t cpu_features = android_getCpuFeatures();
 404
 405    if (cpu_family == ANDROID_CPU_FAMILY_ARM) {
 406       if (cpu_features & ANDROID_CPU_ARM_FEATURE_NEON)
 407          util_cpu_caps.has_neon = 1;
 408    }
 409 #elif defined(PIPE_OS_LINUX)
 410     Elf32_auxv_t aux;
 411     int fd;
 412
 413     fd = open("/proc/self/auxv", O_RDONLY | O_CLOEXEC);
 414     if (fd >= 0) {
 415        while (read(fd, &aux, sizeof(Elf32_auxv_t)) == sizeof(Elf32_auxv_t)) {
 416           if (aux.a_type == AT_HWCAP) {
 417              uint32_t hwcap = aux.a_un.a_val;
 418
 419              util_cpu_caps.has_neon = (hwcap >> 12) & 1;
 420              break;
 421           }
 422        }
 423        close (fd);
 424     }
 425 #endif /* PIPE_OS_LINUX */
 426 }
 427
 428 #elif defined(PIPE_ARCH_AARCH64)
 429 static void
 430 check_os_arm_support(void)
 431 {
 432     util_cpu_caps.has_neon = true;
 433 }
 434 #endif /* PIPE_ARCH_ARM || PIPE_ARCH_AARCH64 */
 435
 436 static void
 437 get_cpu_topology(void)
 438 {
 439    /* Default. This is correct if L3 is not present or there is only one. */
 440    util_cpu_caps.cores_per_L3 = util_cpu_caps.nr_cpus;
 441
 442 #if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
 443    /* AMD Zen */
 444    if (util_cpu_caps.x86_cpu_type == 0x17) {
 445       uint32_t regs[4];
 446
 447       /* Query the L3 cache topology information. */
 448       cpuid_count(0x8000001D, 3, regs);
 449       unsigned cache_level = (regs[0] >> 5) & 0x7;
 450       unsigned cores_per_cache = ((regs[0] >> 14) & 0xfff) + 1;
 451
 452       if (cache_level == 3)
 453          util_cpu_caps.cores_per_L3 = cores_per_cache;
 454    }
 455 #endif
 456 }
 457
 458 static void
 459 util_cpu_detect_once(void)
 460 {
 461    memset(&util_cpu_caps, 0, sizeof util_cpu_caps);
 462
 463    /* Count the number of CPUs in system */
 464 #if defined(PIPE_OS_WINDOWS)
 465    {
 466       SYSTEM_INFO system_info;
 467       GetSystemInfo(&system_info);
 468       util_cpu_caps.nr_cpus = system_info.dwNumberOfProcessors;
 469    }
 470 #elif defined(PIPE_OS_UNIX) && defined(_SC_NPROCESSORS_ONLN)
 471    util_cpu_caps.nr_cpus = sysconf(_SC_NPROCESSORS_ONLN);
 472    if (util_cpu_caps.nr_cpus == ~0)
 473       util_cpu_caps.nr_cpus = 1;
 474 #elif defined(PIPE_OS_BSD)
 475    {
 476       int mib[2], ncpu;
 477       int len;
 478
 479       mib[0] = CTL_HW;
 480       mib[1] = HW_NCPU;
 481
 482       len = sizeof (ncpu);
 483       sysctl(mib, 2, &ncpu, &len, NULL, 0);
 484       util_cpu_caps.nr_cpus = ncpu;
 485    }
 486 #else
 487    util_cpu_caps.nr_cpus = 1;
 488 #endif
 489
 490    /* Make the fallback cacheline size nonzero so that it can be
 491     * safely passed to align().
 492     */
 493    util_cpu_caps.cacheline = sizeof(void *);
 494
 495 #if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
 496    if (has_cpuid()) {
 497       uint32_t regs[4];
 498       uint32_t regs2[4];
 499
 500       util_cpu_caps.cacheline = 32;
 501
 502       /* Get max cpuid level */
 503       cpuid(0x00000000, regs);
 504
 505       if (regs[0] >= 0x00000001) {
 506          unsigned int cacheline;
 507
 508          cpuid (0x00000001, regs2);
 509
 510          util_cpu_caps.x86_cpu_type = (regs2[0] >> 8) & 0xf;
 511          /* Add "extended family". */
 512          if (util_cpu_caps.x86_cpu_type == 0xf)
 513              util_cpu_caps.x86_cpu_type += ((regs2[0] >> 20) & 0xff);
 514
 515          /* general feature flags */
 516          util_cpu_caps.has_tsc    = (regs2[3] >>  4) & 1; /* 0x0000010 */
 517          util_cpu_caps.has_mmx    = (regs2[3] >> 23) & 1; /* 0x0800000 */
 518          util_cpu_caps.has_sse    = (regs2[3] >> 25) & 1; /* 0x2000000 */
 519          util_cpu_caps.has_sse2   = (regs2[3] >> 26) & 1; /* 0x4000000 */
 520          util_cpu_caps.has_sse3   = (regs2[2] >>  0) & 1; /* 0x0000001 */
 521          util_cpu_caps.has_ssse3  = (regs2[2] >>  9) & 1; /* 0x0000020 */
 522          util_cpu_caps.has_sse4_1 = (regs2[2] >> 19) & 1;
 523          util_cpu_caps.has_sse4_2 = (regs2[2] >> 20) & 1;
 524          util_cpu_caps.has_popcnt = (regs2[2] >> 23) & 1;
 525          util_cpu_caps.has_avx    = ((regs2[2] >> 28) & 1) && // AVX
 526                                     ((regs2[2] >> 27) & 1) && // OSXSAVE
 527                                     ((xgetbv() & 6) == 6);    // XMM & YMM
 528          util_cpu_caps.has_f16c   = ((regs2[2] >> 29) & 1) && util_cpu_caps.has_avx;
 529          util_cpu_caps.has_fma    = ((regs2[2] >> 12) & 1) && util_cpu_caps.has_avx;
 530          util_cpu_caps.has_mmx2   = util_cpu_caps.has_sse; /* SSE cpus supports mmxext too */
 531 #if defined(PIPE_ARCH_X86_64)
 532          util_cpu_caps.has_daz = 1;
 533 #else
 534          util_cpu_caps.has_daz = util_cpu_caps.has_sse3 ||
 535             (util_cpu_caps.has_sse2 && sse2_has_daz());
 536 #endif
 537
 538          cacheline = ((regs2[1] >> 8) & 0xFF) * 8;
 539          if (cacheline > 0)
 540             util_cpu_caps.cacheline = cacheline;
 541       }
 542       if (util_cpu_caps.has_avx && regs[0] >= 0x00000007) {
 543          uint32_t regs7[4];
 544          cpuid_count(0x00000007, 0x00000000, regs7);
 545          util_cpu_caps.has_avx2 = (regs7[1] >> 5) & 1;
 546       }
 547
 548       // check for avx512
 549       if (((regs2[2] >> 27) & 1) && // OSXSAVE
 550           (xgetbv() & (0x7 << 5)) && // OPMASK: upper-256 enabled by OS
 551           ((xgetbv() & 6) == 6)) { // XMM/YMM enabled by OS
 552          uint32_t regs3[4];
 553          cpuid_count(0x00000007, 0x00000000, regs3);
 554          util_cpu_caps.has_avx512f    = (regs3[1] >> 16) & 1;
 555          util_cpu_caps.has_avx512dq   = (regs3[1] >> 17) & 1;
 556          util_cpu_caps.has_avx512ifma = (regs3[1] >> 21) & 1;
 557          util_cpu_caps.has_avx512pf   = (regs3[1] >> 26) & 1;
 558          util_cpu_caps.has_avx512er   = (regs3[1] >> 27) & 1;
 559          util_cpu_caps.has_avx512cd   = (regs3[1] >> 28) & 1;
 560          util_cpu_caps.has_avx512bw   = (regs3[1] >> 30) & 1;
 561          util_cpu_caps.has_avx512vl   = (regs3[1] >> 31) & 1;
 562          util_cpu_caps.has_avx512vbmi = (regs3[2] >>  1) & 1;
 563       }
 564
 565       if (regs[1] == 0x756e6547 && regs[2] == 0x6c65746e && regs[3] == 0x49656e69) {
 566          /* GenuineIntel */
 567          util_cpu_caps.has_intel = 1;
 568       }
 569
 570       cpuid(0x80000000, regs);
 571
 572       if (regs[0] >= 0x80000001) {
 573
 574          cpuid(0x80000001, regs2);
 575
 576          util_cpu_caps.has_mmx  |= (regs2[3] >> 23) & 1;
 577          util_cpu_caps.has_mmx2 |= (regs2[3] >> 22) & 1;
 578          util_cpu_caps.has_3dnow = (regs2[3] >> 31) & 1;
 579          util_cpu_caps.has_3dnow_ext = (regs2[3] >> 30) & 1;
 580
 581          util_cpu_caps.has_xop = util_cpu_caps.has_avx &&
 582                                  ((regs2[2] >> 11) & 1);
 583       }
 584
 585       if (regs[0] >= 0x80000006) {
 586          /* should we really do this if the clflush size above worked? */
 587          unsigned int cacheline;
 588          cpuid(0x80000006, regs2);
 589          cacheline = regs2[2] & 0xFF;
 590          if (cacheline > 0)
 591             util_cpu_caps.cacheline = cacheline;
 592       }
 593
 594       if (!util_cpu_caps.has_sse) {
 595          util_cpu_caps.has_sse2 = 0;
 596          util_cpu_caps.has_sse3 = 0;
 597          util_cpu_caps.has_ssse3 = 0;
 598          util_cpu_caps.has_sse4_1 = 0;
 599       }
 600    }
 601 #endif /* PIPE_ARCH_X86 || PIPE_ARCH_X86_64 */
 602
 603 #if defined(PIPE_ARCH_ARM) || defined(PIPE_ARCH_AARCH64)
 604    check_os_arm_support();
 605 #endif
 606
 607 #if defined(PIPE_ARCH_PPC)
 608    check_os_altivec_support();
 609 #endif /* PIPE_ARCH_PPC */
 610
 611    get_cpu_topology();
 612
 613 #ifdef DEBUG
 614    if (debug_get_option_dump_cpu()) {
 615       debug_printf("util_cpu_caps.nr_cpus = %u\n", util_cpu_caps.nr_cpus);
 616
 617       debug_printf("util_cpu_caps.x86_cpu_type = %u\n", util_cpu_caps.x86_cpu_type);
 618       debug_printf("util_cpu_caps.cacheline = %u\n", util_cpu_caps.cacheline);
 619
 620       debug_printf("util_cpu_caps.has_tsc = %u\n", util_cpu_caps.has_tsc);
 621       debug_printf("util_cpu_caps.has_mmx = %u\n", util_cpu_caps.has_mmx);
 622       debug_printf("util_cpu_caps.has_mmx2 = %u\n", util_cpu_caps.has_mmx2);
 623       debug_printf("util_cpu_caps.has_sse = %u\n", util_cpu_caps.has_sse);
 624       debug_printf("util_cpu_caps.has_sse2 = %u\n", util_cpu_caps.has_sse2);
 625       debug_printf("util_cpu_caps.has_sse3 = %u\n", util_cpu_caps.has_sse3);
 626       debug_printf("util_cpu_caps.has_ssse3 = %u\n", util_cpu_caps.has_ssse3);
 627       debug_printf("util_cpu_caps.has_sse4_1 = %u\n", util_cpu_caps.has_sse4_1);
 628       debug_printf("util_cpu_caps.has_sse4_2 = %u\n", util_cpu_caps.has_sse4_2);
 629       debug_printf("util_cpu_caps.has_avx = %u\n", util_cpu_caps.has_avx);
 630       debug_printf("util_cpu_caps.has_avx2 = %u\n", util_cpu_caps.has_avx2);
 631       debug_printf("util_cpu_caps.has_f16c = %u\n", util_cpu_caps.has_f16c);
 632       debug_printf("util_cpu_caps.has_popcnt = %u\n", util_cpu_caps.has_popcnt);
 633       debug_printf("util_cpu_caps.has_3dnow = %u\n", util_cpu_caps.has_3dnow);
 634       debug_printf("util_cpu_caps.has_3dnow_ext = %u\n", util_cpu_caps.has_3dnow_ext);
 635       debug_printf("util_cpu_caps.has_xop = %u\n", util_cpu_caps.has_xop);
 636       debug_printf("util_cpu_caps.has_altivec = %u\n", util_cpu_caps.has_altivec);
 637       debug_printf("util_cpu_caps.has_vsx = %u\n", util_cpu_caps.has_vsx);
 638       debug_printf("util_cpu_caps.has_neon = %u\n", util_cpu_caps.has_neon);
 639       debug_printf("util_cpu_caps.has_daz = %u\n", util_cpu_caps.has_daz);
 640       debug_printf("util_cpu_caps.has_avx512f = %u\n", util_cpu_caps.has_avx512f);
 641       debug_printf("util_cpu_caps.has_avx512dq = %u\n", util_cpu_caps.has_avx512dq);
 642       debug_printf("util_cpu_caps.has_avx512ifma = %u\n", util_cpu_caps.has_avx512ifma);
 643       debug_printf("util_cpu_caps.has_avx512pf = %u\n", util_cpu_caps.has_avx512pf);
 644       debug_printf("util_cpu_caps.has_avx512er = %u\n", util_cpu_caps.has_avx512er);
 645       debug_printf("util_cpu_caps.has_avx512cd = %u\n", util_cpu_caps.has_avx512cd);
 646       debug_printf("util_cpu_caps.has_avx512bw = %u\n", util_cpu_caps.has_avx512bw);
 647       debug_printf("util_cpu_caps.has_avx512vl = %u\n", util_cpu_caps.has_avx512vl);
 648       debug_printf("util_cpu_caps.has_avx512vbmi = %u\n", util_cpu_caps.has_avx512vbmi);
 649    }
 650 #endif
 651 }
 652
 653 static once_flag cpu_once_flag = ONCE_FLAG_INIT;
 654
 655 void
 656 util_cpu_detect(void)
 657 {
 658    call_once(&cpu_once_flag, util_cpu_detect_once);
 659 }