+/*
+ * PowerPC / POWER assembly stuff. The special BR_POWER_ASM_MACROS macro
+ * must be defined before including this file; this is done by source
+ * files that use some inline assembly for PowerPC / POWER machines.
+ */
+
+#if BR_POWER_ASM_MACROS
+
+#define lxvw4x(xt, ra, rb) lxvw4x_(xt, ra, rb)
+#define stxvw4x(xt, ra, rb) stxvw4x_(xt, ra, rb)
+
+#define bdnz(foo) bdnz_(foo)
+#define beq(foo) beq_(foo)
+
+#define li(rx, value) li_(rx, value)
+#define addi(rx, ra, imm) addi_(rx, ra, imm)
+#define cmpldi(rx, imm) cmpldi_(rx, imm)
+#define mtctr(rx) mtctr_(rx)
+#define vspltb(vrt, vrb, uim) vspltb_(vrt, vrb, uim)
+#define vspltw(vrt, vrb, uim) vspltw_(vrt, vrb, uim)
+#define vspltisb(vrt, imm) vspltisb_(vrt, imm)
+#define vspltisw(vrt, imm) vspltisw_(vrt, imm)
+#define vrlw(vrt, vra, vrb) vrlw_(vrt, vra, vrb)
+#define vsbox(vrt, vra) vsbox_(vrt, vra)
+#define vxor(vrt, vra, vrb) vxor_(vrt, vra, vrb)
+#define vand(vrt, vra, vrb) vand_(vrt, vra, vrb)
+#define vsro(vrt, vra, vrb) vsro_(vrt, vra, vrb)
+#define vsl(vrt, vra, vrb) vsl_(vrt, vra, vrb)
+#define vsldoi(vt, va, vb, sh) vsldoi_(vt, va, vb, sh)
+#define vsr(vrt, vra, vrb) vsr_(vrt, vra, vrb)
+#define vadduwm(vrt, vra, vrb) vadduwm_(vrt, vra, vrb)
+#define vsububm(vrt, vra, vrb) vsububm_(vrt, vra, vrb)
+#define vsubuwm(vrt, vra, vrb) vsubuwm_(vrt, vra, vrb)
+#define vsrw(vrt, vra, vrb) vsrw_(vrt, vra, vrb)
+#define vcipher(vt, va, vb) vcipher_(vt, va, vb)
+#define vcipherlast(vt, va, vb) vcipherlast_(vt, va, vb)
+#define vncipher(vt, va, vb) vncipher_(vt, va, vb)
+#define vncipherlast(vt, va, vb) vncipherlast_(vt, va, vb)
+#define vperm(vt, va, vb, vc) vperm_(vt, va, vb, vc)
+#define vpmsumd(vt, va, vb) vpmsumd_(vt, va, vb)
+#define xxpermdi(vt, va, vb, d) xxpermdi_(vt, va, vb, d)
+
+#define lxvw4x_(xt, ra, rb) "\tlxvw4x\t" #xt "," #ra "," #rb "\n"
+#define stxvw4x_(xt, ra, rb) "\tstxvw4x\t" #xt "," #ra "," #rb "\n"
+
+#define label(foo) #foo "%=:\n"
+#define bdnz_(foo) "\tbdnz\t" #foo "%=\n"
+#define beq_(foo) "\tbeq\t" #foo "%=\n"
+
+#define li_(rx, value) "\tli\t" #rx "," #value "\n"
+#define addi_(rx, ra, imm) "\taddi\t" #rx "," #ra "," #imm "\n"
+#define cmpldi_(rx, imm) "\tcmpldi\t" #rx "," #imm "\n"
+#define mtctr_(rx) "\tmtctr\t" #rx "\n"
+#define vspltb_(vrt, vrb, uim) "\tvspltb\t" #vrt "," #vrb "," #uim "\n"
+#define vspltw_(vrt, vrb, uim) "\tvspltw\t" #vrt "," #vrb "," #uim "\n"
+#define vspltisb_(vrt, imm) "\tvspltisb\t" #vrt "," #imm "\n"
+#define vspltisw_(vrt, imm) "\tvspltisw\t" #vrt "," #imm "\n"
+#define vrlw_(vrt, vra, vrb) "\tvrlw\t" #vrt "," #vra "," #vrb "\n"
+#define vsbox_(vrt, vra) "\tvsbox\t" #vrt "," #vra "\n"
+#define vxor_(vrt, vra, vrb) "\tvxor\t" #vrt "," #vra "," #vrb "\n"
+#define vand_(vrt, vra, vrb) "\tvand\t" #vrt "," #vra "," #vrb "\n"
+#define vsro_(vrt, vra, vrb) "\tvsro\t" #vrt "," #vra "," #vrb "\n"
+#define vsl_(vrt, vra, vrb) "\tvsl\t" #vrt "," #vra "," #vrb "\n"
+#define vsldoi_(vt, va, vb, sh) "\tvsldoi\t" #vt "," #va "," #vb "," #sh "\n"
+#define vsr_(vrt, vra, vrb) "\tvsr\t" #vrt "," #vra "," #vrb "\n"
+#define vadduwm_(vrt, vra, vrb) "\tvadduwm\t" #vrt "," #vra "," #vrb "\n"
+#define vsububm_(vrt, vra, vrb) "\tvsububm\t" #vrt "," #vra "," #vrb "\n"
+#define vsubuwm_(vrt, vra, vrb) "\tvsubuwm\t" #vrt "," #vra "," #vrb "\n"
+#define vsrw_(vrt, vra, vrb) "\tvsrw\t" #vrt "," #vra "," #vrb "\n"
+#define vcipher_(vt, va, vb) "\tvcipher\t" #vt "," #va "," #vb "\n"
+#define vcipherlast_(vt, va, vb) "\tvcipherlast\t" #vt "," #va "," #vb "\n"
+#define vncipher_(vt, va, vb) "\tvncipher\t" #vt "," #va "," #vb "\n"
+#define vncipherlast_(vt, va, vb) "\tvncipherlast\t" #vt "," #va "," #vb "\n"
+#define vperm_(vt, va, vb, vc) "\tvperm\t" #vt "," #va "," #vb "," #vc "\n"
+#define vpmsumd_(vt, va, vb) "\tvpmsumd\t" #vt "," #va "," #vb "\n"
+#define xxpermdi_(vt, va, vb, d) "\txxpermdi\t" #vt "," #va "," #vb "," #d "\n"
+
+#endif
+
+/* ==================================================================== */
+/*
+ * Special "activate intrinsics" code, needed for some compiler versions.
+ * This is defined at the end of this file, so that it won't impact any
+ * of the inline functions defined previously; and it is controlled by
+ * a specific macro defined in the caller code.
+ *
+ * Calling code conventions:
+ *
+ * - Caller must define BR_ENABLE_INTRINSICS before including "inner.h".
+ * - Functions that use intrinsics must be enclosed in an "enabled"
+ * region (between BR_TARGETS_X86_UP and BR_TARGETS_X86_DOWN).
+ * - Functions that use intrinsics must be tagged with the appropriate
+ * BR_TARGET().
+ */
+
+#if BR_ENABLE_INTRINSICS && (BR_GCC_4_4 || BR_CLANG_3_7 || BR_MSC_2005)
+
+/*
+ * x86 intrinsics (both 32-bit and 64-bit).
+ */
+#if BR_i386 || BR_amd64
+
+/*
+ * On GCC before version 5.0, we need to use the pragma to enable the
+ * target options globally, because the 'target' function attribute
+ * appears to be unreliable. Before 4.6 we must also avoid the
+ * push_options / pop_options mechanism, because it tends to trigger
+ * some internal compiler errors.
+ */
+#if BR_GCC && !BR_GCC_5_0
+#if BR_GCC_4_6
+#define BR_TARGETS_X86_UP \
+ _Pragma("GCC push_options") \
+ _Pragma("GCC target(\"sse2,ssse3,sse4.1,aes,pclmul,rdrnd\")")
+#define BR_TARGETS_X86_DOWN \
+ _Pragma("GCC pop_options")
+#else
+#define BR_TARGETS_X86_UP \
+ _Pragma("GCC target(\"sse2,ssse3,sse4.1,aes,pclmul\")")
+#endif
+#define BR_TARGETS_X86_DOWN
+#pragma GCC diagnostic ignored "-Wpsabi"
+#endif
+
+#if BR_CLANG && !BR_CLANG_3_8
+#undef __SSE2__
+#undef __SSE3__
+#undef __SSSE3__
+#undef __SSE4_1__
+#undef __AES__
+#undef __PCLMUL__
+#undef __RDRND__
+#define __SSE2__ 1
+#define __SSE3__ 1
+#define __SSSE3__ 1
+#define __SSE4_1__ 1
+#define __AES__ 1
+#define __PCLMUL__ 1
+#define __RDRND__ 1
+#endif
+
+#ifndef BR_TARGETS_X86_UP
+#define BR_TARGETS_X86_UP
+#endif
+#ifndef BR_TARGETS_X86_DOWN
+#define BR_TARGETS_X86_DOWN
+#endif
+
+#if BR_GCC || BR_CLANG
+BR_TARGETS_X86_UP
+#include <x86intrin.h>
+#include <cpuid.h>
+#define br_bswap32 __builtin_bswap32
+BR_TARGETS_X86_DOWN
+#endif
+
+#if BR_MSC
+#include <stdlib.h>
+#include <intrin.h>
+#include <immintrin.h>
+#define br_bswap32 _byteswap_ulong
+#endif
+
+static inline int
+br_cpuid(uint32_t mask_eax, uint32_t mask_ebx,
+ uint32_t mask_ecx, uint32_t mask_edx)
+{
+#if BR_GCC || BR_CLANG
+ unsigned eax, ebx, ecx, edx;
+
+ if (__get_cpuid(1, &eax, &ebx, &ecx, &edx)) {
+ if ((eax & mask_eax) == mask_eax
+ && (ebx & mask_ebx) == mask_ebx
+ && (ecx & mask_ecx) == mask_ecx
+ && (edx & mask_edx) == mask_edx)
+ {
+ return 1;
+ }
+ }
+#elif BR_MSC
+ int info[4];
+
+ __cpuid(info, 1);
+ if (((uint32_t)info[0] & mask_eax) == mask_eax
+ && ((uint32_t)info[1] & mask_ebx) == mask_ebx
+ && ((uint32_t)info[2] & mask_ecx) == mask_ecx
+ && ((uint32_t)info[3] & mask_edx) == mask_edx)
+ {
+ return 1;
+ }
+#endif
+ return 0;
+}
+
+#endif
+
+#endif
+
+/* ==================================================================== */
+