X-Git-Url: https://bearssl.org/gitweb//home/git/?p=BearSSL;a=blobdiff_plain;f=src%2Finner.h;h=fb49d0ef99d73b91d3d036b9fd5acc865eefe4a4;hp=9123a421fd01ef9b8c387e1dd6314de1c638dc88;hb=8e94ad2fcb11794c559025277e56f3fbeb676f5d;hpb=2f9c953af45299f8546df8984d5262e767a7d943;ds=inline diff --git a/src/inner.h b/src/inner.h index 9123a42..fb49d0e 100644 --- a/src/inner.h +++ b/src/inner.h @@ -31,6 +31,15 @@ #include "config.h" #include "bearssl.h" +/* + * On MSVC, disable the warning about applying unary minus on an + * unsigned type: it is standard, we do it all the time, and for + * good reasons. + */ +#if _MSC_VER +#pragma warning( disable : 4146 ) +#endif + /* * Maximum size for a RSA modulus (in bits). Allocated stack buffers * depend on that size, so this value should be kept small. Currently, @@ -96,58 +105,409 @@ #endif #endif +/* + * Set BR_LOMUL on platforms where it makes sense. + */ +#ifndef BR_LOMUL +#if BR_ARMEL_CORTEXM_GCC +#define BR_LOMUL 1 +#endif +#endif + +/* + * Architecture detection. + */ +#ifndef BR_i386 +#if __i386__ || _M_IX86 +#define BR_i386 1 +#endif +#endif + +#ifndef BR_amd64 +#if __x86_64__ || _M_X64 +#define BR_amd64 1 +#endif +#endif + +/* + * Compiler brand and version. + * + * Implementations that use intrinsics need to detect the compiler type + * and version because some specific actions may be needed to activate + * the corresponding opcodes, both for header inclusion, and when using + * them in a function. + * + * BR_GCC, BR_CLANG and BR_MSC will be set to 1 for, respectively, GCC, + * Clang and MS Visual C. For each of them, sub-macros will be defined + * for versions; each sub-macro is set whenever the compiler version is + * at least as recent as the one corresponding to the macro. + */ + +/* + * GCC thresholds are on versions 4.4 to 4.9 and 5.0. + */ +#ifndef BR_GCC +#if __GNUC__ && !__clang__ +#define BR_GCC 1 + +#if __GNUC__ > 4 +#define BR_GCC_5_0 1 +#elif __GNUC__ == 4 && __GNUC_MINOR__ >= 9 +#define BR_GCC_4_9 1 +#elif __GNUC__ == 4 && __GNUC_MINOR__ >= 8 +#define BR_GCC_4_8 1 +#elif __GNUC__ == 4 && __GNUC_MINOR__ >= 7 +#define BR_GCC_4_7 1 +#elif __GNUC__ == 4 && __GNUC_MINOR__ >= 6 +#define BR_GCC_4_6 1 +#elif __GNUC__ == 4 && __GNUC_MINOR__ >= 5 +#define BR_GCC_4_5 1 +#elif __GNUC__ == 4 && __GNUC_MINOR__ >= 4 +#define BR_GCC_4_4 1 +#endif + +#if BR_GCC_5_0 +#define BR_GCC_4_9 1 +#endif +#if BR_GCC_4_9 +#define BR_GCC_4_8 1 +#endif +#if BR_GCC_4_8 +#define BR_GCC_4_7 1 +#endif +#if BR_GCC_4_7 +#define BR_GCC_4_6 1 +#endif +#if BR_GCC_4_6 +#define BR_GCC_4_5 1 +#endif +#if BR_GCC_4_5 +#define BR_GCC_4_4 1 +#endif + +#endif +#endif + +/* + * Clang thresholds are on versions 3.7.0 and 3.8.0. + */ +#ifndef BR_CLANG +#if __clang__ +#define BR_CLANG 1 + +#if __clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 8) +#define BR_CLANG_3_8 1 +#elif __clang_major__ == 3 && __clang_minor__ >= 7 +#define BR_CLANG_3_7 1 +#endif + +#if BR_CLANG_3_8 +#define BR_CLANG_3_7 1 +#endif + +#endif +#endif + +/* + * MS Visual C thresholds are on Visual Studio 2005 to 2015. + */ +#ifndef BR_MSC +#if _MSC_VER +#define BR_MSC 1 + +#if _MSC_VER >= 1900 +#define BR_MSC_2015 1 +#elif _MSC_VER >= 1800 +#define BR_MSC_2013 1 +#elif _MSC_VER >= 1700 +#define BR_MSC_2012 1 +#elif _MSC_VER >= 1600 +#define BR_MSC_2010 1 +#elif _MSC_VER >= 1500 +#define BR_MSC_2008 1 +#elif _MSC_VER >= 1400 +#define BR_MSC_2005 1 +#endif + +#if BR_MSC_2015 +#define BR_MSC_2013 1 +#endif +#if BR_MSC_2013 +#define BR_MSC_2012 1 +#endif +#if BR_MSC_2012 +#define BR_MSC_2010 1 +#endif +#if BR_MSC_2010 +#define BR_MSC_2008 1 +#endif +#if BR_MSC_2008 +#define BR_MSC_2005 1 +#endif + +#endif +#endif + +/* + * GCC 4.4+ and Clang 3.7+ allow tagging specific functions with a + * 'target' attribute that activates support for specific opcodes. + */ +#if BR_GCC_4_4 || BR_CLANG_3_7 +#define BR_TARGET(x) __attribute__((target(x))) +#else +#define BR_TARGET(x) +#endif + +/* + * AES-NI intrinsics are available on x86 (32-bit and 64-bit) with + * GCC 4.8+, Clang 3.7+ and MSC 2012+. + */ +#ifndef BR_AES_X86NI +#if (BR_i386 || BR_amd64) && (BR_GCC_4_8 || BR_CLANG_3_7 || BR_MSC_2012) +#define BR_AES_X86NI 1 +#endif +#endif + +/* + * SSE2 intrinsics are available on x86 (32-bit and 64-bit) with + * GCC 4.4+, Clang 3.7+ and MSC 2005+. + */ +#ifndef BR_SSE2 +#if (BR_i386 || BR_amd64) && (BR_GCC_4_4 || BR_CLANG_3_7 || BR_MSC_2005) +#define BR_SSE2 1 +#endif +#endif + +/* + * RDRAND intrinsics are available on x86 (32-bit and 64-bit) with + * GCC 4.6+, Clang 3.7+ and MSC 2012+. + */ +#ifndef BR_RDRAND +#if (BR_i386 || BR_amd64) && (BR_GCC_4_6 || BR_CLANG_3_7 || BR_MSC_2012) +#define BR_RDRAND 1 +#endif +#endif + +/* + * Determine type of OS for random number generation. Macro names and + * values are documented on: + * https://sourceforge.net/p/predef/wiki/OperatingSystems/ + * + * TODO: enrich the list of detected system. Also add detection for + * alternate system calls like getentropy(), which are usually + * preferable when available. + */ + +#ifndef BR_USE_URANDOM +#if defined _AIX \ + || defined __ANDROID__ \ + || defined __FreeBSD__ \ + || defined __NetBSD__ \ + || defined __OpenBSD__ \ + || defined __DragonFly__ \ + || defined __linux__ \ + || (defined __sun && (defined __SVR4 || defined __svr4__)) \ + || (defined __APPLE__ && defined __MACH__) +#define BR_USE_URANDOM 1 +#endif +#endif + +#ifndef BR_USE_WIN32_RAND +#if defined _WIN32 || defined _WIN64 +#define BR_USE_WIN32_RAND 1 +#endif +#endif + +/* + * POWER8 crypto support. We rely on compiler macros for the + * architecture, since we do not have a reliable, simple way to detect + * the required support at runtime (we could try running an opcode, and + * trapping the exception or signal on illegal instruction, but this + * induces some non-trivial OS dependencies that we would prefer to + * avoid if possible). + */ +#ifndef BR_POWER8 +#if __GNUC__ && ((_ARCH_PWR8 || _ARCH_PPC) && __CRYPTO__) +#define BR_POWER8 1 +#endif +#endif + +/* + * Detect endinanness on POWER8. + */ +#if BR_POWER8 +#if defined BR_POWER8_LE +#undef BR_POWER8_BE +#if BR_POWER8_LE +#define BR_POWER8_BE 0 +#else +#define BR_POWER8_BE 1 +#endif +#elif defined BR_POWER8_BE +#undef BR_POWER8_LE +#if BR_POWER8_BE +#define BR_POWER8_LE 0 +#else +#define BR_POWER8_LE 1 +#endif +#else +#if __LITTLE_ENDIAN__ +#define BR_POWER8_LE 1 +#define BR_POWER8_BE 0 +#else +#define BR_POWER8_LE 0 +#define BR_POWER8_BE 1 +#endif +#endif +#endif + +/* + * Detect support for 128-bit integers. + */ +#if !defined BR_INT128 && !defined BR_UMUL128 +#ifdef __SIZEOF_INT128__ +#define BR_INT128 1 +#elif _M_X64 +#define BR_UMUL128 1 +#endif +#endif + +/* + * Detect support for unaligned accesses with known endianness. + * + * x86 (both 32-bit and 64-bit) is little-endian and allows unaligned + * accesses. + * + * POWER/PowerPC allows unaligned accesses when big-endian. POWER8 and + * later also allow unaligned accesses when little-endian. + */ +#if !defined BR_LE_UNALIGNED && !defined BR_BE_UNALIGNED + +#if __i386 || __i386__ || __x86_64__ || _M_IX86 || _M_X64 +#define BR_LE_UNALIGNED 1 +#elif BR_POWER8_BE +#define BR_BE_UNALIGNED 1 +#elif BR_POWER8_LE +#define BR_LE_UNALIGNED 1 +#elif (__powerpc__ || __powerpc64__ || _M_PPC || _ARCH_PPC || _ARCH_PPC64) \ + && __BIG_ENDIAN__ +#define BR_BE_UNALIGNED 1 +#endif + +#endif + +/* + * Detect support for an OS-provided time source. + */ + +#ifndef BR_USE_UNIX_TIME +#if defined __unix__ || defined __linux__ \ + || defined _POSIX_SOURCE || defined _POSIX_C_SOURCE \ + || (defined __APPLE__ && defined __MACH__) +#define BR_USE_UNIX_TIME 1 +#endif +#endif + +#ifndef BR_USE_WIN32_TIME +#if defined _WIN32 || defined _WIN64 +#define BR_USE_WIN32_TIME 1 +#endif +#endif + /* ==================================================================== */ /* * Encoding/decoding functions. * * 32-bit and 64-bit decoding, both little-endian and big-endian, is - * implemented with the inline functions below. These functions are - * generic: they don't depend on the architecture natural endianness, - * and they can handle unaligned accesses. Optimized versions for some - * specific architectures may be implemented at a later time. - */ + * implemented with the inline functions below. + * + * When allowed by some compile-time options (autodetected or provided), + * optimised code is used, to perform direct memory access when the + * underlying architecture supports it, both for endianness and + * alignment. This, however, may trigger strict aliasing issues; the + * code below uses unions to perform (supposedly) safe type punning. + * Since the C aliasing rules are relatively complex and were amended, + * or at least re-explained with different phrasing, in all successive + * versions of the C standard, it is always a bit risky to bet that any + * specific version of a C compiler got it right, for some notion of + * "right". + */ + +typedef union { + uint16_t u; + unsigned char b[sizeof(uint16_t)]; +} br_union_u16; + +typedef union { + uint32_t u; + unsigned char b[sizeof(uint32_t)]; +} br_union_u32; + +typedef union { + uint64_t u; + unsigned char b[sizeof(uint64_t)]; +} br_union_u64; static inline void br_enc16le(void *dst, unsigned x) { +#if BR_LE_UNALIGNED + ((br_union_u16 *)dst)->u = x; +#else unsigned char *buf; buf = dst; buf[0] = (unsigned char)x; buf[1] = (unsigned char)(x >> 8); +#endif } static inline void br_enc16be(void *dst, unsigned x) { +#if BR_BE_UNALIGNED + ((br_union_u16 *)dst)->u = x; +#else unsigned char *buf; buf = dst; buf[0] = (unsigned char)(x >> 8); buf[1] = (unsigned char)x; +#endif } static inline unsigned br_dec16le(const void *src) { +#if BR_LE_UNALIGNED + return ((const br_union_u16 *)src)->u; +#else const unsigned char *buf; buf = src; return (unsigned)buf[0] | ((unsigned)buf[1] << 8); +#endif } static inline unsigned br_dec16be(const void *src) { +#if BR_BE_UNALIGNED + return ((const br_union_u16 *)src)->u; +#else const unsigned char *buf; buf = src; return ((unsigned)buf[0] << 8) | (unsigned)buf[1]; +#endif } static inline void br_enc32le(void *dst, uint32_t x) { +#if BR_LE_UNALIGNED + ((br_union_u32 *)dst)->u = x; +#else unsigned char *buf; buf = dst; @@ -155,11 +515,15 @@ br_enc32le(void *dst, uint32_t x) buf[1] = (unsigned char)(x >> 8); buf[2] = (unsigned char)(x >> 16); buf[3] = (unsigned char)(x >> 24); +#endif } static inline void br_enc32be(void *dst, uint32_t x) { +#if BR_BE_UNALIGNED + ((br_union_u32 *)dst)->u = x; +#else unsigned char *buf; buf = dst; @@ -167,11 +531,15 @@ br_enc32be(void *dst, uint32_t x) buf[1] = (unsigned char)(x >> 16); buf[2] = (unsigned char)(x >> 8); buf[3] = (unsigned char)x; +#endif } static inline uint32_t br_dec32le(const void *src) { +#if BR_LE_UNALIGNED + return ((const br_union_u32 *)src)->u; +#else const unsigned char *buf; buf = src; @@ -179,11 +547,15 @@ br_dec32le(const void *src) | ((uint32_t)buf[1] << 8) | ((uint32_t)buf[2] << 16) | ((uint32_t)buf[3] << 24); +#endif } static inline uint32_t br_dec32be(const void *src) { +#if BR_BE_UNALIGNED + return ((const br_union_u32 *)src)->u; +#else const unsigned char *buf; buf = src; @@ -191,46 +563,63 @@ br_dec32be(const void *src) | ((uint32_t)buf[1] << 16) | ((uint32_t)buf[2] << 8) | (uint32_t)buf[3]; +#endif } static inline void br_enc64le(void *dst, uint64_t x) { +#if BR_LE_UNALIGNED + ((br_union_u64 *)dst)->u = x; +#else unsigned char *buf; buf = dst; br_enc32le(buf, (uint32_t)x); br_enc32le(buf + 4, (uint32_t)(x >> 32)); +#endif } static inline void br_enc64be(void *dst, uint64_t x) { +#if BR_BE_UNALIGNED + ((br_union_u64 *)dst)->u = x; +#else unsigned char *buf; buf = dst; br_enc32be(buf, (uint32_t)(x >> 32)); br_enc32be(buf + 4, (uint32_t)x); +#endif } static inline uint64_t br_dec64le(const void *src) { +#if BR_LE_UNALIGNED + return ((const br_union_u64 *)src)->u; +#else const unsigned char *buf; buf = src; return (uint64_t)br_dec32le(buf) | ((uint64_t)br_dec32le(buf + 4) << 32); +#endif } static inline uint64_t br_dec64be(const void *src) { +#if BR_BE_UNALIGNED + return ((const br_union_u64 *)src)->u; +#else const unsigned char *buf; buf = src; return ((uint64_t)br_dec32be(buf) << 32) | (uint64_t)br_dec32be(buf + 4); +#endif } /* @@ -289,8 +678,8 @@ void br_sha2small_round(const unsigned char *buf, uint32_t *val); */ void br_tls_phash(void *dst, size_t len, const br_hash_class *dig, - const void *secret, size_t secret_len, - const char *label, const void *seed, size_t seed_len); + const void *secret, size_t secret_len, const char *label, + size_t seed_num, const br_tls_prf_seed_chunk *seed); /* * Copy all configured hash implementations from a multihash context @@ -300,7 +689,7 @@ static inline void br_multihash_copyimpl(br_multihash_context *dst, const br_multihash_context *src) { - memcpy(dst->impl, src->impl, sizeof src->impl); + memcpy((void *)dst->impl, src->impl, sizeof src->impl); } /* ==================================================================== */ @@ -1053,6 +1442,25 @@ void br_i31_from_monty(uint32_t *x, const uint32_t *m, uint32_t m0i); void br_i31_modpow(uint32_t *x, const unsigned char *e, size_t elen, const uint32_t *m, uint32_t m0i, uint32_t *t1, uint32_t *t2); +/* + * Compute a modular exponentiation. x[] MUST be an integer modulo m[] + * (same announced bit length, lower value). m[] MUST be odd. The + * exponent is in big-endian unsigned notation, over 'elen' bytes. The + * "m0i" parameter is equal to -(1/m0) mod 2^31, where m0 is the least + * significant value word of m[] (this works only if m[] is an odd + * integer). The tmp[] array is used for temporaries, and has size + * 'twlen' words; it must be large enough to accommodate at least two + * temporary values with the same size as m[] (including the leading + * "bit length" word). If there is room for more temporaries, then this + * function may use the extra room for window-based optimisation, + * resulting in faster computations. + * + * Returned value is 1 on success, 0 on error. An error is reported if + * the provided tmp[] array is too short. + */ +uint32_t br_i31_modpow_opt(uint32_t *x, const unsigned char *e, size_t elen, + const uint32_t *m, uint32_t m0i, uint32_t *tmp, size_t twlen); + /* * Compute d+a*b, result in d. The initial announced bit length of d[] * MUST match that of a[]. The d[] array MUST be large enough to @@ -1068,6 +1476,10 @@ void br_i31_mulacc(uint32_t *d, const uint32_t *a, const uint32_t *b); /* ==================================================================== */ +/* + * FIXME: document "i15" functions. + */ + static inline void br_i15_zero(uint16_t *x, uint16_t bit_len) { @@ -1093,6 +1505,9 @@ void br_i15_to_monty(uint16_t *x, const uint16_t *m); void br_i15_modpow(uint16_t *x, const unsigned char *e, size_t elen, const uint16_t *m, uint16_t m0i, uint16_t *t1, uint16_t *t2); +uint32_t br_i15_modpow_opt(uint16_t *x, const unsigned char *e, size_t elen, + const uint16_t *m, uint16_t m0i, uint16_t *tmp, size_t twlen); + void br_i15_encode(void *dst, size_t len, const uint16_t *x); uint32_t br_i15_decode_mod(uint16_t *x, @@ -1113,6 +1528,9 @@ void br_i15_reduce(uint16_t *x, const uint16_t *a, const uint16_t *m); void br_i15_mulacc(uint16_t *d, const uint16_t *a, const uint16_t *b); +uint32_t br_i62_modpow_opt(uint32_t *x31, const unsigned char *e, size_t elen, + const uint32_t *m31, uint32_t m0i31, uint64_t *tmp, size_t twlen); + /* ==================================================================== */ static inline size_t @@ -1412,6 +1830,40 @@ unsigned br_aes_ct64_keysched(uint64_t *comp_skey, void br_aes_ct64_skey_expand(uint64_t *skey, unsigned num_rounds, const uint64_t *comp_skey); +/* + * Test support for AES-NI opcodes. + */ +int br_aes_x86ni_supported(void); + +/* + * AES key schedule, using x86 AES-NI instructions. This yields the + * subkeys in the encryption direction. Number of rounds is returned. + * Key size MUST be 16, 24 or 32 bytes; otherwise, 0 is returned. + */ +unsigned br_aes_x86ni_keysched_enc(unsigned char *skni, + const void *key, size_t len); + +/* + * AES key schedule, using x86 AES-NI instructions. This yields the + * subkeys in the decryption direction. Number of rounds is returned. + * Key size MUST be 16, 24 or 32 bytes; otherwise, 0 is returned. + */ +unsigned br_aes_x86ni_keysched_dec(unsigned char *skni, + const void *key, size_t len); + +/* + * Test support for AES POWER8 opcodes. + */ +int br_aes_pwr8_supported(void); + +/* + * AES key schedule, using POWER8 instructions. This yields the + * subkeys in the encryption direction. Number of rounds is returned. + * Key size MUST be 16, 24 or 32 bytes; otherwise, 0 is returned. + */ +unsigned br_aes_pwr8_keysched(unsigned char *skni, + const void *key, size_t len); + /* ==================================================================== */ /* * RSA. @@ -1456,30 +1908,14 @@ extern const br_ec_curve_def br_secp256r1; extern const br_ec_curve_def br_secp384r1; extern const br_ec_curve_def br_secp521r1; -extern const br_ec_curve_def br_curve25519; - -#if 0 -/* obsolete */ /* - * Type for the parameters for a "prime curve": - * coordinates are in GF(p), with p prime - * curve equation is Y^2 = X^3 - 3*X + b - * b is in Montgomery representation - * curve order is n and is prime - * base point is G (encoded) and has order n + * For Curve25519, the advertised "order" really is 2^255-1, since the + * point multipliction function really works over arbitrary 255-bit + * scalars. This value is only meant as a hint for ECDH key generation; + * only ECDSA uses the exact curve order, and ECDSA is not used with + * that specific curve. */ -typedef struct { - const uint32_t *p; - const uint32_t *b; - const uint32_t p0i; -} br_ec_prime_i31_curve; - -extern const br_ec_prime_i31_curve br_ec_prime_i31_secp256r1; -extern const br_ec_prime_i31_curve br_ec_prime_i31_secp384r1; -extern const br_ec_prime_i31_curve br_ec_prime_i31_secp521r1; - -#define BR_EC_I31_LEN ((BR_MAX_EC_SIZE + 61) / 31) -#endif +extern const br_ec_curve_def br_curve25519; /* * Decode some bytes as an i31 integer, with truncation (corresponding @@ -1705,4 +2141,204 @@ int br_ssl_choose_hash(unsigned bf); /* ==================================================================== */ +/* + * PowerPC / POWER assembly stuff. The special BR_POWER_ASM_MACROS macro + * must be defined before including this file; this is done by source + * files that use some inline assembly for PowerPC / POWER machines. + */ + +#if BR_POWER_ASM_MACROS + +#define lxvw4x(xt, ra, rb) lxvw4x_(xt, ra, rb) +#define stxvw4x(xt, ra, rb) stxvw4x_(xt, ra, rb) + +#define bdnz(foo) bdnz_(foo) +#define beq(foo) beq_(foo) + +#define li(rx, value) li_(rx, value) +#define addi(rx, ra, imm) addi_(rx, ra, imm) +#define cmpldi(rx, imm) cmpldi_(rx, imm) +#define mtctr(rx) mtctr_(rx) +#define vspltb(vrt, vrb, uim) vspltb_(vrt, vrb, uim) +#define vspltw(vrt, vrb, uim) vspltw_(vrt, vrb, uim) +#define vspltisb(vrt, imm) vspltisb_(vrt, imm) +#define vspltisw(vrt, imm) vspltisw_(vrt, imm) +#define vrlw(vrt, vra, vrb) vrlw_(vrt, vra, vrb) +#define vsbox(vrt, vra) vsbox_(vrt, vra) +#define vxor(vrt, vra, vrb) vxor_(vrt, vra, vrb) +#define vand(vrt, vra, vrb) vand_(vrt, vra, vrb) +#define vsro(vrt, vra, vrb) vsro_(vrt, vra, vrb) +#define vsl(vrt, vra, vrb) vsl_(vrt, vra, vrb) +#define vsldoi(vt, va, vb, sh) vsldoi_(vt, va, vb, sh) +#define vsr(vrt, vra, vrb) vsr_(vrt, vra, vrb) +#define vadduwm(vrt, vra, vrb) vadduwm_(vrt, vra, vrb) +#define vsububm(vrt, vra, vrb) vsububm_(vrt, vra, vrb) +#define vsubuwm(vrt, vra, vrb) vsubuwm_(vrt, vra, vrb) +#define vsrw(vrt, vra, vrb) vsrw_(vrt, vra, vrb) +#define vcipher(vt, va, vb) vcipher_(vt, va, vb) +#define vcipherlast(vt, va, vb) vcipherlast_(vt, va, vb) +#define vncipher(vt, va, vb) vncipher_(vt, va, vb) +#define vncipherlast(vt, va, vb) vncipherlast_(vt, va, vb) +#define vperm(vt, va, vb, vc) vperm_(vt, va, vb, vc) +#define vpmsumd(vt, va, vb) vpmsumd_(vt, va, vb) +#define xxpermdi(vt, va, vb, d) xxpermdi_(vt, va, vb, d) + +#define lxvw4x_(xt, ra, rb) "\tlxvw4x\t" #xt "," #ra "," #rb "\n" +#define stxvw4x_(xt, ra, rb) "\tstxvw4x\t" #xt "," #ra "," #rb "\n" + +#define label(foo) #foo "%=:\n" +#define bdnz_(foo) "\tbdnz\t" #foo "%=\n" +#define beq_(foo) "\tbeq\t" #foo "%=\n" + +#define li_(rx, value) "\tli\t" #rx "," #value "\n" +#define addi_(rx, ra, imm) "\taddi\t" #rx "," #ra "," #imm "\n" +#define cmpldi_(rx, imm) "\tcmpldi\t" #rx "," #imm "\n" +#define mtctr_(rx) "\tmtctr\t" #rx "\n" +#define vspltb_(vrt, vrb, uim) "\tvspltb\t" #vrt "," #vrb "," #uim "\n" +#define vspltw_(vrt, vrb, uim) "\tvspltw\t" #vrt "," #vrb "," #uim "\n" +#define vspltisb_(vrt, imm) "\tvspltisb\t" #vrt "," #imm "\n" +#define vspltisw_(vrt, imm) "\tvspltisw\t" #vrt "," #imm "\n" +#define vrlw_(vrt, vra, vrb) "\tvrlw\t" #vrt "," #vra "," #vrb "\n" +#define vsbox_(vrt, vra) "\tvsbox\t" #vrt "," #vra "\n" +#define vxor_(vrt, vra, vrb) "\tvxor\t" #vrt "," #vra "," #vrb "\n" +#define vand_(vrt, vra, vrb) "\tvand\t" #vrt "," #vra "," #vrb "\n" +#define vsro_(vrt, vra, vrb) "\tvsro\t" #vrt "," #vra "," #vrb "\n" +#define vsl_(vrt, vra, vrb) "\tvsl\t" #vrt "," #vra "," #vrb "\n" +#define vsldoi_(vt, va, vb, sh) "\tvsldoi\t" #vt "," #va "," #vb "," #sh "\n" +#define vsr_(vrt, vra, vrb) "\tvsr\t" #vrt "," #vra "," #vrb "\n" +#define vadduwm_(vrt, vra, vrb) "\tvadduwm\t" #vrt "," #vra "," #vrb "\n" +#define vsububm_(vrt, vra, vrb) "\tvsububm\t" #vrt "," #vra "," #vrb "\n" +#define vsubuwm_(vrt, vra, vrb) "\tvsubuwm\t" #vrt "," #vra "," #vrb "\n" +#define vsrw_(vrt, vra, vrb) "\tvsrw\t" #vrt "," #vra "," #vrb "\n" +#define vcipher_(vt, va, vb) "\tvcipher\t" #vt "," #va "," #vb "\n" +#define vcipherlast_(vt, va, vb) "\tvcipherlast\t" #vt "," #va "," #vb "\n" +#define vncipher_(vt, va, vb) "\tvncipher\t" #vt "," #va "," #vb "\n" +#define vncipherlast_(vt, va, vb) "\tvncipherlast\t" #vt "," #va "," #vb "\n" +#define vperm_(vt, va, vb, vc) "\tvperm\t" #vt "," #va "," #vb "," #vc "\n" +#define vpmsumd_(vt, va, vb) "\tvpmsumd\t" #vt "," #va "," #vb "\n" +#define xxpermdi_(vt, va, vb, d) "\txxpermdi\t" #vt "," #va "," #vb "," #d "\n" + +#endif + +/* ==================================================================== */ +/* + * Special "activate intrinsics" code, needed for some compiler versions. + * This is defined at the end of this file, so that it won't impact any + * of the inline functions defined previously; and it is controlled by + * a specific macro defined in the caller code. + * + * Calling code conventions: + * + * - Caller must define BR_ENABLE_INTRINSICS before including "inner.h". + * - Functions that use intrinsics must be enclosed in an "enabled" + * region (between BR_TARGETS_X86_UP and BR_TARGETS_X86_DOWN). + * - Functions that use intrinsics must be tagged with the appropriate + * BR_TARGET(). + */ + +#if BR_ENABLE_INTRINSICS && (BR_GCC_4_4 || BR_CLANG_3_7 || BR_MSC_2005) + +/* + * x86 intrinsics (both 32-bit and 64-bit). + */ +#if BR_i386 || BR_amd64 + +/* + * On GCC before version 5.0, we need to use the pragma to enable the + * target options globally, because the 'target' function attribute + * appears to be unreliable. Before 4.6 we must also avoid the + * push_options / pop_options mechanism, because it tends to trigger + * some internal compiler errors. + */ +#if BR_GCC && !BR_GCC_5_0 +#if BR_GCC_4_6 +#define BR_TARGETS_X86_UP \ + _Pragma("GCC push_options") \ + _Pragma("GCC target(\"sse2,ssse3,sse4.1,aes,pclmul,rdrnd\")") +#define BR_TARGETS_X86_DOWN \ + _Pragma("GCC pop_options") +#else +#define BR_TARGETS_X86_UP \ + _Pragma("GCC target(\"sse2,ssse3,sse4.1,aes,pclmul\")") +#endif +#define BR_TARGETS_X86_DOWN +#pragma GCC diagnostic ignored "-Wpsabi" +#endif + +#if BR_CLANG && !BR_CLANG_3_8 +#undef __SSE2__ +#undef __SSE3__ +#undef __SSSE3__ +#undef __SSE4_1__ +#undef __AES__ +#undef __PCLMUL__ +#undef __RDRND__ +#define __SSE2__ 1 +#define __SSE3__ 1 +#define __SSSE3__ 1 +#define __SSE4_1__ 1 +#define __AES__ 1 +#define __PCLMUL__ 1 +#define __RDRND__ 1 +#endif + +#ifndef BR_TARGETS_X86_UP +#define BR_TARGETS_X86_UP +#endif +#ifndef BR_TARGETS_X86_DOWN +#define BR_TARGETS_X86_DOWN +#endif + +#if BR_GCC || BR_CLANG +BR_TARGETS_X86_UP +#include +#include +#define br_bswap32 __builtin_bswap32 +BR_TARGETS_X86_DOWN +#endif + +#if BR_MSC +#include +#include +#include +#define br_bswap32 _byteswap_ulong +#endif + +static inline int +br_cpuid(uint32_t mask_eax, uint32_t mask_ebx, + uint32_t mask_ecx, uint32_t mask_edx) +{ +#if BR_GCC || BR_CLANG + unsigned eax, ebx, ecx, edx; + + if (__get_cpuid(1, &eax, &ebx, &ecx, &edx)) { + if ((eax & mask_eax) == mask_eax + && (ebx & mask_ebx) == mask_ebx + && (ecx & mask_ecx) == mask_ecx + && (edx & mask_edx) == mask_edx) + { + return 1; + } + } +#elif BR_MSC + int info[4]; + + __cpuid(info, 1); + if (((uint32_t)info[0] & mask_eax) == mask_eax + && ((uint32_t)info[1] & mask_ebx) == mask_ebx + && ((uint32_t)info[2] & mask_ecx) == mask_ecx + && ((uint32_t)info[3] & mask_edx) == mask_edx) + { + return 1; + } +#endif + return 0; +} + +#endif + +#endif + +/* ==================================================================== */ + #endif