| // This file is part of Eigen, a lightweight C++ template library |
| // for linear algebra. |
| // |
| // Copyright (C) 2014 Benoit Steiner (benoit.steiner.goog@gmail.com) |
| // |
| // This Source Code Form is subject to the terms of the Mozilla |
| // Public License v. 2.0. If a copy of the MPL was not distributed |
| // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. |
| |
| #ifndef EIGEN_PACKET_MATH_AVX512_H |
| #define EIGEN_PACKET_MATH_AVX512_H |
| |
| namespace Eigen { |
| |
| namespace internal { |
| |
| #ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD |
| #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8 |
| #endif |
| |
| #ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS |
| #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS (2*sizeof(void*)) |
| #endif |
| |
| #ifdef __FMA__ |
| #ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD |
| #define EIGEN_HAS_SINGLE_INSTRUCTION_MADD |
| #endif |
| #endif |
| |
| typedef __m512 Packet16f; |
| typedef __m512i Packet16i; |
| typedef __m512d Packet8d; |
| |
| template <> |
| struct is_arithmetic<__m512> { |
| enum { value = true }; |
| }; |
| template <> |
| struct is_arithmetic<__m512i> { |
| enum { value = true }; |
| }; |
| template <> |
| struct is_arithmetic<__m512d> { |
| enum { value = true }; |
| }; |
| |
| #define _EIGEN_DECLARE_CONST_Packet16f(NAME, X) \ |
| const Packet16f p16f_##NAME = pset1<Packet16f>(X) |
| |
| #define _EIGEN_DECLARE_CONST_Packet16f_FROM_INT(NAME, X) \ |
| const Packet16f p16f_##NAME = (__m512)pset1<Packet16i>(X) |
| |
| #define _EIGEN_DECLARE_CONST_Packet16i(NAME, X) \ |
| const Packet16i p16i_##NAME = pset1<Packet16i>(X) |
| |
| #define _EIGEN_DECLARE_CONST_Packet8d(NAME, X) \ |
| const Packet8d p8d_##NAME = pset1<Packet8d>(X) |
| |
| #define _EIGEN_DECLARE_CONST_Packet8d_FROM_INT64(NAME, X) \ |
| const Packet8d p8d_##NAME = _mm512_castsi512_pd(_mm512_set1_epi64(X)) |
| |
| template<> |
| struct packet_traits<float> : default_packet_traits |
| { |
| typedef Packet16f type; |
| typedef Packet8f half; |
| enum { |
| Vectorizable = 1, |
| AlignedOnScalar = 1, |
| size = 16, |
| HasHalfPacket = 1, |
| HasLog = 1, |
| HasExp = 1, |
| HasDiv = 1, |
| HasBlend = 1, |
| HasSqrt = 1, |
| HasRsqrt = 1, |
| HasSelect = 1, |
| HasEq = 1, |
| HasTanH = 1 |
| }; |
| }; |
| template<> |
| struct packet_traits<double> : default_packet_traits { |
| typedef Packet8d type; |
| typedef Packet4d half; |
| enum { |
| Vectorizable = 1, |
| AlignedOnScalar = 1, |
| size = 8, |
| HasHalfPacket = 1, |
| HasExp = 1, |
| HasDiv = 1, |
| HasBlend = 1, |
| HasSqrt = 1, |
| HasRsqrt = EIGEN_FAST_MATH, |
| HasSelect = 1, |
| HasEq = 1, |
| }; |
| }; |
| |
| template<> |
| struct packet_traits<int> : default_packet_traits { |
| typedef Packet16i type; |
| typedef Packet8i half; |
| enum { |
| Vectorizable = 1, |
| AlignedOnScalar = 1, |
| size = 16, |
| }; |
| }; |
| |
| template <> |
| struct unpacket_traits<Packet16f> { |
| typedef float type; |
| typedef Packet8f half; |
| enum { size = 16 }; |
| }; |
| template <> |
| struct unpacket_traits<Packet8d> { |
| typedef double type; |
| typedef Packet4d half; |
| enum { size = 8 }; |
| }; |
| template <> |
| struct unpacket_traits<Packet16i> { |
| typedef int type; |
| typedef Packet8i half; |
| enum { size = 16 }; |
| }; |
| |
| template <> |
| EIGEN_STRONG_INLINE Packet16f pset1<Packet16f>(const float& from) { |
| return _mm512_set1_ps(from); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet8d pset1<Packet8d>(const double& from) { |
| return _mm512_set1_pd(from); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet16i pset1<Packet16i>(const int& from) { |
| return _mm512_set1_epi32(from); |
| } |
| |
| template <> |
| EIGEN_STRONG_INLINE Packet16f pload1<Packet16f>(const float* from) { |
| return _mm512_broadcastss_ps(_mm_load_ps1(from)); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet8d pload1<Packet8d>(const double* from) { |
| return _mm512_broadcastsd_pd(_mm_load_pd1(from)); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet16f plset<float>(const float& a) { |
| return _mm512_add_ps( |
| _mm512_set1_ps(a), |
| _mm512_set_ps(15.0f, 14.0f, 13.0f, 12.0f, 11.0f, 10.0f, 9.0f, 8.0f, 7.0f, 6.0f, 5.0f, |
| 4.0f, 3.0f, 2.0f, 1.0f, 0.0f)); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet8d plset<double>(const double& a) { |
| return _mm512_add_pd(_mm512_set1_pd(a), |
| _mm512_set_pd(7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.0)); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet16i plset<int>(const int& a) { |
| return _mm512_add_epi32( |
| _mm512_set1_epi32(a), |
| _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, |
| 4, 3, 2, 1, 0)); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet16i pcast<Packet16f, Packet16i>(const Packet16f& a) { |
| return _mm512_cvtps_epi32(a); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet16i padd<Packet16i>(const Packet16i& a, |
| const Packet16i& b) { |
| return _mm512_add_epi32(a, b); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet16f padd<Packet16f>(const Packet16f& a, |
| const Packet16f& b) { |
| return _mm512_add_ps(a, b); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet8d padd<Packet8d>(const Packet8d& a, |
| const Packet8d& b) { |
| return _mm512_add_pd(a, b); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet16i psub<Packet16i>(const Packet16i& a, |
| const Packet16i& b) { |
| return _mm512_sub_epi32(a, b); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet16f psub<Packet16f>(const Packet16f& a, |
| const Packet16f& b) { |
| return _mm512_sub_ps(a, b); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet8d psub<Packet8d>(const Packet8d& a, |
| const Packet8d& b) { |
| return _mm512_sub_pd(a, b); |
| } |
| |
| // TODO (nishantpatil) Convert between _mmask16 and Packet16f. Templates in |
| // GenericPacketMath need to be modified? |
| |
| template<> |
| EIGEN_STRONG_INLINE Packet16f |
| ple<Packet16f>(const Packet16f& a, const Packet16f& b) { |
| return _mm512_movm_epi32(_mm512_cmp_ps_mask(a, b, _CMP_NGT_UQ)); |
| } |
| template<> |
| EIGEN_STRONG_INLINE Packet8d |
| ple<Packet8d>(const Packet8d& a, const Packet8d& b) { |
| return _mm512_movm_epi64(_mm512_cmp_pd_mask(a, b, _CMP_NGT_UQ)); |
| } |
| |
| template<> |
| EIGEN_STRONG_INLINE Packet16f |
| plt<Packet16f>(const Packet16f& a, const Packet16f& b) { |
| return _mm512_movm_epi32(_mm512_cmp_ps_mask(a, b, _CMP_NGE_UQ)); |
| } |
| template<> |
| EIGEN_STRONG_INLINE Packet8d plt<Packet8d>(const Packet8d& a, |
| const Packet8d& b) { return _mm512_movm_epi64(_mm512_cmp_pd_mask(a, b, _CMP_NGE_UQ)); |
| } |
| |
| template<> |
| EIGEN_STRONG_INLINE Packet16f peq<Packet16f>(const Packet16f& a, |
| const Packet16f& b) { return _mm512_movm_epi32(_mm512_cmp_ps_mask(a, b, _CMP_EQ_UQ)); |
| } |
| template<> |
| EIGEN_STRONG_INLINE Packet8d peq<Packet8d>(const Packet8d& a, |
| const Packet8d& b) { return _mm512_movm_epi64(_mm512_cmp_pd_mask(a, b, _CMP_EQ_UQ)); |
| } |
| |
| template<> |
| EIGEN_STRONG_INLINE Packet16f pselect<Packet16f>(const Packet16f& |
| a, const Packet16f& b, const Packet16f& false_mask) { return |
| _mm512_mask_blend_ps(_mm512_movepi32_mask(false_mask), a, b); |
| } |
| template<> |
| EIGEN_STRONG_INLINE Packet8d pselect<Packet8d>(const Packet8d& a, |
| const Packet8d& b, const Packet8d& false_mask) { return |
| _mm512_mask_blend_pd(_mm512_movepi32_mask(false_mask), a, b); |
| } |
| |
| template <> |
| EIGEN_STRONG_INLINE Packet16i pnegate(const Packet16i& a) { |
| return _mm512_sub_epi32(_mm512_setzero_si512(), a); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet16f pnegate(const Packet16f& a) { |
| return _mm512_sub_ps(_mm512_setzero_ps(), a); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet8d pnegate(const Packet8d& a) { |
| return _mm512_sub_pd(_mm512_setzero_pd(), a); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet16f pconj(const Packet16f& a) { |
| return a; |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet8d pconj(const Packet8d& a) { |
| return a; |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet16i pconj(const Packet16i& a) { |
| return a; |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet16i pmul<Packet16i>(const Packet16i& a, |
| const Packet16i& b) { |
| return _mm512_mullo_epi32(a, b); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet16f pmul<Packet16f>(const Packet16f& a, |
| const Packet16f& b) { |
| return _mm512_mul_ps(a, b); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet8d pmul<Packet8d>(const Packet8d& a, |
| const Packet8d& b) { |
| return _mm512_mul_pd(a, b); |
| } |
| |
| #ifdef __FMA__ |
| template <> |
| EIGEN_STRONG_INLINE Packet16f pmadd(const Packet16f& a, const Packet16f& b, |
| const Packet16f& c) { |
| return _mm512_fmadd_ps(a, b, c); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet8d pmadd(const Packet8d& a, const Packet8d& b, |
| const Packet8d& c) { |
| return _mm512_fmadd_pd(a, b, c); |
| } |
| #endif |
| |
| template <> |
| EIGEN_STRONG_INLINE Packet16i pmin<Packet16i>(const Packet16i& a, |
| const Packet16i& b) { |
| return _mm512_min_epi32(a, b); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet16f pmin<Packet16f>(const Packet16f& a, |
| const Packet16f& b) { |
| return _mm512_min_ps(a, b); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet8d pmin<Packet8d>(const Packet8d& a, |
| const Packet8d& b) { |
| return _mm512_min_pd(a, b); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet16i pmax<Packet16i>(const Packet16i& a, |
| const Packet16i& b) { |
| return _mm512_max_epi32(a, b); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet16f pmax<Packet16f>(const Packet16f& a, |
| const Packet16f& b) { |
| return _mm512_max_ps(a, b); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet8d pmax<Packet8d>(const Packet8d& a, |
| const Packet8d& b) { |
| return _mm512_max_pd(a, b); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet16f pand<Packet16f>(const Packet16f& a, |
| const Packet16f& b) { |
| #ifdef EIGEN_VECTORIZE_AVX512DQ |
| return _mm512_and_ps(a, b); |
| #else |
| Packet16f res = _mm512_undefined_ps(); |
| Packet4f lane0_a = _mm512_extractf32x4_ps(a, 0); |
| Packet4f lane0_b = _mm512_extractf32x4_ps(b, 0); |
| res = _mm512_insertf32x4(res, _mm_and_ps(lane0_a, lane0_b), 0); |
| |
| Packet4f lane1_a = _mm512_extractf32x4_ps(a, 1); |
| Packet4f lane1_b = _mm512_extractf32x4_ps(b, 1); |
| res = _mm512_insertf32x4(res, _mm_and_ps(lane1_a, lane1_b), 1); |
| |
| Packet4f lane2_a = _mm512_extractf32x4_ps(a, 2); |
| Packet4f lane2_b = _mm512_extractf32x4_ps(b, 2); |
| res = _mm512_insertf32x4(res, _mm_and_ps(lane2_a, lane2_b), 2); |
| |
| Packet4f lane3_a = _mm512_extractf32x4_ps(a, 3); |
| Packet4f lane3_b = _mm512_extractf32x4_ps(b, 3); |
| res = _mm512_insertf32x4(res, _mm_and_ps(lane3_a, lane3_b), 3); |
| |
| return res; |
| #endif |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet8d pand<Packet8d>(const Packet8d& a, |
| const Packet8d& b) { |
| #ifdef EIGEN_VECTORIZE_AVX512DQ |
| return _mm512_and_pd(a, b); |
| #else |
| Packet8d res = _mm512_undefined_pd(); |
| Packet4d lane0_a = _mm512_extractf64x4_pd(a, 0); |
| Packet4d lane0_b = _mm512_extractf64x4_pd(b, 0); |
| res = _mm512_insertf64x4(res, _mm256_and_pd(lane0_a, lane0_b), 0); |
| |
| Packet4d lane1_a = _mm512_extractf64x4_pd(a, 1); |
| Packet4d lane1_b = _mm512_extractf64x4_pd(b, 1); |
| res = _mm512_insertf64x4(res, _mm256_and_pd(lane1_a, lane1_b), 1); |
| |
| return res; |
| #endif |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet16f por<Packet16f>(const Packet16f& a, |
| const Packet16f& b) { |
| #ifdef EIGEN_VECTORIZE_AVX512DQ |
| return _mm512_or_ps(a, b); |
| #else |
| Packet16f res = _mm512_undefined_ps(); |
| Packet4f lane0_a = _mm512_extractf32x4_ps(a, 0); |
| Packet4f lane0_b = _mm512_extractf32x4_ps(b, 0); |
| res = _mm512_insertf32x4(res, _mm_or_ps(lane0_a, lane0_b), 0); |
| |
| Packet4f lane1_a = _mm512_extractf32x4_ps(a, 1); |
| Packet4f lane1_b = _mm512_extractf32x4_ps(b, 1); |
| res = _mm512_insertf32x4(res, _mm_or_ps(lane1_a, lane1_b), 1); |
| |
| Packet4f lane2_a = _mm512_extractf32x4_ps(a, 2); |
| Packet4f lane2_b = _mm512_extractf32x4_ps(b, 2); |
| res = _mm512_insertf32x4(res, _mm_or_ps(lane2_a, lane2_b), 2); |
| |
| Packet4f lane3_a = _mm512_extractf32x4_ps(a, 3); |
| Packet4f lane3_b = _mm512_extractf32x4_ps(b, 3); |
| res = _mm512_insertf32x4(res, _mm_or_ps(lane3_a, lane3_b), 3); |
| |
| return res; |
| #endif |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet8d por<Packet8d>(const Packet8d& a, |
| const Packet8d& b) { |
| #ifdef EIGEN_VECTORIZE_AVX512DQ |
| return _mm512_or_pd(a, b); |
| #else |
| Packet8d res = _mm512_undefined_pd(); |
| Packet4d lane0_a = _mm512_extractf64x4_pd(a, 0); |
| Packet4d lane0_b = _mm512_extractf64x4_pd(b, 0); |
| res = _mm512_insertf64x4(res, _mm256_or_pd(lane0_a, lane0_b), 0); |
| |
| Packet4d lane1_a = _mm512_extractf64x4_pd(a, 1); |
| Packet4d lane1_b = _mm512_extractf64x4_pd(b, 1); |
| res = _mm512_insertf64x4(res, _mm256_or_pd(lane1_a, lane1_b), 1); |
| |
| return res; |
| #endif |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet16f pxor<Packet16f>(const Packet16f& a, |
| const Packet16f& b) { |
| #ifdef EIGEN_VECTORIZE_AVX512DQ |
| return _mm512_xor_ps(a, b); |
| #else |
| Packet16f res = _mm512_undefined_ps(); |
| Packet4f lane0_a = _mm512_extractf32x4_ps(a, 0); |
| Packet4f lane0_b = _mm512_extractf32x4_ps(b, 0); |
| res = _mm512_insertf32x4(res, _mm_xor_ps(lane0_a, lane0_b), 0); |
| |
| Packet4f lane1_a = _mm512_extractf32x4_ps(a, 1); |
| Packet4f lane1_b = _mm512_extractf32x4_ps(b, 1); |
| res = _mm512_insertf32x4(res, _mm_xor_ps(lane1_a, lane1_b), 1); |
| |
| Packet4f lane2_a = _mm512_extractf32x4_ps(a, 2); |
| Packet4f lane2_b = _mm512_extractf32x4_ps(b, 2); |
| res = _mm512_insertf32x4(res, _mm_xor_ps(lane2_a, lane2_b), 2); |
| |
| Packet4f lane3_a = _mm512_extractf32x4_ps(a, 3); |
| Packet4f lane3_b = _mm512_extractf32x4_ps(b, 3); |
| res = _mm512_insertf32x4(res, _mm_xor_ps(lane3_a, lane3_b), 3); |
| |
| return res; |
| #endif |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet8d pxor<Packet8d>(const Packet8d& a, |
| const Packet8d& b) { |
| #ifdef EIGEN_VECTORIZE_AVX512DQ |
| return _mm512_xor_pd(a, b); |
| #else |
| Packet8d res = _mm512_undefined_pd(); |
| Packet4d lane0_a = _mm512_extractf64x4_pd(a, 0); |
| Packet4d lane0_b = _mm512_extractf64x4_pd(b, 0); |
| res = _mm512_insertf64x4(res, _mm256_xor_pd(lane0_a, lane0_b), 0); |
| |
| Packet4d lane1_a = _mm512_extractf64x4_pd(a, 1); |
| Packet4d lane1_b = _mm512_extractf64x4_pd(b, 1); |
| res = _mm512_insertf64x4(res, _mm256_xor_pd(lane1_a, lane1_b), 1); |
| |
| return res; |
| #endif |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet16f pandnot<Packet16f>(const Packet16f& a, |
| const Packet16f& b) { |
| #ifdef EIGEN_VECTORIZE_AVX512DQ |
| return _mm512_andnot_ps(a, b); |
| #else |
| Packet16f res = _mm512_undefined_ps(); |
| Packet4f lane0_a = _mm512_extractf32x4_ps(a, 0); |
| Packet4f lane0_b = _mm512_extractf32x4_ps(b, 0); |
| res = _mm512_insertf32x4(res, _mm_andnot_ps(lane0_a, lane0_b), 0); |
| |
| Packet4f lane1_a = _mm512_extractf32x4_ps(a, 1); |
| Packet4f lane1_b = _mm512_extractf32x4_ps(b, 1); |
| res = _mm512_insertf32x4(res, _mm_andnot_ps(lane1_a, lane1_b), 1); |
| |
| Packet4f lane2_a = _mm512_extractf32x4_ps(a, 2); |
| Packet4f lane2_b = _mm512_extractf32x4_ps(b, 2); |
| res = _mm512_insertf32x4(res, _mm_andnot_ps(lane2_a, lane2_b), 2); |
| |
| Packet4f lane3_a = _mm512_extractf32x4_ps(a, 3); |
| Packet4f lane3_b = _mm512_extractf32x4_ps(b, 3); |
| res = _mm512_insertf32x4(res, _mm_andnot_ps(lane3_a, lane3_b), 3); |
| |
| return res; |
| #endif |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet8d pandnot<Packet8d>(const Packet8d& a, |
| const Packet8d& b) { |
| #ifdef EIGEN_VECTORIZE_AVX512DQ |
| return _mm512_andnot_pd(a, b); |
| #else |
| Packet8d res = _mm512_undefined_pd(); |
| Packet4d lane0_a = _mm512_extractf64x4_pd(a, 0); |
| Packet4d lane0_b = _mm512_extractf64x4_pd(b, 0); |
| res = _mm512_insertf64x4(res, _mm256_andnot_pd(lane0_a, lane0_b), 0); |
| |
| Packet4d lane1_a = _mm512_extractf64x4_pd(a, 1); |
| Packet4d lane1_b = _mm512_extractf64x4_pd(b, 1); |
| res = _mm512_insertf64x4(res, _mm256_andnot_pd(lane1_a, lane1_b), 1); |
| |
| return res; |
| #endif |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet16f pload<Packet16f>(const float* from) { |
| EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_ps(from); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet8d pload<Packet8d>(const double* from) { |
| EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_pd(from); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet16i pload<Packet16i>(const int* from) { |
| EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_si512( |
| reinterpret_cast<const __m512i*>(from)); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet16f ploadu<Packet16f>(const float* from) { |
| EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_ps(from); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet8d ploadu<Packet8d>(const double* from) { |
| EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_pd(from); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet16i ploadu<Packet16i>(const int* from) { |
| EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_si512( |
| reinterpret_cast<const __m512i*>(from)); |
| } |
| // Loads 8 floats from memory a returns the packet |
| // {a0, a0 a1, a1, a2, a2, a3, a3, a4, a4, a5, a5, a6, a6, a7, a7} |
| template <> |
| EIGEN_STRONG_INLINE Packet16f ploaddup<Packet16f>(const float* from) { |
| __m256 low_half = _mm256_load_ps(from); |
| __m512 even_elements = _mm512_cvtepu32_epi64(low_half); |
| __m512 pairs = _mm512_permute_ps(even_elements, _MM_SHUFFLE(2, 2, 0, 0)); |
| return pairs; |
| } |
| // Loads 4 doubles from memory a returns the packet {a0, a0 a1, a1, a2, a2, a3, |
| // a3} |
| template <> |
| EIGEN_STRONG_INLINE Packet8d ploaddup<Packet8d>(const double* from) { |
| __m512d x = _mm512_setzero_pd(); |
| x = _mm512_insertf32x4(x, _mm_loaddup_pd(&from[0]), 0); |
| x = _mm512_insertf32x4(x, _mm_loaddup_pd(&from[1]), 1); |
| x = _mm512_insertf32x4(x, _mm_loaddup_pd(&from[2]), 2); |
| x = _mm512_insertf32x4(x, _mm_loaddup_pd(&from[3]), 3); |
| return x; |
| } |
| // Loads 8 ints from memory a returns the packet |
| // {a0, a0 a1, a1, a2, a2, a3, a3, a4, a4, a5, a5, a6, a6, a7, a7} |
| template <> |
| EIGEN_STRONG_INLINE Packet16i ploaddup<Packet16i>(const int* from) { |
| __m256i low_half = _mm256_load_si256((const __m256i*)from); |
| __m512 even_elements = _mm512_cvtepu32_epi64(low_half); |
| __m512i pairs = _mm512_shuffle_epi32(even_elements, _MM_SHUFFLE(2, 2, 0, 0)); |
| return pairs; |
| } |
| // Loads 4 ints from memory a returns the packet |
| // {a0, a0 a0, a0, a1, a1, a1, a1, a2, a2, a2, a2, a3, a3, a3, a3} |
| template <> |
| EIGEN_STRONG_INLINE Packet16i ploadquad<Packet16i>(const int* from) { |
| Packet16i tmp = _mm512_undefined_ps(); |
| //= _mm512_castps128_ps512(_mm_load_ps1(from)); |
| tmp = _mm512_insertf32x4(tmp, _mm_load_ps1((const float*)(from)), 0); |
| tmp = _mm512_insertf32x4(tmp, _mm_load_ps1((const float*)(from + 1)), 1); |
| tmp = _mm512_insertf32x4(tmp, _mm_load_ps1((const float*)(from + 2)), 2); |
| tmp = _mm512_insertf32x4(tmp, _mm_load_ps1((const float*)(from + 3)), 3); |
| return tmp; |
| } |
| // Loads 4 floats from memory a returns the packet |
| // {a0, a0 a0, a0, a1, a1, a1, a1, a2, a2, a2, a2, a3, a3, a3, a3} |
| template <> |
| EIGEN_STRONG_INLINE Packet16f ploadquad<Packet16f>(const float* from) { |
| Packet16f tmp = _mm512_undefined_ps(); |
| //= _mm512_castps128_ps512(_mm_load_ps1(from)); |
| tmp = _mm512_insertf32x4(tmp, _mm_load_ps1(from), 0); |
| tmp = _mm512_insertf32x4(tmp, _mm_load_ps1(from + 1), 1); |
| tmp = _mm512_insertf32x4(tmp, _mm_load_ps1(from + 2), 2); |
| tmp = _mm512_insertf32x4(tmp, _mm_load_ps1(from + 3), 3); |
| return tmp; |
| } |
| // Loads 4 doubles from memory a returns the packet |
| // {a0, a0 a0, a0, a1, a1, a1, a1} |
| template <> |
| EIGEN_STRONG_INLINE Packet8d ploadquad<Packet8d>(const double* from) { |
| Packet8d tmp = _mm512_undefined_pd(); |
| Packet2d tmp0 = _mm_load_pd1(from); |
| Packet2d tmp1 = _mm_load_pd1(from + 1); |
| Packet4d lane0 = _mm256_broadcastsd_pd(tmp0); |
| Packet4d lane1 = _mm256_broadcastsd_pd(tmp1); |
| tmp = _mm512_insertf64x4(tmp, lane0, 0); |
| return _mm512_insertf64x4(tmp, lane1, 1); |
| } |
| template <> |
| EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet16f& from) { |
| EIGEN_DEBUG_ALIGNED_STORE _mm512_store_ps(to, from); |
| } |
| template <> |
| EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet8d& from) { |
| EIGEN_DEBUG_ALIGNED_STORE _mm512_store_pd(to, from); |
| } |
| template <> |
| EIGEN_STRONG_INLINE void pstore<int>(int* to, const Packet16i& from) { |
| EIGEN_DEBUG_ALIGNED_STORE _mm512_storeu_si512(reinterpret_cast<__m512i*>(to), |
| from); |
| } |
| template <> |
| EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet16f& from) { |
| EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_ps(to, from); |
| } |
| template <> |
| EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet8d& from) { |
| EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_pd(to, from); |
| } |
| template <> |
| EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet16i& from) { |
| EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_si512( |
| reinterpret_cast<__m512i*>(to), from); |
| } |
| template <> |
| EIGEN_DEVICE_FUNC inline Packet16i pgather<int, Packet16i>(const int* from, |
| int stride) { |
| Packet16i stride_vector = _mm512_set1_epi32(stride); |
| Packet16i stride_multiplier = |
| _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); |
| Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier); |
| |
| return _mm512_i32gather_epi32(indices, from, 4); |
| } |
| template <> |
| EIGEN_DEVICE_FUNC inline Packet16f pgather<float, Packet16f>(const float* from, |
| int stride) { |
| Packet16i stride_vector = _mm512_set1_epi32(stride); |
| Packet16i stride_multiplier = |
| _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); |
| Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier); |
| |
| return _mm512_i32gather_ps(indices, from, 4); |
| } |
| template <> |
| EIGEN_DEVICE_FUNC inline Packet8d pgather<double, Packet8d>(const double* from, |
| int stride) { |
| Packet8i stride_vector = _mm256_set1_epi32(stride); |
| Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); |
| Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier); |
| |
| return _mm512_i32gather_pd(indices, from, 8); |
| } |
| template <> |
| EIGEN_DEVICE_FUNC inline void pscatter<int, Packet16i>(int* to, |
| const Packet16i& from, |
| int stride) { |
| Packet16i stride_vector = _mm512_set1_epi32(stride); |
| Packet16i stride_multiplier = |
| _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); |
| Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier); |
| _mm512_i32scatter_epi32(to, indices, from, 4); |
| } |
| template <> |
| EIGEN_DEVICE_FUNC inline void pscatter<float, Packet16f>(float* to, |
| const Packet16f& from, |
| int stride) { |
| Packet16i stride_vector = _mm512_set1_epi32(stride); |
| Packet16i stride_multiplier = |
| _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); |
| Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier); |
| _mm512_i32scatter_ps(to, indices, from, 4); |
| } |
| template <> |
| EIGEN_DEVICE_FUNC inline void pscatter<double, Packet8d>(double* to, |
| const Packet8d& from, |
| int stride) { |
| Packet8i stride_vector = _mm256_set1_epi32(stride); |
| Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); |
| Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier); |
| _mm512_i32scatter_pd(to, indices, from, 8); |
| } |
| template <> |
| EIGEN_STRONG_INLINE void pstore1<Packet16f>(float* to, const float& a) { |
| Packet16f pa = pset1<Packet16f>(a); |
| pstore(to, pa); |
| } |
| template <> |
| EIGEN_STRONG_INLINE void pstore1<Packet8d>(double* to, const double& a) { |
| Packet8d pa = pset1<Packet8d>(a); |
| pstore(to, pa); |
| } |
| template <> |
| EIGEN_STRONG_INLINE void pstore1<Packet16i>(int* to, const int& a) { |
| Packet16i pa = pset1<Packet16i>(a); |
| pstore(to, pa); |
| } |
| template<> |
| EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { |
| _mm_prefetch((const char*)(addr), _MM_HINT_T0); |
| } |
| template<> |
| EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { |
| _mm_prefetch((const char*)(addr), _MM_HINT_T0); |
| } |
| template<> |
| EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) { |
| _mm_prefetch((const char*)(addr), _MM_HINT_T0); |
| } |
| template <> |
| EIGEN_STRONG_INLINE float pfirst<Packet16f>(const Packet16f& a) { |
| return _mm_cvtss_f32(_mm512_extractf32x4_ps(a, 0)); |
| } |
| template <> |
| EIGEN_STRONG_INLINE double pfirst<Packet8d>(const Packet8d& a) { |
| return _mm_cvtsd_f64(_mm256_extractf128_pd(_mm512_extractf64x4_pd(a, 0), 0)); |
| } |
| template <> |
| EIGEN_STRONG_INLINE int pfirst<Packet16i>(const Packet16i& a) { |
| return _mm_extract_epi32(_mm512_extracti32x4_epi32(a, 0), 0); |
| } |
| |
| template<> |
| EIGEN_STRONG_INLINE Packet16i preverse(const Packet16i& a) |
| { |
| return _mm512_permutexvar_epi32( |
| _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), |
| a); |
| } |
| |
| template<> |
| EIGEN_STRONG_INLINE Packet16f preverse(const Packet16f& a) |
| { |
| return _mm512_permutexvar_ps( |
| _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), |
| a); |
| } |
| |
| template<> |
| EIGEN_STRONG_INLINE Packet8d preverse(const Packet8d& a) |
| { |
| return _mm512_permutexvar_pd( |
| _mm512_set_epi32(0, 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7), a); |
| } |
| |
| template<> |
| EIGEN_STRONG_INLINE Packet16i pabs(const Packet16i& a) |
| { |
| return _mm512_abs_epi32(a); |
| } |
| template<> |
| EIGEN_STRONG_INLINE Packet16f pabs(const Packet16f& a) |
| { |
| return _mm512_abs_ps(a); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet8d pabs(const Packet8d& a) { |
| return _mm512_abs_pd(a); |
| } |
| |
| #ifdef EIGEN_VECTORIZE_AVX512DQ |
| // AVX512F does not define _mm512_extractf32x8_ps to extract _m256 from _m512 |
| #define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT) \ |
| __m256 OUTPUT##_0 = _mm512_extractf32x8_ps(INPUT, 0); \ |
| __m256 OUTPUT##_1 = _mm512_extractf32x8_ps(INPUT, 1) |
| #else |
| #define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT) \ |
| __m256 OUTPUT##_0 = _mm256_insertf128_ps( \ |
| _mm256_castps128_ps256(_mm512_extractf32x4_ps(INPUT, 0)), \ |
| _mm512_extractf32x4_ps(INPUT, 1), 1); \ |
| __m256 OUTPUT##_1 = _mm256_insertf128_ps( \ |
| _mm256_castps128_ps256(_mm512_extractf32x4_ps(INPUT, 2)), \ |
| _mm512_extractf32x4_ps(INPUT, 3), 1); |
| #endif |
| |
| #ifdef EIGEN_VECTORIZE_AVX512DQ |
| #define EIGEN_INSERT_8f_INTO_16f(OUTPUT, INPUTA, INPUTB) \ |
| OUTPUT = _mm512_insertf32x8(OUTPUT, INPUTA, 0); \ |
| OUTPUT = _mm512_insertf32x8(OUTPUT, INPUTB, 1); |
| #else |
| #define EIGEN_INSERT_8f_INTO_16f(OUTPUT, INPUTA, INPUTB) \ |
| OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTA, 0), 0); \ |
| OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTA, 1), 1); \ |
| OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTB, 0), 2); \ |
| OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTB, 1), 3); |
| #endif |
| |
| template<> |
| EIGEN_STRONG_INLINE Packet16i preduxp<Packet16i>(const Packet16i* vecs) |
| { |
| __m512i quarters[4]; |
| for (int i = 0; i < 4; ++i) { |
| auto v0 = vecs[i + 0]; |
| auto v8 = vecs[i + 8]; |
| auto t0 = _mm512_add_epi32( |
| _mm512_shuffle_i32x4(v0, v8, _MM_SHUFFLE(1, 0, 1, 0)), |
| _mm512_shuffle_i32x4(v0, v8, _MM_SHUFFLE(3, 2, 3, 2))); |
| auto v4 = vecs[i + 4]; |
| auto v12 = vecs[i + 12]; |
| auto t4 = _mm512_add_epi32( |
| _mm512_shuffle_i32x4(v4, v12, _MM_SHUFFLE(1, 0, 1, 0)), |
| _mm512_shuffle_i32x4(v4, v12, _MM_SHUFFLE(3, 2, 3, 2))); |
| // 128-bit lanes for add: |
| // top = t0[0] t4[1] t0[2] t4[3] |
| // bot = t0[1] t4[0] t0[3] t4[2] |
| auto top = _mm512_mask_blend_epi64(0xcc, t0, t4); |
| auto bot = _mm512_permutex_epi64( |
| _mm512_mask_blend_epi64(0x33, t0, t4), |
| _MM_SHUFFLE(1, 0, 3, 2)); |
| auto t = _mm512_add_epi32(top, bot); |
| quarters[i] = t; |
| } |
| // Now, sum(j=0..3, quarters[i & 3][j + (i & 12)]) = sum(j=0..15, vecs[i][j]) |
| // In order to use horizontal add operations, we need to work on 256-bit data |
| // only; remember that the adds are only within 128-bit lanes and interleave |
| // the sums from the two input vectors |
| __m256i result0 = |
| _mm256_hadd_epi32( |
| _mm256_hadd_epi32( |
| _mm512_castsi512_si256(quarters[0]), |
| _mm512_castsi512_si256(quarters[1])), |
| _mm256_hadd_epi32( |
| _mm512_castsi512_si256(quarters[2]), |
| _mm512_castsi512_si256(quarters[3]))); |
| __m256i result1 = |
| _mm256_hadd_epi32( |
| _mm256_hadd_epi32( |
| _mm512_extracti64x4_epi64(quarters[0], 1), |
| _mm512_extracti64x4_epi64(quarters[1], 1)), |
| _mm256_hadd_epi32( |
| _mm512_extracti64x4_epi64(quarters[2], 1), |
| _mm512_extracti64x4_epi64(quarters[3], 1))); |
| return _mm512_inserti64x4(_mm512_castsi256_si512(result0), result1, 1); |
| } |
| |
| template<> |
| EIGEN_STRONG_INLINE Packet16f |
| preduxp<Packet16f>(const Packet16f* vecs) { |
| EIGEN_EXTRACT_8f_FROM_16f(vecs[0], vecs0); |
| EIGEN_EXTRACT_8f_FROM_16f(vecs[1], vecs1); |
| EIGEN_EXTRACT_8f_FROM_16f(vecs[2], vecs2); |
| EIGEN_EXTRACT_8f_FROM_16f(vecs[3], vecs3); |
| EIGEN_EXTRACT_8f_FROM_16f(vecs[4], vecs4); |
| EIGEN_EXTRACT_8f_FROM_16f(vecs[5], vecs5); |
| EIGEN_EXTRACT_8f_FROM_16f(vecs[6], vecs6); |
| EIGEN_EXTRACT_8f_FROM_16f(vecs[7], vecs7); |
| EIGEN_EXTRACT_8f_FROM_16f(vecs[8], vecs8); |
| EIGEN_EXTRACT_8f_FROM_16f(vecs[9], vecs9); |
| EIGEN_EXTRACT_8f_FROM_16f(vecs[10], vecs10); |
| EIGEN_EXTRACT_8f_FROM_16f(vecs[11], vecs11); |
| EIGEN_EXTRACT_8f_FROM_16f(vecs[12], vecs12); |
| EIGEN_EXTRACT_8f_FROM_16f(vecs[13], vecs13); |
| EIGEN_EXTRACT_8f_FROM_16f(vecs[14], vecs14); |
| EIGEN_EXTRACT_8f_FROM_16f(vecs[15], vecs15); |
| |
| __m256 hsum1 = _mm256_hadd_ps(vecs0_0, vecs1_0); |
| __m256 hsum2 = _mm256_hadd_ps(vecs2_0, vecs3_0); |
| __m256 hsum3 = _mm256_hadd_ps(vecs4_0, vecs5_0); |
| __m256 hsum4 = _mm256_hadd_ps(vecs6_0, vecs7_0); |
| |
| __m256 hsum5 = _mm256_hadd_ps(hsum1, hsum1); |
| __m256 hsum6 = _mm256_hadd_ps(hsum2, hsum2); |
| __m256 hsum7 = _mm256_hadd_ps(hsum3, hsum3); |
| __m256 hsum8 = _mm256_hadd_ps(hsum4, hsum4); |
| |
| __m256 perm1 = _mm256_permute2f128_ps(hsum5, hsum5, 0x23); |
| __m256 perm2 = _mm256_permute2f128_ps(hsum6, hsum6, 0x23); |
| __m256 perm3 = _mm256_permute2f128_ps(hsum7, hsum7, 0x23); |
| __m256 perm4 = _mm256_permute2f128_ps(hsum8, hsum8, 0x23); |
| |
| __m256 sum1 = _mm256_add_ps(perm1, hsum5); |
| __m256 sum2 = _mm256_add_ps(perm2, hsum6); |
| __m256 sum3 = _mm256_add_ps(perm3, hsum7); |
| __m256 sum4 = _mm256_add_ps(perm4, hsum8); |
| |
| __m256 blend1 = _mm256_blend_ps(sum1, sum2, 0xcc); |
| __m256 blend2 = _mm256_blend_ps(sum3, sum4, 0xcc); |
| |
| __m256 final = _mm256_blend_ps(blend1, blend2, 0xf0); |
| |
| hsum1 = _mm256_hadd_ps(vecs0_1, vecs1_1); |
| hsum2 = _mm256_hadd_ps(vecs2_1, vecs3_1); |
| hsum3 = _mm256_hadd_ps(vecs4_1, vecs5_1); |
| hsum4 = _mm256_hadd_ps(vecs6_1, vecs7_1); |
| |
| hsum5 = _mm256_hadd_ps(hsum1, hsum1); |
| hsum6 = _mm256_hadd_ps(hsum2, hsum2); |
| hsum7 = _mm256_hadd_ps(hsum3, hsum3); |
| hsum8 = _mm256_hadd_ps(hsum4, hsum4); |
| |
| perm1 = _mm256_permute2f128_ps(hsum5, hsum5, 0x23); |
| perm2 = _mm256_permute2f128_ps(hsum6, hsum6, 0x23); |
| perm3 = _mm256_permute2f128_ps(hsum7, hsum7, 0x23); |
| perm4 = _mm256_permute2f128_ps(hsum8, hsum8, 0x23); |
| |
| sum1 = _mm256_add_ps(perm1, hsum5); |
| sum2 = _mm256_add_ps(perm2, hsum6); |
| sum3 = _mm256_add_ps(perm3, hsum7); |
| sum4 = _mm256_add_ps(perm4, hsum8); |
| |
| blend1 = _mm256_blend_ps(sum1, sum2, 0xcc); |
| blend2 = _mm256_blend_ps(sum3, sum4, 0xcc); |
| |
| final = padd(final, _mm256_blend_ps(blend1, blend2, 0xf0)); |
| |
| hsum1 = _mm256_hadd_ps(vecs8_0, vecs9_0); |
| hsum2 = _mm256_hadd_ps(vecs10_0, vecs11_0); |
| hsum3 = _mm256_hadd_ps(vecs12_0, vecs13_0); |
| hsum4 = _mm256_hadd_ps(vecs14_0, vecs15_0); |
| |
| hsum5 = _mm256_hadd_ps(hsum1, hsum1); |
| hsum6 = _mm256_hadd_ps(hsum2, hsum2); |
| hsum7 = _mm256_hadd_ps(hsum3, hsum3); |
| hsum8 = _mm256_hadd_ps(hsum4, hsum4); |
| |
| perm1 = _mm256_permute2f128_ps(hsum5, hsum5, 0x23); |
| perm2 = _mm256_permute2f128_ps(hsum6, hsum6, 0x23); |
| perm3 = _mm256_permute2f128_ps(hsum7, hsum7, 0x23); |
| perm4 = _mm256_permute2f128_ps(hsum8, hsum8, 0x23); |
| |
| sum1 = _mm256_add_ps(perm1, hsum5); |
| sum2 = _mm256_add_ps(perm2, hsum6); |
| sum3 = _mm256_add_ps(perm3, hsum7); |
| sum4 = _mm256_add_ps(perm4, hsum8); |
| |
| blend1 = _mm256_blend_ps(sum1, sum2, 0xcc); |
| blend2 = _mm256_blend_ps(sum3, sum4, 0xcc); |
| |
| __m256 final_1 = _mm256_blend_ps(blend1, blend2, 0xf0); |
| |
| hsum1 = _mm256_hadd_ps(vecs8_1, vecs9_1); |
| hsum2 = _mm256_hadd_ps(vecs10_1, vecs11_1); |
| hsum3 = _mm256_hadd_ps(vecs12_1, vecs13_1); |
| hsum4 = _mm256_hadd_ps(vecs14_1, vecs15_1); |
| |
| hsum5 = _mm256_hadd_ps(hsum1, hsum1); |
| hsum6 = _mm256_hadd_ps(hsum2, hsum2); |
| hsum7 = _mm256_hadd_ps(hsum3, hsum3); |
| hsum8 = _mm256_hadd_ps(hsum4, hsum4); |
| |
| perm1 = _mm256_permute2f128_ps(hsum5, hsum5, 0x23); |
| perm2 = _mm256_permute2f128_ps(hsum6, hsum6, 0x23); |
| perm3 = _mm256_permute2f128_ps(hsum7, hsum7, 0x23); |
| perm4 = _mm256_permute2f128_ps(hsum8, hsum8, 0x23); |
| |
| sum1 = _mm256_add_ps(perm1, hsum5); |
| sum2 = _mm256_add_ps(perm2, hsum6); |
| sum3 = _mm256_add_ps(perm3, hsum7); |
| sum4 = _mm256_add_ps(perm4, hsum8); |
| |
| blend1 = _mm256_blend_ps(sum1, sum2, 0xcc); |
| blend2 = _mm256_blend_ps(sum3, sum4, 0xcc); |
| |
| final_1 = padd(final_1, _mm256_blend_ps(blend1, blend2, 0xf0)); |
| |
| __m512 final_output = _mm512_undefined_ps(); |
| |
| EIGEN_INSERT_8f_INTO_16f(final_output, final, final_1); |
| return final_output; |
| } |
| |
| template<> |
| EIGEN_STRONG_INLINE Packet8d |
| preduxp<Packet8d>(const Packet8d* vecs) { |
| Packet4d vecs0_0 = _mm512_extractf64x4_pd(vecs[0], 0); |
| Packet4d vecs0_1 = _mm512_extractf64x4_pd(vecs[0], 1); |
| |
| Packet4d vecs1_0 = _mm512_extractf64x4_pd(vecs[1], 0); |
| Packet4d vecs1_1 = _mm512_extractf64x4_pd(vecs[1], 1); |
| |
| Packet4d vecs2_0 = _mm512_extractf64x4_pd(vecs[2], 0); |
| Packet4d vecs2_1 = _mm512_extractf64x4_pd(vecs[2], 1); |
| |
| Packet4d vecs3_0 = _mm512_extractf64x4_pd(vecs[3], 0); |
| Packet4d vecs3_1 = _mm512_extractf64x4_pd(vecs[3], 1); |
| |
| Packet4d vecs4_0 = _mm512_extractf64x4_pd(vecs[4], 0); |
| Packet4d vecs4_1 = _mm512_extractf64x4_pd(vecs[4], 1); |
| |
| Packet4d vecs5_0 = _mm512_extractf64x4_pd(vecs[5], 0); |
| Packet4d vecs5_1 = _mm512_extractf64x4_pd(vecs[5], 1); |
| |
| Packet4d vecs6_0 = _mm512_extractf64x4_pd(vecs[6], 0); |
| Packet4d vecs6_1 = _mm512_extractf64x4_pd(vecs[6], 1); |
| |
| Packet4d vecs7_0 = _mm512_extractf64x4_pd(vecs[7], 0); |
| Packet4d vecs7_1 = _mm512_extractf64x4_pd(vecs[7], 1); |
| |
| Packet4d tmp0, tmp1; |
| |
| tmp0 = _mm256_hadd_pd(vecs0_0, vecs1_0); |
| tmp0 = _mm256_add_pd(tmp0, _mm256_permute2f128_pd(tmp0, tmp0, 1)); |
| |
| tmp1 = _mm256_hadd_pd(vecs2_0, vecs3_0); |
| tmp1 = _mm256_add_pd(tmp1, _mm256_permute2f128_pd(tmp1, tmp1, 1)); |
| |
| __m256d final_0 = _mm256_blend_pd(tmp0, tmp1, 0xC); |
| |
| tmp0 = _mm256_hadd_pd(vecs0_1, vecs1_1); |
| tmp0 = _mm256_add_pd(tmp0, _mm256_permute2f128_pd(tmp0, tmp0, 1)); |
| |
| tmp1 = _mm256_hadd_pd(vecs2_1, vecs3_1); |
| tmp1 = _mm256_add_pd(tmp1, _mm256_permute2f128_pd(tmp1, tmp1, 1)); |
| |
| final_0 = padd(final_0, _mm256_blend_pd(tmp0, tmp1, 0xC)); |
| |
| tmp0 = _mm256_hadd_pd(vecs4_0, vecs5_0); |
| tmp0 = _mm256_add_pd(tmp0, _mm256_permute2f128_pd(tmp0, tmp0, 1)); |
| |
| tmp1 = _mm256_hadd_pd(vecs6_0, vecs7_0); |
| tmp1 = _mm256_add_pd(tmp1, _mm256_permute2f128_pd(tmp1, tmp1, 1)); |
| |
| __m256d final_1 = _mm256_blend_pd(tmp0, tmp1, 0xC); |
| |
| tmp0 = _mm256_hadd_pd(vecs4_1, vecs5_1); |
| tmp0 = _mm256_add_pd(tmp0, _mm256_permute2f128_pd(tmp0, tmp0, 1)); |
| |
| tmp1 = _mm256_hadd_pd(vecs6_1, vecs7_1); |
| tmp1 = _mm256_add_pd(tmp1, _mm256_permute2f128_pd(tmp1, tmp1, 1)); |
| |
| final_1 = padd(final_1, _mm256_blend_pd(tmp0, tmp1, 0xC)); |
| |
| __m512d final_output = _mm512_undefined_pd(); |
| final_output = _mm512_insertf64x4(final_output, final_0, 0); |
| |
| return _mm512_insertf64x4(final_output, final_1, 1); |
| } |
| |
| template <> |
| EIGEN_STRONG_INLINE float predux<Packet16f>(const Packet16f& a) { |
| Packet8f x1 = |
| _mm256_add_ps(_mm512_castps512_ps256(a), _mm512_extractf64x4_pd(a, 1)); |
| return predux<Packet8f>(x1); |
| } |
| template <> |
| EIGEN_STRONG_INLINE int predux<Packet16i>(const Packet16i& a) { |
| Packet8i x1 = _mm256_add_epi32( |
| _mm512_castsi512_si256(a), |
| _mm512_extracti64x4_epi64(a, 1)); |
| Packet4i x2 = _mm_add_epi32( |
| _mm256_castsi256_si128(x1), |
| _mm256_extracti32x4_epi32(x1, 1)); |
| return predux<Packet4i>(x2); |
| } |
| template <> |
| EIGEN_STRONG_INLINE double predux<Packet8d>(const Packet8d& a) { |
| Packet4d x1 = _mm256_add_pd( |
| _mm512_castpd512_pd256(a), |
| _mm512_extractf64x4_pd(a, 1)); |
| return predux<Packet4d>(x1); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet8i predux4<Packet16i>(const Packet16i& a) { |
| // Cast not found? |
| // Packet4f lane0 = _mm512_castps512_ps128(a); |
| #ifdef EIGEN_VECTORIZE_AVX512DQ |
| Packet8i lane0 = _mm512_extracti32x8_epi32(a, 0); |
| Packet8i lane1 = _mm512_extracti32x8_epi32(a, 1); |
| return padd(lane0, lane1); |
| #else |
| Packet4i lane0 = _mm512_extracti32x4_epi32(a, 0); |
| Packet4i lane1 = _mm512_extracti32x4_epi32(a, 1); |
| Packet4i lane2 = _mm512_extracti32x4_epi32(a, 2); |
| Packet4i lane3 = _mm512_extracti32x4_epi32(a, 3); |
| Packet4i sum0 = padd(lane0, lane2); |
| Packet4i sum1 = padd(lane1, lane3); |
| return _mm256_inserti128_si256(_mm256_castsi128_si256(sum0), sum1, 1); |
| #endif |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet8f predux4<Packet16f>(const Packet16f& a) { |
| // Cast not found? |
| // Packet4f lane0 = _mm512_castps512_ps128(a); |
| #ifdef EIGEN_VECTORIZE_AVX512DQ |
| Packet8f lane0 = _mm512_extractf32x8_ps(a, 0); |
| Packet8f lane1 = _mm512_extractf32x8_ps(a, 1); |
| return padd(lane0, lane1); |
| #else |
| Packet4f lane0 = _mm512_extractf32x4_ps(a, 0); |
| Packet4f lane1 = _mm512_extractf32x4_ps(a, 1); |
| Packet4f lane2 = _mm512_extractf32x4_ps(a, 2); |
| Packet4f lane3 = _mm512_extractf32x4_ps(a, 3); |
| Packet4f sum0 = padd(lane0, lane2); |
| Packet4f sum1 = padd(lane1, lane3); |
| return _mm256_insertf128_ps(_mm256_castps128_ps256(sum0), sum1, 1); |
| #endif |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet4d predux4<Packet8d>(const Packet8d& a) { |
| Packet4d lane0 = _mm512_extractf64x4_pd(a, 0); |
| Packet4d lane1 = _mm512_extractf64x4_pd(a, 1); |
| Packet4d res = padd(lane0, lane1); |
| return res; |
| } |
| template <> |
| EIGEN_STRONG_INLINE int predux_mul<Packet16i>(const Packet16i& a) { |
| #ifdef EIGEN_VECTORIZE_AVX512DQ |
| Packet8i lane0 = _mm512_castsi512_si256(a); |
| Packet8i lane1 = _mm512_extracti32x8_epi32(a, 1); |
| Packet8i res = _mm256_mullo_epi32(lane0, lane1); |
| Packet4i res128 = _mm_mullo_epi32( |
| _mm256_castsi256_si128(res), |
| _mm256_extracti128_si256(res, 1)); |
| res128 = _mm_mullo_epi32( |
| res128, |
| _mm_shuffle_epi32(res128, _MM_SHUFFLE(0, 0, 3, 2))); |
| return pfirst( |
| _mm_mullo_epi32( |
| res128, |
| _mm_shuffle_epi32(res128, _MM_SHUFFLE(0, 0, 0, 1)))); |
| #else |
| Packet4i lane0 = _mm512_extracti32x4_epi32(a, 0); |
| Packet4i lane1 = _mm512_extracti32x4_epi32(a, 1); |
| Packet4i lane2 = _mm512_extracti32x4_epi32(a, 2); |
| Packet4i lane3 = _mm512_extracti32x4_epi32(a, 3); |
| Packet4i res = pmul(pmul(lane0, lane1), pmul(lane2, lane3)); |
| res = pmul(res, _mm_shuffle_epi32(res, _MM_SHUFFLE(0, 0, 3, 2))); |
| return pfirst(pmul(res, _mm_shuffle_epi32(res, _MM_SHUFFLE(0, 0, 0, 1)))); |
| #endif |
| } |
| template <> |
| EIGEN_STRONG_INLINE float predux_mul<Packet16f>(const Packet16f& a) { |
| #ifdef EIGEN_VECTORIZE_AVX512DQ |
| Packet8f lane0 = _mm512_extractf32x8_ps(a, 0); |
| Packet8f lane1 = _mm512_extractf32x8_ps(a, 1); |
| Packet8f res = pmul(lane0, lane1); |
| Packet4f res128 = |
| _mm256_castps256_ps128(pmul(res, _mm256_permute2f128_ps(res, res, 1))); |
| res128 = pmul(res128, _mm_permute_ps(res128, _MM_SHUFFLE(0, 0, 3, 2))); |
| return pfirst(pmul(res128, _mm_permute_ps(res128, _MM_SHUFFLE(0, 0, 0, 1)))); |
| #else |
| Packet4f lane0 = _mm512_extractf32x4_ps(a, 0); |
| Packet4f lane1 = _mm512_extractf32x4_ps(a, 1); |
| Packet4f lane2 = _mm512_extractf32x4_ps(a, 2); |
| Packet4f lane3 = _mm512_extractf32x4_ps(a, 3); |
| Packet4f res = pmul(pmul(lane0, lane1), pmul(lane2, lane3)); |
| res = pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2))); |
| return pfirst(pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1)))); |
| #endif |
| } |
| template <> |
| EIGEN_STRONG_INLINE double predux_mul<Packet8d>(const Packet8d& a) { |
| Packet4d lane0 = _mm512_extractf64x4_pd(a, 0); |
| Packet4d lane1 = _mm512_extractf64x4_pd(a, 1); |
| Packet4d res = pmul(lane0, lane1); |
| res = pmul(res, _mm256_permute2f128_pd(res, res, 1)); |
| return pfirst(pmul(res, _mm256_shuffle_pd(res, res, 1))); |
| } |
| template <> |
| EIGEN_STRONG_INLINE int predux_min<Packet16i>(const Packet16i& a) { |
| Packet4i lane0 = _mm512_extracti32x4_epi32(a, 0); |
| Packet4i lane1 = _mm512_extracti32x4_epi32(a, 1); |
| Packet4i lane2 = _mm512_extracti32x4_epi32(a, 2); |
| Packet4i lane3 = _mm512_extracti32x4_epi32(a, 3); |
| Packet4i res = |
| _mm_min_epi32(_mm_min_epi32(lane0, lane1), _mm_min_epi32(lane2, lane3)); |
| res = _mm_min_epi32(res, _mm_shuffle_epi32(res, _MM_SHUFFLE(0, 0, 3, 2))); |
| return pfirst( |
| _mm_min_epi32( |
| res, |
| _mm_shuffle_epi32(res, _MM_SHUFFLE(0, 0, 0, 1)))); |
| } |
| template <> |
| EIGEN_STRONG_INLINE float predux_min<Packet16f>(const Packet16f& a) { |
| Packet4f lane0 = _mm512_extractf32x4_ps(a, 0); |
| Packet4f lane1 = _mm512_extractf32x4_ps(a, 1); |
| Packet4f lane2 = _mm512_extractf32x4_ps(a, 2); |
| Packet4f lane3 = _mm512_extractf32x4_ps(a, 3); |
| Packet4f res = _mm_min_ps(_mm_min_ps(lane0, lane1), _mm_min_ps(lane2, lane3)); |
| res = _mm_min_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2))); |
| return pfirst(_mm_min_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1)))); |
| } |
| template <> |
| EIGEN_STRONG_INLINE double predux_min<Packet8d>(const Packet8d& a) { |
| Packet4d lane0 = _mm512_extractf64x4_pd(a, 0); |
| Packet4d lane1 = _mm512_extractf64x4_pd(a, 1); |
| Packet4d res = _mm256_min_pd(lane0, lane1); |
| res = _mm256_min_pd(res, _mm256_permute2f128_pd(res, res, 1)); |
| return pfirst(_mm256_min_pd(res, _mm256_shuffle_pd(res, res, 1))); |
| } |
| template <> |
| EIGEN_STRONG_INLINE int predux_max<Packet16i>(const Packet16i& a) { |
| Packet4i lane0 = _mm512_extracti32x4_epi32(a, 0); |
| Packet4i lane1 = _mm512_extracti32x4_epi32(a, 1); |
| Packet4i lane2 = _mm512_extracti32x4_epi32(a, 2); |
| Packet4i lane3 = _mm512_extracti32x4_epi32(a, 3); |
| Packet4i res = |
| _mm_max_epi32(_mm_max_epi32(lane0, lane1), _mm_max_epi32(lane2, lane3)); |
| res = _mm_max_epi32(res, _mm_shuffle_epi32(res, _MM_SHUFFLE(0, 0, 3, 2))); |
| return pfirst( |
| _mm_max_epi32( |
| res, |
| _mm_shuffle_epi32(res, _MM_SHUFFLE(0, 0, 0, 1)))); |
| } |
| template <> |
| EIGEN_STRONG_INLINE float predux_max<Packet16f>(const Packet16f& a) { |
| Packet4f lane0 = _mm512_extractf32x4_ps(a, 0); |
| Packet4f lane1 = _mm512_extractf32x4_ps(a, 1); |
| Packet4f lane2 = _mm512_extractf32x4_ps(a, 2); |
| Packet4f lane3 = _mm512_extractf32x4_ps(a, 3); |
| Packet4f res = _mm_max_ps(_mm_max_ps(lane0, lane1), _mm_max_ps(lane2, lane3)); |
| res = _mm_max_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2))); |
| return pfirst(_mm_max_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1)))); |
| } |
| template <> |
| EIGEN_STRONG_INLINE double predux_max<Packet8d>(const Packet8d& a) { |
| Packet4d lane0 = _mm512_extractf64x4_pd(a, 0); |
| Packet4d lane1 = _mm512_extractf64x4_pd(a, 1); |
| Packet4d res = _mm256_max_pd(lane0, lane1); |
| res = _mm256_max_pd(res, _mm256_permute2f128_pd(res, res, 1)); |
| return pfirst(_mm256_max_pd(res, _mm256_shuffle_pd(res, res, 1))); |
| } |
| |
| template <int Offset> |
| struct palign_impl<Offset, Packet16i> { |
| static EIGEN_STRONG_INLINE |
| void run(Packet16i& first, const Packet16i& second) { |
| first = _mm512_alignr_epi32(second, first, Offset); |
| } |
| }; |
| template <int Offset> |
| struct palign_impl<Offset, Packet16f> { |
| static EIGEN_STRONG_INLINE |
| void run(Packet16f& first, const Packet16f& second) { |
| first = _mm512_alignr_epi32(second, first, Offset); |
| } |
| }; |
| template <int Offset> |
| struct palign_impl<Offset, Packet8d> { |
| static EIGEN_STRONG_INLINE |
| void run(Packet8d& first, const Packet8d& second) { |
| first = _mm512_alignr_epi64(second, first, Offset); |
| } |
| }; |
| |
| #define PACK_OUTPUT(OUTPUT, INPUT, INDEX, STRIDE) \ |
| EIGEN_INSERT_8f_INTO_16f(OUTPUT[INDEX], INPUT[INDEX], INPUT[INDEX + STRIDE]); |
| |
| // For types implicitly convertible to and from __m512 |
| template <typename PacketType> |
| EIGEN_DEVICE_FUNC inline void |
| ptranspose_16_base(PacketBlock<PacketType>& kernel) { |
| __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]); |
| __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]); |
| __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]); |
| __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]); |
| __m512 T4 = _mm512_unpacklo_ps(kernel.packet[4], kernel.packet[5]); |
| __m512 T5 = _mm512_unpackhi_ps(kernel.packet[4], kernel.packet[5]); |
| __m512 T6 = _mm512_unpacklo_ps(kernel.packet[6], kernel.packet[7]); |
| __m512 T7 = _mm512_unpackhi_ps(kernel.packet[6], kernel.packet[7]); |
| __m512 T8 = _mm512_unpacklo_ps(kernel.packet[8], kernel.packet[9]); |
| __m512 T9 = _mm512_unpackhi_ps(kernel.packet[8], kernel.packet[9]); |
| __m512 T10 = _mm512_unpacklo_ps(kernel.packet[10], kernel.packet[11]); |
| __m512 T11 = _mm512_unpackhi_ps(kernel.packet[10], kernel.packet[11]); |
| __m512 T12 = _mm512_unpacklo_ps(kernel.packet[12], kernel.packet[13]); |
| __m512 T13 = _mm512_unpackhi_ps(kernel.packet[12], kernel.packet[13]); |
| __m512 T14 = _mm512_unpacklo_ps(kernel.packet[14], kernel.packet[15]); |
| __m512 T15 = _mm512_unpackhi_ps(kernel.packet[14], kernel.packet[15]); |
| __m512 S0 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(1, 0, 1, 0)); |
| __m512 S1 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(3, 2, 3, 2)); |
| __m512 S2 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(1, 0, 1, 0)); |
| __m512 S3 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(3, 2, 3, 2)); |
| __m512 S4 = _mm512_shuffle_ps(T4, T6, _MM_SHUFFLE(1, 0, 1, 0)); |
| __m512 S5 = _mm512_shuffle_ps(T4, T6, _MM_SHUFFLE(3, 2, 3, 2)); |
| __m512 S6 = _mm512_shuffle_ps(T5, T7, _MM_SHUFFLE(1, 0, 1, 0)); |
| __m512 S7 = _mm512_shuffle_ps(T5, T7, _MM_SHUFFLE(3, 2, 3, 2)); |
| __m512 S8 = _mm512_shuffle_ps(T8, T10, _MM_SHUFFLE(1, 0, 1, 0)); |
| __m512 S9 = _mm512_shuffle_ps(T8, T10, _MM_SHUFFLE(3, 2, 3, 2)); |
| __m512 S10 = _mm512_shuffle_ps(T9, T11, _MM_SHUFFLE(1, 0, 1, 0)); |
| __m512 S11 = _mm512_shuffle_ps(T9, T11, _MM_SHUFFLE(3, 2, 3, 2)); |
| __m512 S12 = _mm512_shuffle_ps(T12, T14, _MM_SHUFFLE(1, 0, 1, 0)); |
| __m512 S13 = _mm512_shuffle_ps(T12, T14, _MM_SHUFFLE(3, 2, 3, 2)); |
| __m512 S14 = _mm512_shuffle_ps(T13, T15, _MM_SHUFFLE(1, 0, 1, 0)); |
| __m512 S15 = _mm512_shuffle_ps(T13, T15, _MM_SHUFFLE(3, 2, 3, 2)); |
| |
| EIGEN_EXTRACT_8f_FROM_16f(S0, S0); |
| EIGEN_EXTRACT_8f_FROM_16f(S1, S1); |
| EIGEN_EXTRACT_8f_FROM_16f(S2, S2); |
| EIGEN_EXTRACT_8f_FROM_16f(S3, S3); |
| EIGEN_EXTRACT_8f_FROM_16f(S4, S4); |
| EIGEN_EXTRACT_8f_FROM_16f(S5, S5); |
| EIGEN_EXTRACT_8f_FROM_16f(S6, S6); |
| EIGEN_EXTRACT_8f_FROM_16f(S7, S7); |
| EIGEN_EXTRACT_8f_FROM_16f(S8, S8); |
| EIGEN_EXTRACT_8f_FROM_16f(S9, S9); |
| EIGEN_EXTRACT_8f_FROM_16f(S10, S10); |
| EIGEN_EXTRACT_8f_FROM_16f(S11, S11); |
| EIGEN_EXTRACT_8f_FROM_16f(S12, S12); |
| EIGEN_EXTRACT_8f_FROM_16f(S13, S13); |
| EIGEN_EXTRACT_8f_FROM_16f(S14, S14); |
| EIGEN_EXTRACT_8f_FROM_16f(S15, S15); |
| |
| PacketBlock<Packet8f, 32> tmp; |
| |
| tmp.packet[0] = _mm256_permute2f128_ps(S0_0, S4_0, 0x20); |
| tmp.packet[1] = _mm256_permute2f128_ps(S1_0, S5_0, 0x20); |
| tmp.packet[2] = _mm256_permute2f128_ps(S2_0, S6_0, 0x20); |
| tmp.packet[3] = _mm256_permute2f128_ps(S3_0, S7_0, 0x20); |
| tmp.packet[4] = _mm256_permute2f128_ps(S0_0, S4_0, 0x31); |
| tmp.packet[5] = _mm256_permute2f128_ps(S1_0, S5_0, 0x31); |
| tmp.packet[6] = _mm256_permute2f128_ps(S2_0, S6_0, 0x31); |
| tmp.packet[7] = _mm256_permute2f128_ps(S3_0, S7_0, 0x31); |
| |
| tmp.packet[8] = _mm256_permute2f128_ps(S0_1, S4_1, 0x20); |
| tmp.packet[9] = _mm256_permute2f128_ps(S1_1, S5_1, 0x20); |
| tmp.packet[10] = _mm256_permute2f128_ps(S2_1, S6_1, 0x20); |
| tmp.packet[11] = _mm256_permute2f128_ps(S3_1, S7_1, 0x20); |
| tmp.packet[12] = _mm256_permute2f128_ps(S0_1, S4_1, 0x31); |
| tmp.packet[13] = _mm256_permute2f128_ps(S1_1, S5_1, 0x31); |
| tmp.packet[14] = _mm256_permute2f128_ps(S2_1, S6_1, 0x31); |
| tmp.packet[15] = _mm256_permute2f128_ps(S3_1, S7_1, 0x31); |
| |
| // Second set of _m256 outputs |
| tmp.packet[16] = _mm256_permute2f128_ps(S8_0, S12_0, 0x20); |
| tmp.packet[17] = _mm256_permute2f128_ps(S9_0, S13_0, 0x20); |
| tmp.packet[18] = _mm256_permute2f128_ps(S10_0, S14_0, 0x20); |
| tmp.packet[19] = _mm256_permute2f128_ps(S11_0, S15_0, 0x20); |
| tmp.packet[20] = _mm256_permute2f128_ps(S8_0, S12_0, 0x31); |
| tmp.packet[21] = _mm256_permute2f128_ps(S9_0, S13_0, 0x31); |
| tmp.packet[22] = _mm256_permute2f128_ps(S10_0, S14_0, 0x31); |
| tmp.packet[23] = _mm256_permute2f128_ps(S11_0, S15_0, 0x31); |
| |
| tmp.packet[24] = _mm256_permute2f128_ps(S8_1, S12_1, 0x20); |
| tmp.packet[25] = _mm256_permute2f128_ps(S9_1, S13_1, 0x20); |
| tmp.packet[26] = _mm256_permute2f128_ps(S10_1, S14_1, 0x20); |
| tmp.packet[27] = _mm256_permute2f128_ps(S11_1, S15_1, 0x20); |
| tmp.packet[28] = _mm256_permute2f128_ps(S8_1, S12_1, 0x31); |
| tmp.packet[29] = _mm256_permute2f128_ps(S9_1, S13_1, 0x31); |
| tmp.packet[30] = _mm256_permute2f128_ps(S10_1, S14_1, 0x31); |
| tmp.packet[31] = _mm256_permute2f128_ps(S11_1, S15_1, 0x31); |
| |
| // Pack them into the output |
| PACK_OUTPUT(kernel.packet, tmp.packet, 0, 16); |
| PACK_OUTPUT(kernel.packet, tmp.packet, 1, 16); |
| PACK_OUTPUT(kernel.packet, tmp.packet, 2, 16); |
| PACK_OUTPUT(kernel.packet, tmp.packet, 3, 16); |
| |
| PACK_OUTPUT(kernel.packet, tmp.packet, 4, 16); |
| PACK_OUTPUT(kernel.packet, tmp.packet, 5, 16); |
| PACK_OUTPUT(kernel.packet, tmp.packet, 6, 16); |
| PACK_OUTPUT(kernel.packet, tmp.packet, 7, 16); |
| |
| PACK_OUTPUT(kernel.packet, tmp.packet, 8, 16); |
| PACK_OUTPUT(kernel.packet, tmp.packet, 9, 16); |
| PACK_OUTPUT(kernel.packet, tmp.packet, 10, 16); |
| PACK_OUTPUT(kernel.packet, tmp.packet, 11, 16); |
| |
| PACK_OUTPUT(kernel.packet, tmp.packet, 12, 16); |
| PACK_OUTPUT(kernel.packet, tmp.packet, 13, 16); |
| PACK_OUTPUT(kernel.packet, tmp.packet, 14, 16); |
| PACK_OUTPUT(kernel.packet, tmp.packet, 15, 16); |
| } |
| |
| template <> |
| EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16f, 16>& kernel) { |
| ptranspose_16_base(kernel); |
| } |
| template <> |
| EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16i, 16>& kernel) { |
| ptranspose_16_base(kernel); |
| } |
| #define PACK_OUTPUT_2(OUTPUT, INPUT, INDEX, STRIDE) \ |
| EIGEN_INSERT_8f_INTO_16f(OUTPUT[INDEX], INPUT[2 * INDEX], \ |
| INPUT[2 * INDEX + STRIDE]); |
| |
| template <typename PacketType> |
| EIGEN_DEVICE_FUNC inline void |
| ptranspose_4_base(PacketBlock<PacketType, 4>& kernel) { |
| __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]); |
| __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]); |
| __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]); |
| __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]); |
| |
| __m512 S0 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(1, 0, 1, 0)); |
| __m512 S1 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(3, 2, 3, 2)); |
| __m512 S2 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(1, 0, 1, 0)); |
| __m512 S3 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(3, 2, 3, 2)); |
| |
| EIGEN_EXTRACT_8f_FROM_16f(S0, S0); |
| EIGEN_EXTRACT_8f_FROM_16f(S1, S1); |
| EIGEN_EXTRACT_8f_FROM_16f(S2, S2); |
| EIGEN_EXTRACT_8f_FROM_16f(S3, S3); |
| |
| PacketBlock<Packet8f, 8> tmp; |
| |
| tmp.packet[0] = _mm256_permute2f128_ps(S0_0, S1_0, 0x20); |
| tmp.packet[1] = _mm256_permute2f128_ps(S2_0, S3_0, 0x20); |
| tmp.packet[2] = _mm256_permute2f128_ps(S0_0, S1_0, 0x31); |
| tmp.packet[3] = _mm256_permute2f128_ps(S2_0, S3_0, 0x31); |
| |
| tmp.packet[4] = _mm256_permute2f128_ps(S0_1, S1_1, 0x20); |
| tmp.packet[5] = _mm256_permute2f128_ps(S2_1, S3_1, 0x20); |
| tmp.packet[6] = _mm256_permute2f128_ps(S0_1, S1_1, 0x31); |
| tmp.packet[7] = _mm256_permute2f128_ps(S2_1, S3_1, 0x31); |
| |
| PACK_OUTPUT_2(kernel.packet, tmp.packet, 0, 1); |
| PACK_OUTPUT_2(kernel.packet, tmp.packet, 1, 1); |
| PACK_OUTPUT_2(kernel.packet, tmp.packet, 2, 1); |
| PACK_OUTPUT_2(kernel.packet, tmp.packet, 3, 1); |
| } |
| template <> |
| EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16f, 4>& kernel) { |
| ptranspose_4_base(kernel); |
| } |
| template <> |
| EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16i, 4>& kernel) { |
| ptranspose_4_base(kernel); |
| } |
| |
| #define PACK_OUTPUT_SQ_D(OUTPUT, INPUT, INDEX, STRIDE) \ |
| OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[INDEX], 0); \ |
| OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[INDEX + STRIDE], 1); |
| |
| #define PACK_OUTPUT_D(OUTPUT, INPUT, INDEX, STRIDE) \ |
| OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[(2 * INDEX)], 0); \ |
| OUTPUT[INDEX] = \ |
| _mm512_insertf64x4(OUTPUT[INDEX], INPUT[(2 * INDEX) + STRIDE], 1); |
| |
| template <> |
| EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8d, 4>& kernel) { |
| __m512d T0 = _mm512_shuffle_pd(kernel.packet[0], kernel.packet[1], 0); |
| __m512d T1 = _mm512_shuffle_pd(kernel.packet[0], kernel.packet[1], 0xff); |
| __m512d T2 = _mm512_shuffle_pd(kernel.packet[2], kernel.packet[3], 0); |
| __m512d T3 = _mm512_shuffle_pd(kernel.packet[2], kernel.packet[3], 0xff); |
| |
| PacketBlock<Packet4d, 8> tmp; |
| |
| tmp.packet[0] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0), |
| _mm512_extractf64x4_pd(T2, 0), 0x20); |
| tmp.packet[1] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0), |
| _mm512_extractf64x4_pd(T3, 0), 0x20); |
| tmp.packet[2] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0), |
| _mm512_extractf64x4_pd(T2, 0), 0x31); |
| tmp.packet[3] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0), |
| _mm512_extractf64x4_pd(T3, 0), 0x31); |
| |
| tmp.packet[4] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1), |
| _mm512_extractf64x4_pd(T2, 1), 0x20); |
| tmp.packet[5] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1), |
| _mm512_extractf64x4_pd(T3, 1), 0x20); |
| tmp.packet[6] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1), |
| _mm512_extractf64x4_pd(T2, 1), 0x31); |
| tmp.packet[7] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1), |
| _mm512_extractf64x4_pd(T3, 1), 0x31); |
| |
| PACK_OUTPUT_D(kernel.packet, tmp.packet, 0, 1); |
| PACK_OUTPUT_D(kernel.packet, tmp.packet, 1, 1); |
| PACK_OUTPUT_D(kernel.packet, tmp.packet, 2, 1); |
| PACK_OUTPUT_D(kernel.packet, tmp.packet, 3, 1); |
| } |
| template <> |
| EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8d, 8>& kernel) { |
| __m512d T0 = _mm512_unpacklo_pd(kernel.packet[0], kernel.packet[1]); |
| __m512d T1 = _mm512_unpackhi_pd(kernel.packet[0], kernel.packet[1]); |
| __m512d T2 = _mm512_unpacklo_pd(kernel.packet[2], kernel.packet[3]); |
| __m512d T3 = _mm512_unpackhi_pd(kernel.packet[2], kernel.packet[3]); |
| __m512d T4 = _mm512_unpacklo_pd(kernel.packet[4], kernel.packet[5]); |
| __m512d T5 = _mm512_unpackhi_pd(kernel.packet[4], kernel.packet[5]); |
| __m512d T6 = _mm512_unpacklo_pd(kernel.packet[6], kernel.packet[7]); |
| __m512d T7 = _mm512_unpackhi_pd(kernel.packet[6], kernel.packet[7]); |
| |
| PacketBlock<Packet4d, 16> tmp; |
| |
| tmp.packet[0] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0), |
| _mm512_extractf64x4_pd(T2, 0), 0x20); |
| tmp.packet[1] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0), |
| _mm512_extractf64x4_pd(T3, 0), 0x20); |
| tmp.packet[2] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0), |
| _mm512_extractf64x4_pd(T2, 0), 0x31); |
| tmp.packet[3] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0), |
| _mm512_extractf64x4_pd(T3, 0), 0x31); |
| |
| tmp.packet[4] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1), |
| _mm512_extractf64x4_pd(T2, 1), 0x20); |
| tmp.packet[5] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1), |
| _mm512_extractf64x4_pd(T3, 1), 0x20); |
| tmp.packet[6] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1), |
| _mm512_extractf64x4_pd(T2, 1), 0x31); |
| tmp.packet[7] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1), |
| _mm512_extractf64x4_pd(T3, 1), 0x31); |
| |
| tmp.packet[8] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 0), |
| _mm512_extractf64x4_pd(T6, 0), 0x20); |
| tmp.packet[9] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 0), |
| _mm512_extractf64x4_pd(T7, 0), 0x20); |
| tmp.packet[10] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 0), |
| _mm512_extractf64x4_pd(T6, 0), 0x31); |
| tmp.packet[11] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 0), |
| _mm512_extractf64x4_pd(T7, 0), 0x31); |
| |
| tmp.packet[12] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 1), |
| _mm512_extractf64x4_pd(T6, 1), 0x20); |
| tmp.packet[13] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 1), |
| _mm512_extractf64x4_pd(T7, 1), 0x20); |
| tmp.packet[14] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 1), |
| _mm512_extractf64x4_pd(T6, 1), 0x31); |
| tmp.packet[15] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 1), |
| _mm512_extractf64x4_pd(T7, 1), 0x31); |
| |
| PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 0, 8); |
| PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 1, 8); |
| PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 2, 8); |
| PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 3, 8); |
| |
| PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 4, 8); |
| PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 5, 8); |
| PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 6, 8); |
| PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 7, 8); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet16f pblend(const Selector<16>& ifPacket, |
| const Packet16f& thenPacket, |
| const Packet16f& elsePacket) { |
| const __m512 zero = _mm512_setzero_ps(); |
| const __m512 select = _mm512_set_ps( |
| ifPacket.select[15], ifPacket.select[14], ifPacket.select[13], |
| ifPacket.select[12], ifPacket.select[11], ifPacket.select[10], |
| ifPacket.select[9], ifPacket.select[8], ifPacket.select[7], |
| ifPacket.select[6], ifPacket.select[5], ifPacket.select[4], |
| ifPacket.select[3], ifPacket.select[2], ifPacket.select[1], |
| ifPacket.select[0]); |
| __mmask16 false_mask = _mm512_cmp_ps_mask(select, zero, _CMP_EQ_UQ); |
| // TODO(nishantpatil) generate false_mask from Selector |
| return _mm512_mask_blend_ps(false_mask, thenPacket, elsePacket); |
| } |
| template <> |
| EIGEN_STRONG_INLINE Packet8d pblend(const Selector<8>& ifPacket, |
| const Packet8d& thenPacket, |
| const Packet8d& elsePacket) { |
| const __m512d zero = _mm512_setzero_pd(); |
| const __m512d select = |
| _mm512_set_pd(ifPacket.select[7], ifPacket.select[6], ifPacket.select[5], |
| ifPacket.select[4], ifPacket.select[3], ifPacket.select[2], |
| ifPacket.select[1], ifPacket.select[0]); |
| __mmask16 false_mask = _mm512_cmp_pd_mask(select, zero, _CMP_EQ_UQ); |
| // TODO(nishantpatil) generate false_mask from Selector |
| return _mm512_mask_blend_pd(false_mask, thenPacket, elsePacket); |
| } |
| // Functions to print vectors of different types, makes debugging much easier. |
| namespace{ |
| void print16f(char* name, __m512 val) { |
| float temp[16] __attribute__((aligned(32))); |
| _mm512_store_ps(temp, val); |
| printf("%s: ", name); |
| for (int k = 0; k < 16; k++) printf("%.8e ", temp[k]); |
| printf("\n"); |
| } |
| // void print16i(char* name, __m512i val) { |
| // int temp[16] __attribute__((aligned(32))); |
| // _mm512_store_si512((__m512i*)temp, val); |
| // printf("%s: ", name); |
| // for (int k = 0; k < 16; k++) printf("%i ", temp[k]); |
| // printf("\n"); |
| //} |
| }; |
| |
| } // end namespace internal |
| |
| } // end namespace Eigen |
| |
| #endif // EIGEN_PACKET_MATH_AVX_H |