Eigen/src/Core/arch/NEON/PacketMath.h

// This file is part of Eigen, a lightweight C++ template library
// for linear algebra.
//
// Copyright (C) 2008-2009 Gael Guennebaud <gael.guennebaud@inria.fr>
// Copyright (C) 2010 Konstantinos Margaritis <markos@freevec.org>
// Heavily based on Gael's SSE version.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.

#ifndef EIGEN_PACKET_MATH_NEON_H
#define EIGEN_PACKET_MATH_NEON_H

// IWYU pragma: private
#include "../../InternalHeaderCheck.h"

namespace Eigen {

namespace internal {

#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
#endif

#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
#endif

#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
#if EIGEN_ARCH_ARM64
#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
#else
#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 16
#endif
#endif

#if EIGEN_COMP_MSVC_STRICT

// In MSVC's arm_neon.h header file, all NEON vector types
// are aliases to the same underlying type __n128.
// We thus have to wrap them to make them different C++ types.
// (See also bug 1428)
typedef eigen_packet_wrapper<float32x2_t, 0> Packet2f;
typedef eigen_packet_wrapper<float32x4_t, 1> Packet4f;
typedef eigen_packet_wrapper<int32_t, 2> Packet4c;
typedef eigen_packet_wrapper<int8x8_t, 3> Packet8c;
typedef eigen_packet_wrapper<int8x16_t, 4> Packet16c;
typedef eigen_packet_wrapper<uint32_t, 5> Packet4uc;
typedef eigen_packet_wrapper<uint8x8_t, 6> Packet8uc;
typedef eigen_packet_wrapper<uint8x16_t, 7> Packet16uc;
typedef eigen_packet_wrapper<int16x4_t, 8> Packet4s;
typedef eigen_packet_wrapper<int16x8_t, 9> Packet8s;
typedef eigen_packet_wrapper<uint16x4_t, 10> Packet4us;
typedef eigen_packet_wrapper<uint16x8_t, 11> Packet8us;
typedef eigen_packet_wrapper<int32x2_t, 12> Packet2i;
typedef eigen_packet_wrapper<int32x4_t, 13> Packet4i;
typedef eigen_packet_wrapper<uint32x2_t, 14> Packet2ui;
typedef eigen_packet_wrapper<uint32x4_t, 15> Packet4ui;
typedef eigen_packet_wrapper<int64x2_t, 16> Packet2l;
typedef eigen_packet_wrapper<uint64x2_t, 17> Packet2ul;

EIGEN_ALWAYS_INLINE Packet4f make_packet4f(float a, float b, float c, float d) {
  float from[4] = {a, b, c, d};
  return vld1q_f32(from);
}

EIGEN_ALWAYS_INLINE Packet2f make_packet2f(float a, float b) {
  float from[2] = {a, b};
  return vld1_f32(from);
}

#else

typedef float32x2_t Packet2f;
typedef float32x4_t Packet4f;
typedef eigen_packet_wrapper<int32_t, 2> Packet4c;
typedef int8x8_t Packet8c;
typedef int8x16_t Packet16c;
typedef eigen_packet_wrapper<uint32_t, 5> Packet4uc;
typedef uint8x8_t Packet8uc;
typedef uint8x16_t Packet16uc;
typedef int16x4_t Packet4s;
typedef int16x8_t Packet8s;
typedef uint16x4_t Packet4us;
typedef uint16x8_t Packet8us;
typedef int32x2_t Packet2i;
typedef int32x4_t Packet4i;
typedef uint32x2_t Packet2ui;
typedef uint32x4_t Packet4ui;
typedef int64x2_t Packet2l;
typedef uint64x2_t Packet2ul;

EIGEN_ALWAYS_INLINE Packet4f make_packet4f(float a, float b, float c, float d) { return Packet4f{a, b, c, d}; }
EIGEN_ALWAYS_INLINE Packet2f make_packet2f(float a, float b) { return Packet2f{a, b}; }

#endif  // EIGEN_COMP_MSVC_STRICT

EIGEN_STRONG_INLINE Packet4f shuffle1(const Packet4f& m, int mask) {
  const float* a = reinterpret_cast<const float*>(&m);
  Packet4f res =
      make_packet4f(*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3)), *(a + ((mask >> 6) & 3)));
  return res;
}

// fuctionally equivalent to _mm_shuffle_ps in SSE when interleave
// == false (i.e. shuffle<false>(m, n, mask) equals _mm_shuffle_ps(m, n, mask)),
// interleave m and n when interleave == true. Currently used in LU/arch/InverseSize4.h
// to enable a shared implementation for fast inversion of matrices of size 4.
template <bool interleave>
EIGEN_STRONG_INLINE Packet4f shuffle2(const Packet4f& m, const Packet4f& n, int mask) {
  const float* a = reinterpret_cast<const float*>(&m);
  const float* b = reinterpret_cast<const float*>(&n);
  Packet4f res =
      make_packet4f(*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(b + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3)));
  return res;
}

template <>
EIGEN_STRONG_INLINE Packet4f shuffle2<true>(const Packet4f& m, const Packet4f& n, int mask) {
  const float* a = reinterpret_cast<const float*>(&m);
  const float* b = reinterpret_cast<const float*>(&n);
  Packet4f res =
      make_packet4f(*(a + (mask & 3)), *(b + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3)));
  return res;
}

EIGEN_STRONG_INLINE static int eigen_neon_shuffle_mask(int p, int q, int r, int s) {
  return ((s) << 6 | (r) << 4 | (q) << 2 | (p));
}

EIGEN_STRONG_INLINE Packet4f vec4f_swizzle1(const Packet4f& a, int p, int q, int r, int s) {
  return shuffle1(a, eigen_neon_shuffle_mask(p, q, r, s));
}
EIGEN_STRONG_INLINE Packet4f vec4f_swizzle2(const Packet4f& a, const Packet4f& b, int p, int q, int r, int s) {
  return shuffle2<false>(a, b, eigen_neon_shuffle_mask(p, q, r, s));
}
EIGEN_STRONG_INLINE Packet4f vec4f_movelh(const Packet4f& a, const Packet4f& b) {
  return shuffle2<false>(a, b, eigen_neon_shuffle_mask(0, 1, 0, 1));
}
EIGEN_STRONG_INLINE Packet4f vec4f_movehl(const Packet4f& a, const Packet4f& b) {
  return shuffle2<false>(b, a, eigen_neon_shuffle_mask(2, 3, 2, 3));
}
EIGEN_STRONG_INLINE Packet4f vec4f_unpacklo(const Packet4f& a, const Packet4f& b) {
  return shuffle2<true>(a, b, eigen_neon_shuffle_mask(0, 0, 1, 1));
}
EIGEN_STRONG_INLINE Packet4f vec4f_unpackhi(const Packet4f& a, const Packet4f& b) {
  return shuffle2<true>(a, b, eigen_neon_shuffle_mask(2, 2, 3, 3));
}
#define vec4f_duplane(a, p) Packet4f(vdupq_lane_f32(vget_low_f32(a), p))

#define EIGEN_DECLARE_CONST_Packet4f(NAME, X) const Packet4f p4f_##NAME = pset1<Packet4f>(X)

#define EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME, X) \
  const Packet4f p4f_##NAME = vreinterpretq_f32_u32(pset1<int32_t>(X))

#define EIGEN_DECLARE_CONST_Packet4i(NAME, X) const Packet4i p4i_##NAME = pset1<Packet4i>(X)

#if EIGEN_ARCH_ARM64 && EIGEN_COMP_GNUC
// __builtin_prefetch tends to do nothing on ARM64 compilers because the
// prefetch instructions there are too detailed for __builtin_prefetch to map
// meaningfully to them.
#define EIGEN_ARM_PREFETCH(ADDR) __asm__ __volatile__("prfm pldl1keep, [%[addr]]\n" ::[addr] "r"(ADDR) :);
#elif EIGEN_HAS_BUILTIN(__builtin_prefetch) || EIGEN_COMP_GNUC
#define EIGEN_ARM_PREFETCH(ADDR) __builtin_prefetch(ADDR);
#elif defined __pld
#define EIGEN_ARM_PREFETCH(ADDR) __pld(ADDR)
#elif EIGEN_ARCH_ARM
#define EIGEN_ARM_PREFETCH(ADDR) __asm__ __volatile__("pld [%[addr]]\n" ::[addr] "r"(ADDR) :);
#else
// by default no explicit prefetching
#define EIGEN_ARM_PREFETCH(ADDR)
#endif

template <>
struct packet_traits<float> : default_packet_traits {
  typedef Packet4f type;
  typedef Packet2f half;
  enum {
    Vectorizable = 1,
    AlignedOnScalar = 1,
    size = 4,

    HasCmp = 1,
    HasAdd = 1,
    HasSub = 1,
    HasShift = 1,
    HasMul = 1,
    HasNegate = 1,
    HasAbs = 1,
    HasArg = 0,
    HasAbs2 = 1,
    HasAbsDiff = 1,
    HasMin = 1,
    HasMax = 1,
    HasConj = 1,
    HasSetLinear = 1,
    HasBlend = 0,

    HasDiv = 1,
    HasFloor = 1,
    HasCeil = 1,
    HasRint = 1,

    HasSin = EIGEN_FAST_MATH,
    HasCos = EIGEN_FAST_MATH,
    HasACos = 1,
    HasASin = 1,
    HasATan = 1,
    HasATanh = 1,
    HasLog = 1,
    HasExp = 1,
    HasSqrt = 1,
    HasRsqrt = 1,
    HasTanh = EIGEN_FAST_MATH,
    HasErf = EIGEN_FAST_MATH,
    HasBessel = 0,  // Issues with accuracy.
    HasNdtri = 0
  };
};

template <>
struct packet_traits<int8_t> : default_packet_traits {
  typedef Packet16c type;
  typedef Packet8c half;
  enum {
    Vectorizable = 1,
    AlignedOnScalar = 1,
    size = 16,

    HasCmp = 1,
    HasAdd = 1,
    HasSub = 1,
    HasShift = 1,
    HasMul = 1,
    HasNegate = 1,
    HasAbs = 1,
    HasAbsDiff = 1,
    HasArg = 0,
    HasAbs2 = 1,
    HasMin = 1,
    HasMax = 1,
    HasConj = 1,
    HasSetLinear = 1,
    HasBlend = 0
  };
};

template <>
struct packet_traits<uint8_t> : default_packet_traits {
  typedef Packet16uc type;
  typedef Packet8uc half;
  enum {
    Vectorizable = 1,
    AlignedOnScalar = 1,
    size = 16,

    HasCmp = 1,
    HasAdd = 1,
    HasSub = 1,
    HasShift = 1,
    HasMul = 1,
    HasNegate = 0,
    HasAbs = 1,
    HasAbsDiff = 1,
    HasArg = 0,
    HasAbs2 = 1,
    HasMin = 1,
    HasMax = 1,
    HasConj = 1,
    HasSetLinear = 1,
    HasBlend = 0,

    HasSqrt = 1
  };
};

template <>
struct packet_traits<int16_t> : default_packet_traits {
  typedef Packet8s type;
  typedef Packet4s half;
  enum {
    Vectorizable = 1,
    AlignedOnScalar = 1,
    size = 8,

    HasCmp = 1,
    HasAdd = 1,
    HasSub = 1,
    HasShift = 1,
    HasMul = 1,
    HasNegate = 1,
    HasAbs = 1,
    HasAbsDiff = 1,
    HasArg = 0,
    HasAbs2 = 1,
    HasMin = 1,
    HasMax = 1,
    HasConj = 1,
    HasSetLinear = 1,
    HasBlend = 0
  };
};

template <>
struct packet_traits<uint16_t> : default_packet_traits {
  typedef Packet8us type;
  typedef Packet4us half;
  enum {
    Vectorizable = 1,
    AlignedOnScalar = 1,
    size = 8,

    HasCmp = 1,
    HasAdd = 1,
    HasSub = 1,
    HasShift = 1,
    HasMul = 1,
    HasNegate = 0,
    HasAbs = 1,
    HasAbsDiff = 1,
    HasArg = 0,
    HasAbs2 = 1,
    HasMin = 1,
    HasMax = 1,
    HasConj = 1,
    HasSetLinear = 1,
    HasBlend = 0,
    HasSqrt = 1
  };
};

template <>
struct packet_traits<int32_t> : default_packet_traits {
  typedef Packet4i type;
  typedef Packet2i half;
  enum {
    Vectorizable = 1,
    AlignedOnScalar = 1,
    size = 4,

    HasCmp = 1,
    HasAdd = 1,
    HasSub = 1,
    HasShift = 1,
    HasMul = 1,
    HasNegate = 1,
    HasAbs = 1,
    HasArg = 0,
    HasAbs2 = 1,
    HasAbsDiff = 1,
    HasMin = 1,
    HasMax = 1,
    HasConj = 1,
    HasSetLinear = 1,
    HasBlend = 0
  };
};

template <>
struct packet_traits<uint32_t> : default_packet_traits {
  typedef Packet4ui type;
  typedef Packet2ui half;
  enum {
    Vectorizable = 1,
    AlignedOnScalar = 1,
    size = 4,

    HasCmp = 1,
    HasAdd = 1,
    HasSub = 1,
    HasShift = 1,
    HasMul = 1,
    HasNegate = 0,
    HasAbs = 1,
    HasArg = 0,
    HasAbs2 = 1,
    HasAbsDiff = 1,
    HasMin = 1,
    HasMax = 1,
    HasConj = 1,
    HasSetLinear = 1,
    HasBlend = 0,

    HasSqrt = 1
  };
};

template <>
struct packet_traits<int64_t> : default_packet_traits {
  typedef Packet2l type;
  typedef Packet2l half;
  enum {
    Vectorizable = 1,
    AlignedOnScalar = 1,
    size = 2,

    HasCmp = 1,
    HasAdd = 1,
    HasSub = 1,
    HasShift = 1,
    HasMul = 1,
    HasNegate = 1,
    HasAbs = 1,
    HasArg = 0,
    HasAbs2 = 1,
    HasAbsDiff = 1,
    HasMin = 1,
    HasMax = 1,
    HasConj = 1,
    HasSetLinear = 1,
    HasBlend = 0
  };
};

template <>
struct packet_traits<uint64_t> : default_packet_traits {
  typedef Packet2ul type;
  typedef Packet2ul half;
  enum {
    Vectorizable = 1,
    AlignedOnScalar = 1,
    size = 2,

    HasCmp = 1,
    HasAdd = 1,
    HasSub = 1,
    HasShift = 1,
    HasMul = 1,
    HasNegate = 0,
    HasAbs = 1,
    HasArg = 0,
    HasAbs2 = 1,
    HasAbsDiff = 1,
    HasMin = 1,
    HasMax = 1,
    HasConj = 1,
    HasSetLinear = 1,
    HasBlend = 0
  };
};

template <>
struct unpacket_traits<Packet2f> {
  typedef float type;
  typedef Packet2f half;
  typedef Packet2i integer_packet;
  enum {
    size = 2,
    alignment = Aligned16,
    vectorizable = true,
    masked_load_available = false,
    masked_store_available = false
  };
};
template <>
struct unpacket_traits<Packet4f> {
  typedef float type;
  typedef Packet2f half;
  typedef Packet4i integer_packet;
  enum {
    size = 4,
    alignment = Aligned16,
    vectorizable = true,
    masked_load_available = false,
    masked_store_available = false
  };
};
template <>
struct unpacket_traits<Packet4c> {
  typedef int8_t type;
  typedef Packet4c half;
  enum {
    size = 4,
    alignment = Unaligned,
    vectorizable = true,
    masked_load_available = false,
    masked_store_available = false
  };
};
template <>
struct unpacket_traits<Packet8c> {
  typedef int8_t type;
  typedef Packet4c half;
  enum {
    size = 8,
    alignment = Aligned16,
    vectorizable = true,
    masked_load_available = false,
    masked_store_available = false
  };
};
template <>
struct unpacket_traits<Packet16c> {
  typedef int8_t type;
  typedef Packet8c half;
  enum {
    size = 16,
    alignment = Aligned16,
    vectorizable = true,
    masked_load_available = false,
    masked_store_available = false
  };
};
template <>
struct unpacket_traits<Packet4uc> {
  typedef uint8_t type;
  typedef Packet4uc half;
  enum {
    size = 4,
    alignment = Unaligned,
    vectorizable = true,
    masked_load_available = false,
    masked_store_available = false
  };
};
template <>
struct unpacket_traits<Packet8uc> {
  typedef uint8_t type;
  typedef Packet4uc half;
  enum {
    size = 8,
    alignment = Aligned16,
    vectorizable = true,
    masked_load_available = false,
    masked_store_available = false
  };
};
template <>
struct unpacket_traits<Packet16uc> {
  typedef uint8_t type;
  typedef Packet8uc half;
  enum {
    size = 16,
    alignment = Aligned16,
    vectorizable = true,
    masked_load_available = false,
    masked_store_available = false
  };
};
template <>
struct unpacket_traits<Packet4s> {
  typedef int16_t type;
  typedef Packet4s half;
  enum {
    size = 4,
    alignment = Aligned16,
    vectorizable = true,
    masked_load_available = false,
    masked_store_available = false
  };
};
template <>
struct unpacket_traits<Packet8s> {
  typedef int16_t type;
  typedef Packet4s half;
  enum {
    size = 8,
    alignment = Aligned16,
    vectorizable = true,
    masked_load_available = false,
    masked_store_available = false
  };
};
template <>
struct unpacket_traits<Packet4us> {
  typedef uint16_t type;
  typedef Packet4us half;
  enum {
    size = 4,
    alignment = Aligned16,
    vectorizable = true,
    masked_load_available = false,
    masked_store_available = false
  };
};
template <>
struct unpacket_traits<Packet8us> {
  typedef uint16_t type;
  typedef Packet4us half;
  enum {
    size = 8,
    alignment = Aligned16,
    vectorizable = true,
    masked_load_available = false,
    masked_store_available = false
  };
};
template <>
struct unpacket_traits<Packet2i> {
  typedef int32_t type;
  typedef Packet2i half;
  enum {
    size = 2,
    alignment = Aligned16,
    vectorizable = true,
    masked_load_available = false,
    masked_store_available = false
  };
};
template <>
struct unpacket_traits<Packet4i> {
  typedef int32_t type;
  typedef Packet2i half;
  enum {
    size = 4,
    alignment = Aligned16,
    vectorizable = true,
    masked_load_available = false,
    masked_store_available = false
  };
};
template <>
struct unpacket_traits<Packet2ui> {
  typedef uint32_t type;
  typedef Packet2ui half;
  enum {
    size = 2,
    alignment = Aligned16,
    vectorizable = true,
    masked_load_available = false,
    masked_store_available = false
  };
};
template <>
struct unpacket_traits<Packet4ui> {
  typedef uint32_t type;
  typedef Packet2ui half;
  enum {
    size = 4,
    alignment = Aligned16,
    vectorizable = true,
    masked_load_available = false,
    masked_store_available = false
  };
};
template <>
struct unpacket_traits<Packet2l> {
  typedef int64_t type;
  typedef Packet2l half;
  enum {
    size = 2,
    alignment = Aligned16,
    vectorizable = true,
    masked_load_available = false,
    masked_store_available = false
  };
};
template <>
struct unpacket_traits<Packet2ul> {
  typedef uint64_t type;
  typedef Packet2ul half;
  enum {
    size = 2,
    alignment = Aligned16,
    vectorizable = true,
    masked_load_available = false,
    masked_store_available = false
  };
};

template <>
EIGEN_STRONG_INLINE Packet2f pset1<Packet2f>(const float& from) {
  return vdup_n_f32(from);
}
template <>
EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) {
  return vdupq_n_f32(from);
}
template <>
EIGEN_STRONG_INLINE Packet4c pset1<Packet4c>(const int8_t& from) {
  return vget_lane_s32(vreinterpret_s32_s8(vdup_n_s8(from)), 0);
}
template <>
EIGEN_STRONG_INLINE Packet8c pset1<Packet8c>(const int8_t& from) {
  return vdup_n_s8(from);
}
template <>
EIGEN_STRONG_INLINE Packet16c pset1<Packet16c>(const int8_t& from) {
  return vdupq_n_s8(from);
}
template <>
EIGEN_STRONG_INLINE Packet4uc pset1<Packet4uc>(const uint8_t& from) {
  return vget_lane_u32(vreinterpret_u32_u8(vdup_n_u8(from)), 0);
}
template <>
EIGEN_STRONG_INLINE Packet8uc pset1<Packet8uc>(const uint8_t& from) {
  return vdup_n_u8(from);
}
template <>
EIGEN_STRONG_INLINE Packet16uc pset1<Packet16uc>(const uint8_t& from) {
  return vdupq_n_u8(from);
}
template <>
EIGEN_STRONG_INLINE Packet4s pset1<Packet4s>(const int16_t& from) {
  return vdup_n_s16(from);
}
template <>
EIGEN_STRONG_INLINE Packet8s pset1<Packet8s>(const int16_t& from) {
  return vdupq_n_s16(from);
}
template <>
EIGEN_STRONG_INLINE Packet4us pset1<Packet4us>(const uint16_t& from) {
  return vdup_n_u16(from);
}
template <>
EIGEN_STRONG_INLINE Packet8us pset1<Packet8us>(const uint16_t& from) {
  return vdupq_n_u16(from);
}
template <>
EIGEN_STRONG_INLINE Packet2i pset1<Packet2i>(const int32_t& from) {
  return vdup_n_s32(from);
}
template <>
EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int32_t& from) {
  return vdupq_n_s32(from);
}
template <>
EIGEN_STRONG_INLINE Packet2ui pset1<Packet2ui>(const uint32_t& from) {
  return vdup_n_u32(from);
}
template <>
EIGEN_STRONG_INLINE Packet4ui pset1<Packet4ui>(const uint32_t& from) {
  return vdupq_n_u32(from);
}
template <>
EIGEN_STRONG_INLINE Packet2l pset1<Packet2l>(const int64_t& from) {
  return vdupq_n_s64(from);
}
template <>
EIGEN_STRONG_INLINE Packet2ul pset1<Packet2ul>(const uint64_t& from) {
  return vdupq_n_u64(from);
}

template <>
EIGEN_STRONG_INLINE Packet2f pset1frombits<Packet2f>(uint32_t from) {
  return vreinterpret_f32_u32(vdup_n_u32(from));
}
template <>
EIGEN_STRONG_INLINE Packet4f pset1frombits<Packet4f>(uint32_t from) {
  return vreinterpretq_f32_u32(vdupq_n_u32(from));
}

template <>
EIGEN_STRONG_INLINE Packet2f plset<Packet2f>(const float& a) {
  const float c[] = {0.0f, 1.0f};
  return vadd_f32(pset1<Packet2f>(a), vld1_f32(c));
}
template <>
EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(const float& a) {
  const float c[] = {0.0f, 1.0f, 2.0f, 3.0f};
  return vaddq_f32(pset1<Packet4f>(a), vld1q_f32(c));
}
template <>
EIGEN_STRONG_INLINE Packet4c plset<Packet4c>(const int8_t& a) {
  return vget_lane_s32(vreinterpret_s32_s8(vadd_s8(vreinterpret_s8_u32(vdup_n_u32(0x03020100)), vdup_n_s8(a))), 0);
}
template <>
EIGEN_STRONG_INLINE Packet8c plset<Packet8c>(const int8_t& a) {
  const int8_t c[] = {0, 1, 2, 3, 4, 5, 6, 7};
  return vadd_s8(pset1<Packet8c>(a), vld1_s8(c));
}
template <>
EIGEN_STRONG_INLINE Packet16c plset<Packet16c>(const int8_t& a) {
  const int8_t c[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
  return vaddq_s8(pset1<Packet16c>(a), vld1q_s8(c));
}
template <>
EIGEN_STRONG_INLINE Packet4uc plset<Packet4uc>(const uint8_t& a) {
  return vget_lane_u32(vreinterpret_u32_u8(vadd_u8(vreinterpret_u8_u32(vdup_n_u32(0x03020100)), vdup_n_u8(a))), 0);
}
template <>
EIGEN_STRONG_INLINE Packet8uc plset<Packet8uc>(const uint8_t& a) {
  const uint8_t c[] = {0, 1, 2, 3, 4, 5, 6, 7};
  return vadd_u8(pset1<Packet8uc>(a), vld1_u8(c));
}
template <>
EIGEN_STRONG_INLINE Packet16uc plset<Packet16uc>(const uint8_t& a) {
  const uint8_t c[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
  return vaddq_u8(pset1<Packet16uc>(a), vld1q_u8(c));
}
template <>
EIGEN_STRONG_INLINE Packet4s plset<Packet4s>(const int16_t& a) {
  const int16_t c[] = {0, 1, 2, 3};
  return vadd_s16(pset1<Packet4s>(a), vld1_s16(c));
}
template <>
EIGEN_STRONG_INLINE Packet4us plset<Packet4us>(const uint16_t& a) {
  const uint16_t c[] = {0, 1, 2, 3};
  return vadd_u16(pset1<Packet4us>(a), vld1_u16(c));
}
template <>
EIGEN_STRONG_INLINE Packet8s plset<Packet8s>(const int16_t& a) {
  const int16_t c[] = {0, 1, 2, 3, 4, 5, 6, 7};
  return vaddq_s16(pset1<Packet8s>(a), vld1q_s16(c));
}
template <>
EIGEN_STRONG_INLINE Packet8us plset<Packet8us>(const uint16_t& a) {
  const uint16_t c[] = {0, 1, 2, 3, 4, 5, 6, 7};
  return vaddq_u16(pset1<Packet8us>(a), vld1q_u16(c));
}
template <>
EIGEN_STRONG_INLINE Packet2i plset<Packet2i>(const int32_t& a) {
  const int32_t c[] = {0, 1};
  return vadd_s32(pset1<Packet2i>(a), vld1_s32(c));
}
template <>
EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(const int32_t& a) {
  const int32_t c[] = {0, 1, 2, 3};
  return vaddq_s32(pset1<Packet4i>(a), vld1q_s32(c));
}
template <>
EIGEN_STRONG_INLINE Packet2ui plset<Packet2ui>(const uint32_t& a) {
  const uint32_t c[] = {0, 1};
  return vadd_u32(pset1<Packet2ui>(a), vld1_u32(c));
}
template <>
EIGEN_STRONG_INLINE Packet4ui plset<Packet4ui>(const uint32_t& a) {
  const uint32_t c[] = {0, 1, 2, 3};
  return vaddq_u32(pset1<Packet4ui>(a), vld1q_u32(c));
}
template <>
EIGEN_STRONG_INLINE Packet2l plset<Packet2l>(const int64_t& a) {
  const int64_t c[] = {0, 1};
  return vaddq_s64(pset1<Packet2l>(a), vld1q_s64(c));
}
template <>
EIGEN_STRONG_INLINE Packet2ul plset<Packet2ul>(const uint64_t& a) {
  const uint64_t c[] = {0, 1};
  return vaddq_u64(pset1<Packet2ul>(a), vld1q_u64(c));
}

template <>
EIGEN_STRONG_INLINE Packet2f padd<Packet2f>(const Packet2f& a, const Packet2f& b) {
  return vadd_f32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) {
  return vaddq_f32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4c padd<Packet4c>(const Packet4c& a, const Packet4c& b) {
  return vget_lane_s32(
      vreinterpret_s32_s8(vadd_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
}
template <>
EIGEN_STRONG_INLINE Packet8c padd<Packet8c>(const Packet8c& a, const Packet8c& b) {
  return vadd_s8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet16c padd<Packet16c>(const Packet16c& a, const Packet16c& b) {
  return vaddq_s8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4uc padd<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
  return vget_lane_u32(
      vreinterpret_u32_u8(vadd_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
}
template <>
EIGEN_STRONG_INLINE Packet8uc padd<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
  return vadd_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet16uc padd<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
  return vaddq_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4s padd<Packet4s>(const Packet4s& a, const Packet4s& b) {
  return vadd_s16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet8s padd<Packet8s>(const Packet8s& a, const Packet8s& b) {
  return vaddq_s16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4us padd<Packet4us>(const Packet4us& a, const Packet4us& b) {
  return vadd_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet8us padd<Packet8us>(const Packet8us& a, const Packet8us& b) {
  return vaddq_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2i padd<Packet2i>(const Packet2i& a, const Packet2i& b) {
  return vadd_s32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) {
  return vaddq_s32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2ui padd<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
  return vadd_u32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4ui padd<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
  return vaddq_u32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2l padd<Packet2l>(const Packet2l& a, const Packet2l& b) {
  return vaddq_s64(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2ul padd<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
  return vaddq_u64(a, b);
}

template <>
EIGEN_STRONG_INLINE Packet2f psub<Packet2f>(const Packet2f& a, const Packet2f& b) {
  return vsub_f32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) {
  return vsubq_f32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4c psub<Packet4c>(const Packet4c& a, const Packet4c& b) {
  return vget_lane_s32(
      vreinterpret_s32_s8(vsub_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
}
template <>
EIGEN_STRONG_INLINE Packet8c psub<Packet8c>(const Packet8c& a, const Packet8c& b) {
  return vsub_s8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet16c psub<Packet16c>(const Packet16c& a, const Packet16c& b) {
  return vsubq_s8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4uc psub<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
  return vget_lane_u32(
      vreinterpret_u32_u8(vsub_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
}
template <>
EIGEN_STRONG_INLINE Packet8uc psub<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
  return vsub_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet16uc psub<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
  return vsubq_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4s psub<Packet4s>(const Packet4s& a, const Packet4s& b) {
  return vsub_s16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet8s psub<Packet8s>(const Packet8s& a, const Packet8s& b) {
  return vsubq_s16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4us psub<Packet4us>(const Packet4us& a, const Packet4us& b) {
  return vsub_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet8us psub<Packet8us>(const Packet8us& a, const Packet8us& b) {
  return vsubq_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2i psub<Packet2i>(const Packet2i& a, const Packet2i& b) {
  return vsub_s32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) {
  return vsubq_s32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2ui psub<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
  return vsub_u32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4ui psub<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
  return vsubq_u32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2l psub<Packet2l>(const Packet2l& a, const Packet2l& b) {
  return vsubq_s64(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2ul psub<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
  return vsubq_u64(a, b);
}

template <>
EIGEN_STRONG_INLINE Packet2f pxor<Packet2f>(const Packet2f& a, const Packet2f& b);
template <>
EIGEN_STRONG_INLINE Packet2f paddsub<Packet2f>(const Packet2f& a, const Packet2f& b) {
  Packet2f mask = make_packet2f(numext::bit_cast<float>(0x80000000u), 0.0f);
  return padd(a, pxor(mask, b));
}
template <>
EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b);
template <>
EIGEN_STRONG_INLINE Packet4f paddsub<Packet4f>(const Packet4f& a, const Packet4f& b) {
  Packet4f mask = make_packet4f(numext::bit_cast<float>(0x80000000u), 0.0f, numext::bit_cast<float>(0x80000000u), 0.0f);
  return padd(a, pxor(mask, b));
}

template <>
EIGEN_STRONG_INLINE Packet2f pnegate(const Packet2f& a) {
  return vneg_f32(a);
}
template <>
EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) {
  return vnegq_f32(a);
}
template <>
EIGEN_STRONG_INLINE Packet4c pnegate(const Packet4c& a) {
  return vget_lane_s32(vreinterpret_s32_s8(vneg_s8(vreinterpret_s8_s32(vdup_n_s32(a)))), 0);
}
template <>
EIGEN_STRONG_INLINE Packet8c pnegate(const Packet8c& a) {
  return vneg_s8(a);
}
template <>
EIGEN_STRONG_INLINE Packet16c pnegate(const Packet16c& a) {
  return vnegq_s8(a);
}
template <>
EIGEN_STRONG_INLINE Packet4s pnegate(const Packet4s& a) {
  return vneg_s16(a);
}
template <>
EIGEN_STRONG_INLINE Packet8s pnegate(const Packet8s& a) {
  return vnegq_s16(a);
}
template <>
EIGEN_STRONG_INLINE Packet2i pnegate(const Packet2i& a) {
  return vneg_s32(a);
}
template <>
EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) {
  return vnegq_s32(a);
}
template <>
EIGEN_STRONG_INLINE Packet2l pnegate(const Packet2l& a) {
#if EIGEN_ARCH_ARM64
  return vnegq_s64(a);
#else
  return vcombine_s64(vdup_n_s64(-vgetq_lane_s64(a, 0)), vdup_n_s64(-vgetq_lane_s64(a, 1)));
#endif
}

template <>
EIGEN_STRONG_INLINE Packet2f pconj(const Packet2f& a) {
  return a;
}
template <>
EIGEN_STRONG_INLINE Packet4f pconj(const Packet4f& a) {
  return a;
}
template <>
EIGEN_STRONG_INLINE Packet4c pconj(const Packet4c& a) {
  return a;
}
template <>
EIGEN_STRONG_INLINE Packet8c pconj(const Packet8c& a) {
  return a;
}
template <>
EIGEN_STRONG_INLINE Packet16c pconj(const Packet16c& a) {
  return a;
}
template <>
EIGEN_STRONG_INLINE Packet4uc pconj(const Packet4uc& a) {
  return a;
}
template <>
EIGEN_STRONG_INLINE Packet8uc pconj(const Packet8uc& a) {
  return a;
}
template <>
EIGEN_STRONG_INLINE Packet16uc pconj(const Packet16uc& a) {
  return a;
}
template <>
EIGEN_STRONG_INLINE Packet4s pconj(const Packet4s& a) {
  return a;
}
template <>
EIGEN_STRONG_INLINE Packet8s pconj(const Packet8s& a) {
  return a;
}
template <>
EIGEN_STRONG_INLINE Packet4us pconj(const Packet4us& a) {
  return a;
}
template <>
EIGEN_STRONG_INLINE Packet8us pconj(const Packet8us& a) {
  return a;
}
template <>
EIGEN_STRONG_INLINE Packet2i pconj(const Packet2i& a) {
  return a;
}
template <>
EIGEN_STRONG_INLINE Packet4i pconj(const Packet4i& a) {
  return a;
}
template <>
EIGEN_STRONG_INLINE Packet2ui pconj(const Packet2ui& a) {
  return a;
}
template <>
EIGEN_STRONG_INLINE Packet4ui pconj(const Packet4ui& a) {
  return a;
}
template <>
EIGEN_STRONG_INLINE Packet2l pconj(const Packet2l& a) {
  return a;
}
template <>
EIGEN_STRONG_INLINE Packet2ul pconj(const Packet2ul& a) {
  return a;
}

template <>
EIGEN_STRONG_INLINE Packet2f pmul<Packet2f>(const Packet2f& a, const Packet2f& b) {
  return vmul_f32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) {
  return vmulq_f32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4c pmul<Packet4c>(const Packet4c& a, const Packet4c& b) {
  return vget_lane_s32(
      vreinterpret_s32_s8(vmul_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
}
template <>
EIGEN_STRONG_INLINE Packet8c pmul<Packet8c>(const Packet8c& a, const Packet8c& b) {
  return vmul_s8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet16c pmul<Packet16c>(const Packet16c& a, const Packet16c& b) {
  return vmulq_s8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4uc pmul<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
  return vget_lane_u32(
      vreinterpret_u32_u8(vmul_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
}
template <>
EIGEN_STRONG_INLINE Packet8uc pmul<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
  return vmul_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet16uc pmul<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
  return vmulq_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4s pmul<Packet4s>(const Packet4s& a, const Packet4s& b) {
  return vmul_s16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet8s pmul<Packet8s>(const Packet8s& a, const Packet8s& b) {
  return vmulq_s16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4us pmul<Packet4us>(const Packet4us& a, const Packet4us& b) {
  return vmul_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet8us pmul<Packet8us>(const Packet8us& a, const Packet8us& b) {
  return vmulq_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2i pmul<Packet2i>(const Packet2i& a, const Packet2i& b) {
  return vmul_s32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b) {
  return vmulq_s32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2ui pmul<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
  return vmul_u32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4ui pmul<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
  return vmulq_u32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2l pmul<Packet2l>(const Packet2l& a, const Packet2l& b) {
  return vcombine_s64(vdup_n_s64(vgetq_lane_s64(a, 0) * vgetq_lane_s64(b, 0)),
                      vdup_n_s64(vgetq_lane_s64(a, 1) * vgetq_lane_s64(b, 1)));
}
template <>
EIGEN_STRONG_INLINE Packet2ul pmul<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
  return vcombine_u64(vdup_n_u64(vgetq_lane_u64(a, 0) * vgetq_lane_u64(b, 0)),
                      vdup_n_u64(vgetq_lane_u64(a, 1) * vgetq_lane_u64(b, 1)));
}

template <>
EIGEN_STRONG_INLINE Packet4c pdiv<Packet4c>(const Packet4c& /*a*/, const Packet4c& /*b*/) {
  eigen_assert(false && "packet integer division are not supported by NEON");
  return pset1<Packet4c>(0);
}
template <>
EIGEN_STRONG_INLINE Packet8c pdiv<Packet8c>(const Packet8c& /*a*/, const Packet8c& /*b*/) {
  eigen_assert(false && "packet integer division are not supported by NEON");
  return pset1<Packet8c>(0);
}
template <>
EIGEN_STRONG_INLINE Packet16c pdiv<Packet16c>(const Packet16c& /*a*/, const Packet16c& /*b*/) {
  eigen_assert(false && "packet integer division are not supported by NEON");
  return pset1<Packet16c>(0);
}
template <>
EIGEN_STRONG_INLINE Packet4uc pdiv<Packet4uc>(const Packet4uc& /*a*/, const Packet4uc& /*b*/) {
  eigen_assert(false && "packet integer division are not supported by NEON");
  return pset1<Packet4uc>(0);
}
template <>
EIGEN_STRONG_INLINE Packet8uc pdiv<Packet8uc>(const Packet8uc& /*a*/, const Packet8uc& /*b*/) {
  eigen_assert(false && "packet integer division are not supported by NEON");
  return pset1<Packet8uc>(0);
}
template <>
EIGEN_STRONG_INLINE Packet16uc pdiv<Packet16uc>(const Packet16uc& /*a*/, const Packet16uc& /*b*/) {
  eigen_assert(false && "packet integer division are not supported by NEON");
  return pset1<Packet16uc>(0);
}
template <>
EIGEN_STRONG_INLINE Packet4s pdiv<Packet4s>(const Packet4s& /*a*/, const Packet4s& /*b*/) {
  eigen_assert(false && "packet integer division are not supported by NEON");
  return pset1<Packet4s>(0);
}
template <>
EIGEN_STRONG_INLINE Packet8s pdiv<Packet8s>(const Packet8s& /*a*/, const Packet8s& /*b*/) {
  eigen_assert(false && "packet integer division are not supported by NEON");
  return pset1<Packet8s>(0);
}
template <>
EIGEN_STRONG_INLINE Packet4us pdiv<Packet4us>(const Packet4us& /*a*/, const Packet4us& /*b*/) {
  eigen_assert(false && "packet integer division are not supported by NEON");
  return pset1<Packet4us>(0);
}
template <>
EIGEN_STRONG_INLINE Packet8us pdiv<Packet8us>(const Packet8us& /*a*/, const Packet8us& /*b*/) {
  eigen_assert(false && "packet integer division are not supported by NEON");
  return pset1<Packet8us>(0);
}
template <>
EIGEN_STRONG_INLINE Packet2i pdiv<Packet2i>(const Packet2i& /*a*/, const Packet2i& /*b*/) {
  eigen_assert(false && "packet integer division are not supported by NEON");
  return pset1<Packet2i>(0);
}
template <>
EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(const Packet4i& /*a*/, const Packet4i& /*b*/) {
  eigen_assert(false && "packet integer division are not supported by NEON");
  return pset1<Packet4i>(0);
}
template <>
EIGEN_STRONG_INLINE Packet2ui pdiv<Packet2ui>(const Packet2ui& /*a*/, const Packet2ui& /*b*/) {
  eigen_assert(false && "packet integer division are not supported by NEON");
  return pset1<Packet2ui>(0);
}
template <>
EIGEN_STRONG_INLINE Packet4ui pdiv<Packet4ui>(const Packet4ui& /*a*/, const Packet4ui& /*b*/) {
  eigen_assert(false && "packet integer division are not supported by NEON");
  return pset1<Packet4ui>(0);
}
template <>
EIGEN_STRONG_INLINE Packet2l pdiv<Packet2l>(const Packet2l& /*a*/, const Packet2l& /*b*/) {
  eigen_assert(false && "packet integer division are not supported by NEON");
  return pset1<Packet2l>(0LL);
}
template <>
EIGEN_STRONG_INLINE Packet2ul pdiv<Packet2ul>(const Packet2ul& /*a*/, const Packet2ul& /*b*/) {
  eigen_assert(false && "packet integer division are not supported by NEON");
  return pset1<Packet2ul>(0ULL);
}

#ifdef EIGEN_VECTORIZE_FMA
template <>
EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
  return vfmaq_f32(c, a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2f pmadd(const Packet2f& a, const Packet2f& b, const Packet2f& c) {
  return vfma_f32(c, a, b);
}
#else
template <>
EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
  return vmlaq_f32(c, a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2f pmadd(const Packet2f& a, const Packet2f& b, const Packet2f& c) {
  return vmla_f32(c, a, b);
}
#endif

// No FMA instruction for int, so use MLA unconditionally.
template <>
EIGEN_STRONG_INLINE Packet4c pmadd(const Packet4c& a, const Packet4c& b, const Packet4c& c) {
  return vget_lane_s32(
      vreinterpret_s32_s8(vmla_s8(vreinterpret_s8_s32(vdup_n_s32(c)), vreinterpret_s8_s32(vdup_n_s32(a)),
                                  vreinterpret_s8_s32(vdup_n_s32(b)))),
      0);
}
template <>
EIGEN_STRONG_INLINE Packet8c pmadd(const Packet8c& a, const Packet8c& b, const Packet8c& c) {
  return vmla_s8(c, a, b);
}
template <>
EIGEN_STRONG_INLINE Packet16c pmadd(const Packet16c& a, const Packet16c& b, const Packet16c& c) {
  return vmlaq_s8(c, a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4uc pmadd(const Packet4uc& a, const Packet4uc& b, const Packet4uc& c) {
  return vget_lane_u32(
      vreinterpret_u32_u8(vmla_u8(vreinterpret_u8_u32(vdup_n_u32(c)), vreinterpret_u8_u32(vdup_n_u32(a)),
                                  vreinterpret_u8_u32(vdup_n_u32(b)))),
      0);
}
template <>
EIGEN_STRONG_INLINE Packet8uc pmadd(const Packet8uc& a, const Packet8uc& b, const Packet8uc& c) {
  return vmla_u8(c, a, b);
}
template <>
EIGEN_STRONG_INLINE Packet16uc pmadd(const Packet16uc& a, const Packet16uc& b, const Packet16uc& c) {
  return vmlaq_u8(c, a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4s pmadd(const Packet4s& a, const Packet4s& b, const Packet4s& c) {
  return vmla_s16(c, a, b);
}
template <>
EIGEN_STRONG_INLINE Packet8s pmadd(const Packet8s& a, const Packet8s& b, const Packet8s& c) {
  return vmlaq_s16(c, a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4us pmadd(const Packet4us& a, const Packet4us& b, const Packet4us& c) {
  return vmla_u16(c, a, b);
}
template <>
EIGEN_STRONG_INLINE Packet8us pmadd(const Packet8us& a, const Packet8us& b, const Packet8us& c) {
  return vmlaq_u16(c, a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2i pmadd(const Packet2i& a, const Packet2i& b, const Packet2i& c) {
  return vmla_s32(c, a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) {
  return vmlaq_s32(c, a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2ui pmadd(const Packet2ui& a, const Packet2ui& b, const Packet2ui& c) {
  return vmla_u32(c, a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4ui pmadd(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c) {
  return vmlaq_u32(c, a, b);
}

template <>
EIGEN_STRONG_INLINE Packet2f pabsdiff<Packet2f>(const Packet2f& a, const Packet2f& b) {
  return vabd_f32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4f pabsdiff<Packet4f>(const Packet4f& a, const Packet4f& b) {
  return vabdq_f32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4c pabsdiff<Packet4c>(const Packet4c& a, const Packet4c& b) {
  return vget_lane_s32(
      vreinterpret_s32_s8(vabd_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
}
template <>
EIGEN_STRONG_INLINE Packet8c pabsdiff<Packet8c>(const Packet8c& a, const Packet8c& b) {
  return vabd_s8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet16c pabsdiff<Packet16c>(const Packet16c& a, const Packet16c& b) {
  return vabdq_s8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4uc pabsdiff<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
  return vget_lane_u32(
      vreinterpret_u32_u8(vabd_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
}
template <>
EIGEN_STRONG_INLINE Packet8uc pabsdiff<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
  return vabd_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet16uc pabsdiff<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
  return vabdq_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4s pabsdiff<Packet4s>(const Packet4s& a, const Packet4s& b) {
  return vabd_s16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet8s pabsdiff<Packet8s>(const Packet8s& a, const Packet8s& b) {
  return vabdq_s16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4us pabsdiff<Packet4us>(const Packet4us& a, const Packet4us& b) {
  return vabd_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet8us pabsdiff<Packet8us>(const Packet8us& a, const Packet8us& b) {
  return vabdq_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2i pabsdiff<Packet2i>(const Packet2i& a, const Packet2i& b) {
  return vabd_s32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4i pabsdiff<Packet4i>(const Packet4i& a, const Packet4i& b) {
  return vabdq_s32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2ui pabsdiff<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
  return vabd_u32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4ui pabsdiff<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
  return vabdq_u32(a, b);
}

template <>
EIGEN_STRONG_INLINE Packet2f pmin<Packet2f>(const Packet2f& a, const Packet2f& b) {
  return vmin_f32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) {
  return vminq_f32(a, b);
}

#ifdef __ARM_FEATURE_NUMERIC_MAXMIN
// numeric max and min are only available if ARM_FEATURE_NUMERIC_MAXMIN is defined (which can only be the case for Armv8
// systems).
template <>
EIGEN_STRONG_INLINE Packet4f pmin<PropagateNumbers, Packet4f>(const Packet4f& a, const Packet4f& b) {
  return vminnmq_f32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2f pmin<PropagateNumbers, Packet2f>(const Packet2f& a, const Packet2f& b) {
  return vminnm_f32(a, b);
}
#endif

template <>
EIGEN_STRONG_INLINE Packet4f pmin<PropagateNaN, Packet4f>(const Packet4f& a, const Packet4f& b) {
  return pmin<Packet4f>(a, b);
}

template <>
EIGEN_STRONG_INLINE Packet2f pmin<PropagateNaN, Packet2f>(const Packet2f& a, const Packet2f& b) {
  return pmin<Packet2f>(a, b);
}

template <>
EIGEN_STRONG_INLINE Packet4c pmin<Packet4c>(const Packet4c& a, const Packet4c& b) {
  return vget_lane_s32(
      vreinterpret_s32_s8(vmin_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
}
template <>
EIGEN_STRONG_INLINE Packet8c pmin<Packet8c>(const Packet8c& a, const Packet8c& b) {
  return vmin_s8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet16c pmin<Packet16c>(const Packet16c& a, const Packet16c& b) {
  return vminq_s8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4uc pmin<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
  return vget_lane_u32(
      vreinterpret_u32_u8(vmin_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
}
template <>
EIGEN_STRONG_INLINE Packet8uc pmin<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
  return vmin_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet16uc pmin<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
  return vminq_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4s pmin<Packet4s>(const Packet4s& a, const Packet4s& b) {
  return vmin_s16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet8s pmin<Packet8s>(const Packet8s& a, const Packet8s& b) {
  return vminq_s16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4us pmin<Packet4us>(const Packet4us& a, const Packet4us& b) {
  return vmin_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet8us pmin<Packet8us>(const Packet8us& a, const Packet8us& b) {
  return vminq_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2i pmin<Packet2i>(const Packet2i& a, const Packet2i& b) {
  return vmin_s32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b) {
  return vminq_s32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2ui pmin<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
  return vmin_u32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4ui pmin<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
  return vminq_u32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2l pmin<Packet2l>(const Packet2l& a, const Packet2l& b) {
  return vcombine_s64(vdup_n_s64((std::min)(vgetq_lane_s64(a, 0), vgetq_lane_s64(b, 0))),
                      vdup_n_s64((std::min)(vgetq_lane_s64(a, 1), vgetq_lane_s64(b, 1))));
}
template <>
EIGEN_STRONG_INLINE Packet2ul pmin<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
  return vcombine_u64(vdup_n_u64((std::min)(vgetq_lane_u64(a, 0), vgetq_lane_u64(b, 0))),
                      vdup_n_u64((std::min)(vgetq_lane_u64(a, 1), vgetq_lane_u64(b, 1))));
}

template <>
EIGEN_STRONG_INLINE Packet2f pmax<Packet2f>(const Packet2f& a, const Packet2f& b) {
  return vmax_f32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) {
  return vmaxq_f32(a, b);
}

#ifdef __ARM_FEATURE_NUMERIC_MAXMIN
// numeric max and min are only available if ARM_FEATURE_NUMERIC_MAXMIN is defined (which can only be the case for Armv8
// systems).
template <>
EIGEN_STRONG_INLINE Packet4f pmax<PropagateNumbers, Packet4f>(const Packet4f& a, const Packet4f& b) {
  return vmaxnmq_f32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2f pmax<PropagateNumbers, Packet2f>(const Packet2f& a, const Packet2f& b) {
  return vmaxnm_f32(a, b);
}
#endif

template <>
EIGEN_STRONG_INLINE Packet4f pmax<PropagateNaN, Packet4f>(const Packet4f& a, const Packet4f& b) {
  return pmax<Packet4f>(a, b);
}

template <>
EIGEN_STRONG_INLINE Packet2f pmax<PropagateNaN, Packet2f>(const Packet2f& a, const Packet2f& b) {
  return pmax<Packet2f>(a, b);
}

template <>
EIGEN_STRONG_INLINE Packet4c pmax<Packet4c>(const Packet4c& a, const Packet4c& b) {
  return vget_lane_s32(
      vreinterpret_s32_s8(vmax_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
}
template <>
EIGEN_STRONG_INLINE Packet8c pmax<Packet8c>(const Packet8c& a, const Packet8c& b) {
  return vmax_s8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet16c pmax<Packet16c>(const Packet16c& a, const Packet16c& b) {
  return vmaxq_s8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4uc pmax<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
  return vget_lane_u32(
      vreinterpret_u32_u8(vmax_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
}
template <>
EIGEN_STRONG_INLINE Packet8uc pmax<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
  return vmax_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet16uc pmax<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
  return vmaxq_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4s pmax<Packet4s>(const Packet4s& a, const Packet4s& b) {
  return vmax_s16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet8s pmax<Packet8s>(const Packet8s& a, const Packet8s& b) {
  return vmaxq_s16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4us pmax<Packet4us>(const Packet4us& a, const Packet4us& b) {
  return vmax_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet8us pmax<Packet8us>(const Packet8us& a, const Packet8us& b) {
  return vmaxq_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2i pmax<Packet2i>(const Packet2i& a, const Packet2i& b) {
  return vmax_s32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b) {
  return vmaxq_s32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2ui pmax<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
  return vmax_u32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4ui pmax<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
  return vmaxq_u32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2l pmax<Packet2l>(const Packet2l& a, const Packet2l& b) {
  return vcombine_s64(vdup_n_s64((std::max)(vgetq_lane_s64(a, 0), vgetq_lane_s64(b, 0))),
                      vdup_n_s64((std::max)(vgetq_lane_s64(a, 1), vgetq_lane_s64(b, 1))));
}
template <>
EIGEN_STRONG_INLINE Packet2ul pmax<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
  return vcombine_u64(vdup_n_u64((std::max)(vgetq_lane_u64(a, 0), vgetq_lane_u64(b, 0))),
                      vdup_n_u64((std::max)(vgetq_lane_u64(a, 1), vgetq_lane_u64(b, 1))));
}

template <>
EIGEN_STRONG_INLINE Packet2f pcmp_le<Packet2f>(const Packet2f& a, const Packet2f& b) {
  return vreinterpret_f32_u32(vcle_f32(a, b));
}
template <>
EIGEN_STRONG_INLINE Packet4f pcmp_le<Packet4f>(const Packet4f& a, const Packet4f& b) {
  return vreinterpretq_f32_u32(vcleq_f32(a, b));
}
template <>
EIGEN_STRONG_INLINE Packet4c pcmp_le<Packet4c>(const Packet4c& a, const Packet4c& b) {
  return vget_lane_s32(
      vreinterpret_s32_u8(vcle_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
}
template <>
EIGEN_STRONG_INLINE Packet8c pcmp_le<Packet8c>(const Packet8c& a, const Packet8c& b) {
  return vreinterpret_s8_u8(vcle_s8(a, b));
}
template <>
EIGEN_STRONG_INLINE Packet16c pcmp_le<Packet16c>(const Packet16c& a, const Packet16c& b) {
  return vreinterpretq_s8_u8(vcleq_s8(a, b));
}
template <>
EIGEN_STRONG_INLINE Packet4uc pcmp_le<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
  return vget_lane_u32(
      vreinterpret_u32_u8(vcle_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
}
template <>
EIGEN_STRONG_INLINE Packet8uc pcmp_le<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
  return vcle_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet16uc pcmp_le<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
  return vcleq_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4s pcmp_le<Packet4s>(const Packet4s& a, const Packet4s& b) {
  return vreinterpret_s16_u16(vcle_s16(a, b));
}
template <>
EIGEN_STRONG_INLINE Packet8s pcmp_le<Packet8s>(const Packet8s& a, const Packet8s& b) {
  return vreinterpretq_s16_u16(vcleq_s16(a, b));
}
template <>
EIGEN_STRONG_INLINE Packet4us pcmp_le<Packet4us>(const Packet4us& a, const Packet4us& b) {
  return vcle_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet8us pcmp_le<Packet8us>(const Packet8us& a, const Packet8us& b) {
  return vcleq_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2i pcmp_le<Packet2i>(const Packet2i& a, const Packet2i& b) {
  return vreinterpret_s32_u32(vcle_s32(a, b));
}
template <>
EIGEN_STRONG_INLINE Packet4i pcmp_le<Packet4i>(const Packet4i& a, const Packet4i& b) {
  return vreinterpretq_s32_u32(vcleq_s32(a, b));
}
template <>
EIGEN_STRONG_INLINE Packet2ui pcmp_le<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
  return vcle_u32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4ui pcmp_le<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
  return vcleq_u32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2l pcmp_le<Packet2l>(const Packet2l& a, const Packet2l& b) {
#if EIGEN_ARCH_ARM64
  return vreinterpretq_s64_u64(vcleq_s64(a, b));
#else
  return vcombine_s64(vdup_n_s64(vgetq_lane_s64(a, 0) <= vgetq_lane_s64(b, 0) ? numext::int64_t(-1) : 0),
                      vdup_n_s64(vgetq_lane_s64(a, 1) <= vgetq_lane_s64(b, 1) ? numext::int64_t(-1) : 0));
#endif
}
template <>
EIGEN_STRONG_INLINE Packet2ul pcmp_le<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
#if EIGEN_ARCH_ARM64
  return vcleq_u64(a, b);
#else
  return vcombine_u64(vdup_n_u64(vgetq_lane_u64(a, 0) <= vgetq_lane_u64(b, 0) ? numext::uint64_t(-1) : 0),
                      vdup_n_u64(vgetq_lane_u64(a, 1) <= vgetq_lane_u64(b, 1) ? numext::uint64_t(-1) : 0));
#endif
}

template <>
EIGEN_STRONG_INLINE Packet2f pcmp_lt<Packet2f>(const Packet2f& a, const Packet2f& b) {
  return vreinterpret_f32_u32(vclt_f32(a, b));
}
template <>
EIGEN_STRONG_INLINE Packet4f pcmp_lt<Packet4f>(const Packet4f& a, const Packet4f& b) {
  return vreinterpretq_f32_u32(vcltq_f32(a, b));
}
template <>
EIGEN_STRONG_INLINE Packet4c pcmp_lt<Packet4c>(const Packet4c& a, const Packet4c& b) {
  return vget_lane_s32(
      vreinterpret_s32_u8(vclt_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
}
template <>
EIGEN_STRONG_INLINE Packet8c pcmp_lt<Packet8c>(const Packet8c& a, const Packet8c& b) {
  return vreinterpret_s8_u8(vclt_s8(a, b));
}
template <>
EIGEN_STRONG_INLINE Packet16c pcmp_lt<Packet16c>(const Packet16c& a, const Packet16c& b) {
  return vreinterpretq_s8_u8(vcltq_s8(a, b));
}
template <>
EIGEN_STRONG_INLINE Packet4uc pcmp_lt<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
  return vget_lane_u32(
      vreinterpret_u32_u8(vclt_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
}
template <>
EIGEN_STRONG_INLINE Packet8uc pcmp_lt<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
  return vclt_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet16uc pcmp_lt<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
  return vcltq_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4s pcmp_lt<Packet4s>(const Packet4s& a, const Packet4s& b) {
  return vreinterpret_s16_u16(vclt_s16(a, b));
}
template <>
EIGEN_STRONG_INLINE Packet8s pcmp_lt<Packet8s>(const Packet8s& a, const Packet8s& b) {
  return vreinterpretq_s16_u16(vcltq_s16(a, b));
}
template <>
EIGEN_STRONG_INLINE Packet4us pcmp_lt<Packet4us>(const Packet4us& a, const Packet4us& b) {
  return vclt_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet8us pcmp_lt<Packet8us>(const Packet8us& a, const Packet8us& b) {
  return vcltq_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2i pcmp_lt<Packet2i>(const Packet2i& a, const Packet2i& b) {
  return vreinterpret_s32_u32(vclt_s32(a, b));
}
template <>
EIGEN_STRONG_INLINE Packet4i pcmp_lt<Packet4i>(const Packet4i& a, const Packet4i& b) {
  return vreinterpretq_s32_u32(vcltq_s32(a, b));
}
template <>
EIGEN_STRONG_INLINE Packet2ui pcmp_lt<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
  return vclt_u32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4ui pcmp_lt<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
  return vcltq_u32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2l pcmp_lt<Packet2l>(const Packet2l& a, const Packet2l& b) {
#if EIGEN_ARCH_ARM64
  return vreinterpretq_s64_u64(vcltq_s64(a, b));
#else
  return vcombine_s64(vdup_n_s64(vgetq_lane_s64(a, 0) < vgetq_lane_s64(b, 0) ? numext::int64_t(-1) : 0),
                      vdup_n_s64(vgetq_lane_s64(a, 1) < vgetq_lane_s64(b, 1) ? numext::int64_t(-1) : 0));
#endif
}
template <>
EIGEN_STRONG_INLINE Packet2ul pcmp_lt<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
#if EIGEN_ARCH_ARM64
  return vcltq_u64(a, b);
#else
  return vcombine_u64(vdup_n_u64(vgetq_lane_u64(a, 0) < vgetq_lane_u64(b, 0) ? numext::uint64_t(-1) : 0),
                      vdup_n_u64(vgetq_lane_u64(a, 1) < vgetq_lane_u64(b, 1) ? numext::uint64_t(-1) : 0));
#endif
}

template <>
EIGEN_STRONG_INLINE Packet2f pcmp_eq<Packet2f>(const Packet2f& a, const Packet2f& b) {
  return vreinterpret_f32_u32(vceq_f32(a, b));
}
template <>
EIGEN_STRONG_INLINE Packet4f pcmp_eq<Packet4f>(const Packet4f& a, const Packet4f& b) {
  return vreinterpretq_f32_u32(vceqq_f32(a, b));
}
template <>
EIGEN_STRONG_INLINE Packet4c pcmp_eq<Packet4c>(const Packet4c& a, const Packet4c& b) {
  return vget_lane_s32(
      vreinterpret_s32_u8(vceq_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
}
template <>
EIGEN_STRONG_INLINE Packet8c pcmp_eq<Packet8c>(const Packet8c& a, const Packet8c& b) {
  return vreinterpret_s8_u8(vceq_s8(a, b));
}
template <>
EIGEN_STRONG_INLINE Packet16c pcmp_eq<Packet16c>(const Packet16c& a, const Packet16c& b) {
  return vreinterpretq_s8_u8(vceqq_s8(a, b));
}
template <>
EIGEN_STRONG_INLINE Packet4uc pcmp_eq<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
  return vget_lane_u32(
      vreinterpret_u32_u8(vceq_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
}
template <>
EIGEN_STRONG_INLINE Packet8uc pcmp_eq<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
  return vceq_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet16uc pcmp_eq<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
  return vceqq_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4s pcmp_eq<Packet4s>(const Packet4s& a, const Packet4s& b) {
  return vreinterpret_s16_u16(vceq_s16(a, b));
}
template <>
EIGEN_STRONG_INLINE Packet8s pcmp_eq<Packet8s>(const Packet8s& a, const Packet8s& b) {
  return vreinterpretq_s16_u16(vceqq_s16(a, b));
}
template <>
EIGEN_STRONG_INLINE Packet4us pcmp_eq<Packet4us>(const Packet4us& a, const Packet4us& b) {
  return vceq_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet8us pcmp_eq<Packet8us>(const Packet8us& a, const Packet8us& b) {
  return vceqq_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2i pcmp_eq<Packet2i>(const Packet2i& a, const Packet2i& b) {
  return vreinterpret_s32_u32(vceq_s32(a, b));
}
template <>
EIGEN_STRONG_INLINE Packet4i pcmp_eq<Packet4i>(const Packet4i& a, const Packet4i& b) {
  return vreinterpretq_s32_u32(vceqq_s32(a, b));
}
template <>
EIGEN_STRONG_INLINE Packet2ui pcmp_eq<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
  return vceq_u32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4ui pcmp_eq<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
  return vceqq_u32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2l pcmp_eq<Packet2l>(const Packet2l& a, const Packet2l& b) {
#if EIGEN_ARCH_ARM64
  return vreinterpretq_s64_u64(vceqq_s64(a, b));
#else
  return vcombine_s64(vdup_n_s64(vgetq_lane_s64(a, 0) == vgetq_lane_s64(b, 0) ? numext::int64_t(-1) : 0),
                      vdup_n_s64(vgetq_lane_s64(a, 1) == vgetq_lane_s64(b, 1) ? numext::int64_t(-1) : 0));
#endif
}
template <>
EIGEN_STRONG_INLINE Packet2ul pcmp_eq<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
#if EIGEN_ARCH_ARM64
  return vceqq_u64(a, b);
#else
  return vcombine_u64(vdup_n_u64(vgetq_lane_u64(a, 0) == vgetq_lane_u64(b, 0) ? numext::uint64_t(-1) : 0),
                      vdup_n_u64(vgetq_lane_u64(a, 1) == vgetq_lane_u64(b, 1) ? numext::uint64_t(-1) : 0));
#endif
}

template <>
EIGEN_STRONG_INLINE Packet2f pcmp_lt_or_nan<Packet2f>(const Packet2f& a, const Packet2f& b) {
  return vreinterpret_f32_u32(vmvn_u32(vcge_f32(a, b)));
}
template <>
EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan<Packet4f>(const Packet4f& a, const Packet4f& b) {
  return vreinterpretq_f32_u32(vmvnq_u32(vcgeq_f32(a, b)));
}

// Logical Operations are not supported for float, so we have to reinterpret casts using NEON intrinsics
template <>
EIGEN_STRONG_INLINE Packet2f pand<Packet2f>(const Packet2f& a, const Packet2f& b) {
  return vreinterpret_f32_u32(vand_u32(vreinterpret_u32_f32(a), vreinterpret_u32_f32(b)));
}
template <>
EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b) {
  return vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(a), vreinterpretq_u32_f32(b)));
}
template <>
EIGEN_STRONG_INLINE Packet4c pand<Packet4c>(const Packet4c& a, const Packet4c& b) {
  return a & b;
}
template <>
EIGEN_STRONG_INLINE Packet8c pand<Packet8c>(const Packet8c& a, const Packet8c& b) {
  return vand_s8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet16c pand<Packet16c>(const Packet16c& a, const Packet16c& b) {
  return vandq_s8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4uc pand<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
  return a & b;
}
template <>
EIGEN_STRONG_INLINE Packet8uc pand<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
  return vand_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet16uc pand<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
  return vandq_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4s pand<Packet4s>(const Packet4s& a, const Packet4s& b) {
  return vand_s16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet8s pand<Packet8s>(const Packet8s& a, const Packet8s& b) {
  return vandq_s16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4us pand<Packet4us>(const Packet4us& a, const Packet4us& b) {
  return vand_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet8us pand<Packet8us>(const Packet8us& a, const Packet8us& b) {
  return vandq_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2i pand<Packet2i>(const Packet2i& a, const Packet2i& b) {
  return vand_s32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(const Packet4i& a, const Packet4i& b) {
  return vandq_s32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2ui pand<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
  return vand_u32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4ui pand<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
  return vandq_u32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2l pand<Packet2l>(const Packet2l& a, const Packet2l& b) {
  return vandq_s64(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2ul pand<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
  return vandq_u64(a, b);
}

template <>
EIGEN_STRONG_INLINE Packet2f por<Packet2f>(const Packet2f& a, const Packet2f& b) {
  return vreinterpret_f32_u32(vorr_u32(vreinterpret_u32_f32(a), vreinterpret_u32_f32(b)));
}
template <>
EIGEN_STRONG_INLINE Packet4f por<Packet4f>(const Packet4f& a, const Packet4f& b) {
  return vreinterpretq_f32_u32(vorrq_u32(vreinterpretq_u32_f32(a), vreinterpretq_u32_f32(b)));
}
template <>
EIGEN_STRONG_INLINE Packet4c por<Packet4c>(const Packet4c& a, const Packet4c& b) {
  return a | b;
}
template <>
EIGEN_STRONG_INLINE Packet8c por<Packet8c>(const Packet8c& a, const Packet8c& b) {
  return vorr_s8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet16c por<Packet16c>(const Packet16c& a, const Packet16c& b) {
  return vorrq_s8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4uc por<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
  return a | b;
}
template <>
EIGEN_STRONG_INLINE Packet8uc por<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
  return vorr_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet16uc por<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
  return vorrq_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4s por<Packet4s>(const Packet4s& a, const Packet4s& b) {
  return vorr_s16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet8s por<Packet8s>(const Packet8s& a, const Packet8s& b) {
  return vorrq_s16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4us por<Packet4us>(const Packet4us& a, const Packet4us& b) {
  return vorr_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet8us por<Packet8us>(const Packet8us& a, const Packet8us& b) {
  return vorrq_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2i por<Packet2i>(const Packet2i& a, const Packet2i& b) {
  return vorr_s32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4i por<Packet4i>(const Packet4i& a, const Packet4i& b) {
  return vorrq_s32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2ui por<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
  return vorr_u32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4ui por<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
  return vorrq_u32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2l por<Packet2l>(const Packet2l& a, const Packet2l& b) {
  return vorrq_s64(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2ul por<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
  return vorrq_u64(a, b);
}

template <>
EIGEN_STRONG_INLINE Packet2f pxor<Packet2f>(const Packet2f& a, const Packet2f& b) {
  return vreinterpret_f32_u32(veor_u32(vreinterpret_u32_f32(a), vreinterpret_u32_f32(b)));
}
template <>
EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b) {
  return vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(a), vreinterpretq_u32_f32(b)));
}
template <>
EIGEN_STRONG_INLINE Packet4c pxor<Packet4c>(const Packet4c& a, const Packet4c& b) {
  return a ^ b;
}
template <>
EIGEN_STRONG_INLINE Packet8c pxor<Packet8c>(const Packet8c& a, const Packet8c& b) {
  return veor_s8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet16c pxor<Packet16c>(const Packet16c& a, const Packet16c& b) {
  return veorq_s8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4uc pxor<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
  return a ^ b;
}
template <>
EIGEN_STRONG_INLINE Packet8uc pxor<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
  return veor_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet16uc pxor<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
  return veorq_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4s pxor<Packet4s>(const Packet4s& a, const Packet4s& b) {
  return veor_s16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet8s pxor<Packet8s>(const Packet8s& a, const Packet8s& b) {
  return veorq_s16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4us pxor<Packet4us>(const Packet4us& a, const Packet4us& b) {
  return veor_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet8us pxor<Packet8us>(const Packet8us& a, const Packet8us& b) {
  return veorq_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2i pxor<Packet2i>(const Packet2i& a, const Packet2i& b) {
  return veor_s32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(const Packet4i& a, const Packet4i& b) {
  return veorq_s32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2ui pxor<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
  return veor_u32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4ui pxor<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
  return veorq_u32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2l pxor<Packet2l>(const Packet2l& a, const Packet2l& b) {
  return veorq_s64(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2ul pxor<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
  return veorq_u64(a, b);
}

template <>
EIGEN_STRONG_INLINE Packet2f pandnot<Packet2f>(const Packet2f& a, const Packet2f& b) {
  return vreinterpret_f32_u32(vbic_u32(vreinterpret_u32_f32(a), vreinterpret_u32_f32(b)));
}
template <>
EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) {
  return vreinterpretq_f32_u32(vbicq_u32(vreinterpretq_u32_f32(a), vreinterpretq_u32_f32(b)));
}
template <>
EIGEN_STRONG_INLINE Packet4c pandnot<Packet4c>(const Packet4c& a, const Packet4c& b) {
  return a & ~b;
}
template <>
EIGEN_STRONG_INLINE Packet8c pandnot<Packet8c>(const Packet8c& a, const Packet8c& b) {
  return vbic_s8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet16c pandnot<Packet16c>(const Packet16c& a, const Packet16c& b) {
  return vbicq_s8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4uc pandnot<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
  return a & ~b;
}
template <>
EIGEN_STRONG_INLINE Packet8uc pandnot<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
  return vbic_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet16uc pandnot<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
  return vbicq_u8(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4s pandnot<Packet4s>(const Packet4s& a, const Packet4s& b) {
  return vbic_s16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet8s pandnot<Packet8s>(const Packet8s& a, const Packet8s& b) {
  return vbicq_s16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4us pandnot<Packet4us>(const Packet4us& a, const Packet4us& b) {
  return vbic_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet8us pandnot<Packet8us>(const Packet8us& a, const Packet8us& b) {
  return vbicq_u16(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2i pandnot<Packet2i>(const Packet2i& a, const Packet2i& b) {
  return vbic_s32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) {
  return vbicq_s32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2ui pandnot<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
  return vbic_u32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet4ui pandnot<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
  return vbicq_u32(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2l pandnot<Packet2l>(const Packet2l& a, const Packet2l& b) {
  return vbicq_s64(a, b);
}
template <>
EIGEN_STRONG_INLINE Packet2ul pandnot<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
  return vbicq_u64(a, b);
}

template <int N>
EIGEN_STRONG_INLINE Packet4c parithmetic_shift_right(Packet4c& a) {
  return vget_lane_s32(vreinterpret_s32_s8(vshr_n_s8(vreinterpret_s8_s32(vdup_n_s32(a)), N)), 0);
}
template <int N>
EIGEN_STRONG_INLINE Packet8c parithmetic_shift_right(Packet8c a) {
  return vshr_n_s8(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet16c parithmetic_shift_right(Packet16c a) {
  return vshrq_n_s8(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet4uc parithmetic_shift_right(Packet4uc& a) {
  return vget_lane_u32(vreinterpret_u32_u8(vshr_n_u8(vreinterpret_u8_u32(vdup_n_u32(a)), N)), 0);
}
template <int N>
EIGEN_STRONG_INLINE Packet8uc parithmetic_shift_right(Packet8uc a) {
  return vshr_n_u8(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet16uc parithmetic_shift_right(Packet16uc a) {
  return vshrq_n_u8(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet4s parithmetic_shift_right(Packet4s a) {
  return vshr_n_s16(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet8s parithmetic_shift_right(Packet8s a) {
  return vshrq_n_s16(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet4us parithmetic_shift_right(Packet4us a) {
  return vshr_n_u16(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet8us parithmetic_shift_right(Packet8us a) {
  return vshrq_n_u16(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet2i parithmetic_shift_right(Packet2i a) {
  return vshr_n_s32(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet4i parithmetic_shift_right(Packet4i a) {
  return vshrq_n_s32(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet2ui parithmetic_shift_right(Packet2ui a) {
  return vshr_n_u32(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet4ui parithmetic_shift_right(Packet4ui a) {
  return vshrq_n_u32(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet2l parithmetic_shift_right(Packet2l a) {
  return vshrq_n_s64(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet2ul parithmetic_shift_right(Packet2ul a) {
  return vshrq_n_u64(a, N);
}

template <int N>
EIGEN_STRONG_INLINE Packet4c plogical_shift_right(Packet4c& a) {
  return vget_lane_s32(vreinterpret_s32_u8(vshr_n_u8(vreinterpret_u8_s32(vdup_n_s32(a)), N)), 0);
}
template <int N>
EIGEN_STRONG_INLINE Packet8c plogical_shift_right(Packet8c a) {
  return vreinterpret_s8_u8(vshr_n_u8(vreinterpret_u8_s8(a), N));
}
template <int N>
EIGEN_STRONG_INLINE Packet16c plogical_shift_right(Packet16c a) {
  return vreinterpretq_s8_u8(vshrq_n_u8(vreinterpretq_u8_s8(a), N));
}
template <int N>
EIGEN_STRONG_INLINE Packet4uc plogical_shift_right(Packet4uc& a) {
  return vget_lane_u32(vreinterpret_u32_s8(vshr_n_s8(vreinterpret_s8_u32(vdup_n_u32(a)), N)), 0);
}
template <int N>
EIGEN_STRONG_INLINE Packet8uc plogical_shift_right(Packet8uc a) {
  return vshr_n_u8(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet16uc plogical_shift_right(Packet16uc a) {
  return vshrq_n_u8(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet4s plogical_shift_right(Packet4s a) {
  return vreinterpret_s16_u16(vshr_n_u16(vreinterpret_u16_s16(a), N));
}
template <int N>
EIGEN_STRONG_INLINE Packet8s plogical_shift_right(Packet8s a) {
  return vreinterpretq_s16_u16(vshrq_n_u16(vreinterpretq_u16_s16(a), N));
}
template <int N>
EIGEN_STRONG_INLINE Packet4us plogical_shift_right(Packet4us a) {
  return vshr_n_u16(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet8us plogical_shift_right(Packet8us a) {
  return vshrq_n_u16(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet2i plogical_shift_right(Packet2i a) {
  return vreinterpret_s32_u32(vshr_n_u32(vreinterpret_u32_s32(a), N));
}
template <int N>
EIGEN_STRONG_INLINE Packet4i plogical_shift_right(Packet4i a) {
  return vreinterpretq_s32_u32(vshrq_n_u32(vreinterpretq_u32_s32(a), N));
}
template <int N>
EIGEN_STRONG_INLINE Packet2ui plogical_shift_right(Packet2ui a) {
  return vshr_n_u32(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet4ui plogical_shift_right(Packet4ui a) {
  return vshrq_n_u32(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet2l plogical_shift_right(Packet2l a) {
  return vreinterpretq_s64_u64(vshrq_n_u64(vreinterpretq_u64_s64(a), N));
}
template <int N>
EIGEN_STRONG_INLINE Packet2ul plogical_shift_right(Packet2ul a) {
  return vshrq_n_u64(a, N);
}

template <int N>
EIGEN_STRONG_INLINE Packet4c plogical_shift_left(Packet4c& a) {
  return vget_lane_s32(vreinterpret_s32_s8(vshl_n_s8(vreinterpret_s8_s32(vdup_n_s32(a)), N)), 0);
}
template <int N>
EIGEN_STRONG_INLINE Packet8c plogical_shift_left(Packet8c a) {
  return vshl_n_s8(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet16c plogical_shift_left(Packet16c a) {
  return vshlq_n_s8(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet4uc plogical_shift_left(Packet4uc& a) {
  return vget_lane_u32(vreinterpret_u32_u8(vshl_n_u8(vreinterpret_u8_u32(vdup_n_u32(a)), N)), 0);
}
template <int N>
EIGEN_STRONG_INLINE Packet8uc plogical_shift_left(Packet8uc a) {
  return vshl_n_u8(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet16uc plogical_shift_left(Packet16uc a) {
  return vshlq_n_u8(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet4s plogical_shift_left(Packet4s a) {
  return vshl_n_s16(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet8s plogical_shift_left(Packet8s a) {
  return vshlq_n_s16(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet4us plogical_shift_left(Packet4us a) {
  return vshl_n_u16(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet8us plogical_shift_left(Packet8us a) {
  return vshlq_n_u16(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet2i plogical_shift_left(Packet2i a) {
  return vshl_n_s32(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet4i plogical_shift_left(Packet4i a) {
  return vshlq_n_s32(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet2ui plogical_shift_left(Packet2ui a) {
  return vshl_n_u32(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet4ui plogical_shift_left(Packet4ui a) {
  return vshlq_n_u32(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet2l plogical_shift_left(Packet2l a) {
  return vshlq_n_s64(a, N);
}
template <int N>
EIGEN_STRONG_INLINE Packet2ul plogical_shift_left(Packet2ul a) {
  return vshlq_n_u64(a, N);
}

template <>
EIGEN_STRONG_INLINE Packet2f pload<Packet2f>(const float* from) {
  EIGEN_DEBUG_ALIGNED_LOAD return vld1_f32(from);
}
template <>
EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) {
  EIGEN_DEBUG_ALIGNED_LOAD return vld1q_f32(from);
}
template <>
EIGEN_STRONG_INLINE Packet4c pload<Packet4c>(const int8_t* from) {
  Packet4c res;
  memcpy(&res, from, sizeof(Packet4c));
  return res;
}
template <>
EIGEN_STRONG_INLINE Packet8c pload<Packet8c>(const int8_t* from) {
  EIGEN_DEBUG_ALIGNED_LOAD return vld1_s8(from);
}
template <>
EIGEN_STRONG_INLINE Packet16c pload<Packet16c>(const int8_t* from) {
  EIGEN_DEBUG_ALIGNED_LOAD return vld1q_s8(from);
}
template <>
EIGEN_STRONG_INLINE Packet4uc pload<Packet4uc>(const uint8_t* from) {
  Packet4uc res;
  memcpy(&res, from, sizeof(Packet4uc));
  return res;
}
template <>
EIGEN_STRONG_INLINE Packet8uc pload<Packet8uc>(const uint8_t* from) {
  EIGEN_DEBUG_ALIGNED_LOAD return vld1_u8(from);
}
template <>
EIGEN_STRONG_INLINE Packet16uc pload<Packet16uc>(const uint8_t* from) {
  EIGEN_DEBUG_ALIGNED_LOAD return vld1q_u8(from);
}
template <>
EIGEN_STRONG_INLINE Packet4s pload<Packet4s>(const int16_t* from) {
  EIGEN_DEBUG_ALIGNED_LOAD return vld1_s16(from);
}
template <>
EIGEN_STRONG_INLINE Packet8s pload<Packet8s>(const int16_t* from) {
  EIGEN_DEBUG_ALIGNED_LOAD return vld1q_s16(from);
}
template <>
EIGEN_STRONG_INLINE Packet4us pload<Packet4us>(const uint16_t* from) {
  EIGEN_DEBUG_ALIGNED_LOAD return vld1_u16(from);
}
template <>
EIGEN_STRONG_INLINE Packet8us pload<Packet8us>(const uint16_t* from) {
  EIGEN_DEBUG_ALIGNED_LOAD return vld1q_u16(from);
}
template <>
EIGEN_STRONG_INLINE Packet2i pload<Packet2i>(const int32_t* from) {
  EIGEN_DEBUG_ALIGNED_LOAD return vld1_s32(from);
}
template <>
EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int32_t* from) {
  EIGEN_DEBUG_ALIGNED_LOAD return vld1q_s32(from);
}
template <>
EIGEN_STRONG_INLINE Packet2ui pload<Packet2ui>(const uint32_t* from) {
  EIGEN_DEBUG_ALIGNED_LOAD return vld1_u32(from);
}
template <>
EIGEN_STRONG_INLINE Packet4ui pload<Packet4ui>(const uint32_t* from) {
  EIGEN_DEBUG_ALIGNED_LOAD return vld1q_u32(from);
}
template <>
EIGEN_STRONG_INLINE Packet2l pload<Packet2l>(const int64_t* from) {
  EIGEN_DEBUG_ALIGNED_LOAD return vld1q_s64(from);
}
template <>
EIGEN_STRONG_INLINE Packet2ul pload<Packet2ul>(const uint64_t* from) {
  EIGEN_DEBUG_ALIGNED_LOAD return vld1q_u64(from);
}

template <>
EIGEN_STRONG_INLINE Packet2f ploadu<Packet2f>(const float* from) {
  EIGEN_DEBUG_UNALIGNED_LOAD return vld1_f32(from);
}
template <>
EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from) {
  EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_f32(from);
}
template <>
EIGEN_STRONG_INLINE Packet4c ploadu<Packet4c>(const int8_t* from) {
  Packet4c res;
  memcpy(&res, from, sizeof(Packet4c));
  return res;
}
template <>
EIGEN_STRONG_INLINE Packet8c ploadu<Packet8c>(const int8_t* from) {
  EIGEN_DEBUG_UNALIGNED_LOAD return vld1_s8(from);
}
template <>
EIGEN_STRONG_INLINE Packet16c ploadu<Packet16c>(const int8_t* from) {
  EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_s8(from);
}
template <>
EIGEN_STRONG_INLINE Packet4uc ploadu<Packet4uc>(const uint8_t* from) {
  Packet4uc res;
  memcpy(&res, from, sizeof(Packet4uc));
  return res;
}
template <>
EIGEN_STRONG_INLINE Packet8uc ploadu<Packet8uc>(const uint8_t* from) {
  EIGEN_DEBUG_UNALIGNED_LOAD return vld1_u8(from);
}
template <>
EIGEN_STRONG_INLINE Packet16uc ploadu<Packet16uc>(const uint8_t* from) {
  EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_u8(from);
}
template <>
EIGEN_STRONG_INLINE Packet4s ploadu<Packet4s>(const int16_t* from) {
  EIGEN_DEBUG_UNALIGNED_LOAD return vld1_s16(from);
}
template <>
EIGEN_STRONG_INLINE Packet8s ploadu<Packet8s>(const int16_t* from) {
  EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_s16(from);
}
template <>
EIGEN_STRONG_INLINE Packet4us ploadu<Packet4us>(const uint16_t* from) {
  EIGEN_DEBUG_UNALIGNED_LOAD return vld1_u16(from);
}
template <>
EIGEN_STRONG_INLINE Packet8us ploadu<Packet8us>(const uint16_t* from) {
  EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_u16(from);
}
template <>
EIGEN_STRONG_INLINE Packet2i ploadu<Packet2i>(const int32_t* from) {
  EIGEN_DEBUG_UNALIGNED_LOAD return vld1_s32(from);
}
template <>
EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int32_t* from) {
  EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_s32(from);
}
template <>
EIGEN_STRONG_INLINE Packet2ui ploadu<Packet2ui>(const uint32_t* from) {
  EIGEN_DEBUG_UNALIGNED_LOAD return vld1_u32(from);
}
template <>
EIGEN_STRONG_INLINE Packet4ui ploadu<Packet4ui>(const uint32_t* from) {
  EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_u32(from);
}
template <>
EIGEN_STRONG_INLINE Packet2l ploadu<Packet2l>(const int64_t* from) {
  EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_s64(from);
}
template <>
EIGEN_STRONG_INLINE Packet2ul ploadu<Packet2ul>(const uint64_t* from) {
  EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_u64(from);
}

template <>
EIGEN_STRONG_INLINE Packet2f ploaddup<Packet2f>(const float* from) {
  return vld1_dup_f32(from);
}
template <>
EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from) {
  return vcombine_f32(vld1_dup_f32(from), vld1_dup_f32(from + 1));
}
template <>
EIGEN_STRONG_INLINE Packet4c ploaddup<Packet4c>(const int8_t* from) {
  const int8x8_t a = vreinterpret_s8_s32(vdup_n_s32(pload<Packet4c>(from)));
  return vget_lane_s32(vreinterpret_s32_s8(vzip_s8(a, a).val[0]), 0);
}
template <>
EIGEN_STRONG_INLINE Packet8c ploaddup<Packet8c>(const int8_t* from) {
  const int8x8_t a = vld1_s8(from);
  return vzip_s8(a, a).val[0];
}
template <>
EIGEN_STRONG_INLINE Packet16c ploaddup<Packet16c>(const int8_t* from) {
  const int8x8_t a = vld1_s8(from);
  const int8x8x2_t b = vzip_s8(a, a);
  return vcombine_s8(b.val[0], b.val[1]);
}
template <>
EIGEN_STRONG_INLINE Packet4uc ploaddup<Packet4uc>(const uint8_t* from) {
  const uint8x8_t a = vreinterpret_u8_u32(vdup_n_u32(pload<Packet4uc>(from)));
  return vget_lane_u32(vreinterpret_u32_u8(vzip_u8(a, a).val[0]), 0);
}
template <>
EIGEN_STRONG_INLINE Packet8uc ploaddup<Packet8uc>(const uint8_t* from) {
  const uint8x8_t