Eigen/src/Core/arch/SSE/Reductions.h

// This file is part of Eigen, a lightweight C++ template library
// for linear algebra.
//
// Copyright (C) 2025 Charlie Schlosser <cs.schlosser@gmail.com>
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.

#ifndef EIGEN_REDUCTIONS_SSE_H
#define EIGEN_REDUCTIONS_SSE_H

// IWYU pragma: private
#include "../../InternalHeaderCheck.h"

namespace Eigen {

namespace internal {

template <typename Packet>
struct sse_add_wrapper {
  static EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) { return padd<Packet>(a, b); }
};

template <typename Packet>
struct sse_mul_wrapper {
  static EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) { return pmul<Packet>(a, b); }
};

template <typename Packet>
struct sse_min_wrapper {
  static EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) { return pmin<Packet>(a, b); }
};

template <int NaNPropagation, typename Packet>
struct sse_min_prop_wrapper {
  static EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) {
    return pmin<NaNPropagation, Packet>(a, b);
  }
};

template <typename Packet>
struct sse_max_wrapper {
  static EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) { return pmax<Packet>(a, b); }
};

template <int NaNPropagation, typename Packet>
struct sse_max_prop_wrapper {
  static EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) {
    return pmax<NaNPropagation, Packet>(a, b);
  }
};

template <typename Packet, typename Op>
struct sse_predux_common;

template <typename Packet>
struct sse_predux_impl : sse_predux_common<Packet, sse_add_wrapper<Packet>> {};

template <typename Packet>
struct sse_predux_mul_impl : sse_predux_common<Packet, sse_mul_wrapper<Packet>> {};

template <typename Packet>
struct sse_predux_min_impl : sse_predux_common<Packet, sse_min_wrapper<Packet>> {};

template <int NaNPropagation, typename Packet>
struct sse_predux_min_prop_impl : sse_predux_common<Packet, sse_min_prop_wrapper<NaNPropagation, Packet>> {};

template <typename Packet>
struct sse_predux_max_impl : sse_predux_common<Packet, sse_max_wrapper<Packet>> {};

template <int NaNPropagation, typename Packet>
struct sse_predux_max_prop_impl : sse_predux_common<Packet, sse_max_prop_wrapper<NaNPropagation, Packet>> {};

/* -- -- -- -- -- -- -- -- -- -- -- -- Packet16b -- -- -- -- -- -- -- -- -- -- -- -- */

template <>
EIGEN_STRONG_INLINE bool predux(const Packet16b& a) {
  Packet4i tmp = _mm_or_si128(a, _mm_unpackhi_epi64(a, a));
  return (pfirst(tmp) != 0) || (pfirst<Packet4i>(_mm_shuffle_epi32(tmp, 1)) != 0);
}

template <>
EIGEN_STRONG_INLINE bool predux_mul(const Packet16b& a) {
  Packet4i tmp = _mm_and_si128(a, _mm_unpackhi_epi64(a, a));
  return ((pfirst<Packet4i>(tmp) == 0x01010101) && (pfirst<Packet4i>(_mm_shuffle_epi32(tmp, 1)) == 0x01010101));
}

template <>
EIGEN_STRONG_INLINE bool predux_min(const Packet16b& a) {
  return predux_mul(a);
}

template <>
EIGEN_STRONG_INLINE bool predux_max(const Packet16b& a) {
  return predux(a);
}

template <>
EIGEN_STRONG_INLINE bool predux_any(const Packet16b& a) {
  return predux(a);
}

/* -- -- -- -- -- -- -- -- -- -- -- -- Packet4i -- -- -- -- -- -- -- -- -- -- -- -- */

template <typename Op>
struct sse_predux_common<Packet4i, Op> {
  static EIGEN_STRONG_INLINE int run(const Packet4i& a) {
    Packet4i tmp;
    tmp = Op::packetOp(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(0, 1, 2, 3)));
    tmp = Op::packetOp(tmp, _mm_unpackhi_epi32(tmp, tmp));
    return _mm_cvtsi128_si32(tmp);
  }
};

template <>
EIGEN_STRONG_INLINE int predux(const Packet4i& a) {
  return sse_predux_impl<Packet4i>::run(a);
}

template <>
EIGEN_STRONG_INLINE int predux_mul(const Packet4i& a) {
  return sse_predux_mul_impl<Packet4i>::run(a);
}

#ifdef EIGEN_VECTORIZE_SSE4_1
template <>
EIGEN_STRONG_INLINE int predux_min(const Packet4i& a) {
  return sse_predux_min_impl<Packet4i>::run(a);
}

template <>
EIGEN_STRONG_INLINE int predux_max(const Packet4i& a) {
  return sse_predux_max_impl<Packet4i>::run(a);
}
#endif

template <>
EIGEN_STRONG_INLINE bool predux_any(const Packet4i& a) {
  return _mm_movemask_ps(_mm_castsi128_ps(a)) != 0x0;
}

/* -- -- -- -- -- -- -- -- -- -- -- -- Packet4ui -- -- -- -- -- -- -- -- -- -- -- -- */

template <typename Op>
struct sse_predux_common<Packet4ui, Op> {
  static EIGEN_STRONG_INLINE uint32_t run(const Packet4ui& a) {
    Packet4ui tmp;
    tmp = Op::packetOp(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(0, 1, 2, 3)));
    tmp = Op::packetOp(tmp, _mm_unpackhi_epi32(tmp, tmp));
    return static_cast<uint32_t>(_mm_cvtsi128_si32(tmp));
  }
};

template <>
EIGEN_STRONG_INLINE uint32_t predux(const Packet4ui& a) {
  return sse_predux_impl<Packet4ui>::run(a);
}

template <>
EIGEN_STRONG_INLINE uint32_t predux_mul(const Packet4ui& a) {
  return sse_predux_mul_impl<Packet4ui>::run(a);
}

#ifdef EIGEN_VECTORIZE_SSE4_1
template <>
EIGEN_STRONG_INLINE uint32_t predux_min(const Packet4ui& a) {
  return sse_predux_min_impl<Packet4ui>::run(a);
}

template <>
EIGEN_STRONG_INLINE uint32_t predux_max(const Packet4ui& a) {
  return sse_predux_max_impl<Packet4ui>::run(a);
}
#endif

template <>
EIGEN_STRONG_INLINE bool predux_any(const Packet4ui& a) {
  return _mm_movemask_ps(_mm_castsi128_ps(a)) != 0x0;
}

/* -- -- -- -- -- -- -- -- -- -- -- -- Packet2l -- -- -- -- -- -- -- -- -- -- -- -- */

template <typename Op>
struct sse_predux_common<Packet2l, Op> {
  static EIGEN_STRONG_INLINE int64_t run(const Packet2l& a) {
    Packet2l tmp;
    tmp = Op::packetOp(a, _mm_unpackhi_epi64(a, a));
    return pfirst(tmp);
  }
};

template <>
EIGEN_STRONG_INLINE int64_t predux(const Packet2l& a) {
  return sse_predux_impl<Packet2l>::run(a);
}

template <>
EIGEN_STRONG_INLINE bool predux_any(const Packet2l& a) {
  return _mm_movemask_pd(_mm_castsi128_pd(a)) != 0x0;
}

/* -- -- -- -- -- -- -- -- -- -- -- -- Packet4f -- -- -- -- -- -- -- -- -- -- -- -- */

template <typename Op>
struct sse_predux_common<Packet4f, Op> {
  static EIGEN_STRONG_INLINE float run(const Packet4f& a) {
    Packet4f tmp;
    tmp = Op::packetOp(a, _mm_movehl_ps(a, a));
#ifdef EIGEN_VECTORIZE_SSE3
    tmp = Op::packetOp(tmp, _mm_movehdup_ps(tmp));
#else
    tmp = Op::packetOp(tmp, _mm_shuffle_ps(tmp, tmp, 1));
#endif
    return _mm_cvtss_f32(tmp);
  }
};

template <>
EIGEN_STRONG_INLINE float predux(const Packet4f& a) {
  return sse_predux_impl<Packet4f>::run(a);
}

template <>
EIGEN_STRONG_INLINE float predux_mul(const Packet4f& a) {
  return sse_predux_mul_impl<Packet4f>::run(a);
}

template <>
EIGEN_STRONG_INLINE float predux_min(const Packet4f& a) {
  return sse_predux_min_impl<Packet4f>::run(a);
}

template <>
EIGEN_STRONG_INLINE float predux_min<PropagateNumbers>(const Packet4f& a) {
  return sse_predux_min_prop_impl<PropagateNumbers, Packet4f>::run(a);
}

template <>
EIGEN_STRONG_INLINE float predux_min<PropagateNaN>(const Packet4f& a) {
  return sse_predux_min_prop_impl<PropagateNaN, Packet4f>::run(a);
}

template <>
EIGEN_STRONG_INLINE float predux_max(const Packet4f& a) {
  return sse_predux_max_impl<Packet4f>::run(a);
}

template <>
EIGEN_STRONG_INLINE float predux_max<PropagateNumbers>(const Packet4f& a) {
  return sse_predux_max_prop_impl<PropagateNumbers, Packet4f>::run(a);
}

template <>
EIGEN_STRONG_INLINE float predux_max<PropagateNaN>(const Packet4f& a) {
  return sse_predux_max_prop_impl<PropagateNaN, Packet4f>::run(a);
}

template <>
EIGEN_STRONG_INLINE bool predux_any(const Packet4f& a) {
  return _mm_movemask_ps(a) != 0x0;
}

/* -- -- -- -- -- -- -- -- -- -- -- -- Packet2d -- -- -- -- -- -- -- -- -- -- -- -- */

template <typename Op>
struct sse_predux_common<Packet2d, Op> {
  static EIGEN_STRONG_INLINE double run(const Packet2d& a) {
    Packet2d tmp;
    tmp = Op::packetOp(a, _mm_unpackhi_pd(a, a));
    return _mm_cvtsd_f64(tmp);
  }
};

template <>
EIGEN_STRONG_INLINE double predux(const Packet2d& a) {
  return sse_predux_impl<Packet2d>::run(a);
}

template <>
EIGEN_STRONG_INLINE double predux_mul(const Packet2d& a) {
  return sse_predux_mul_impl<Packet2d>::run(a);
}

template <>
EIGEN_STRONG_INLINE double predux_min(const Packet2d& a) {
  return sse_predux_min_impl<Packet2d>::run(a);
}

template <>
EIGEN_STRONG_INLINE double predux_min<PropagateNumbers>(const Packet2d& a) {
  return sse_predux_min_prop_impl<PropagateNumbers, Packet2d>::run(a);
}

template <>
EIGEN_STRONG_INLINE double predux_min<PropagateNaN>(const Packet2d& a) {
  return sse_predux_min_prop_impl<PropagateNaN, Packet2d>::run(a);
}

template <>
EIGEN_STRONG_INLINE double predux_max(const Packet2d& a) {
  return sse_predux_max_impl<Packet2d>::run(a);
}

template <>
EIGEN_STRONG_INLINE double predux_max<PropagateNumbers>(const Packet2d& a) {
  return sse_predux_max_prop_impl<PropagateNumbers, Packet2d>::run(a);
}

template <>
EIGEN_STRONG_INLINE double predux_max<PropagateNaN>(const Packet2d& a) {
  return sse_predux_max_prop_impl<PropagateNaN, Packet2d>::run(a);
}

template <>
EIGEN_STRONG_INLINE bool predux_any(const Packet2d& a) {
  return _mm_movemask_pd(a) != 0x0;
}

}  // end namespace internal

}  // end namespace Eigen

#endif  // EIGEN_REDUCTIONS_SSE_H