No public description PiperOrigin-RevId: 654364383 Change-Id: I4c964c0eb6ee2518f5211e0638a559a2ecb41bd8
diff --git a/Eigen/src/Core/util/ConfigureVectorization.h b/Eigen/src/Core/util/ConfigureVectorization.h index 1c72173..6c53b87 100644 --- a/Eigen/src/Core/util/ConfigureVectorization.h +++ b/Eigen/src/Core/util/ConfigureVectorization.h
@@ -38,6 +38,16 @@ #define EIGEN_ALIGNOF(x) alignof(x) #endif +// Align to the boundary that avoids false sharing. +// https://en.cppreference.com/w/cpp/thread/hardware_destructive_interference_size +#ifdef __cpp_lib_hardware_interference_size +#define EIGEN_ALIGN_TO_AVOID_FALSE_SHARING \ + EIGEN_ALIGN_TO_BOUNDARY(std::hardware_destructive_interference_size) +#else +#define EIGEN_ALIGN_TO_AVOID_FALSE_SHARING \ + EIGEN_ALIGN_TO_BOUNDARY(128) +#endif + // If the user explicitly disable vectorization, then we also disable alignment #if defined(EIGEN_DONT_VECTORIZE) #if defined(EIGEN_GPUCC)
diff --git a/Eigen/src/ThreadPool/EventCount.h b/Eigen/src/ThreadPool/EventCount.h index 0117b4b..b3f2a18 100644 --- a/Eigen/src/ThreadPool/EventCount.h +++ b/Eigen/src/ThreadPool/EventCount.h
@@ -157,9 +157,7 @@ class Waiter { friend class EventCount; - // Align to 128 byte boundary to prevent false sharing with other Waiter - // objects in the same vector. - EIGEN_ALIGN_TO_BOUNDARY(128) std::atomic<uint64_t> next; + EIGEN_ALIGN_TO_AVOID_FALSE_SHARING std::atomic<uint64_t> next; EIGEN_MUTEX mu; EIGEN_CONDVAR cv; uint64_t epoch = 0;
diff --git a/Eigen/src/ThreadPool/NonBlockingThreadPool.h b/Eigen/src/ThreadPool/NonBlockingThreadPool.h index 66fd63c..0939dde 100644 --- a/Eigen/src/ThreadPool/NonBlockingThreadPool.h +++ b/Eigen/src/ThreadPool/NonBlockingThreadPool.h
@@ -211,10 +211,6 @@ ThreadPoolTempl* pool; // Parent pool, or null for normal threads. uint64_t rand; // Random generator state. int thread_id; // Worker thread index in pool. -#ifndef EIGEN_THREAD_LOCAL - // Prevent false sharing. - char pad_[128]; -#endif }; struct ThreadData {
diff --git a/Eigen/src/ThreadPool/RunQueue.h b/Eigen/src/ThreadPool/RunQueue.h index 9f40e9d..9b1fb9e 100644 --- a/Eigen/src/ThreadPool/RunQueue.h +++ b/Eigen/src/ThreadPool/RunQueue.h
@@ -45,7 +45,8 @@ eigen_plain_assert((kSize & (kSize - 1)) == 0); eigen_plain_assert(kSize > 2); // why would you do this? eigen_plain_assert(kSize <= (64 << 10)); // leave enough space for counter - for (unsigned i = 0; i < kSize; i++) array_[i].state.store(kEmpty, std::memory_order_relaxed); + for (unsigned i = 0; i < kSize; i++) + array_[i].state.store(kEmpty, std::memory_order_relaxed); } ~RunQueue() { eigen_plain_assert(Size() == 0); } @@ -56,7 +57,9 @@ unsigned front = front_.load(std::memory_order_relaxed); Elem* e = &array_[front & kMask]; uint8_t s = e->state.load(std::memory_order_relaxed); - if (s != kEmpty || !e->state.compare_exchange_strong(s, kBusy, std::memory_order_acquire)) return w; + if (s != kEmpty || + !e->state.compare_exchange_strong(s, kBusy, std::memory_order_acquire)) + return w; front_.store(front + 1 + (kSize << 1), std::memory_order_relaxed); e->w = std::move(w); e->state.store(kReady, std::memory_order_release); @@ -69,7 +72,9 @@ unsigned front = front_.load(std::memory_order_relaxed); Elem* e = &array_[(front - 1) & kMask]; uint8_t s = e->state.load(std::memory_order_relaxed); - if (s != kReady || !e->state.compare_exchange_strong(s, kBusy, std::memory_order_acquire)) return Work(); + if (s != kReady || + !e->state.compare_exchange_strong(s, kBusy, std::memory_order_acquire)) + return Work(); Work w = std::move(e->w); e->state.store(kEmpty, std::memory_order_release); front = ((front - 1) & kMask2) | (front & ~kMask2); @@ -84,7 +89,9 @@ unsigned back = back_.load(std::memory_order_relaxed); Elem* e = &array_[(back - 1) & kMask]; uint8_t s = e->state.load(std::memory_order_relaxed); - if (s != kEmpty || !e->state.compare_exchange_strong(s, kBusy, std::memory_order_acquire)) return w; + if (s != kEmpty || + !e->state.compare_exchange_strong(s, kBusy, std::memory_order_acquire)) + return w; back = ((back - 1) & kMask2) | (back & ~kMask2); back_.store(back, std::memory_order_relaxed); e->w = std::move(w); @@ -99,7 +106,9 @@ unsigned back = back_.load(std::memory_order_relaxed); Elem* e = &array_[back & kMask]; uint8_t s = e->state.load(std::memory_order_relaxed); - if (s != kReady || !e->state.compare_exchange_strong(s, kBusy, std::memory_order_acquire)) return Work(); + if (s != kReady || + !e->state.compare_exchange_strong(s, kBusy, std::memory_order_acquire)) + return Work(); Work w = std::move(e->w); e->state.store(kEmpty, std::memory_order_release); back_.store(back + 1 + (kSize << 1), std::memory_order_relaxed); @@ -121,7 +130,9 @@ Elem* e = &array_[mid & kMask]; uint8_t s = e->state.load(std::memory_order_relaxed); if (n == 0) { - if (s != kReady || !e->state.compare_exchange_strong(s, kBusy, std::memory_order_acquire)) continue; + if (s != kReady || !e->state.compare_exchange_strong( + s, kBusy, std::memory_order_acquire)) + continue; start = mid; } else { // Note: no need to store temporal kBusy, we exclusively own these @@ -132,7 +143,8 @@ e->state.store(kEmpty, std::memory_order_release); n++; } - if (n != 0) back_.store(start + 1 + (kSize << 1), std::memory_order_relaxed); + if (n != 0) + back_.store(start + 1 + (kSize << 1), std::memory_order_relaxed); return n; } @@ -154,16 +166,18 @@ private: static const unsigned kMask = kSize - 1; static const unsigned kMask2 = (kSize << 1) - 1; - struct Elem { - std::atomic<uint8_t> state; - Work w; - }; - enum { + + enum State { kEmpty, kBusy, kReady, }; - EIGEN_MUTEX mutex_; + + struct Elem { + std::atomic<uint8_t> state; + Work w; + }; + // Low log(kSize) + 1 bits in front_ and back_ contain rolling index of // front/back, respectively. The remaining bits contain modification counters // that are incremented on Push operations. This allows us to (1) distinguish @@ -171,9 +185,11 @@ // position, these conditions would be indistinguishable); (2) obtain // consistent snapshot of front_/back_ for Size operation using the // modification counters. - std::atomic<unsigned> front_; - std::atomic<unsigned> back_; - Elem array_[kSize]; + EIGEN_ALIGN_TO_AVOID_FALSE_SHARING std::atomic<unsigned> front_; + EIGEN_ALIGN_TO_AVOID_FALSE_SHARING std::atomic<unsigned> back_; + EIGEN_MUTEX mutex_; // guards `PushBack` and `PopBack` (accesses `back_`) + + EIGEN_ALIGN_TO_AVOID_FALSE_SHARING Elem array_[kSize]; // SizeOrNotEmpty returns current queue size; if NeedSizeEstimate is false, // only whether the size is 0 is guaranteed to be correct. @@ -205,15 +221,16 @@ } } - EIGEN_ALWAYS_INLINE unsigned CalculateSize(unsigned front, unsigned back) const { + EIGEN_ALWAYS_INLINE unsigned CalculateSize(unsigned front, + unsigned back) const { int size = (front & kMask2) - (back & kMask2); // Fix overflow. - if (size < 0) size += 2 * kSize; + if (EIGEN_PREDICT_FALSE(size < 0)) size += 2 * kSize; // Order of modification in push/pop is crafted to make the queue look // larger than it is during concurrent modifications. E.g. push can // increment size before the corresponding pop has decremented it. // So the computed size can be up to kSize + 1, fix it. - if (size > static_cast<int>(kSize)) size = kSize; + if (EIGEN_PREDICT_FALSE(size > static_cast<int>(kSize))) size = kSize; return static_cast<unsigned>(size); }