Please, help us to better know about our user community by answering the following short survey: https://forms.gle/wpyrxWi18ox9Z5ae9
Eigen  3.4.0
 
Loading...
Searching...
No Matches
AVX512/PacketMath.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2016 Benoit Steiner (benoit.steiner.goog@gmail.com)
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9
10#ifndef EIGEN_PACKET_MATH_AVX512_H
11#define EIGEN_PACKET_MATH_AVX512_H
12
13namespace Eigen {
14
15namespace internal {
16
17#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
18#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
19#endif
20
21#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
22#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
23#endif
24
25#ifdef EIGEN_VECTORIZE_FMA
26#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
27#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
28#endif
29#endif
30
31typedef __m512 Packet16f;
32typedef __m512i Packet16i;
33typedef __m512d Packet8d;
34typedef eigen_packet_wrapper<__m256i, 1> Packet16h;
35typedef eigen_packet_wrapper<__m256i, 2> Packet16bf;
36
37template <>
38struct is_arithmetic<__m512> {
39 enum { value = true };
40};
41template <>
42struct is_arithmetic<__m512i> {
43 enum { value = true };
44};
45template <>
46struct is_arithmetic<__m512d> {
47 enum { value = true };
48};
49
50template<> struct is_arithmetic<Packet16h> { enum { value = true }; };
51
52template <>
53struct packet_traits<half> : default_packet_traits {
54 typedef Packet16h type;
55 // There is no half-size packet for Packet16h.
56 typedef Packet16h half;
57 enum {
58 Vectorizable = 1,
59 AlignedOnScalar = 1,
60 size = 16,
61 HasHalfPacket = 1,
62
63 HasCmp = 1,
64 HasAdd = 1,
65 HasSub = 1,
66 HasMul = 1,
67 HasDiv = 1,
68 HasNegate = 1,
69 HasAbs = 1,
70 HasAbs2 = 0,
71 HasMin = 1,
72 HasMax = 1,
73 HasConj = 1,
74 HasSetLinear = 0,
75 HasLog = 1,
76 HasLog1p = 1,
77 HasExpm1 = 1,
78 HasExp = 1,
79 HasSqrt = 1,
80 HasRsqrt = 1,
81 HasSin = EIGEN_FAST_MATH,
82 HasCos = EIGEN_FAST_MATH,
83 HasTanh = EIGEN_FAST_MATH,
84 HasErf = EIGEN_FAST_MATH,
85 HasBlend = 0,
86 HasRound = 1,
87 HasFloor = 1,
88 HasCeil = 1,
89 HasRint = 1,
90 HasBessel = 1,
91 HasNdtri = 1
92 };
93};
94
95template<> struct packet_traits<float> : default_packet_traits
96{
97 typedef Packet16f type;
98 typedef Packet8f half;
99 enum {
100 Vectorizable = 1,
101 AlignedOnScalar = 1,
102 size = 16,
103 HasHalfPacket = 1,
104
105 HasAbs = 1,
106 HasMin = 1,
107 HasMax = 1,
108 HasConj = 1,
109 HasBlend = 0,
110 HasSin = EIGEN_FAST_MATH,
111 HasCos = EIGEN_FAST_MATH,
112#if EIGEN_GNUC_AT_LEAST(5, 3) || (!EIGEN_COMP_GNUC_STRICT)
113 HasLog = 1,
114 HasLog1p = 1,
115 HasExpm1 = 1,
116 HasNdtri = 1,
117 HasBessel = 1,
118 HasExp = 1,
119 HasSqrt = EIGEN_FAST_MATH,
120 HasRsqrt = EIGEN_FAST_MATH,
121 HasTanh = EIGEN_FAST_MATH,
122 HasErf = EIGEN_FAST_MATH,
123#endif
124 HasCmp = 1,
125 HasDiv = 1,
126 HasRound = 1,
127 HasFloor = 1,
128 HasCeil = 1,
129 HasRint = 1
130 };
131 };
132template<> struct packet_traits<double> : default_packet_traits
133{
134 typedef Packet8d type;
135 typedef Packet4d half;
136 enum {
137 Vectorizable = 1,
138 AlignedOnScalar = 1,
139 size = 8,
140 HasHalfPacket = 1,
141#if EIGEN_GNUC_AT_LEAST(5, 3) || (!EIGEN_COMP_GNUC_STRICT)
142 HasLog = 1,
143 HasExp = 1,
144 HasSqrt = EIGEN_FAST_MATH,
145 HasRsqrt = EIGEN_FAST_MATH,
146#endif
147 HasCmp = 1,
148 HasDiv = 1,
149 HasRound = 1,
150 HasFloor = 1,
151 HasCeil = 1,
152 HasRint = 1
153 };
154};
155
156/* TODO Implement AVX512 for integers
157template<> struct packet_traits<int> : default_packet_traits
158{
159 typedef Packet16i type;
160 enum {
161 Vectorizable = 1,
162 AlignedOnScalar = 1,
163 size=8
164 };
165};
166*/
167
168template <>
169struct unpacket_traits<Packet16f> {
170 typedef float type;
171 typedef Packet8f half;
172 typedef Packet16i integer_packet;
173 typedef uint16_t mask_t;
174 enum { size = 16, alignment=Aligned64, vectorizable=true, masked_load_available=true, masked_store_available=true };
175};
176template <>
177struct unpacket_traits<Packet8d> {
178 typedef double type;
179 typedef Packet4d half;
180 enum { size = 8, alignment=Aligned64, vectorizable=true, masked_load_available=false, masked_store_available=false };
181};
182template <>
183struct unpacket_traits<Packet16i> {
184 typedef int type;
185 typedef Packet8i half;
186 enum { size = 16, alignment=Aligned64, vectorizable=false, masked_load_available=false, masked_store_available=false };
187};
188
189template<>
190struct unpacket_traits<Packet16h> {
191 typedef Eigen::half type;
192 typedef Packet8h half;
193 enum {size=16, alignment=Aligned32, vectorizable=true, masked_load_available=false, masked_store_available=false};
194};
195
196template <>
197EIGEN_STRONG_INLINE Packet16f pset1<Packet16f>(const float& from) {
198 return _mm512_set1_ps(from);
199}
200template <>
201EIGEN_STRONG_INLINE Packet8d pset1<Packet8d>(const double& from) {
202 return _mm512_set1_pd(from);
203}
204template <>
205EIGEN_STRONG_INLINE Packet16i pset1<Packet16i>(const int& from) {
206 return _mm512_set1_epi32(from);
207}
208
209template <>
210EIGEN_STRONG_INLINE Packet16f pset1frombits<Packet16f>(unsigned int from) {
211 return _mm512_castsi512_ps(_mm512_set1_epi32(from));
212}
213
214template <>
215EIGEN_STRONG_INLINE Packet8d pset1frombits<Packet8d>(const numext::uint64_t from) {
216 return _mm512_castsi512_pd(_mm512_set1_epi64(from));
217}
218
219template<> EIGEN_STRONG_INLINE Packet16f pzero(const Packet16f& /*a*/) { return _mm512_setzero_ps(); }
220template<> EIGEN_STRONG_INLINE Packet8d pzero(const Packet8d& /*a*/) { return _mm512_setzero_pd(); }
221template<> EIGEN_STRONG_INLINE Packet16i pzero(const Packet16i& /*a*/) { return _mm512_setzero_si512(); }
222
223template<> EIGEN_STRONG_INLINE Packet16f peven_mask(const Packet16f& /*a*/) {
224 return _mm512_castsi512_ps(_mm512_set_epi32(0, -1, 0, -1, 0, -1, 0, -1,
225 0, -1, 0, -1, 0, -1, 0, -1));
226}
227template<> EIGEN_STRONG_INLINE Packet16i peven_mask(const Packet16i& /*a*/) {
228 return _mm512_set_epi32(0, -1, 0, -1, 0, -1, 0, -1,
229 0, -1, 0, -1, 0, -1, 0, -1);
230}
231template<> EIGEN_STRONG_INLINE Packet8d peven_mask(const Packet8d& /*a*/) {
232 return _mm512_castsi512_pd(_mm512_set_epi32(0, 0, -1, -1, 0, 0, -1, -1,
233 0, 0, -1, -1, 0, 0, -1, -1));
234}
235
236template <>
237EIGEN_STRONG_INLINE Packet16f pload1<Packet16f>(const float* from) {
238 return _mm512_broadcastss_ps(_mm_load_ps1(from));
239}
240template <>
241EIGEN_STRONG_INLINE Packet8d pload1<Packet8d>(const double* from) {
242 return _mm512_set1_pd(*from);
243}
244
245template <>
246EIGEN_STRONG_INLINE Packet16f plset<Packet16f>(const float& a) {
247 return _mm512_add_ps(
248 _mm512_set1_ps(a),
249 _mm512_set_ps(15.0f, 14.0f, 13.0f, 12.0f, 11.0f, 10.0f, 9.0f, 8.0f, 7.0f, 6.0f, 5.0f,
250 4.0f, 3.0f, 2.0f, 1.0f, 0.0f));
251}
252template <>
253EIGEN_STRONG_INLINE Packet8d plset<Packet8d>(const double& a) {
254 return _mm512_add_pd(_mm512_set1_pd(a),
255 _mm512_set_pd(7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.0));
256}
257
258template <>
259EIGEN_STRONG_INLINE Packet16f padd<Packet16f>(const Packet16f& a,
260 const Packet16f& b) {
261 return _mm512_add_ps(a, b);
262}
263template <>
264EIGEN_STRONG_INLINE Packet8d padd<Packet8d>(const Packet8d& a,
265 const Packet8d& b) {
266 return _mm512_add_pd(a, b);
267}
268template <>
269EIGEN_STRONG_INLINE Packet16i padd<Packet16i>(const Packet16i& a,
270 const Packet16i& b) {
271 return _mm512_add_epi32(a, b);
272}
273
274template <>
275EIGEN_STRONG_INLINE Packet16f psub<Packet16f>(const Packet16f& a,
276 const Packet16f& b) {
277 return _mm512_sub_ps(a, b);
278}
279template <>
280EIGEN_STRONG_INLINE Packet8d psub<Packet8d>(const Packet8d& a,
281 const Packet8d& b) {
282 return _mm512_sub_pd(a, b);
283}
284template <>
285EIGEN_STRONG_INLINE Packet16i psub<Packet16i>(const Packet16i& a,
286 const Packet16i& b) {
287 return _mm512_sub_epi32(a, b);
288}
289
290template <>
291EIGEN_STRONG_INLINE Packet16f pnegate(const Packet16f& a) {
292 return _mm512_sub_ps(_mm512_set1_ps(0.0), a);
293}
294template <>
295EIGEN_STRONG_INLINE Packet8d pnegate(const Packet8d& a) {
296 return _mm512_sub_pd(_mm512_set1_pd(0.0), a);
297}
298
299template <>
300EIGEN_STRONG_INLINE Packet16f pconj(const Packet16f& a) {
301 return a;
302}
303template <>
304EIGEN_STRONG_INLINE Packet8d pconj(const Packet8d& a) {
305 return a;
306}
307template <>
308EIGEN_STRONG_INLINE Packet16i pconj(const Packet16i& a) {
309 return a;
310}
311
312template <>
313EIGEN_STRONG_INLINE Packet16f pmul<Packet16f>(const Packet16f& a,
314 const Packet16f& b) {
315 return _mm512_mul_ps(a, b);
316}
317template <>
318EIGEN_STRONG_INLINE Packet8d pmul<Packet8d>(const Packet8d& a,
319 const Packet8d& b) {
320 return _mm512_mul_pd(a, b);
321}
322template <>
323EIGEN_STRONG_INLINE Packet16i pmul<Packet16i>(const Packet16i& a,
324 const Packet16i& b) {
325 return _mm512_mullo_epi32(a, b);
326}
327
328template <>
329EIGEN_STRONG_INLINE Packet16f pdiv<Packet16f>(const Packet16f& a,
330 const Packet16f& b) {
331 return _mm512_div_ps(a, b);
332}
333template <>
334EIGEN_STRONG_INLINE Packet8d pdiv<Packet8d>(const Packet8d& a,
335 const Packet8d& b) {
336 return _mm512_div_pd(a, b);
337}
338
339#ifdef EIGEN_VECTORIZE_FMA
340template <>
341EIGEN_STRONG_INLINE Packet16f pmadd(const Packet16f& a, const Packet16f& b,
342 const Packet16f& c) {
343 return _mm512_fmadd_ps(a, b, c);
344}
345template <>
346EIGEN_STRONG_INLINE Packet8d pmadd(const Packet8d& a, const Packet8d& b,
347 const Packet8d& c) {
348 return _mm512_fmadd_pd(a, b, c);
349}
350#endif
351
352template <>
353EIGEN_DEVICE_FUNC inline Packet16f pselect(const Packet16f& mask,
354 const Packet16f& a,
355 const Packet16f& b) {
356 __mmask16 mask16 = _mm512_cmp_epi32_mask(
357 _mm512_castps_si512(mask), _mm512_setzero_epi32(), _MM_CMPINT_EQ);
358 return _mm512_mask_blend_ps(mask16, a, b);
359}
360
361template <>
362EIGEN_DEVICE_FUNC inline Packet8d pselect(const Packet8d& mask,
363 const Packet8d& a,
364 const Packet8d& b) {
365 __mmask8 mask8 = _mm512_cmp_epi64_mask(_mm512_castpd_si512(mask),
366 _mm512_setzero_epi32(), _MM_CMPINT_EQ);
367 return _mm512_mask_blend_pd(mask8, a, b);
368}
369
370template <>
371EIGEN_STRONG_INLINE Packet16f pmin<Packet16f>(const Packet16f& a,
372 const Packet16f& b) {
373 // Arguments are reversed to match NaN propagation behavior of std::min.
374 return _mm512_min_ps(b, a);
375}
376template <>
377EIGEN_STRONG_INLINE Packet8d pmin<Packet8d>(const Packet8d& a,
378 const Packet8d& b) {
379 // Arguments are reversed to match NaN propagation behavior of std::min.
380 return _mm512_min_pd(b, a);
381}
382
383template <>
384EIGEN_STRONG_INLINE Packet16f pmax<Packet16f>(const Packet16f& a,
385 const Packet16f& b) {
386 // Arguments are reversed to match NaN propagation behavior of std::max.
387 return _mm512_max_ps(b, a);
388}
389template <>
390EIGEN_STRONG_INLINE Packet8d pmax<Packet8d>(const Packet8d& a,
391 const Packet8d& b) {
392 // Arguments are reversed to match NaN propagation behavior of std::max.
393 return _mm512_max_pd(b, a);
394}
395
396// Add specializations for min/max with prescribed NaN progation.
397template<>
398EIGEN_STRONG_INLINE Packet16f pmin<PropagateNumbers, Packet16f>(const Packet16f& a, const Packet16f& b) {
399 return pminmax_propagate_numbers(a, b, pmin<Packet16f>);
400}
401template<>
402EIGEN_STRONG_INLINE Packet8d pmin<PropagateNumbers, Packet8d>(const Packet8d& a, const Packet8d& b) {
403 return pminmax_propagate_numbers(a, b, pmin<Packet8d>);
404}
405template<>
406EIGEN_STRONG_INLINE Packet16f pmax<PropagateNumbers, Packet16f>(const Packet16f& a, const Packet16f& b) {
407 return pminmax_propagate_numbers(a, b, pmax<Packet16f>);
408}
409template<>
410EIGEN_STRONG_INLINE Packet8d pmax<PropagateNumbers, Packet8d>(const Packet8d& a, const Packet8d& b) {
411 return pminmax_propagate_numbers(a, b, pmax<Packet8d>);
412}
413template<>
414EIGEN_STRONG_INLINE Packet16f pmin<PropagateNaN, Packet16f>(const Packet16f& a, const Packet16f& b) {
415 return pminmax_propagate_nan(a, b, pmin<Packet16f>);
416}
417template<>
418EIGEN_STRONG_INLINE Packet8d pmin<PropagateNaN, Packet8d>(const Packet8d& a, const Packet8d& b) {
419 return pminmax_propagate_nan(a, b, pmin<Packet8d>);
420}
421template<>
422EIGEN_STRONG_INLINE Packet16f pmax<PropagateNaN, Packet16f>(const Packet16f& a, const Packet16f& b) {
423 return pminmax_propagate_nan(a, b, pmax<Packet16f>);
424}
425template<>
426EIGEN_STRONG_INLINE Packet8d pmax<PropagateNaN, Packet8d>(const Packet8d& a, const Packet8d& b) {
427 return pminmax_propagate_nan(a, b, pmax<Packet8d>);
428}
429
430
431#ifdef EIGEN_VECTORIZE_AVX512DQ
432template<int I_> EIGEN_STRONG_INLINE Packet8f extract256(Packet16f x) { return _mm512_extractf32x8_ps(x,I_); }
433template<int I_> EIGEN_STRONG_INLINE Packet2d extract128(Packet8d x) { return _mm512_extractf64x2_pd(x,I_); }
434EIGEN_STRONG_INLINE Packet16f cat256(Packet8f a, Packet8f b) { return _mm512_insertf32x8(_mm512_castps256_ps512(a),b,1); }
435#else
436// AVX512F does not define _mm512_extractf32x8_ps to extract _m256 from _m512
437template<int I_> EIGEN_STRONG_INLINE Packet8f extract256(Packet16f x) {
438 return _mm256_castsi256_ps(_mm512_extracti64x4_epi64( _mm512_castps_si512(x),I_));
439}
440
441// AVX512F does not define _mm512_extractf64x2_pd to extract _m128 from _m512
442template<int I_> EIGEN_STRONG_INLINE Packet2d extract128(Packet8d x) {
443 return _mm_castsi128_pd(_mm512_extracti32x4_epi32( _mm512_castpd_si512(x),I_));
444}
445
446EIGEN_STRONG_INLINE Packet16f cat256(Packet8f a, Packet8f b) {
447 return _mm512_castsi512_ps(_mm512_inserti64x4(_mm512_castsi256_si512(_mm256_castps_si256(a)),
448 _mm256_castps_si256(b),1));
449}
450#endif
451
452// Helper function for bit packing snippet of low precision comparison.
453// It packs the flags from 32x16 to 16x16.
454EIGEN_STRONG_INLINE __m256i Pack32To16(Packet16f rf) {
455 // Split data into small pieces and handle with AVX instructions
456 // to guarantee internal order of vector.
457 // Operation:
458 // dst[15:0] := Saturate16(rf[31:0])
459 // dst[31:16] := Saturate16(rf[63:32])
460 // ...
461 // dst[255:240] := Saturate16(rf[255:224])
462 __m256i lo = _mm256_castps_si256(extract256<0>(rf));
463 __m256i hi = _mm256_castps_si256(extract256<1>(rf));
464 __m128i result_lo = _mm_packs_epi32(_mm256_extractf128_si256(lo, 0),
465 _mm256_extractf128_si256(lo, 1));
466 __m128i result_hi = _mm_packs_epi32(_mm256_extractf128_si256(hi, 0),
467 _mm256_extractf128_si256(hi, 1));
468 return _mm256_insertf128_si256(_mm256_castsi128_si256(result_lo), result_hi, 1);
469}
470
471template <>
472EIGEN_STRONG_INLINE Packet16f pcmp_eq(const Packet16f& a, const Packet16f& b) {
473 __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_EQ_OQ);
474 return _mm512_castsi512_ps(
475 _mm512_mask_set1_epi32(_mm512_set1_epi32(0), mask, 0xffffffffu));
476}
477template<> EIGEN_STRONG_INLINE Packet16f pcmp_le(const Packet16f& a, const Packet16f& b) {
478 __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_LE_OQ);
479 return _mm512_castsi512_ps(
480 _mm512_mask_set1_epi32(_mm512_set1_epi32(0), mask, 0xffffffffu));
481}
482
483template<> EIGEN_STRONG_INLINE Packet16f pcmp_lt(const Packet16f& a, const Packet16f& b) {
484 __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_LT_OQ);
485 return _mm512_castsi512_ps(
486 _mm512_mask_set1_epi32(_mm512_set1_epi32(0), mask, 0xffffffffu));
487}
488
489template<> EIGEN_STRONG_INLINE Packet16f pcmp_lt_or_nan(const Packet16f& a, const Packet16f& b) {
490 __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_NGE_UQ);
491 return _mm512_castsi512_ps(
492 _mm512_mask_set1_epi32(_mm512_set1_epi32(0), mask, 0xffffffffu));
493}
494
495template<> EIGEN_STRONG_INLINE Packet16i pcmp_eq(const Packet16i& a, const Packet16i& b) {
496 __mmask16 mask = _mm512_cmp_epi32_mask(a, b, _CMP_EQ_OQ);
497 return _mm512_mask_set1_epi32(_mm512_set1_epi32(0), mask, 0xffffffffu);
498}
499
500
501template <>
502EIGEN_STRONG_INLINE Packet8d pcmp_eq(const Packet8d& a, const Packet8d& b) {
503 __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_EQ_OQ);
504 return _mm512_castsi512_pd(
505 _mm512_mask_set1_epi64(_mm512_set1_epi64(0), mask, 0xffffffffffffffffu));
506}
507template <>
508EIGEN_STRONG_INLINE Packet8d pcmp_le(const Packet8d& a, const Packet8d& b) {
509 __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_LE_OQ);
510 return _mm512_castsi512_pd(
511 _mm512_mask_set1_epi64(_mm512_set1_epi64(0), mask, 0xffffffffffffffffu));
512}
513template <>
514EIGEN_STRONG_INLINE Packet8d pcmp_lt(const Packet8d& a, const Packet8d& b) {
515 __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_LT_OQ);
516 return _mm512_castsi512_pd(
517 _mm512_mask_set1_epi64(_mm512_set1_epi64(0), mask, 0xffffffffffffffffu));
518}
519template <>
520EIGEN_STRONG_INLINE Packet8d pcmp_lt_or_nan(const Packet8d& a, const Packet8d& b) {
521 __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_NGE_UQ);
522 return _mm512_castsi512_pd(
523 _mm512_mask_set1_epi64(_mm512_set1_epi64(0), mask, 0xffffffffffffffffu));
524}
525
526template<> EIGEN_STRONG_INLINE Packet16f print<Packet16f>(const Packet16f& a) { return _mm512_roundscale_ps(a, _MM_FROUND_CUR_DIRECTION); }
527template<> EIGEN_STRONG_INLINE Packet8d print<Packet8d>(const Packet8d& a) { return _mm512_roundscale_pd(a, _MM_FROUND_CUR_DIRECTION); }
528
529template<> EIGEN_STRONG_INLINE Packet16f pceil<Packet16f>(const Packet16f& a) { return _mm512_roundscale_ps(a, _MM_FROUND_TO_POS_INF); }
530template<> EIGEN_STRONG_INLINE Packet8d pceil<Packet8d>(const Packet8d& a) { return _mm512_roundscale_pd(a, _MM_FROUND_TO_POS_INF); }
531
532template<> EIGEN_STRONG_INLINE Packet16f pfloor<Packet16f>(const Packet16f& a) { return _mm512_roundscale_ps(a, _MM_FROUND_TO_NEG_INF); }
533template<> EIGEN_STRONG_INLINE Packet8d pfloor<Packet8d>(const Packet8d& a) { return _mm512_roundscale_pd(a, _MM_FROUND_TO_NEG_INF); }
534
535template <>
536EIGEN_STRONG_INLINE Packet16i ptrue<Packet16i>(const Packet16i& /*a*/) {
537 return _mm512_set1_epi32(0xffffffffu);
538}
539
540template <>
541EIGEN_STRONG_INLINE Packet16f ptrue<Packet16f>(const Packet16f& a) {
542 return _mm512_castsi512_ps(ptrue<Packet16i>(_mm512_castps_si512(a)));
543}
544
545template <>
546EIGEN_STRONG_INLINE Packet8d ptrue<Packet8d>(const Packet8d& a) {
547 return _mm512_castsi512_pd(ptrue<Packet16i>(_mm512_castpd_si512(a)));
548}
549
550template <>
551EIGEN_STRONG_INLINE Packet16i pand<Packet16i>(const Packet16i& a,
552 const Packet16i& b) {
553 return _mm512_and_si512(a,b);
554}
555
556template <>
557EIGEN_STRONG_INLINE Packet16f pand<Packet16f>(const Packet16f& a,
558 const Packet16f& b) {
559#ifdef EIGEN_VECTORIZE_AVX512DQ
560 return _mm512_and_ps(a, b);
561#else
562 return _mm512_castsi512_ps(pand(_mm512_castps_si512(a),_mm512_castps_si512(b)));
563#endif
564}
565template <>
566EIGEN_STRONG_INLINE Packet8d pand<Packet8d>(const Packet8d& a,
567 const Packet8d& b) {
568#ifdef EIGEN_VECTORIZE_AVX512DQ
569 return _mm512_and_pd(a, b);
570#else
571 Packet8d res = _mm512_undefined_pd();
572 Packet4d lane0_a = _mm512_extractf64x4_pd(a, 0);
573 Packet4d lane0_b = _mm512_extractf64x4_pd(b, 0);
574 res = _mm512_insertf64x4(res, _mm256_and_pd(lane0_a, lane0_b), 0);
575
576 Packet4d lane1_a = _mm512_extractf64x4_pd(a, 1);
577 Packet4d lane1_b = _mm512_extractf64x4_pd(b, 1);
578 return _mm512_insertf64x4(res, _mm256_and_pd(lane1_a, lane1_b), 1);
579#endif
580}
581
582template <>
583EIGEN_STRONG_INLINE Packet16i por<Packet16i>(const Packet16i& a, const Packet16i& b) {
584 return _mm512_or_si512(a, b);
585}
586
587template <>
588EIGEN_STRONG_INLINE Packet16f por<Packet16f>(const Packet16f& a, const Packet16f& b) {
589#ifdef EIGEN_VECTORIZE_AVX512DQ
590 return _mm512_or_ps(a, b);
591#else
592 return _mm512_castsi512_ps(por(_mm512_castps_si512(a),_mm512_castps_si512(b)));
593#endif
594}
595
596template <>
597EIGEN_STRONG_INLINE Packet8d por<Packet8d>(const Packet8d& a,
598 const Packet8d& b) {
599#ifdef EIGEN_VECTORIZE_AVX512DQ
600 return _mm512_or_pd(a, b);
601#else
602 return _mm512_castsi512_pd(por(_mm512_castpd_si512(a),_mm512_castpd_si512(b)));
603#endif
604}
605
606template <>
607EIGEN_STRONG_INLINE Packet16i pxor<Packet16i>(const Packet16i& a, const Packet16i& b) {
608 return _mm512_xor_si512(a, b);
609}
610
611template <>
612EIGEN_STRONG_INLINE Packet16f pxor<Packet16f>(const Packet16f& a, const Packet16f& b) {
613#ifdef EIGEN_VECTORIZE_AVX512DQ
614 return _mm512_xor_ps(a, b);
615#else
616 return _mm512_castsi512_ps(pxor(_mm512_castps_si512(a),_mm512_castps_si512(b)));
617#endif
618}
619
620template <>
621EIGEN_STRONG_INLINE Packet8d pxor<Packet8d>(const Packet8d& a, const Packet8d& b) {
622#ifdef EIGEN_VECTORIZE_AVX512DQ
623 return _mm512_xor_pd(a, b);
624#else
625 return _mm512_castsi512_pd(pxor(_mm512_castpd_si512(a),_mm512_castpd_si512(b)));
626#endif
627}
628
629template <>
630EIGEN_STRONG_INLINE Packet16i pandnot<Packet16i>(const Packet16i& a, const Packet16i& b) {
631 return _mm512_andnot_si512(b, a);
632}
633
634template <>
635EIGEN_STRONG_INLINE Packet16f pandnot<Packet16f>(const Packet16f& a, const Packet16f& b) {
636#ifdef EIGEN_VECTORIZE_AVX512DQ
637 return _mm512_andnot_ps(b, a);
638#else
639 return _mm512_castsi512_ps(pandnot(_mm512_castps_si512(a),_mm512_castps_si512(b)));
640#endif
641}
642template <>
643EIGEN_STRONG_INLINE Packet8d pandnot<Packet8d>(const Packet8d& a,const Packet8d& b) {
644#ifdef EIGEN_VECTORIZE_AVX512DQ
645 return _mm512_andnot_pd(b, a);
646#else
647 return _mm512_castsi512_pd(pandnot(_mm512_castpd_si512(a),_mm512_castpd_si512(b)));
648#endif
649}
650
651template<> EIGEN_STRONG_INLINE Packet16f pround<Packet16f>(const Packet16f& a)
652{
653 // Work-around for default std::round rounding mode.
654 const Packet16f mask = pset1frombits<Packet16f>(static_cast<numext::uint32_t>(0x80000000u));
655 const Packet16f prev0dot5 = pset1frombits<Packet16f>(static_cast<numext::uint32_t>(0x3EFFFFFFu));
656 return _mm512_roundscale_ps(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO);
657}
658template<> EIGEN_STRONG_INLINE Packet8d pround<Packet8d>(const Packet8d& a)
659{
660 // Work-around for default std::round rounding mode.
661 const Packet8d mask = pset1frombits<Packet8d>(static_cast<numext::uint64_t>(0x8000000000000000ull));
662 const Packet8d prev0dot5 = pset1frombits<Packet8d>(static_cast<numext::uint64_t>(0x3FDFFFFFFFFFFFFFull));
663 return _mm512_roundscale_pd(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO);
664}
665
666template<int N> EIGEN_STRONG_INLINE Packet16i parithmetic_shift_right(Packet16i a) {
667 return _mm512_srai_epi32(a, N);
668}
669
670template<int N> EIGEN_STRONG_INLINE Packet16i plogical_shift_right(Packet16i a) {
671 return _mm512_srli_epi32(a, N);
672}
673
674template<int N> EIGEN_STRONG_INLINE Packet16i plogical_shift_left(Packet16i a) {
675 return _mm512_slli_epi32(a, N);
676}
677
678template <>
679EIGEN_STRONG_INLINE Packet16f pload<Packet16f>(const float* from) {
680 EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_ps(from);
681}
682template <>
683EIGEN_STRONG_INLINE Packet8d pload<Packet8d>(const double* from) {
684 EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_pd(from);
685}
686template <>
687EIGEN_STRONG_INLINE Packet16i pload<Packet16i>(const int* from) {
688 EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_si512(
689 reinterpret_cast<const __m512i*>(from));
690}
691
692template <>
693EIGEN_STRONG_INLINE Packet16f ploadu<Packet16f>(const float* from) {
694 EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_ps(from);
695}
696template <>
697EIGEN_STRONG_INLINE Packet8d ploadu<Packet8d>(const double* from) {
698 EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_pd(from);
699}
700template <>
701EIGEN_STRONG_INLINE Packet16i ploadu<Packet16i>(const int* from) {
702 EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_si512(
703 reinterpret_cast<const __m512i*>(from));
704}
705
706template <>
707EIGEN_STRONG_INLINE Packet16f ploadu<Packet16f>(const float* from, uint16_t umask) {
708 __mmask16 mask = static_cast<__mmask16>(umask);
709 EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_maskz_loadu_ps(mask, from);
710}
711
712// Loads 8 floats from memory a returns the packet
713// {a0, a0 a1, a1, a2, a2, a3, a3, a4, a4, a5, a5, a6, a6, a7, a7}
714template <>
715EIGEN_STRONG_INLINE Packet16f ploaddup<Packet16f>(const float* from) {
716 // an unaligned load is required here as there is no requirement
717 // on the alignment of input pointer 'from'
718 __m256i low_half = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from));
719 __m512 even_elements = _mm512_castsi512_ps(_mm512_cvtepu32_epi64(low_half));
720 __m512 pairs = _mm512_permute_ps(even_elements, _MM_SHUFFLE(2, 2, 0, 0));
721 return pairs;
722}
723
724#ifdef EIGEN_VECTORIZE_AVX512DQ
725// FIXME: this does not look optimal, better load a Packet4d and shuffle...
726// Loads 4 doubles from memory a returns the packet {a0, a0 a1, a1, a2, a2, a3,
727// a3}
728template <>
729EIGEN_STRONG_INLINE Packet8d ploaddup<Packet8d>(const double* from) {
730 __m512d x = _mm512_setzero_pd();
731 x = _mm512_insertf64x2(x, _mm_loaddup_pd(&from[0]), 0);
732 x = _mm512_insertf64x2(x, _mm_loaddup_pd(&from[1]), 1);
733 x = _mm512_insertf64x2(x, _mm_loaddup_pd(&from[2]), 2);
734 x = _mm512_insertf64x2(x, _mm_loaddup_pd(&from[3]), 3);
735 return x;
736}
737#else
738template <>
739EIGEN_STRONG_INLINE Packet8d ploaddup<Packet8d>(const double* from) {
740 __m512d x = _mm512_setzero_pd();
741 x = _mm512_mask_broadcastsd_pd(x, 0x3<<0, _mm_load_sd(from+0));
742 x = _mm512_mask_broadcastsd_pd(x, 0x3<<2, _mm_load_sd(from+1));
743 x = _mm512_mask_broadcastsd_pd(x, 0x3<<4, _mm_load_sd(from+2));
744 x = _mm512_mask_broadcastsd_pd(x, 0x3<<6, _mm_load_sd(from+3));
745 return x;
746}
747#endif
748
749// Loads 4 floats from memory a returns the packet
750// {a0, a0 a0, a0, a1, a1, a1, a1, a2, a2, a2, a2, a3, a3, a3, a3}
751template <>
752EIGEN_STRONG_INLINE Packet16f ploadquad<Packet16f>(const float* from) {
753 Packet16f tmp = _mm512_castps128_ps512(ploadu<Packet4f>(from));
754 const Packet16i scatter_mask = _mm512_set_epi32(3,3,3,3, 2,2,2,2, 1,1,1,1, 0,0,0,0);
755 return _mm512_permutexvar_ps(scatter_mask, tmp);
756}
757
758// Loads 2 doubles from memory a returns the packet
759// {a0, a0 a0, a0, a1, a1, a1, a1}
760template <>
761EIGEN_STRONG_INLINE Packet8d ploadquad<Packet8d>(const double* from) {
762 __m256d lane0 = _mm256_set1_pd(*from);
763 __m256d lane1 = _mm256_set1_pd(*(from+1));
764 __m512d tmp = _mm512_undefined_pd();
765 tmp = _mm512_insertf64x4(tmp, lane0, 0);
766 return _mm512_insertf64x4(tmp, lane1, 1);
767}
768
769template <>
770EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet16f& from) {
771 EIGEN_DEBUG_ALIGNED_STORE _mm512_store_ps(to, from);
772}
773template <>
774EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet8d& from) {
775 EIGEN_DEBUG_ALIGNED_STORE _mm512_store_pd(to, from);
776}
777template <>
778EIGEN_STRONG_INLINE void pstore<int>(int* to, const Packet16i& from) {
779 EIGEN_DEBUG_ALIGNED_STORE _mm512_storeu_si512(reinterpret_cast<__m512i*>(to),
780 from);
781}
782
783template <>
784EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet16f& from) {
785 EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_ps(to, from);
786}
787template <>
788EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet8d& from) {
789 EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_pd(to, from);
790}
791template <>
792EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet16i& from) {
793 EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_si512(
794 reinterpret_cast<__m512i*>(to), from);
795}
796template <>
797EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet16f& from, uint16_t umask) {
798 __mmask16 mask = static_cast<__mmask16>(umask);
799 EIGEN_DEBUG_UNALIGNED_STORE return _mm512_mask_storeu_ps(to, mask, from);
800}
801
802template <>
803EIGEN_DEVICE_FUNC inline Packet16f pgather<float, Packet16f>(const float* from,
804 Index stride) {
805 Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
806 Packet16i stride_multiplier =
807 _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
808 Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
809
810 return _mm512_i32gather_ps(indices, from, 4);
811}
812template <>
813EIGEN_DEVICE_FUNC inline Packet8d pgather<double, Packet8d>(const double* from,
814 Index stride) {
815 Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
816 Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
817 Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
818
819 return _mm512_i32gather_pd(indices, from, 8);
820}
821
822template <>
823EIGEN_DEVICE_FUNC inline void pscatter<float, Packet16f>(float* to,
824 const Packet16f& from,
825 Index stride) {
826 Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
827 Packet16i stride_multiplier =
828 _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
829 Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
830 _mm512_i32scatter_ps(to, indices, from, 4);
831}
832template <>
833EIGEN_DEVICE_FUNC inline void pscatter<double, Packet8d>(double* to,
834 const Packet8d& from,
835 Index stride) {
836 Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
837 Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
838 Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
839 _mm512_i32scatter_pd(to, indices, from, 8);
840}
841
842template <>
843EIGEN_STRONG_INLINE void pstore1<Packet16f>(float* to, const float& a) {
844 Packet16f pa = pset1<Packet16f>(a);
845 pstore(to, pa);
846}
847template <>
848EIGEN_STRONG_INLINE void pstore1<Packet8d>(double* to, const double& a) {
849 Packet8d pa = pset1<Packet8d>(a);
850 pstore(to, pa);
851}
852template <>
853EIGEN_STRONG_INLINE void pstore1<Packet16i>(int* to, const int& a) {
854 Packet16i pa = pset1<Packet16i>(a);
855 pstore(to, pa);
856}
857
858template<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); }
859template<> EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); }
860template<> EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); }
861
862template <>
863EIGEN_STRONG_INLINE float pfirst<Packet16f>(const Packet16f& a) {
864 return _mm_cvtss_f32(_mm512_extractf32x4_ps(a, 0));
865}
866template <>
867EIGEN_STRONG_INLINE double pfirst<Packet8d>(const Packet8d& a) {
868 return _mm_cvtsd_f64(_mm256_extractf128_pd(_mm512_extractf64x4_pd(a, 0), 0));
869}
870template <>
871EIGEN_STRONG_INLINE int pfirst<Packet16i>(const Packet16i& a) {
872 return _mm_extract_epi32(_mm512_extracti32x4_epi32(a, 0), 0);
873}
874
875template<> EIGEN_STRONG_INLINE Packet16f preverse(const Packet16f& a)
876{
877 return _mm512_permutexvar_ps(_mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), a);
878}
879
880template<> EIGEN_STRONG_INLINE Packet8d preverse(const Packet8d& a)
881{
882 return _mm512_permutexvar_pd(_mm512_set_epi32(0, 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7), a);
883}
884
885template<> EIGEN_STRONG_INLINE Packet16f pabs(const Packet16f& a)
886{
887 // _mm512_abs_ps intrinsic not found, so hack around it
888 return _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(a), _mm512_set1_epi32(0x7fffffff)));
889}
890template <>
891EIGEN_STRONG_INLINE Packet8d pabs(const Packet8d& a) {
892 // _mm512_abs_ps intrinsic not found, so hack around it
893 return _mm512_castsi512_pd(_mm512_and_si512(_mm512_castpd_si512(a),
894 _mm512_set1_epi64(0x7fffffffffffffff)));
895}
896
897template<>
898EIGEN_STRONG_INLINE Packet16f pfrexp<Packet16f>(const Packet16f& a, Packet16f& exponent){
899 return pfrexp_generic(a, exponent);
900}
901
902// Extract exponent without existence of Packet8l.
903template<>
904EIGEN_STRONG_INLINE
905Packet8d pfrexp_generic_get_biased_exponent(const Packet8d& a) {
906 const Packet8d cst_exp_mask = pset1frombits<Packet8d>(static_cast<uint64_t>(0x7ff0000000000000ull));
907 #ifdef EIGEN_VECTORIZE_AVX512DQ
908 return _mm512_cvtepi64_pd(_mm512_srli_epi64(_mm512_castpd_si512(pand(a, cst_exp_mask)), 52));
909 #else
910 return _mm512_cvtepi32_pd(_mm512_cvtepi64_epi32(_mm512_srli_epi64(_mm512_castpd_si512(pand(a, cst_exp_mask)), 52)));
911 #endif
912}
913
914template<>
915EIGEN_STRONG_INLINE Packet8d pfrexp<Packet8d>(const Packet8d& a, Packet8d& exponent) {
916 return pfrexp_generic(a, exponent);
917}
918
919template<> EIGEN_STRONG_INLINE Packet16f pldexp<Packet16f>(const Packet16f& a, const Packet16f& exponent) {
920 return pldexp_generic(a, exponent);
921}
922
923template<> EIGEN_STRONG_INLINE Packet8d pldexp<Packet8d>(const Packet8d& a, const Packet8d& exponent) {
924 // Clamp exponent to [-2099, 2099]
925 const Packet8d max_exponent = pset1<Packet8d>(2099.0);
926 const Packet8i e = _mm512_cvtpd_epi32(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent));
927
928 // Split 2^e into four factors and multiply.
929 const Packet8i bias = pset1<Packet8i>(1023);
930 Packet8i b = parithmetic_shift_right<2>(e); // floor(e/4)
931
932 // 2^b
933 const Packet8i permute_idx = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7);
934 Packet8i hi = _mm256_permutevar8x32_epi32(padd(b, bias), permute_idx);
935 Packet8i lo = _mm256_slli_epi64(hi, 52);
936 hi = _mm256_slli_epi64(_mm256_srli_epi64(hi, 32), 52);
937 Packet8d c = _mm512_castsi512_pd(_mm512_inserti64x4(_mm512_castsi256_si512(lo), hi, 1));
938 Packet8d out = pmul(pmul(pmul(a, c), c), c); // a * 2^(3b)
939
940 // 2^(e - 3b)
941 b = psub(psub(psub(e, b), b), b); // e - 3b
942 hi = _mm256_permutevar8x32_epi32(padd(b, bias), permute_idx);
943 lo = _mm256_slli_epi64(hi, 52);
944 hi = _mm256_slli_epi64(_mm256_srli_epi64(hi, 32), 52);
945 c = _mm512_castsi512_pd(_mm512_inserti64x4(_mm512_castsi256_si512(lo), hi, 1));
946 out = pmul(out, c); // a * 2^e
947 return out;
948}
949
950#ifdef EIGEN_VECTORIZE_AVX512DQ
951// AVX512F does not define _mm512_extractf32x8_ps to extract _m256 from _m512
952#define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT) \
953 __m256 OUTPUT##_0 = _mm512_extractf32x8_ps(INPUT, 0); \
954 __m256 OUTPUT##_1 = _mm512_extractf32x8_ps(INPUT, 1)
955#else
956#define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT) \
957 __m256 OUTPUT##_0 = _mm256_insertf128_ps( \
958 _mm256_castps128_ps256(_mm512_extractf32x4_ps(INPUT, 0)), \
959 _mm512_extractf32x4_ps(INPUT, 1), 1); \
960 __m256 OUTPUT##_1 = _mm256_insertf128_ps( \
961 _mm256_castps128_ps256(_mm512_extractf32x4_ps(INPUT, 2)), \
962 _mm512_extractf32x4_ps(INPUT, 3), 1);
963#endif
964
965#ifdef EIGEN_VECTORIZE_AVX512DQ
966#define EIGEN_INSERT_8f_INTO_16f(OUTPUT, INPUTA, INPUTB) \
967 OUTPUT = _mm512_insertf32x8(_mm512_castps256_ps512(INPUTA), INPUTB, 1);
968#else
969#define EIGEN_INSERT_8f_INTO_16f(OUTPUT, INPUTA, INPUTB) \
970 OUTPUT = _mm512_undefined_ps(); \
971 OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTA, 0), 0); \
972 OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTA, 1), 1); \
973 OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTB, 0), 2); \
974 OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTB, 1), 3);
975#endif
976
977template <>
978EIGEN_STRONG_INLINE float predux<Packet16f>(const Packet16f& a) {
979#ifdef EIGEN_VECTORIZE_AVX512DQ
980 __m256 lane0 = _mm512_extractf32x8_ps(a, 0);
981 __m256 lane1 = _mm512_extractf32x8_ps(a, 1);
982 Packet8f x = _mm256_add_ps(lane0, lane1);
983 return predux<Packet8f>(x);
984#else
985 __m128 lane0 = _mm512_extractf32x4_ps(a, 0);
986 __m128 lane1 = _mm512_extractf32x4_ps(a, 1);
987 __m128 lane2 = _mm512_extractf32x4_ps(a, 2);
988 __m128 lane3 = _mm512_extractf32x4_ps(a, 3);
989 __m128 sum = _mm_add_ps(_mm_add_ps(lane0, lane1), _mm_add_ps(lane2, lane3));
990 sum = _mm_hadd_ps(sum, sum);
991 sum = _mm_hadd_ps(sum, _mm_permute_ps(sum, 1));
992 return _mm_cvtss_f32(sum);
993#endif
994}
995template <>
996EIGEN_STRONG_INLINE double predux<Packet8d>(const Packet8d& a) {
997 __m256d lane0 = _mm512_extractf64x4_pd(a, 0);
998 __m256d lane1 = _mm512_extractf64x4_pd(a, 1);
999 __m256d sum = _mm256_add_pd(lane0, lane1);
1000 __m256d tmp0 = _mm256_hadd_pd(sum, _mm256_permute2f128_pd(sum, sum, 1));
1001 return _mm_cvtsd_f64(_mm256_castpd256_pd128(_mm256_hadd_pd(tmp0, tmp0)));
1002}
1003
1004template <>
1005EIGEN_STRONG_INLINE Packet8f predux_half_dowto4<Packet16f>(const Packet16f& a) {
1006#ifdef EIGEN_VECTORIZE_AVX512DQ
1007 __m256 lane0 = _mm512_extractf32x8_ps(a, 0);
1008 __m256 lane1 = _mm512_extractf32x8_ps(a, 1);
1009 return _mm256_add_ps(lane0, lane1);
1010#else
1011 __m128 lane0 = _mm512_extractf32x4_ps(a, 0);
1012 __m128 lane1 = _mm512_extractf32x4_ps(a, 1);
1013 __m128 lane2 = _mm512_extractf32x4_ps(a, 2);
1014 __m128 lane3 = _mm512_extractf32x4_ps(a, 3);
1015 __m128 sum0 = _mm_add_ps(lane0, lane2);
1016 __m128 sum1 = _mm_add_ps(lane1, lane3);
1017 return _mm256_insertf128_ps(_mm256_castps128_ps256(sum0), sum1, 1);
1018#endif
1019}
1020template <>
1021EIGEN_STRONG_INLINE Packet4d predux_half_dowto4<Packet8d>(const Packet8d& a) {
1022 __m256d lane0 = _mm512_extractf64x4_pd(a, 0);
1023 __m256d lane1 = _mm512_extractf64x4_pd(a, 1);
1024 return _mm256_add_pd(lane0, lane1);
1025}
1026
1027template <>
1028EIGEN_STRONG_INLINE float predux_mul<Packet16f>(const Packet16f& a) {
1029//#ifdef EIGEN_VECTORIZE_AVX512DQ
1030#if 0
1031 Packet8f lane0 = _mm512_extractf32x8_ps(a, 0);
1032 Packet8f lane1 = _mm512_extractf32x8_ps(a, 1);
1033 Packet8f res = pmul(lane0, lane1);
1034 res = pmul(res, _mm256_permute2f128_ps(res, res, 1));
1035 res = pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2)));
1036 return pfirst(pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1))));
1037#else
1038 __m128 lane0 = _mm512_extractf32x4_ps(a, 0);
1039 __m128 lane1 = _mm512_extractf32x4_ps(a, 1);
1040 __m128 lane2 = _mm512_extractf32x4_ps(a, 2);
1041 __m128 lane3 = _mm512_extractf32x4_ps(a, 3);
1042 __m128 res = pmul(pmul(lane0, lane1), pmul(lane2, lane3));
1043 res = pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2)));
1044 return pfirst(pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1))));
1045#endif
1046}
1047template <>
1048EIGEN_STRONG_INLINE double predux_mul<Packet8d>(const Packet8d& a) {
1049 __m256d lane0 = _mm512_extractf64x4_pd(a, 0);
1050 __m256d lane1 = _mm512_extractf64x4_pd(a, 1);
1051 __m256d res = pmul(lane0, lane1);
1052 res = pmul(res, _mm256_permute2f128_pd(res, res, 1));
1053 return pfirst(pmul(res, _mm256_shuffle_pd(res, res, 1)));
1054}
1055
1056template <>
1057EIGEN_STRONG_INLINE float predux_min<Packet16f>(const Packet16f& a) {
1058 __m128 lane0 = _mm512_extractf32x4_ps(a, 0);
1059 __m128 lane1 = _mm512_extractf32x4_ps(a, 1);
1060 __m128 lane2 = _mm512_extractf32x4_ps(a, 2);
1061 __m128 lane3 = _mm512_extractf32x4_ps(a, 3);
1062 __m128 res = _mm_min_ps(_mm_min_ps(lane0, lane1), _mm_min_ps(lane2, lane3));
1063 res = _mm_min_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2)));
1064 return pfirst(_mm_min_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1))));
1065}
1066template <>
1067EIGEN_STRONG_INLINE double predux_min<Packet8d>(const Packet8d& a) {
1068 __m256d lane0 = _mm512_extractf64x4_pd(a, 0);
1069 __m256d lane1 = _mm512_extractf64x4_pd(a, 1);
1070 __m256d res = _mm256_min_pd(lane0, lane1);
1071 res = _mm256_min_pd(res, _mm256_permute2f128_pd(res, res, 1));
1072 return pfirst(_mm256_min_pd(res, _mm256_shuffle_pd(res, res, 1)));
1073}
1074
1075template <>
1076EIGEN_STRONG_INLINE float predux_max<Packet16f>(const Packet16f& a) {
1077 __m128 lane0 = _mm512_extractf32x4_ps(a, 0);
1078 __m128 lane1 = _mm512_extractf32x4_ps(a, 1);
1079 __m128 lane2 = _mm512_extractf32x4_ps(a, 2);
1080 __m128 lane3 = _mm512_extractf32x4_ps(a, 3);
1081 __m128 res = _mm_max_ps(_mm_max_ps(lane0, lane1), _mm_max_ps(lane2, lane3));
1082 res = _mm_max_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2)));
1083 return pfirst(_mm_max_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1))));
1084}
1085
1086template <>
1087EIGEN_STRONG_INLINE double predux_max<Packet8d>(const Packet8d& a) {
1088 __m256d lane0 = _mm512_extractf64x4_pd(a, 0);
1089 __m256d lane1 = _mm512_extractf64x4_pd(a, 1);
1090 __m256d res = _mm256_max_pd(lane0, lane1);
1091 res = _mm256_max_pd(res, _mm256_permute2f128_pd(res, res, 1));
1092 return pfirst(_mm256_max_pd(res, _mm256_shuffle_pd(res, res, 1)));
1093}
1094
1095template<> EIGEN_STRONG_INLINE bool predux_any(const Packet16f& x)
1096{
1097 Packet16i xi = _mm512_castps_si512(x);
1098 __mmask16 tmp = _mm512_test_epi32_mask(xi,xi);
1099 return !_mm512_kortestz(tmp,tmp);
1100}
1101
1102
1103
1104#define PACK_OUTPUT(OUTPUT, INPUT, INDEX, STRIDE) \
1105 EIGEN_INSERT_8f_INTO_16f(OUTPUT[INDEX], INPUT[INDEX], INPUT[INDEX + STRIDE]);
1106
1107EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16f, 16>& kernel) {
1108 __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
1109 __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
1110 __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
1111 __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
1112 __m512 T4 = _mm512_unpacklo_ps(kernel.packet[4], kernel.packet[5]);
1113 __m512 T5 = _mm512_unpackhi_ps(kernel.packet[4], kernel.packet[5]);
1114 __m512 T6 = _mm512_unpacklo_ps(kernel.packet[6], kernel.packet[7]);
1115 __m512 T7 = _mm512_unpackhi_ps(kernel.packet[6], kernel.packet[7]);
1116 __m512 T8 = _mm512_unpacklo_ps(kernel.packet[8], kernel.packet[9]);
1117 __m512 T9 = _mm512_unpackhi_ps(kernel.packet[8], kernel.packet[9]);
1118 __m512 T10 = _mm512_unpacklo_ps(kernel.packet[10], kernel.packet[11]);
1119 __m512 T11 = _mm512_unpackhi_ps(kernel.packet[10], kernel.packet[11]);
1120 __m512 T12 = _mm512_unpacklo_ps(kernel.packet[12], kernel.packet[13]);
1121 __m512 T13 = _mm512_unpackhi_ps(kernel.packet[12], kernel.packet[13]);
1122 __m512 T14 = _mm512_unpacklo_ps(kernel.packet[14], kernel.packet[15]);
1123 __m512 T15 = _mm512_unpackhi_ps(kernel.packet[14], kernel.packet[15]);
1124 __m512 S0 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
1125 __m512 S1 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
1126 __m512 S2 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
1127 __m512 S3 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
1128 __m512 S4 = _mm512_shuffle_ps(T4, T6, _MM_SHUFFLE(1, 0, 1, 0));
1129 __m512 S5 = _mm512_shuffle_ps(T4, T6, _MM_SHUFFLE(3, 2, 3, 2));
1130 __m512 S6 = _mm512_shuffle_ps(T5, T7, _MM_SHUFFLE(1, 0, 1, 0));
1131 __m512 S7 = _mm512_shuffle_ps(T5, T7, _MM_SHUFFLE(3, 2, 3, 2));
1132 __m512 S8 = _mm512_shuffle_ps(T8, T10, _MM_SHUFFLE(1, 0, 1, 0));
1133 __m512 S9 = _mm512_shuffle_ps(T8, T10, _MM_SHUFFLE(3, 2, 3, 2));
1134 __m512 S10 = _mm512_shuffle_ps(T9, T11, _MM_SHUFFLE(1, 0, 1, 0));
1135 __m512 S11 = _mm512_shuffle_ps(T9, T11, _MM_SHUFFLE(3, 2, 3, 2));
1136 __m512 S12 = _mm512_shuffle_ps(T12, T14, _MM_SHUFFLE(1, 0, 1, 0));
1137 __m512 S13 = _mm512_shuffle_ps(T12, T14, _MM_SHUFFLE(3, 2, 3, 2));
1138 __m512 S14 = _mm512_shuffle_ps(T13, T15, _MM_SHUFFLE(1, 0, 1, 0));
1139 __m512 S15 = _mm512_shuffle_ps(T13, T15, _MM_SHUFFLE(3, 2, 3, 2));
1140
1141 EIGEN_EXTRACT_8f_FROM_16f(S0, S0);
1142 EIGEN_EXTRACT_8f_FROM_16f(S1, S1);
1143 EIGEN_EXTRACT_8f_FROM_16f(S2, S2);
1144 EIGEN_EXTRACT_8f_FROM_16f(S3, S3);
1145 EIGEN_EXTRACT_8f_FROM_16f(S4, S4);
1146 EIGEN_EXTRACT_8f_FROM_16f(S5, S5);
1147 EIGEN_EXTRACT_8f_FROM_16f(S6, S6);
1148 EIGEN_EXTRACT_8f_FROM_16f(S7, S7);
1149 EIGEN_EXTRACT_8f_FROM_16f(S8, S8);
1150 EIGEN_EXTRACT_8f_FROM_16f(S9, S9);
1151 EIGEN_EXTRACT_8f_FROM_16f(S10, S10);
1152 EIGEN_EXTRACT_8f_FROM_16f(S11, S11);
1153 EIGEN_EXTRACT_8f_FROM_16f(S12, S12);
1154 EIGEN_EXTRACT_8f_FROM_16f(S13, S13);
1155 EIGEN_EXTRACT_8f_FROM_16f(S14, S14);
1156 EIGEN_EXTRACT_8f_FROM_16f(S15, S15);
1157
1158 PacketBlock<Packet8f, 32> tmp;
1159
1160 tmp.packet[0] = _mm256_permute2f128_ps(S0_0, S4_0, 0x20);
1161 tmp.packet[1] = _mm256_permute2f128_ps(S1_0, S5_0, 0x20);
1162 tmp.packet[2] = _mm256_permute2f128_ps(S2_0, S6_0, 0x20);
1163 tmp.packet[3] = _mm256_permute2f128_ps(S3_0, S7_0, 0x20);
1164 tmp.packet[4] = _mm256_permute2f128_ps(S0_0, S4_0, 0x31);
1165 tmp.packet[5] = _mm256_permute2f128_ps(S1_0, S5_0, 0x31);
1166 tmp.packet[6] = _mm256_permute2f128_ps(S2_0, S6_0, 0x31);
1167 tmp.packet[7] = _mm256_permute2f128_ps(S3_0, S7_0, 0x31);
1168
1169 tmp.packet[8] = _mm256_permute2f128_ps(S0_1, S4_1, 0x20);
1170 tmp.packet[9] = _mm256_permute2f128_ps(S1_1, S5_1, 0x20);
1171 tmp.packet[10] = _mm256_permute2f128_ps(S2_1, S6_1, 0x20);
1172 tmp.packet[11] = _mm256_permute2f128_ps(S3_1, S7_1, 0x20);
1173 tmp.packet[12] = _mm256_permute2f128_ps(S0_1, S4_1, 0x31);
1174 tmp.packet[13] = _mm256_permute2f128_ps(S1_1, S5_1, 0x31);
1175 tmp.packet[14] = _mm256_permute2f128_ps(S2_1, S6_1, 0x31);
1176 tmp.packet[15] = _mm256_permute2f128_ps(S3_1, S7_1, 0x31);
1177
1178 // Second set of _m256 outputs
1179 tmp.packet[16] = _mm256_permute2f128_ps(S8_0, S12_0, 0x20);
1180 tmp.packet[17] = _mm256_permute2f128_ps(S9_0, S13_0, 0x20);
1181 tmp.packet[18] = _mm256_permute2f128_ps(S10_0, S14_0, 0x20);
1182 tmp.packet[19] = _mm256_permute2f128_ps(S11_0, S15_0, 0x20);
1183 tmp.packet[20] = _mm256_permute2f128_ps(S8_0, S12_0, 0x31);
1184 tmp.packet[21] = _mm256_permute2f128_ps(S9_0, S13_0, 0x31);
1185 tmp.packet[22] = _mm256_permute2f128_ps(S10_0, S14_0, 0x31);
1186 tmp.packet[23] = _mm256_permute2f128_ps(S11_0, S15_0, 0x31);
1187
1188 tmp.packet[24] = _mm256_permute2f128_ps(S8_1, S12_1, 0x20);
1189 tmp.packet[25] = _mm256_permute2f128_ps(S9_1, S13_1, 0x20);
1190 tmp.packet[26] = _mm256_permute2f128_ps(S10_1, S14_1, 0x20);
1191 tmp.packet[27] = _mm256_permute2f128_ps(S11_1, S15_1, 0x20);
1192 tmp.packet[28] = _mm256_permute2f128_ps(S8_1, S12_1, 0x31);
1193 tmp.packet[29] = _mm256_permute2f128_ps(S9_1, S13_1, 0x31);
1194 tmp.packet[30] = _mm256_permute2f128_ps(S10_1, S14_1, 0x31);
1195 tmp.packet[31] = _mm256_permute2f128_ps(S11_1, S15_1, 0x31);
1196
1197 // Pack them into the output
1198 PACK_OUTPUT(kernel.packet, tmp.packet, 0, 16);
1199 PACK_OUTPUT(kernel.packet, tmp.packet, 1, 16);
1200 PACK_OUTPUT(kernel.packet, tmp.packet, 2, 16);
1201 PACK_OUTPUT(kernel.packet, tmp.packet, 3, 16);
1202
1203 PACK_OUTPUT(kernel.packet, tmp.packet, 4, 16);
1204 PACK_OUTPUT(kernel.packet, tmp.packet, 5, 16);
1205 PACK_OUTPUT(kernel.packet, tmp.packet, 6, 16);
1206 PACK_OUTPUT(kernel.packet, tmp.packet, 7, 16);
1207
1208 PACK_OUTPUT(kernel.packet, tmp.packet, 8, 16);
1209 PACK_OUTPUT(kernel.packet, tmp.packet, 9, 16);
1210 PACK_OUTPUT(kernel.packet, tmp.packet, 10, 16);
1211 PACK_OUTPUT(kernel.packet, tmp.packet, 11, 16);
1212
1213 PACK_OUTPUT(kernel.packet, tmp.packet, 12, 16);
1214 PACK_OUTPUT(kernel.packet, tmp.packet, 13, 16);
1215 PACK_OUTPUT(kernel.packet, tmp.packet, 14, 16);
1216 PACK_OUTPUT(kernel.packet, tmp.packet, 15, 16);
1217}
1218#define PACK_OUTPUT_2(OUTPUT, INPUT, INDEX, STRIDE) \
1219 EIGEN_INSERT_8f_INTO_16f(OUTPUT[INDEX], INPUT[2 * INDEX], \
1220 INPUT[2 * INDEX + STRIDE]);
1221
1222EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16f, 4>& kernel) {
1223 __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
1224 __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
1225 __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
1226 __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
1227
1228 __m512 S0 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
1229 __m512 S1 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
1230 __m512 S2 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
1231 __m512 S3 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
1232
1233 EIGEN_EXTRACT_8f_FROM_16f(S0, S0);
1234 EIGEN_EXTRACT_8f_FROM_16f(S1, S1);
1235 EIGEN_EXTRACT_8f_FROM_16f(S2, S2);
1236 EIGEN_EXTRACT_8f_FROM_16f(S3, S3);
1237
1238 PacketBlock<Packet8f, 8> tmp;
1239
1240 tmp.packet[0] = _mm256_permute2f128_ps(S0_0, S1_0, 0x20);
1241 tmp.packet[1] = _mm256_permute2f128_ps(S2_0, S3_0, 0x20);
1242 tmp.packet[2] = _mm256_permute2f128_ps(S0_0, S1_0, 0x31);
1243 tmp.packet[3] = _mm256_permute2f128_ps(S2_0, S3_0, 0x31);
1244
1245 tmp.packet[4] = _mm256_permute2f128_ps(S0_1, S1_1, 0x20);
1246 tmp.packet[5] = _mm256_permute2f128_ps(S2_1, S3_1, 0x20);
1247 tmp.packet[6] = _mm256_permute2f128_ps(S0_1, S1_1, 0x31);
1248 tmp.packet[7] = _mm256_permute2f128_ps(S2_1, S3_1, 0x31);
1249
1250 PACK_OUTPUT_2(kernel.packet, tmp.packet, 0, 1);
1251 PACK_OUTPUT_2(kernel.packet, tmp.packet, 1, 1);
1252 PACK_OUTPUT_2(kernel.packet, tmp.packet, 2, 1);
1253 PACK_OUTPUT_2(kernel.packet, tmp.packet, 3, 1);
1254}
1255
1256#define PACK_OUTPUT_SQ_D(OUTPUT, INPUT, INDEX, STRIDE) \
1257 OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[INDEX], 0); \
1258 OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[INDEX + STRIDE], 1);
1259
1260#define PACK_OUTPUT_D(OUTPUT, INPUT, INDEX, STRIDE) \
1261 OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[(2 * INDEX)], 0); \
1262 OUTPUT[INDEX] = \
1263 _mm512_insertf64x4(OUTPUT[INDEX], INPUT[(2 * INDEX) + STRIDE], 1);
1264
1265EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8d, 4>& kernel) {
1266 __m512d T0 = _mm512_shuffle_pd(kernel.packet[0], kernel.packet[1], 0);
1267 __m512d T1 = _mm512_shuffle_pd(kernel.packet[0], kernel.packet[1], 0xff);
1268 __m512d T2 = _mm512_shuffle_pd(kernel.packet[2], kernel.packet[3], 0);
1269 __m512d T3 = _mm512_shuffle_pd(kernel.packet[2], kernel.packet[3], 0xff);
1270
1271 PacketBlock<Packet4d, 8> tmp;
1272
1273 tmp.packet[0] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0),
1274 _mm512_extractf64x4_pd(T2, 0), 0x20);
1275 tmp.packet[1] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0),
1276 _mm512_extractf64x4_pd(T3, 0), 0x20);
1277 tmp.packet[2] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0),
1278 _mm512_extractf64x4_pd(T2, 0), 0x31);
1279 tmp.packet[3] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0),
1280 _mm512_extractf64x4_pd(T3, 0), 0x31);
1281
1282 tmp.packet[4] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1),
1283 _mm512_extractf64x4_pd(T2, 1), 0x20);
1284 tmp.packet[5] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1),
1285 _mm512_extractf64x4_pd(T3, 1), 0x20);
1286 tmp.packet[6] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1),
1287 _mm512_extractf64x4_pd(T2, 1), 0x31);
1288 tmp.packet[7] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1),
1289 _mm512_extractf64x4_pd(T3, 1), 0x31);
1290
1291 PACK_OUTPUT_D(kernel.packet, tmp.packet, 0, 1);
1292 PACK_OUTPUT_D(kernel.packet, tmp.packet, 1, 1);
1293 PACK_OUTPUT_D(kernel.packet, tmp.packet, 2, 1);
1294 PACK_OUTPUT_D(kernel.packet, tmp.packet, 3, 1);
1295}
1296
1297EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8d, 8>& kernel) {
1298 __m512d T0 = _mm512_unpacklo_pd(kernel.packet[0], kernel.packet[1]);
1299 __m512d T1 = _mm512_unpackhi_pd(kernel.packet[0], kernel.packet[1]);
1300 __m512d T2 = _mm512_unpacklo_pd(kernel.packet[2], kernel.packet[3]);
1301 __m512d T3 = _mm512_unpackhi_pd(kernel.packet[2], kernel.packet[3]);
1302 __m512d T4 = _mm512_unpacklo_pd(kernel.packet[4], kernel.packet[5]);
1303 __m512d T5 = _mm512_unpackhi_pd(kernel.packet[4], kernel.packet[5]);
1304 __m512d T6 = _mm512_unpacklo_pd(kernel.packet[6], kernel.packet[7]);
1305 __m512d T7 = _mm512_unpackhi_pd(kernel.packet[6], kernel.packet[7]);
1306
1307 PacketBlock<Packet4d, 16> tmp;
1308
1309 tmp.packet[0] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0),
1310 _mm512_extractf64x4_pd(T2, 0), 0x20);
1311 tmp.packet[1] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0),
1312 _mm512_extractf64x4_pd(T3, 0), 0x20);
1313 tmp.packet[2] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0),
1314 _mm512_extractf64x4_pd(T2, 0), 0x31);
1315 tmp.packet[3] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0),
1316 _mm512_extractf64x4_pd(T3, 0), 0x31);
1317
1318 tmp.packet[4] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1),
1319 _mm512_extractf64x4_pd(T2, 1), 0x20);
1320 tmp.packet[5] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1),
1321 _mm512_extractf64x4_pd(T3, 1), 0x20);
1322 tmp.packet[6] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1),
1323 _mm512_extractf64x4_pd(T2, 1), 0x31);
1324 tmp.packet[7] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1),
1325 _mm512_extractf64x4_pd(T3, 1), 0x31);
1326
1327 tmp.packet[8] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 0),
1328 _mm512_extractf64x4_pd(T6, 0), 0x20);
1329 tmp.packet[9] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 0),
1330 _mm512_extractf64x4_pd(T7, 0), 0x20);
1331 tmp.packet[10] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 0),
1332 _mm512_extractf64x4_pd(T6, 0), 0x31);
1333 tmp.packet[11] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 0),
1334 _mm512_extractf64x4_pd(T7, 0), 0x31);
1335
1336 tmp.packet[12] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 1),
1337 _mm512_extractf64x4_pd(T6, 1), 0x20);
1338 tmp.packet[13] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 1),
1339 _mm512_extractf64x4_pd(T7, 1), 0x20);
1340 tmp.packet[14] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 1),
1341 _mm512_extractf64x4_pd(T6, 1), 0x31);
1342 tmp.packet[15] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 1),
1343 _mm512_extractf64x4_pd(T7, 1), 0x31);
1344
1345 PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 0, 8);
1346 PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 1, 8);
1347 PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 2, 8);
1348 PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 3, 8);
1349
1350 PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 4, 8);
1351 PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 5, 8);
1352 PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 6, 8);
1353 PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 7, 8);
1354}
1355template <>
1356EIGEN_STRONG_INLINE Packet16f pblend(const Selector<16>& /*ifPacket*/,
1357 const Packet16f& /*thenPacket*/,
1358 const Packet16f& /*elsePacket*/) {
1359 assert(false && "To be implemented");
1360 return Packet16f();
1361}
1362template <>
1363EIGEN_STRONG_INLINE Packet8d pblend(const Selector<8>& ifPacket,
1364 const Packet8d& thenPacket,
1365 const Packet8d& elsePacket) {
1366 __mmask8 m = (ifPacket.select[0] )
1367 | (ifPacket.select[1]<<1)
1368 | (ifPacket.select[2]<<2)
1369 | (ifPacket.select[3]<<3)
1370 | (ifPacket.select[4]<<4)
1371 | (ifPacket.select[5]<<5)
1372 | (ifPacket.select[6]<<6)
1373 | (ifPacket.select[7]<<7);
1374 return _mm512_mask_blend_pd(m, elsePacket, thenPacket);
1375}
1376
1377// Packet math for Eigen::half
1378template<> EIGEN_STRONG_INLINE Packet16h pset1<Packet16h>(const Eigen::half& from) {
1379 return _mm256_set1_epi16(from.x);
1380}
1381
1382template<> EIGEN_STRONG_INLINE Eigen::half pfirst<Packet16h>(const Packet16h& from) {
1383 return half_impl::raw_uint16_to_half(static_cast<unsigned short>(_mm256_extract_epi16(from, 0)));
1384}
1385
1386template<> EIGEN_STRONG_INLINE Packet16h pload<Packet16h>(const Eigen::half* from) {
1387 return _mm256_load_si256(reinterpret_cast<const __m256i*>(from));
1388}
1389
1390template<> EIGEN_STRONG_INLINE Packet16h ploadu<Packet16h>(const Eigen::half* from) {
1391 return _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from));
1392}
1393
1394template<> EIGEN_STRONG_INLINE void pstore<half>(Eigen::half* to, const Packet16h& from) {
1395 // (void*) -> workaround clang warning:
1396 // cast from 'Eigen::half *' to '__m256i *' increases required alignment from 2 to 32
1397 _mm256_store_si256((__m256i*)(void*)to, from);
1398}
1399
1400template<> EIGEN_STRONG_INLINE void pstoreu<half>(Eigen::half* to, const Packet16h& from) {
1401 // (void*) -> workaround clang warning:
1402 // cast from 'Eigen::half *' to '__m256i *' increases required alignment from 2 to 32
1403 _mm256_storeu_si256((__m256i*)(void*)to, from);
1404}
1405
1406template<> EIGEN_STRONG_INLINE Packet16h
1407ploaddup<Packet16h>(const Eigen::half* from) {
1408 unsigned short a = from[0].x;
1409 unsigned short b = from[1].x;
1410 unsigned short c = from[2].x;
1411 unsigned short d = from[3].x;
1412 unsigned short e = from[4].x;
1413 unsigned short f = from[5].x;
1414 unsigned short g = from[6].x;
1415 unsigned short h = from[7].x;
1416 return _mm256_set_epi16(h, h, g, g, f, f, e, e, d, d, c, c, b, b, a, a);
1417}
1418
1419template<> EIGEN_STRONG_INLINE Packet16h
1420ploadquad(const Eigen::half* from) {
1421 unsigned short a = from[0].x;
1422 unsigned short b = from[1].x;
1423 unsigned short c = from[2].x;
1424 unsigned short d = from[3].x;
1425 return _mm256_set_epi16(d, d, d, d, c, c, c, c, b, b, b, b, a, a, a, a);
1426}
1427
1428EIGEN_STRONG_INLINE Packet16f half2float(const Packet16h& a) {
1429#ifdef EIGEN_HAS_FP16_C
1430 return _mm512_cvtph_ps(a);
1431#else
1432 EIGEN_ALIGN64 half aux[16];
1433 pstore(aux, a);
1434 float f0(aux[0]);
1435 float f1(aux[1]);
1436 float f2(aux[2]);
1437 float f3(aux[3]);
1438 float f4(aux[4]);
1439 float f5(aux[5]);
1440 float f6(aux[6]);
1441 float f7(aux[7]);
1442 float f8(aux[8]);
1443 float f9(aux[9]);
1444 float fa(aux[10]);
1445 float fb(aux[11]);
1446 float fc(aux[12]);
1447 float fd(aux[13]);
1448 float fe(aux[14]);
1449 float ff(aux[15]);
1450
1451 return _mm512_set_ps(
1452 ff, fe, fd, fc, fb, fa, f9, f8, f7, f6, f5, f4, f3, f2, f1, f0);
1453#endif
1454}
1455
1456EIGEN_STRONG_INLINE Packet16h float2half(const Packet16f& a) {
1457#ifdef EIGEN_HAS_FP16_C
1458 return _mm512_cvtps_ph(a, _MM_FROUND_TO_NEAREST_INT|_MM_FROUND_NO_EXC);
1459#else
1460 EIGEN_ALIGN64 float aux[16];
1461 pstore(aux, a);
1462 half h0(aux[0]);
1463 half h1(aux[1]);
1464 half h2(aux[2]);
1465 half h3(aux[3]);
1466 half h4(aux[4]);
1467 half h5(aux[5]);
1468 half h6(aux[6]);
1469 half h7(aux[7]);
1470 half h8(aux[8]);
1471 half h9(aux[9]);
1472 half ha(aux[10]);
1473 half hb(aux[11]);
1474 half hc(aux[12]);
1475 half hd(aux[13]);
1476 half he(aux[14]);
1477 half hf(aux[15]);
1478
1479 return _mm256_set_epi16(
1480 hf.x, he.x, hd.x, hc.x, hb.x, ha.x, h9.x, h8.x,
1481 h7.x, h6.x, h5.x, h4.x, h3.x, h2.x, h1.x, h0.x);
1482#endif
1483}
1484
1485template<> EIGEN_STRONG_INLINE Packet16h ptrue(const Packet16h& a) {
1486 return ptrue(Packet8i(a));
1487}
1488
1489template <>
1490EIGEN_STRONG_INLINE Packet16h pabs(const Packet16h& a) {
1491 const __m256i sign_mask = _mm256_set1_epi16(static_cast<numext::uint16_t>(0x8000));
1492 return _mm256_andnot_si256(sign_mask, a);
1493}
1494
1495template <>
1496EIGEN_STRONG_INLINE Packet16h pmin<Packet16h>(const Packet16h& a,
1497 const Packet16h& b) {
1498 return float2half(pmin<Packet16f>(half2float(a), half2float(b)));
1499}
1500
1501template <>
1502EIGEN_STRONG_INLINE Packet16h pmax<Packet16h>(const Packet16h& a,
1503 const Packet16h& b) {
1504 return float2half(pmax<Packet16f>(half2float(a), half2float(b)));
1505}
1506
1507template <>
1508EIGEN_STRONG_INLINE Packet16h plset<Packet16h>(const half& a) {
1509 return float2half(plset<Packet16f>(static_cast<float>(a)));
1510}
1511
1512template<> EIGEN_STRONG_INLINE Packet16h por(const Packet16h& a,const Packet16h& b) {
1513 // in some cases Packet8i is a wrapper around __m256i, so we need to
1514 // cast to Packet8i to call the correct overload.
1515 return por(Packet8i(a),Packet8i(b));
1516}
1517template<> EIGEN_STRONG_INLINE Packet16h pxor(const Packet16h& a,const Packet16h& b) {
1518 return pxor(Packet8i(a),Packet8i(b));
1519}
1520template<> EIGEN_STRONG_INLINE Packet16h pand(const Packet16h& a,const Packet16h& b) {
1521 return pand(Packet8i(a),Packet8i(b));
1522}
1523template<> EIGEN_STRONG_INLINE Packet16h pandnot(const Packet16h& a,const Packet16h& b) {
1524 return pandnot(Packet8i(a),Packet8i(b));
1525}
1526
1527template<> EIGEN_STRONG_INLINE Packet16h pselect(const Packet16h& mask, const Packet16h& a, const Packet16h& b) {
1528 return _mm256_blendv_epi8(b, a, mask);
1529}
1530
1531template<> EIGEN_STRONG_INLINE Packet16h pround<Packet16h>(const Packet16h& a) {
1532 return float2half(pround<Packet16f>(half2float(a)));
1533}
1534
1535template<> EIGEN_STRONG_INLINE Packet16h print<Packet16h>(const Packet16h& a) {
1536 return float2half(print<Packet16f>(half2float(a)));
1537}
1538
1539template<> EIGEN_STRONG_INLINE Packet16h pceil<Packet16h>(const Packet16h& a) {
1540 return float2half(pceil<Packet16f>(half2float(a)));
1541}
1542
1543template<> EIGEN_STRONG_INLINE Packet16h pfloor<Packet16h>(const Packet16h& a) {
1544 return float2half(pfloor<Packet16f>(half2float(a)));
1545}
1546
1547template<> EIGEN_STRONG_INLINE Packet16h pcmp_eq(const Packet16h& a,const Packet16h& b) {
1548 Packet16f af = half2float(a);
1549 Packet16f bf = half2float(b);
1550 return Pack32To16(pcmp_eq(af, bf));
1551}
1552
1553template<> EIGEN_STRONG_INLINE Packet16h pcmp_le(const Packet16h& a,const Packet16h& b) {
1554 return Pack32To16(pcmp_le(half2float(a), half2float(b)));
1555}
1556
1557template<> EIGEN_STRONG_INLINE Packet16h pcmp_lt(const Packet16h& a,const Packet16h& b) {
1558 return Pack32To16(pcmp_lt(half2float(a), half2float(b)));
1559}
1560
1561template<> EIGEN_STRONG_INLINE Packet16h pcmp_lt_or_nan(const Packet16h& a,const Packet16h& b) {
1562 return Pack32To16(pcmp_lt_or_nan(half2float(a), half2float(b)));
1563}
1564
1565template<> EIGEN_STRONG_INLINE Packet16h pconj(const Packet16h& a) { return a; }
1566
1567template<> EIGEN_STRONG_INLINE Packet16h pnegate(const Packet16h& a) {
1568 Packet16h sign_mask = _mm256_set1_epi16(static_cast<unsigned short>(0x8000));
1569 return _mm256_xor_si256(a, sign_mask);
1570}
1571
1572template<> EIGEN_STRONG_INLINE Packet16h padd<Packet16h>(const Packet16h& a, const Packet16h& b) {
1573 Packet16f af = half2float(a);
1574 Packet16f bf = half2float(b);
1575 Packet16f rf = padd(af, bf);
1576 return float2half(rf);
1577}
1578
1579template<> EIGEN_STRONG_INLINE Packet16h psub<Packet16h>(const Packet16h& a, const Packet16h& b) {
1580 Packet16f af = half2float(a);
1581 Packet16f bf = half2float(b);
1582 Packet16f rf = psub(af, bf);
1583 return float2half(rf);
1584}
1585
1586template<> EIGEN_STRONG_INLINE Packet16h pmul<Packet16h>(const Packet16h& a, const Packet16h& b) {
1587 Packet16f af = half2float(a);
1588 Packet16f bf = half2float(b);
1589 Packet16f rf = pmul(af, bf);
1590 return float2half(rf);
1591}
1592
1593template<> EIGEN_STRONG_INLINE Packet16h pdiv<Packet16h>(const Packet16h& a, const Packet16h& b) {
1594 Packet16f af = half2float(a);
1595 Packet16f bf = half2float(b);
1596 Packet16f rf = pdiv(af, bf);
1597 return float2half(rf);
1598}
1599
1600template<> EIGEN_STRONG_INLINE half predux<Packet16h>(const Packet16h& from) {
1601 Packet16f from_float = half2float(from);
1602 return half(predux(from_float));
1603}
1604
1605template <>
1606EIGEN_STRONG_INLINE Packet8h predux_half_dowto4<Packet16h>(const Packet16h& a) {
1607 Packet8h lane0 = _mm256_extractf128_si256(a, 0);
1608 Packet8h lane1 = _mm256_extractf128_si256(a, 1);
1609 return padd<Packet8h>(lane0, lane1);
1610}
1611
1612template<> EIGEN_STRONG_INLINE Eigen::half predux_max<Packet16h>(const Packet16h& a) {
1613 Packet16f af = half2float(a);
1614 float reduced = predux_max<Packet16f>(af);
1615 return Eigen::half(reduced);
1616}
1617
1618template<> EIGEN_STRONG_INLINE Eigen::half predux_min<Packet16h>(const Packet16h& a) {
1619 Packet16f af = half2float(a);
1620 float reduced = predux_min<Packet16f>(af);
1621 return Eigen::half(reduced);
1622}
1623
1624template<> EIGEN_STRONG_INLINE half predux_mul<Packet16h>(const Packet16h& from) {
1625 Packet16f from_float = half2float(from);
1626 return half(predux_mul(from_float));
1627}
1628
1629template<> EIGEN_STRONG_INLINE Packet16h preverse(const Packet16h& a)
1630{
1631 __m128i m = _mm_setr_epi8(14,15,12,13,10,11,8,9,6,7,4,5,2,3,0,1);
1632 return _mm256_insertf128_si256(
1633 _mm256_castsi128_si256(_mm_shuffle_epi8(_mm256_extractf128_si256(a,1),m)),
1634 _mm_shuffle_epi8(_mm256_extractf128_si256(a,0),m), 1);
1635}
1636
1637template<> EIGEN_STRONG_INLINE Packet16h pgather<Eigen::half, Packet16h>(const Eigen::half* from, Index stride)
1638{
1639 return _mm256_set_epi16(
1640 from[15*stride].x, from[14*stride].x, from[13*stride].x, from[12*stride].x,
1641 from[11*stride].x, from[10*stride].x, from[9*stride].x, from[8*stride].x,
1642 from[7*stride].x, from[6*stride].x, from[5*stride].x, from[4*stride].x,
1643 from[3*stride].x, from[2*stride].x, from[1*stride].x, from[0*stride].x);
1644}
1645
1646template<> EIGEN_STRONG_INLINE void pscatter<half, Packet16h>(half* to, const Packet16h& from, Index stride)
1647{
1648 EIGEN_ALIGN64 half aux[16];
1649 pstore(aux, from);
1650 to[stride*0] = aux[0];
1651 to[stride*1] = aux[1];
1652 to[stride*2] = aux[2];
1653 to[stride*3] = aux[3];
1654 to[stride*4] = aux[4];
1655 to[stride*5] = aux[5];
1656 to[stride*6] = aux[6];
1657 to[stride*7] = aux[7];
1658 to[stride*8] = aux[8];
1659 to[stride*9] = aux[9];
1660 to[stride*10] = aux[10];
1661 to[stride*11] = aux[11];
1662 to[stride*12] = aux[12];
1663 to[stride*13] = aux[13];
1664 to[stride*14] = aux[14];
1665 to[stride*15] = aux[15];
1666}
1667
1668EIGEN_STRONG_INLINE void
1669ptranspose(PacketBlock<Packet16h,16>& kernel) {
1670 __m256i a = kernel.packet[0];
1671 __m256i b = kernel.packet[1];
1672 __m256i c = kernel.packet[2];
1673 __m256i d = kernel.packet[3];
1674 __m256i e = kernel.packet[4];
1675 __m256i f = kernel.packet[5];
1676 __m256i g = kernel.packet[6];
1677 __m256i h = kernel.packet[7];
1678 __m256i i = kernel.packet[8];
1679 __m256i j = kernel.packet[9];
1680 __m256i k = kernel.packet[10];
1681 __m256i l = kernel.packet[11];
1682 __m256i m = kernel.packet[12];
1683 __m256i n = kernel.packet[13];
1684 __m256i o = kernel.packet[14];
1685 __m256i p = kernel.packet[15];
1686
1687 __m256i ab_07 = _mm256_unpacklo_epi16(a, b);
1688 __m256i cd_07 = _mm256_unpacklo_epi16(c, d);
1689 __m256i ef_07 = _mm256_unpacklo_epi16(e, f);
1690 __m256i gh_07 = _mm256_unpacklo_epi16(g, h);
1691 __m256i ij_07 = _mm256_unpacklo_epi16(i, j);
1692 __m256i kl_07 = _mm256_unpacklo_epi16(k, l);
1693 __m256i mn_07 = _mm256_unpacklo_epi16(m, n);
1694 __m256i op_07 = _mm256_unpacklo_epi16(o, p);
1695
1696 __m256i ab_8f = _mm256_unpackhi_epi16(a, b);
1697 __m256i cd_8f = _mm256_unpackhi_epi16(c, d);
1698 __m256i ef_8f = _mm256_unpackhi_epi16(e, f);
1699 __m256i gh_8f = _mm256_unpackhi_epi16(g, h);
1700 __m256i ij_8f = _mm256_unpackhi_epi16(i, j);
1701 __m256i kl_8f = _mm256_unpackhi_epi16(k, l);
1702 __m256i mn_8f = _mm256_unpackhi_epi16(m, n);
1703 __m256i op_8f = _mm256_unpackhi_epi16(o, p);
1704
1705 __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07);
1706 __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07);
1707 __m256i efgh_03 = _mm256_unpacklo_epi32(ef_07, gh_07);
1708 __m256i efgh_47 = _mm256_unpackhi_epi32(ef_07, gh_07);
1709 __m256i ijkl_03 = _mm256_unpacklo_epi32(ij_07, kl_07);
1710 __m256i ijkl_47 = _mm256_unpackhi_epi32(ij_07, kl_07);
1711 __m256i mnop_03 = _mm256_unpacklo_epi32(mn_07, op_07);
1712 __m256i mnop_47 = _mm256_unpackhi_epi32(mn_07, op_07);
1713
1714 __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f);
1715 __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f);
1716 __m256i efgh_8b = _mm256_unpacklo_epi32(ef_8f, gh_8f);
1717 __m256i efgh_cf = _mm256_unpackhi_epi32(ef_8f, gh_8f);
1718 __m256i ijkl_8b = _mm256_unpacklo_epi32(ij_8f, kl_8f);
1719 __m256i ijkl_cf = _mm256_unpackhi_epi32(ij_8f, kl_8f);
1720 __m256i mnop_8b = _mm256_unpacklo_epi32(mn_8f, op_8f);
1721 __m256i mnop_cf = _mm256_unpackhi_epi32(mn_8f, op_8f);
1722
1723 __m256i abcdefgh_01 = _mm256_unpacklo_epi64(abcd_03, efgh_03);
1724 __m256i abcdefgh_23 = _mm256_unpackhi_epi64(abcd_03, efgh_03);
1725 __m256i ijklmnop_01 = _mm256_unpacklo_epi64(ijkl_03, mnop_03);
1726 __m256i ijklmnop_23 = _mm256_unpackhi_epi64(ijkl_03, mnop_03);
1727 __m256i abcdefgh_45 = _mm256_unpacklo_epi64(abcd_47, efgh_47);
1728 __m256i abcdefgh_67 = _mm256_unpackhi_epi64(abcd_47, efgh_47);
1729 __m256i ijklmnop_45 = _mm256_unpacklo_epi64(ijkl_47, mnop_47);
1730 __m256i ijklmnop_67 = _mm256_unpackhi_epi64(ijkl_47, mnop_47);
1731 __m256i abcdefgh_89 = _mm256_unpacklo_epi64(abcd_8b, efgh_8b);
1732 __m256i abcdefgh_ab = _mm256_unpackhi_epi64(abcd_8b, efgh_8b);
1733 __m256i ijklmnop_89 = _mm256_unpacklo_epi64(ijkl_8b, mnop_8b);
1734 __m256i ijklmnop_ab = _mm256_unpackhi_epi64(ijkl_8b, mnop_8b);
1735 __m256i abcdefgh_cd = _mm256_unpacklo_epi64(abcd_cf, efgh_cf);
1736 __m256i abcdefgh_ef = _mm256_unpackhi_epi64(abcd_cf, efgh_cf);
1737 __m256i ijklmnop_cd = _mm256_unpacklo_epi64(ijkl_cf, mnop_cf);
1738 __m256i ijklmnop_ef = _mm256_unpackhi_epi64(ijkl_cf, mnop_cf);
1739
1740 // NOTE: no unpacklo/hi instr in this case, so using permute instr.
1741 __m256i a_p_0 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x20);
1742 __m256i a_p_1 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x20);
1743 __m256i a_p_2 = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x20);
1744 __m256i a_p_3 = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x20);
1745 __m256i a_p_4 = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x20);
1746 __m256i a_p_5 = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x20);
1747 __m256i a_p_6 = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x20);
1748 __m256i a_p_7 = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x20);
1749 __m256i a_p_8 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x31);
1750 __m256i a_p_9 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x31);
1751 __m256i a_p_a = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x31);
1752 __m256i a_p_b = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x31);
1753 __m256i a_p_c = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x31);
1754 __m256i a_p_d = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x31);
1755 __m256i a_p_e = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x31);
1756 __m256i a_p_f = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x31);
1757
1758 kernel.packet[0] = a_p_0;
1759 kernel.packet[1] = a_p_1;
1760 kernel.packet[2] = a_p_2;
1761 kernel.packet[3] = a_p_3;
1762 kernel.packet[4] = a_p_4;
1763 kernel.packet[5] = a_p_5;
1764 kernel.packet[6] = a_p_6;
1765 kernel.packet[7] = a_p_7;
1766 kernel.packet[8] = a_p_8;
1767 kernel.packet[9] = a_p_9;
1768 kernel.packet[10] = a_p_a;
1769 kernel.packet[11] = a_p_b;
1770 kernel.packet[12] = a_p_c;
1771 kernel.packet[13] = a_p_d;
1772 kernel.packet[14] = a_p_e;
1773 kernel.packet[15] = a_p_f;
1774}
1775
1776EIGEN_STRONG_INLINE void
1777ptranspose(PacketBlock<Packet16h,8>& kernel) {
1778 EIGEN_ALIGN64 half in[8][16];
1779 pstore<half>(in[0], kernel.packet[0]);
1780 pstore<half>(in[1], kernel.packet[1]);
1781 pstore<half>(in[2], kernel.packet[2]);
1782 pstore<half>(in[3], kernel.packet[3]);
1783 pstore<half>(in[4], kernel.packet[4]);
1784 pstore<half>(in[5], kernel.packet[5]);
1785 pstore<half>(in[6], kernel.packet[6]);
1786 pstore<half>(in[7], kernel.packet[7]);
1787
1788 EIGEN_ALIGN64 half out[8][16];
1789
1790 for (int i = 0; i < 8; ++i) {
1791 for (int j = 0; j < 8; ++j) {
1792 out[i][j] = in[j][2*i];
1793 }
1794 for (int j = 0; j < 8; ++j) {
1795 out[i][j+8] = in[j][2*i+1];
1796 }
1797 }
1798
1799 kernel.packet[0] = pload<Packet16h>(out[0]);
1800 kernel.packet[1] = pload<Packet16h>(out[1]);
1801 kernel.packet[2] = pload<Packet16h>(out[2]);
1802 kernel.packet[3] = pload<Packet16h>(out[3]);
1803 kernel.packet[4] = pload<Packet16h>(out[4]);
1804 kernel.packet[5] = pload<Packet16h>(out[5]);
1805 kernel.packet[6] = pload<Packet16h>(out[6]);
1806 kernel.packet[7] = pload<Packet16h>(out[7]);
1807}
1808
1809EIGEN_STRONG_INLINE void
1810ptranspose(PacketBlock<Packet16h,4>& kernel) {
1811 EIGEN_ALIGN64 half in[4][16];
1812 pstore<half>(in[0], kernel.packet[0]);
1813 pstore<half>(in[1], kernel.packet[1]);
1814 pstore<half>(in[2], kernel.packet[2]);
1815 pstore<half>(in[3], kernel.packet[3]);
1816
1817 EIGEN_ALIGN64 half out[4][16];
1818
1819 for (int i = 0; i < 4; ++i) {
1820 for (int j = 0; j < 4; ++j) {
1821 out[i][j] = in[j][4*i];
1822 }
1823 for (int j = 0; j < 4; ++j) {
1824 out[i][j+4] = in[j][4*i+1];
1825 }
1826 for (int j = 0; j < 4; ++j) {
1827 out[i][j+8] = in[j][4*i+2];
1828 }
1829 for (int j = 0; j < 4; ++j) {
1830 out[i][j+12] = in[j][4*i+3];
1831 }
1832 }
1833
1834 kernel.packet[0] = pload<Packet16h>(out[0]);
1835 kernel.packet[1] = pload<Packet16h>(out[1]);
1836 kernel.packet[2] = pload<Packet16h>(out[2]);
1837 kernel.packet[3] = pload<Packet16h>(out[3]);
1838}
1839
1840template <> struct is_arithmetic<Packet16bf> { enum { value = true }; };
1841
1842template <>
1843struct packet_traits<bfloat16> : default_packet_traits {
1844 typedef Packet16bf type;
1845 typedef Packet8bf half;
1846 enum {
1847 Vectorizable = 1,
1848 AlignedOnScalar = 1,
1849 size = 16,
1850 HasHalfPacket = 1,
1851 HasBlend = 0,
1852 HasInsert = 1,
1853 HasSin = EIGEN_FAST_MATH,
1854 HasCos = EIGEN_FAST_MATH,
1855#if EIGEN_GNUC_AT_LEAST(5, 3) || (!EIGEN_COMP_GNUC_STRICT)
1856#ifdef EIGEN_VECTORIZE_AVX512DQ
1857 HasLog = 1, // Currently fails test with bad accuracy.
1858 HasLog1p = 1,
1859 HasExpm1 = 1,
1860 HasNdtri = 1,
1861 HasBessel = 1,
1862#endif
1863 HasExp = 1,
1864 HasSqrt = EIGEN_FAST_MATH,
1865 HasRsqrt = EIGEN_FAST_MATH,
1866 HasTanh = EIGEN_FAST_MATH,
1867 HasErf = EIGEN_FAST_MATH,
1868#endif
1869 HasCmp = 1,
1870 HasDiv = 1
1871 };
1872};
1873
1874template <>
1875struct unpacket_traits<Packet16bf>
1876{
1877 typedef bfloat16 type;
1878 enum {size=16, alignment=Aligned32, vectorizable=true, masked_load_available=false, masked_store_available=false};
1879 typedef Packet8bf half;
1880};
1881
1882template <>
1883EIGEN_STRONG_INLINE Packet16bf pset1<Packet16bf>(const bfloat16& from) {
1884 return _mm256_set1_epi16(from.value);
1885}
1886
1887template <>
1888EIGEN_STRONG_INLINE bfloat16 pfirst<Packet16bf>(const Packet16bf& from) {
1889 bfloat16 t;
1890 t.value = static_cast<unsigned short>(_mm256_extract_epi16(from, 0));
1891 return t;
1892}
1893
1894template <>
1895EIGEN_STRONG_INLINE Packet16bf pload<Packet16bf>(const bfloat16* from) {
1896 return _mm256_load_si256(reinterpret_cast<const __m256i*>(from));
1897}
1898
1899template <>
1900EIGEN_STRONG_INLINE Packet16bf ploadu<Packet16bf>(const bfloat16* from) {
1901 return _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from));
1902}
1903
1904template <>
1905EIGEN_STRONG_INLINE void pstore<bfloat16>(bfloat16* to,
1906 const Packet16bf& from) {
1907 _mm256_store_si256(reinterpret_cast<__m256i*>(to), from);
1908}
1909
1910template <>
1911EIGEN_STRONG_INLINE void pstoreu<bfloat16>(bfloat16* to,
1912 const Packet16bf& from) {
1913 _mm256_storeu_si256(reinterpret_cast<__m256i*>(to), from);
1914}
1915
1916template<> EIGEN_STRONG_INLINE Packet16bf
1917ploaddup<Packet16bf>(const bfloat16* from) {
1918 Packet16bf r;
1919 unsigned short a = from[0].value;
1920 unsigned short b = from[1].value;
1921 unsigned short c = from[2].value;
1922 unsigned short d = from[3].value;
1923 unsigned short e = from[4].value;
1924 unsigned short f = from[5].value;
1925 unsigned short g = from[6].value;
1926 unsigned short h = from[7].value;
1927 return _mm256_set_epi16(h, h, g, g, f, f, e, e, d, d, c, c, b, b, a, a);
1928}
1929
1930template<> EIGEN_STRONG_INLINE Packet16bf
1931ploadquad(const bfloat16* from) {
1932 Packet16bf r;
1933 unsigned short a = from[0].value;
1934 unsigned short b = from[1].value;
1935 unsigned short c = from[2].value;
1936 unsigned short d = from[3].value;
1937 return _mm256_set_epi16(d, d, d, d, c, c, c, c, b, b, b, b, a, a, a, a);
1938}
1939
1940EIGEN_STRONG_INLINE Packet16f Bf16ToF32(const Packet16bf& a) {
1941 return _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(a), 16));
1942}
1943
1944// Convert float to bfloat16 according to round-to-nearest-even/denormals algorithm.
1945EIGEN_STRONG_INLINE Packet16bf F32ToBf16(const Packet16f& a) {
1946 Packet16bf r;
1947
1948#if defined(EIGEN_VECTORIZE_AVX512BF16) && EIGEN_GNUC_AT_LEAST(10, 1)
1949 // Since GCC 10.1 supports avx512bf16 and C style explicit cast
1950 // (C++ static_cast is not supported yet), do converion via intrinsic
1951 // and register path for performance.
1952 r = (__m256i)(_mm512_cvtneps_pbh(a));
1953
1954#else
1955 __m512i t;
1956 __m512i input = _mm512_castps_si512(a);
1957 __m512i nan = _mm512_set1_epi32(0x7fc0);
1958
1959 // uint32_t lsb = (input >> 16) & 1;
1960 t = _mm512_and_si512(_mm512_srli_epi32(input, 16), _mm512_set1_epi32(1));
1961 // uint32_t rounding_bias = 0x7fff + lsb;
1962 t = _mm512_add_epi32(t, _mm512_set1_epi32(0x7fff));
1963 // input += rounding_bias;
1964 t = _mm512_add_epi32(t, input);
1965 // input = input >> 16;
1966 t = _mm512_srli_epi32(t, 16);
1967
1968 // Check NaN before converting back to bf16
1969 __mmask16 mask = _mm512_cmp_ps_mask(a, a, _CMP_ORD_Q);
1970
1971 t = _mm512_mask_blend_epi32(mask, nan, t);
1972 // output.value = static_cast<uint16_t>(input);
1973 r = _mm512_cvtepi32_epi16(t);
1974#endif // EIGEN_VECTORIZE_AVX512BF16
1975
1976 return r;
1977}
1978
1979template <>
1980EIGEN_STRONG_INLINE Packet16bf ptrue(const Packet16bf& a) {
1981 return ptrue<Packet8i>(a);
1982}
1983
1984template <>
1985EIGEN_STRONG_INLINE Packet16bf por(const Packet16bf& a, const Packet16bf& b) {
1986 return por<Packet8i>(a, b);
1987}
1988
1989template <>
1990EIGEN_STRONG_INLINE Packet16bf pxor(const Packet16bf& a, const Packet16bf& b) {
1991 return pxor<Packet8i>(a, b);
1992}
1993
1994template <>
1995EIGEN_STRONG_INLINE Packet16bf pand(const Packet16bf& a, const Packet16bf& b) {
1996 return pand<Packet8i>(a, b);
1997}
1998
1999template <>
2000EIGEN_STRONG_INLINE Packet16bf pandnot(const Packet16bf& a,
2001 const Packet16bf& b) {
2002 return pandnot<Packet8i>(a, b);
2003}
2004
2005template <>
2006EIGEN_STRONG_INLINE Packet16bf pselect(const Packet16bf& mask,
2007 const Packet16bf& a,
2008 const Packet16bf& b) {
2009 // Input mask is expected to be all 0/1, handle it with 8-bit
2010 // intrinsic for performance.
2011 return _mm256_blendv_epi8(b, a, mask);
2012}
2013
2014template<> EIGEN_STRONG_INLINE Packet16bf pround<Packet16bf>(const Packet16bf& a)
2015{
2016 return F32ToBf16(pround<Packet16f>(Bf16ToF32(a)));
2017}
2018
2019template<> EIGEN_STRONG_INLINE Packet16bf print<Packet16bf>(const Packet16bf& a) {
2020 return F32ToBf16(print<Packet16f>(Bf16ToF32(a)));
2021}
2022
2023template<> EIGEN_STRONG_INLINE Packet16bf pceil<Packet16bf>(const Packet16bf& a) {
2024 return F32ToBf16(pceil<Packet16f>(Bf16ToF32(a)));
2025}
2026
2027template<> EIGEN_STRONG_INLINE Packet16bf pfloor<Packet16bf>(const Packet16bf& a) {
2028 return F32ToBf16(pfloor<Packet16f>(Bf16ToF32(a)));
2029}
2030
2031template <>
2032EIGEN_STRONG_INLINE Packet16bf pcmp_eq(const Packet16bf& a,
2033 const Packet16bf& b) {
2034 return Pack32To16(pcmp_eq(Bf16ToF32(a), Bf16ToF32(b)));
2035}
2036
2037template <>
2038EIGEN_STRONG_INLINE Packet16bf pcmp_le(const Packet16bf& a,
2039 const Packet16bf& b) {
2040 return Pack32To16(pcmp_le(Bf16ToF32(a), Bf16ToF32(b)));
2041}
2042
2043template <>
2044EIGEN_STRONG_INLINE Packet16bf pcmp_lt(const Packet16bf& a,
2045 const Packet16bf& b) {
2046 return Pack32To16(pcmp_lt(Bf16ToF32(a), Bf16ToF32(b)));
2047}
2048
2049template <>
2050EIGEN_STRONG_INLINE Packet16bf pcmp_lt_or_nan(const Packet16bf& a,
2051 const Packet16bf& b) {
2052 return Pack32To16(pcmp_lt_or_nan(Bf16ToF32(a), Bf16ToF32(b)));
2053}
2054
2055template <>
2056EIGEN_STRONG_INLINE Packet16bf pnegate(const Packet16bf& a) {
2057 Packet16bf sign_mask = _mm256_set1_epi16(static_cast<unsigned short>(0x8000));
2058 return _mm256_xor_si256(a, sign_mask);
2059}
2060
2061template <>
2062EIGEN_STRONG_INLINE Packet16bf pconj(const Packet16bf& a) {
2063 return a;
2064}
2065
2066template <>
2067EIGEN_STRONG_INLINE Packet16bf pabs(const Packet16bf& a) {
2068 const __m256i sign_mask = _mm256_set1_epi16(static_cast<numext::uint16_t>(0x8000));
2069 return _mm256_andnot_si256(sign_mask, a);
2070}
2071
2072template <>
2073EIGEN_STRONG_INLINE Packet16bf padd<Packet16bf>(const Packet16bf& a,
2074 const Packet16bf& b) {
2075 return F32ToBf16(padd<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2076}
2077
2078template <>
2079EIGEN_STRONG_INLINE Packet16bf psub<Packet16bf>(const Packet16bf& a,
2080 const Packet16bf& b) {
2081 return F32ToBf16(psub<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2082}
2083
2084template <>
2085EIGEN_STRONG_INLINE Packet16bf pmul<Packet16bf>(const Packet16bf& a,
2086 const Packet16bf& b) {
2087 return F32ToBf16(pmul<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2088}
2089
2090template <>
2091EIGEN_STRONG_INLINE Packet16bf pdiv<Packet16bf>(const Packet16bf& a,
2092 const Packet16bf& b) {
2093 return F32ToBf16(pdiv<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2094}
2095
2096template <>
2097EIGEN_STRONG_INLINE Packet16bf pmin<Packet16bf>(const Packet16bf& a,
2098 const Packet16bf& b) {
2099 return F32ToBf16(pmin<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2100}
2101
2102template <>
2103EIGEN_STRONG_INLINE Packet16bf pmax<Packet16bf>(const Packet16bf& a,
2104 const Packet16bf& b) {
2105 return F32ToBf16(pmax<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2106}
2107
2108template <>
2109EIGEN_STRONG_INLINE Packet16bf plset<Packet16bf>(const bfloat16& a) {
2110 return F32ToBf16(plset<Packet16f>(static_cast<float>(a)));
2111}
2112
2113template <>
2114EIGEN_STRONG_INLINE Packet8bf predux_half_dowto4<Packet16bf>(const Packet16bf& a) {
2115 Packet8bf lane0 = _mm256_extractf128_si256(a, 0);
2116 Packet8bf lane1 = _mm256_extractf128_si256(a, 1);
2117 return padd<Packet8bf>(lane0, lane1);
2118}
2119
2120template <>
2121EIGEN_STRONG_INLINE bfloat16 predux<Packet16bf>(const Packet16bf& p) {
2122 return static_cast<bfloat16>(predux<Packet16f>(Bf16ToF32(p)));
2123}
2124
2125template <>
2126EIGEN_STRONG_INLINE bfloat16 predux_mul<Packet16bf>(const Packet16bf& from) {
2127 return static_cast<bfloat16>(predux_mul<Packet16f>(Bf16ToF32(from)));
2128}
2129
2130template <>
2131EIGEN_STRONG_INLINE bfloat16 predux_min<Packet16bf>(const Packet16bf& from) {
2132 return static_cast<bfloat16>(predux_min<Packet16f>(Bf16ToF32(from)));
2133}
2134
2135template <>
2136EIGEN_STRONG_INLINE bfloat16 predux_max<Packet16bf>(const Packet16bf& from) {
2137 return static_cast<bfloat16>(predux_max<Packet16f>(Bf16ToF32(from)));
2138}
2139
2140template <>
2141EIGEN_STRONG_INLINE Packet16bf preverse(const Packet16bf& a) {
2142 __m256i m = _mm256_setr_epi8(14,15,12,13,10,11,8,9,6,7,4,5,2,3,0,1,
2143 14,15,12,13,10,11,8,9,6,7,4,5,2,3,0,1);
2144
2145 Packet16bf res;
2146 // Swap hi and lo first because shuffle is in 128-bit lanes.
2147 res = _mm256_permute2x128_si256(a, a, 1);
2148 // Shuffle 8-bit values in src within 2*128-bit lanes.
2149 return _mm256_shuffle_epi8(res, m);
2150}
2151
2152template <>
2153EIGEN_STRONG_INLINE Packet16bf pgather<bfloat16, Packet16bf>(const bfloat16* from,
2154 Index stride) {
2155 return _mm256_set_epi16(
2156 from[15*stride].value, from[14*stride].value, from[13*stride].value, from[12*stride].value,
2157 from[11*stride].value, from[10*stride].value, from[9*stride].value, from[8*stride].value,
2158 from[7*stride].value, from[6*stride].value, from[5*stride].value, from[4*stride].value,
2159 from[3*stride].value, from[2*stride].value, from[1*stride].value, from[0*stride].value);
2160}
2161
2162template <>
2163EIGEN_STRONG_INLINE void pscatter<bfloat16, Packet16bf>(bfloat16* to,
2164 const Packet16bf& from,
2165 Index stride) {
2166 EIGEN_ALIGN64 bfloat16 aux[16];
2167 pstore(aux, from);
2168 to[stride*0] = aux[0];
2169 to[stride*1] = aux[1];
2170 to[stride*2] = aux[2];
2171 to[stride*3] = aux[3];
2172 to[stride*4] = aux[4];
2173 to[stride*5] = aux[5];
2174 to[stride*6] = aux[6];
2175 to[stride*7] = aux[7];
2176 to[stride*8] = aux[8];
2177 to[stride*9] = aux[9];
2178 to[stride*10] = aux[10];
2179 to[stride*11] = aux[11];
2180 to[stride*12] = aux[12];
2181 to[stride*13] = aux[13];
2182 to[stride*14] = aux[14];
2183 to[stride*15] = aux[15];
2184}
2185
2186EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16bf,16>& kernel) {
2187 __m256i a = kernel.packet[0];
2188 __m256i b = kernel.packet[1];
2189 __m256i c = kernel.packet[2];
2190 __m256i d = kernel.packet[3];
2191 __m256i e = kernel.packet[4];
2192 __m256i f = kernel.packet[5];
2193 __m256i g = kernel.packet[6];
2194 __m256i h = kernel.packet[7];
2195 __m256i i = kernel.packet[8];
2196 __m256i j = kernel.packet[9];
2197 __m256i k = kernel.packet[10];
2198 __m256i l = kernel.packet[11];
2199 __m256i m = kernel.packet[12];
2200 __m256i n = kernel.packet[13];
2201 __m256i o = kernel.packet[14];
2202 __m256i p = kernel.packet[15];
2203
2204 __m256i ab_07 = _mm256_unpacklo_epi16(a, b);
2205 __m256i cd_07 = _mm256_unpacklo_epi16(c, d);
2206 __m256i ef_07 = _mm256_unpacklo_epi16(e, f);
2207 __m256i gh_07 = _mm256_unpacklo_epi16(g, h);
2208 __m256i ij_07 = _mm256_unpacklo_epi16(i, j);
2209 __m256i kl_07 = _mm256_unpacklo_epi16(k, l);
2210 __m256i mn_07 = _mm256_unpacklo_epi16(m, n);
2211 __m256i op_07 = _mm256_unpacklo_epi16(o, p);
2212
2213 __m256i ab_8f = _mm256_unpackhi_epi16(a, b);
2214 __m256i cd_8f = _mm256_unpackhi_epi16(c, d);
2215 __m256i ef_8f = _mm256_unpackhi_epi16(e, f);
2216 __m256i gh_8f = _mm256_unpackhi_epi16(g, h);
2217 __m256i ij_8f = _mm256_unpackhi_epi16(i, j);
2218 __m256i kl_8f = _mm256_unpackhi_epi16(k, l);
2219 __m256i mn_8f = _mm256_unpackhi_epi16(m, n);
2220 __m256i op_8f = _mm256_unpackhi_epi16(o, p);
2221
2222 __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07);
2223 __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07);
2224 __m256i efgh_03 = _mm256_unpacklo_epi32(ef_07, gh_07);
2225 __m256i efgh_47 = _mm256_unpackhi_epi32(ef_07, gh_07);
2226 __m256i ijkl_03 = _mm256_unpacklo_epi32(ij_07, kl_07);
2227 __m256i ijkl_47 = _mm256_unpackhi_epi32(ij_07, kl_07);
2228 __m256i mnop_03 = _mm256_unpacklo_epi32(mn_07, op_07);
2229 __m256i mnop_47 = _mm256_unpackhi_epi32(mn_07, op_07);
2230
2231 __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f);
2232 __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f);
2233 __m256i efgh_8b = _mm256_unpacklo_epi32(ef_8f, gh_8f);
2234 __m256i efgh_cf = _mm256_unpackhi_epi32(ef_8f, gh_8f);
2235 __m256i ijkl_8b = _mm256_unpacklo_epi32(ij_8f, kl_8f);
2236 __m256i ijkl_cf = _mm256_unpackhi_epi32(ij_8f, kl_8f);
2237 __m256i mnop_8b = _mm256_unpacklo_epi32(mn_8f, op_8f);
2238 __m256i mnop_cf = _mm256_unpackhi_epi32(mn_8f, op_8f);
2239
2240 __m256i abcdefgh_01 = _mm256_unpacklo_epi64(abcd_03, efgh_03);
2241 __m256i abcdefgh_23 = _mm256_unpackhi_epi64(abcd_03, efgh_03);
2242 __m256i ijklmnop_01 = _mm256_unpacklo_epi64(ijkl_03, mnop_03);
2243 __m256i ijklmnop_23 = _mm256_unpackhi_epi64(ijkl_03, mnop_03);
2244 __m256i abcdefgh_45 = _mm256_unpacklo_epi64(abcd_47, efgh_47);
2245 __m256i abcdefgh_67 = _mm256_unpackhi_epi64(abcd_47, efgh_47);
2246 __m256i ijklmnop_45 = _mm256_unpacklo_epi64(ijkl_47, mnop_47);
2247 __m256i ijklmnop_67 = _mm256_unpackhi_epi64(ijkl_47, mnop_47);
2248 __m256i abcdefgh_89 = _mm256_unpacklo_epi64(abcd_8b, efgh_8b);
2249 __m256i abcdefgh_ab = _mm256_unpackhi_epi64(abcd_8b, efgh_8b);
2250 __m256i ijklmnop_89 = _mm256_unpacklo_epi64(ijkl_8b, mnop_8b);
2251 __m256i ijklmnop_ab = _mm256_unpackhi_epi64(ijkl_8b, mnop_8b);
2252 __m256i abcdefgh_cd = _mm256_unpacklo_epi64(abcd_cf, efgh_cf);
2253 __m256i abcdefgh_ef = _mm256_unpackhi_epi64(abcd_cf, efgh_cf);
2254 __m256i ijklmnop_cd = _mm256_unpacklo_epi64(ijkl_cf, mnop_cf);
2255 __m256i ijklmnop_ef = _mm256_unpackhi_epi64(ijkl_cf, mnop_cf);
2256
2257 // NOTE: no unpacklo/hi instr in this case, so using permute instr.
2258 kernel.packet[0] = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x20);
2259 kernel.packet[1] = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x20);
2260 kernel.packet[2] = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x20);
2261 kernel.packet[3] = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x20);
2262 kernel.packet[4] = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x20);
2263 kernel.packet[5] = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x20);
2264 kernel.packet[6] = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x20);
2265 kernel.packet[7] = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x20);
2266 kernel.packet[8] = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x31);
2267 kernel.packet[9] = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x31);
2268 kernel.packet[10] = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x31);
2269 kernel.packet[11] = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x31);
2270 kernel.packet[12] = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x31);
2271 kernel.packet[13] = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x31);
2272 kernel.packet[14] = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x31);
2273 kernel.packet[15] = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x31);
2274}
2275
2276EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16bf,4>& kernel) {
2277 __m256i a = kernel.packet[0];
2278 __m256i b = kernel.packet[1];
2279 __m256i c = kernel.packet[2];
2280 __m256i d = kernel.packet[3];
2281
2282 __m256i ab_07 = _mm256_unpacklo_epi16(a, b);
2283 __m256i cd_07 = _mm256_unpacklo_epi16(c, d);
2284 __m256i ab_8f = _mm256_unpackhi_epi16(a, b);
2285 __m256i cd_8f = _mm256_unpackhi_epi16(c, d);
2286
2287 __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07);
2288 __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07);
2289 __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f);
2290 __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f);
2291
2292 // NOTE: no unpacklo/hi instr in this case, so using permute instr.
2293 kernel.packet[0] = _mm256_permute2x128_si256(abcd_03, abcd_47, 0x20);
2294 kernel.packet[1] = _mm256_permute2x128_si256(abcd_8b, abcd_cf, 0x20);
2295 kernel.packet[2] = _mm256_permute2x128_si256(abcd_03, abcd_47, 0x31);
2296 kernel.packet[3] = _mm256_permute2x128_si256(abcd_8b, abcd_cf, 0x31);
2297}
2298
2299} // end namespace internal
2300
2301} // end namespace Eigen
2302
2303#endif // EIGEN_PACKET_MATH_AVX512_H
@ Aligned64
Definition: Constants.h:237
@ Aligned32
Definition: Constants.h:236
Namespace containing all symbols from the Eigen library.
Definition: Core:141
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition: Meta.h:74