2#ifdef EIGEN_POWER_USE_PREFETCH
3#define EIGEN_POWER_PREFETCH(p) prefetch(p)
5#define EIGEN_POWER_PREFETCH(p)
12template<
typename Scalar,
typename Packet,
typename DataMapper,
typename Index, const Index accRows>
13EIGEN_STRONG_INLINE
void gemm_extra_col(
14 const DataMapper& res,
15 const Scalar* lhs_base,
16 const Scalar* rhs_base,
24 const Packet& pAlpha);
26template<
typename Scalar,
typename Packet,
typename DataMapper,
typename Index, const Index accRows, const Index accCols>
27EIGEN_STRONG_INLINE
void gemm_extra_row(
28 const DataMapper& res,
29 const Scalar* lhs_base,
30 const Scalar* rhs_base,
42template<
typename Scalar,
typename Packet,
typename DataMapper,
typename Index, const Index accCols>
43EIGEN_STRONG_INLINE
void gemm_unrolled_col(
44 const DataMapper& res,
45 const Scalar* lhs_base,
46 const Scalar* rhs_base,
54 const Packet& pAlpha);
56template<
typename Packet>
57EIGEN_ALWAYS_INLINE Packet bmask(
const int remaining_rows);
59template<
typename Scalar,
typename Packet,
typename Packetc,
typename DataMapper,
typename Index, const Index accRows, const Index accCols,
bool ConjugateLhs,
bool ConjugateRhs,
bool LhsIsReal,
bool RhsIsReal>
60EIGEN_STRONG_INLINE
void gemm_complex_extra_col(
61 const DataMapper& res,
62 const Scalar* lhs_base,
63 const Scalar* rhs_base,
72 const Packet& pAlphaReal,
73 const Packet& pAlphaImag);
75template<
typename Scalar,
typename Packet,
typename Packetc,
typename DataMapper,
typename Index, const Index accRows, const Index accCols,
bool ConjugateLhs,
bool ConjugateRhs,
bool LhsIsReal,
bool RhsIsReal>
76EIGEN_STRONG_INLINE
void gemm_complex_extra_row(
77 const DataMapper& res,
78 const Scalar* lhs_base,
79 const Scalar* rhs_base,
89 const Packet& pAlphaReal,
90 const Packet& pAlphaImag,
93template<
typename Scalar,
typename Packet,
typename Packetc,
typename DataMapper,
typename Index, const Index accCols,
bool ConjugateLhs,
bool ConjugateRhs,
bool LhsIsReal,
bool RhsIsReal>
94EIGEN_STRONG_INLINE
void gemm_complex_unrolled_col(
95 const DataMapper& res,
96 const Scalar* lhs_base,
97 const Scalar* rhs_base,
105 Index remaining_cols,
106 const Packet& pAlphaReal,
107 const Packet& pAlphaImag);
109template<
typename Scalar,
typename Packet>
110EIGEN_ALWAYS_INLINE Packet ploadLhs(
const Scalar* lhs);
112template<
typename DataMapper,
typename Packet,
typename Index, const Index accCols,
int N,
int StorageOrder>
113EIGEN_ALWAYS_INLINE
void bload(PacketBlock<Packet,4>& acc,
const DataMapper& res,
Index row,
Index col);
115template<
typename DataMapper,
typename Packet,
typename Index, const Index accCols,
int N,
int StorageOrder>
116EIGEN_ALWAYS_INLINE
void bload(PacketBlock<Packet,8>& acc,
const DataMapper& res,
Index row,
Index col);
118template<
typename Packet>
119EIGEN_ALWAYS_INLINE
void bscale(PacketBlock<Packet,4>& acc, PacketBlock<Packet,4>& accZ,
const Packet& pAlpha);
121template<
typename Packet,
int N>
122EIGEN_ALWAYS_INLINE
void bscalec(PacketBlock<Packet,N>& aReal, PacketBlock<Packet,N>& aImag,
const Packet& bReal,
const Packet& bImag, PacketBlock<Packet,N>& cReal, PacketBlock<Packet,N>& cImag);
124const static Packet16uc p16uc_SETCOMPLEX32_FIRST = { 0, 1, 2, 3,
129const static Packet16uc p16uc_SETCOMPLEX32_SECOND = { 8, 9, 10, 11,
134const static Packet16uc p16uc_SETCOMPLEX64_FIRST = { 0, 1, 2, 3, 4, 5, 6, 7,
135 16, 17, 18, 19, 20, 21, 22, 23};
138const static Packet16uc p16uc_SETCOMPLEX64_SECOND = { 8, 9, 10, 11, 12, 13, 14, 15,
139 24, 25, 26, 27, 28, 29, 30, 31};
143template<
typename Packet,
typename Packetc>
144EIGEN_ALWAYS_INLINE
void bcouple_common(PacketBlock<Packet,4>& taccReal, PacketBlock<Packet,4>& taccImag, PacketBlock<Packetc, 4>& acc1, PacketBlock<Packetc, 4>& acc2)
146 acc1.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], p16uc_SETCOMPLEX32_FIRST);
147 acc1.packet[1].v = vec_perm(taccReal.packet[1], taccImag.packet[1], p16uc_SETCOMPLEX32_FIRST);
148 acc1.packet[2].v = vec_perm(taccReal.packet[2], taccImag.packet[2], p16uc_SETCOMPLEX32_FIRST);
149 acc1.packet[3].v = vec_perm(taccReal.packet[3], taccImag.packet[3], p16uc_SETCOMPLEX32_FIRST);
151 acc2.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], p16uc_SETCOMPLEX32_SECOND);
152 acc2.packet[1].v = vec_perm(taccReal.packet[1], taccImag.packet[1], p16uc_SETCOMPLEX32_SECOND);
153 acc2.packet[2].v = vec_perm(taccReal.packet[2], taccImag.packet[2], p16uc_SETCOMPLEX32_SECOND);
154 acc2.packet[3].v = vec_perm(taccReal.packet[3], taccImag.packet[3], p16uc_SETCOMPLEX32_SECOND);
157template<
typename Packet,
typename Packetc>
158EIGEN_ALWAYS_INLINE
void bcouple(PacketBlock<Packet,4>& taccReal, PacketBlock<Packet,4>& taccImag, PacketBlock<Packetc,8>& tRes, PacketBlock<Packetc, 4>& acc1, PacketBlock<Packetc, 4>& acc2)
160 bcouple_common<Packet, Packetc>(taccReal, taccImag, acc1, acc2);
162 acc1.packet[0] = padd<Packetc>(tRes.packet[0], acc1.packet[0]);
163 acc1.packet[1] = padd<Packetc>(tRes.packet[1], acc1.packet[1]);
164 acc1.packet[2] = padd<Packetc>(tRes.packet[2], acc1.packet[2]);
165 acc1.packet[3] = padd<Packetc>(tRes.packet[3], acc1.packet[3]);
167 acc2.packet[0] = padd<Packetc>(tRes.packet[4], acc2.packet[0]);
168 acc2.packet[1] = padd<Packetc>(tRes.packet[5], acc2.packet[1]);
169 acc2.packet[2] = padd<Packetc>(tRes.packet[6], acc2.packet[2]);
170 acc2.packet[3] = padd<Packetc>(tRes.packet[7], acc2.packet[3]);
173template<
typename Packet,
typename Packetc>
174EIGEN_ALWAYS_INLINE
void bcouple_common(PacketBlock<Packet,1>& taccReal, PacketBlock<Packet,1>& taccImag, PacketBlock<Packetc, 1>& acc1, PacketBlock<Packetc, 1>& acc2)
176 acc1.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], p16uc_SETCOMPLEX32_FIRST);
178 acc2.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], p16uc_SETCOMPLEX32_SECOND);
181template<
typename Packet,
typename Packetc>
182EIGEN_ALWAYS_INLINE
void bcouple(PacketBlock<Packet,1>& taccReal, PacketBlock<Packet,1>& taccImag, PacketBlock<Packetc,2>& tRes, PacketBlock<Packetc, 1>& acc1, PacketBlock<Packetc, 1>& acc2)
184 bcouple_common<Packet, Packetc>(taccReal, taccImag, acc1, acc2);
186 acc1.packet[0] = padd<Packetc>(tRes.packet[0], acc1.packet[0]);
188 acc2.packet[0] = padd<Packetc>(tRes.packet[1], acc2.packet[0]);
192EIGEN_ALWAYS_INLINE
void bcouple_common<Packet2d, Packet1cd>(PacketBlock<Packet2d,4>& taccReal, PacketBlock<Packet2d,4>& taccImag, PacketBlock<Packet1cd, 4>& acc1, PacketBlock<Packet1cd, 4>& acc2)
194 acc1.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], p16uc_SETCOMPLEX64_FIRST);
195 acc1.packet[1].v = vec_perm(taccReal.packet[1], taccImag.packet[1], p16uc_SETCOMPLEX64_FIRST);
196 acc1.packet[2].v = vec_perm(taccReal.packet[2], taccImag.packet[2], p16uc_SETCOMPLEX64_FIRST);
197 acc1.packet[3].v = vec_perm(taccReal.packet[3], taccImag.packet[3], p16uc_SETCOMPLEX64_FIRST);
199 acc2.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], p16uc_SETCOMPLEX64_SECOND);
200 acc2.packet[1].v = vec_perm(taccReal.packet[1], taccImag.packet[1], p16uc_SETCOMPLEX64_SECOND);
201 acc2.packet[2].v = vec_perm(taccReal.packet[2], taccImag.packet[2], p16uc_SETCOMPLEX64_SECOND);
202 acc2.packet[3].v = vec_perm(taccReal.packet[3], taccImag.packet[3], p16uc_SETCOMPLEX64_SECOND);
206EIGEN_ALWAYS_INLINE
void bcouple_common<Packet2d, Packet1cd>(PacketBlock<Packet2d,1>& taccReal, PacketBlock<Packet2d,1>& taccImag, PacketBlock<Packet1cd, 1>& acc1, PacketBlock<Packet1cd, 1>& acc2)
208 acc1.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], p16uc_SETCOMPLEX64_FIRST);
210 acc2.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], p16uc_SETCOMPLEX64_SECOND);
214template<
typename Scalar,
typename Packet>
215EIGEN_ALWAYS_INLINE Packet ploadRhs(
const Scalar* rhs)
217 return ploadu<Packet>(rhs);
Namespace containing all symbols from the Eigen library.
Definition: Core:141
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition: Meta.h:74