Please, help us to better know about our user community by answering the following short survey: https://forms.gle/wpyrxWi18ox9Z5ae9
Eigen  3.4.0
 
Loading...
Searching...
No Matches
MatrixProductCommon.h
1//#define EIGEN_POWER_USE_PREFETCH // Use prefetching in gemm routines
2#ifdef EIGEN_POWER_USE_PREFETCH
3#define EIGEN_POWER_PREFETCH(p) prefetch(p)
4#else
5#define EIGEN_POWER_PREFETCH(p)
6#endif
7
8namespace Eigen {
9
10namespace internal {
11
12template<typename Scalar, typename Packet, typename DataMapper, typename Index, const Index accRows, const Index accCols>
13EIGEN_ALWAYS_INLINE void gemm_extra_row(
14 const DataMapper& res,
15 const Scalar* lhs_base,
16 const Scalar* rhs_base,
17 Index depth,
18 Index strideA,
19 Index offsetA,
20 Index row,
21 Index col,
22 Index rows,
23 Index cols,
24 Index remaining_rows,
25 const Packet& pAlpha,
26 const Packet& pMask);
27
28template<typename Scalar, typename Packet, typename DataMapper, typename Index, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
29EIGEN_STRONG_INLINE void gemm_extra_cols(
30 const DataMapper& res,
31 const Scalar* blockA,
32 const Scalar* blockB,
33 Index depth,
34 Index strideA,
35 Index offsetA,
36 Index strideB,
37 Index offsetB,
38 Index col,
39 Index rows,
40 Index cols,
41 Index remaining_rows,
42 const Packet& pAlpha,
43 const Packet& pMask);
44
45template<typename Packet>
46EIGEN_ALWAYS_INLINE Packet bmask(const int remaining_rows);
47
48template<typename Scalar, typename Packet, typename Packetc, typename DataMapper, typename Index, const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
49EIGEN_ALWAYS_INLINE void gemm_complex_extra_row(
50 const DataMapper& res,
51 const Scalar* lhs_base,
52 const Scalar* rhs_base,
53 Index depth,
54 Index strideA,
55 Index offsetA,
56 Index strideB,
57 Index row,
58 Index col,
59 Index rows,
60 Index cols,
61 Index remaining_rows,
62 const Packet& pAlphaReal,
63 const Packet& pAlphaImag,
64 const Packet& pMask);
65
66template<typename Scalar, typename Packet, typename Packetc, typename DataMapper, typename Index, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
67EIGEN_STRONG_INLINE void gemm_complex_extra_cols(
68 const DataMapper& res,
69 const Scalar* blockA,
70 const Scalar* blockB,
71 Index depth,
72 Index strideA,
73 Index offsetA,
74 Index strideB,
75 Index offsetB,
76 Index col,
77 Index rows,
78 Index cols,
79 Index remaining_rows,
80 const Packet& pAlphaReal,
81 const Packet& pAlphaImag,
82 const Packet& pMask);
83
84template<typename Scalar, typename Packet>
85EIGEN_ALWAYS_INLINE Packet ploadLhs(const Scalar* lhs);
86
87template<typename DataMapper, typename Packet, typename Index, const Index accCols, int StorageOrder, bool Complex, int N>
88EIGEN_ALWAYS_INLINE void bload(PacketBlock<Packet,N>& acc, const DataMapper& res, Index row, Index col);
89
90template<typename Packet, int N>
91EIGEN_ALWAYS_INLINE void bscale(PacketBlock<Packet,N>& acc, PacketBlock<Packet,N>& accZ, const Packet& pAlpha);
92
93template<typename Packet, int N>
94EIGEN_ALWAYS_INLINE void bscalec(PacketBlock<Packet,N>& aReal, PacketBlock<Packet,N>& aImag, const Packet& bReal, const Packet& bImag, PacketBlock<Packet,N>& cReal, PacketBlock<Packet,N>& cImag);
95
96// Grab two decouples real/imaginary PacketBlocks and return two coupled (real/imaginary pairs) PacketBlocks.
97template<typename Packet, typename Packetc, int N>
98EIGEN_ALWAYS_INLINE void bcouple_common(PacketBlock<Packet,N>& taccReal, PacketBlock<Packet,N>& taccImag, PacketBlock<Packetc, N>& acc1, PacketBlock<Packetc, N>& acc2)
99{
100 acc1.packet[0].v = vec_mergeh(taccReal.packet[0], taccImag.packet[0]);
101 if (N > 1) {
102 acc1.packet[1].v = vec_mergeh(taccReal.packet[1], taccImag.packet[1]);
103 }
104 if (N > 2) {
105 acc1.packet[2].v = vec_mergeh(taccReal.packet[2], taccImag.packet[2]);
106 }
107 if (N > 3) {
108 acc1.packet[3].v = vec_mergeh(taccReal.packet[3], taccImag.packet[3]);
109 }
110
111 acc2.packet[0].v = vec_mergel(taccReal.packet[0], taccImag.packet[0]);
112 if (N > 1) {
113 acc2.packet[1].v = vec_mergel(taccReal.packet[1], taccImag.packet[1]);
114 }
115 if (N > 2) {
116 acc2.packet[2].v = vec_mergel(taccReal.packet[2], taccImag.packet[2]);
117 }
118 if (N > 3) {
119 acc2.packet[3].v = vec_mergel(taccReal.packet[3], taccImag.packet[3]);
120 }
121}
122
123template<typename Packet, typename Packetc, int N>
124EIGEN_ALWAYS_INLINE void bcouple(PacketBlock<Packet,N>& taccReal, PacketBlock<Packet,N>& taccImag, PacketBlock<Packetc,N*2>& tRes, PacketBlock<Packetc, N>& acc1, PacketBlock<Packetc, N>& acc2)
125{
126 bcouple_common<Packet, Packetc, N>(taccReal, taccImag, acc1, acc2);
127
128 acc1.packet[0] = padd<Packetc>(tRes.packet[0], acc1.packet[0]);
129 if (N > 1) {
130 acc1.packet[1] = padd<Packetc>(tRes.packet[1], acc1.packet[1]);
131 }
132 if (N > 2) {
133 acc1.packet[2] = padd<Packetc>(tRes.packet[2], acc1.packet[2]);
134 }
135 if (N > 3) {
136 acc1.packet[3] = padd<Packetc>(tRes.packet[3], acc1.packet[3]);
137 }
138
139 acc2.packet[0] = padd<Packetc>(tRes.packet[0+N], acc2.packet[0]);
140 if (N > 1) {
141 acc2.packet[1] = padd<Packetc>(tRes.packet[1+N], acc2.packet[1]);
142 }
143 if (N > 2) {
144 acc2.packet[2] = padd<Packetc>(tRes.packet[2+N], acc2.packet[2]);
145 }
146 if (N > 3) {
147 acc2.packet[3] = padd<Packetc>(tRes.packet[3+N], acc2.packet[3]);
148 }
149}
150
151// This is necessary because ploadRhs for double returns a pair of vectors when MMA is enabled.
152template<typename Scalar, typename Packet>
153EIGEN_ALWAYS_INLINE Packet ploadRhs(const Scalar* rhs)
154{
155 return ploadu<Packet>(rhs);
156}
157
158} // end namespace internal
159} // end namespace Eigen
Namespace containing all symbols from the Eigen library.
Definition Core:141
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition Meta.h:74