| CUTLASS CUDA Templates for Linear Algebra Subroutines and Solvers

- Main Page
- Modules
- Namespaces
- Classes
- Files

- File List
- File Members

default_gemm_configuration.h
Go to the documentation of this file.
1 /***************************************************************************************************
2 * Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without modification, are permitted
5 * provided that the following conditions are met:
6 * * Redistributions of source code must retain the above copyright notice, this list of
7 * conditions and the following disclaimer.
8 * * Redistributions in binary form must reproduce the above copyright notice, this list of
9 * conditions and the following disclaimer in the documentation and/or other materials
10 * provided with the distribution.
11 * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
12 * to endorse or promote products derived from this software without specific prior written
13 * permission.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
17 * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
19 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
20 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
21 * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
22 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23 *
24 **************************************************************************************************/
29 #pragma once
30
31 #include "cutlass/cutlass.h"
32 #include "cutlass/numeric_types.h"
33 #include "cutlass/arch/arch.h"
34 #include "cutlass/arch/mma.h"
35 #include "cutlass/arch/wmma.h"
36
37 #include "cutlass/gemm/gemm.h"
38 #include "cutlass/epilogue/thread/linear_combination.h"
39 #include "cutlass/epilogue/thread/linear_combination_clamp.h"
40
42
43 namespace cutlass {
44 namespace gemm {
45 namespace device {
46
48
49 template <
50 typename OperatorClass,
51 typename ArchTag,
52 typename ElementA,
53 typename ElementB,
54 typename ElementC,
55 typename ElementAccumulator
56 >
57 struct DefaultGemmConfiguration;
58
60
61 template <
62 typename ArchTag,
63 typename ElementA,
64 typename ElementB,
65 typename ElementC,
66 typename ElementAccumulator>
67 struct DefaultGemmConfiguration<
68 arch::OpClassSimt,
69 ArchTag,
70 ElementA,
71 ElementB,
72 ElementC,
73 ElementAccumulator> {
74
75 static int const kAlignmentA = 1;
76 static int const kAlignmentB = 1;
77 using ThreadblockShape = GemmShape<128, 128, 8>;
78 using WarpShape = GemmShape<32, 64, 8>;
79 using InstructionShape = GemmShape<1, 1, 1>;
80 static int const kStages = 2;
81
82 using EpilogueOutputOp = epilogue::thread::LinearCombination<
83 ElementC,
84 1,
85 ElementAccumulator,
86 ElementAccumulator
87 >;
88
89 using Operator = arch::OpMultiplyAdd;
90 };
91
93
94 template <
95 typename ArchTag,
96 typename ElementC>
97 struct DefaultGemmConfiguration<arch::OpClassSimt, ArchTag, int8_t, int8_t, ElementC, int32_t> {
98
99 static int const kAlignmentA = 4;
100 static int const kAlignmentB = 4;
101 using ThreadblockShape = GemmShape<128, 128, 32>;
102 using WarpShape = GemmShape<32, 64, 32>;
103 using InstructionShape = GemmShape<1, 1, 4>;
104 static int const kStages = 2;
105
106 using EpilogueOutputOp = epilogue::thread::LinearCombinationClamp<
107 ElementC,
108 1,
109 int32_t,
110 float
111 >;
112
113 using Operator = arch::OpMultiplyAdd;
114 };
115
117
118 template <
119 typename ArchTag,
120 typename ElementA,
121 typename ElementB,
122 typename ElementC,
123 typename ElementAccumulator>
124 struct DefaultGemmConfiguration<
125 arch::OpClassWmmaTensorOp,
126 ArchTag,
127 ElementA,
128 ElementB,
129 ElementC,
130 ElementAccumulator> {
131
132 static int const kAlignmentA = 128 / sizeof_bits<ElementA>::value;
133 static int const kAlignmentB = 128 / sizeof_bits<ElementB>::value;
134
135 static int const kStages = 2;
136
137 using EpilogueOutputOp = epilogue::thread::LinearCombination<
138 ElementC,
139 128 / sizeof_bits<ElementC>::value,
140 ElementAccumulator,
141 ElementAccumulator
142 >;
143
144 using Operator = arch::OpMultiplyAdd;
145 };
146
148
149 template <
150 typename ElementA,
151 typename ElementB,
152 typename ElementC,
153 typename ElementAccumulator>
154 struct DefaultGemmConfiguration<
155 arch::OpClassTensorOp,
156 arch::Sm70,
157 ElementA,
158 ElementB,
159 ElementC,
160 ElementAccumulator> {
161
162 static int const kAlignmentA = 128 / sizeof_bits<ElementA>::value;
163 static int const kAlignmentB = 128 / sizeof_bits<ElementB>::value;
164
165 using ThreadblockShape = GemmShape<128, 256, 32>;
166 using WarpShape = GemmShape<64, 64, 32>;
167 using InstructionShape = GemmShape<16, 16, 4>;
168 static int const kStages = 2;
169
170 using EpilogueOutputOp = epilogue::thread::LinearCombination<
171 ElementC,
172 128 / sizeof_bits<ElementC>::value,
173 ElementAccumulator,
174 ElementAccumulator
175 >;
176
177 using Operator = arch::OpMultiplyAdd;
178 };
179
181
182 template <
183 typename ElementA,
184 typename ElementB,
185 typename ElementC,
186 typename ElementAccumulator>
187 struct DefaultGemmConfiguration<
188 arch::OpClassTensorOp,
189 arch::Sm75,
190 ElementA,
191 ElementB,
192 ElementC,
193 ElementAccumulator> {
194
195 static int const kAlignmentA = 128 / sizeof_bits<ElementA>::value;
196 static int const kAlignmentB = 128 / sizeof_bits<ElementA>::value;
197 using ThreadblockShape = GemmShape<128, 256, 32>;
198 using WarpShape = GemmShape<64, 64, 32>;
199 using InstructionShape = GemmShape<16, 8, 8>;
200 static int const kStages = 2;
201
202 using EpilogueOutputOp = epilogue::thread::LinearCombination<
203 ElementC,
204 128 / sizeof_bits<ElementC>::value,
205 ElementAccumulator,
206 ElementAccumulator
207 >;
208
209 using Operator = typename platform::conditional<
210 (platform::is_same<ElementA, int8_t>::value ||
211 platform::is_same<ElementA, int4b_t>::value ||
212 platform::is_same<ElementA, uint8_t>::value ||
213 platform::is_same<ElementA, uint4b_t>::value),
214 arch::OpMultiplyAddSaturate, arch::OpMultiplyAdd>::type;
215 };
216
218
219 template <
220 typename ElementC>
221 struct DefaultGemmConfiguration<
222 arch::OpClassTensorOp,
223 arch::Sm75,
224 int8_t,
225 int8_t,
226 ElementC,
227 int32_t> {
228
229 static int const kAlignmentA = 128 / sizeof_bits<int8_t>::value;
230 static int const kAlignmentB = 128 / sizeof_bits<int8_t>::value;
231
232 using ThreadblockShape = GemmShape<128, 256, 64>;
233 using WarpShape = GemmShape<64, 64, 64>;
234 using InstructionShape = GemmShape<8, 8, 16>;
235 static int const kStages = 2;
236
237 using EpilogueOutputOp = epilogue::thread::LinearCombinationClamp<
238 ElementC, 128 / sizeof_bits<ElementC>::value, int32_t, float>;
239
240 using Operator = arch::OpMultiplyAddSaturate;
241 };
242
244
245 template <
246 typename ElementC>
247 struct DefaultGemmConfiguration<
248 arch::OpClassTensorOp,
249 arch::Sm75,
250 int8_t,
251 uint8_t,
252 ElementC,
253 int32_t> {
254
255 static int const kAlignmentA = 128 / sizeof_bits<int8_t>::value;
256 static int const kAlignmentB = 128 / sizeof_bits<uint8_t>::value;
257
258 using ThreadblockShape = GemmShape<128, 256, 64>;
259 using WarpShape = GemmShape<64, 64, 64>;
260 using InstructionShape = GemmShape<8, 8, 16>;
261 static int const kStages = 2;
262
263 using EpilogueOutputOp = epilogue::thread::LinearCombinationClamp<
264 ElementC, 128 / sizeof_bits<ElementC>::value, int32_t, float>;
265
266 using Operator = arch::OpMultiplyAddSaturate;
267 };
268
270
271 template <
272 typename ElementC>
273 struct DefaultGemmConfiguration<
274 arch::OpClassTensorOp,
275 arch::Sm75,
276 uint8_t,
277 int8_t,
278 ElementC,
279 int32_t> {
280
281 static int const kAlignmentA = 128 / sizeof_bits<uint8_t>::value;
282 static int const kAlignmentB = 128 / sizeof_bits<int8_t>::value;
283
284 using ThreadblockShape = GemmShape<128, 256, 64>;
285 using WarpShape = GemmShape<64, 64, 64>;
286 using InstructionShape = GemmShape<8, 8, 16>;
287 static int const kStages = 2;
288
289 using EpilogueOutputOp = epilogue::thread::LinearCombinationClamp<
290 ElementC, 128 / sizeof_bits<ElementC>::value, int32_t, float>;
291
292 using Operator = arch::OpMultiplyAddSaturate;
293 };
294
296
297 template <
298 typename ElementC>
299 struct DefaultGemmConfiguration<
300 arch::OpClassTensorOp,
301 arch::Sm75,
302 uint8_t,
303 uint8_t,
304 ElementC,
305 int32_t> {
306
307 static int const kAlignmentA = 128 / sizeof_bits<uint8_t>::value;
308 static int const kAlignmentB = 128 / sizeof_bits<uint8_t>::value;
309
310 using ThreadblockShape = GemmShape<128, 256, 64>;
311 using WarpShape = GemmShape<64, 64, 64>;
312 using InstructionShape = GemmShape<8, 8, 16>;
313 static int const kStages = 2;
314
315 using EpilogueOutputOp = epilogue::thread::LinearCombinationClamp<
316 ElementC, 128 / sizeof_bits<ElementC>::value, int32_t, float>;
317
318 using Operator = arch::OpMultiplyAddSaturate;
319 };
320
322
323 template <
324 typename ElementC>
325 struct DefaultGemmConfiguration<
326 arch::OpClassTensorOp,
327 arch::Sm75,
328 int4b_t,
329 int4b_t,
330 ElementC,
331 int32_t> {
332
333 static int const kAlignmentA = 128 / sizeof_bits<int4b_t>::value;
334 static int const kAlignmentB = 128 / sizeof_bits<int4b_t>::value;
335
336 using ThreadblockShape = GemmShape<128, 256, 128>;
337 using WarpShape = GemmShape<64, 64, 128>;
338 using InstructionShape = GemmShape<8, 8, 32>;
339 static int const kStages = 2;
340
341 using EpilogueOutputOp = epilogue::thread::LinearCombinationClamp<
342 ElementC, 128 / sizeof_bits<ElementC>::value, int32_t, float>;
343
344 using Operator = arch::OpMultiplyAddSaturate;
345 };
346
348
349 template <
350 typename ElementC>
351 struct DefaultGemmConfiguration<
352 arch::OpClassTensorOp,
353 arch::Sm75,
354 int4b_t,
355 uint4b_t,
356 ElementC,
357 int32_t> {
358
359 static int const kAlignmentA = 128 / sizeof_bits<int4b_t>::value;
360 static int const kAlignmentB = 128 / sizeof_bits<uint4b_t>::value;
361
362 using ThreadblockShape = GemmShape<128, 256, 128>;
363 using WarpShape = GemmShape<64, 64, 128>;
364 using InstructionShape = GemmShape<8, 8, 32>;
365 static int const kStages = 2;
366
367 using EpilogueOutputOp = epilogue::thread::LinearCombinationClamp<
368 ElementC, 128 / sizeof_bits<ElementC>::value, int32_t, float>;
369
370 using Operator = arch::OpMultiplyAddSaturate;
371 };
372
374
375 template <
376 typename ElementC>
377 struct DefaultGemmConfiguration<
378 arch::OpClassTensorOp,
379 arch::Sm75,
380 uint4b_t,
381 int4b_t,
382 ElementC,
383 int32_t> {
384
385 static int const kAlignmentA = 128 / sizeof_bits<uint4b_t>::value;
386 static int const kAlignmentB = 128 / sizeof_bits<int4b_t>::value;
387
388 using ThreadblockShape = GemmShape<128, 256, 128>;
389 using WarpShape = GemmShape<64, 64, 128>;
390 using InstructionShape = GemmShape<8, 8, 32>;
391 static int const kStages = 2;
392
393 using EpilogueOutputOp = epilogue::thread::LinearCombinationClamp<
394 ElementC, 128 / sizeof_bits<ElementC>::value, int32_t, float>;
395
396 using Operator = arch::OpMultiplyAddSaturate;
397 };
398
400
401 template <
402 typename ElementC>
403 struct DefaultGemmConfiguration<
404 arch::OpClassTensorOp,
405 arch::Sm75,
406 uint4b_t,
407 uint4b_t,
408 ElementC,
409 int32_t> {
410
411 static int const kAlignmentA = 128 / sizeof_bits<uint4b_t>::value;
412 static int const kAlignmentB = 128 / sizeof_bits<uint4b_t>::value;
413
414 using ThreadblockShape = GemmShape<128, 256, 128>;
415 using WarpShape = GemmShape<64, 64, 128>;
416 using InstructionShape = GemmShape<8, 8, 32>;
417 static int const kStages = 2;
418
419 using EpilogueOutputOp = epilogue::thread::LinearCombinationClamp<
420 ElementC, 128 / sizeof_bits<ElementC>::value, int32_t, float>;
421
422 using Operator = arch::OpMultiplyAddSaturate;
423 };
424
426 } // namespace device
427 } // namespace gemm
428 } // namespace cutlass
429
cutlass
Definition: aligned_buffer.h:35
cutlass::gemm::device::DefaultGemmConfiguration< arch::OpClassTensorOp, arch::Sm75, int8_t, int8_t, ElementC, int32_t >::Operator
arch::OpMultiplyAddSaturate Operator
Definition: default_gemm_configuration.h:240
cutlass::epilogue::thread::LinearCombination
Definition: linear_combination.h:56
cutlass::platform::is_same
std::is_same (false specialization)
Definition: platform.h:394
cutlass::gemm::device::DefaultGemmConfiguration< arch::OpClassTensorOp, arch::Sm75, uint4b_t, uint4b_t, ElementC, int32_t >::Operator
arch::OpMultiplyAddSaturate Operator
Definition: default_gemm_configuration.h:422
cutlass::gemm::device::DefaultGemmConfiguration< arch::OpClassTensorOp, arch::Sm75, uint8_t, int8_t, ElementC, int32_t >::Operator
arch::OpMultiplyAddSaturate Operator
Definition: default_gemm_configuration.h:292
cutlass::epilogue::thread::LinearCombinationClamp
Definition: linear_combination_clamp.h:58
cutlass::integer_subbyte
4-bit signed integer type
Definition: integer_subbyte.h:42
linear_combination_clamp.h
Functor performing linear scaling operations used by epilogues. Values are clamped before converting ...
cutlass::arch::Sm70
Definition: arch.h:46
gemm.h
Defines common types used for all GEMM-like operators.
cutlass::gemm::device::DefaultGemmConfiguration< arch::OpClassTensorOp, arch::Sm75, ElementA, ElementB, ElementC, ElementAccumulator >::Operator
typename platform::conditional< (platform::is_same< ElementA, int8_t >::value||platform::is_same< ElementA, int4b_t >::value||platform::is_same< ElementA, uint8_t >::value||platform::is_same< ElementA, uint4b_t >::value), arch::OpMultiplyAddSaturate, arch::OpMultiplyAdd >::type Operator
Definition: default_gemm_configuration.h:214
cutlass::gemm::device::DefaultGemmConfiguration< arch::OpClassTensorOp, arch::Sm75, uint4b_t, int4b_t, ElementC, int32_t >::Operator
arch::OpMultiplyAddSaturate Operator
Definition: default_gemm_configuration.h:396
mma.h
Templates exposing architecture support for multiply-add operations.
cutlass::arch::Sm75
Definition: arch.h:52
cutlass::gemm::device::DefaultGemmConfiguration< arch::OpClassWmmaTensorOp, ArchTag, ElementA, ElementB, ElementC, ElementAccumulator >::Operator
arch::OpMultiplyAdd Operator
Definition: default_gemm_configuration.h:144
linear_combination.h
Functor performing linear combination operations used by epilogues.
cutlass::sizeof_bits
Defines the size of an element in bits.
Definition: numeric_types.h:42
cutlass::gemm::device::DefaultGemmConfiguration< arch::OpClassSimt, ArchTag, int8_t, int8_t, ElementC, int32_t >::Operator
arch::OpMultiplyAdd Operator
Definition: default_gemm_configuration.h:113
cutlass::gemm::device::DefaultGemmConfiguration< arch::OpClassTensorOp, arch::Sm75, int8_t, uint8_t, ElementC, int32_t >::Operator
arch::OpMultiplyAddSaturate Operator
Definition: default_gemm_configuration.h:266
numeric_types.h
Top-level include for all CUTLASS numeric types.
cutlass::gemm::GemmShape
Shape of a matrix multiply-add operation.
Definition: include/cutlass/gemm/gemm.h:57
cutlass::platform::conditional
std::conditional (true specialization)
Definition: platform.h:325
cutlass::gemm::device::DefaultGemmConfiguration< arch::OpClassTensorOp, arch::Sm70, ElementA, ElementB, ElementC, ElementAccumulator >::Operator
arch::OpMultiplyAdd Operator
Definition: default_gemm_configuration.h:177
cutlass::gemm::device::DefaultGemmConfiguration
Definition: default_gemm_configuration.h:57
arch.h
Defines tags for architecture-specific configurations.
wmma.h
Templates exposing architecture support for warp matrix multiply-add (WMMA) operations.
cutlass::gemm::device::DefaultGemmConfiguration< arch::OpClassTensorOp, arch::Sm75, uint8_t, uint8_t, ElementC, int32_t >::Operator
arch::OpMultiplyAddSaturate Operator
Definition: default_gemm_configuration.h:318
cutlass.h
Basic include for CUTLASS.
cutlass::gemm::device::DefaultGemmConfiguration< arch::OpClassTensorOp, arch::Sm75, int4b_t, int4b_t, ElementC, int32_t >::Operator
arch::OpMultiplyAddSaturate Operator
Definition: default_gemm_configuration.h:344
cutlass::gemm::device::DefaultGemmConfiguration< arch::OpClassTensorOp, arch::Sm75, int4b_t, uint4b_t, ElementC, int32_t >::Operator
arch::OpMultiplyAddSaturate Operator
Definition: default_gemm_configuration.h:370
cutlass::gemm::device::DefaultGemmConfiguration< arch::OpClassSimt, ArchTag, ElementA, ElementB, ElementC, ElementAccumulator >::Operator
arch::OpMultiplyAdd Operator
Definition: default_gemm_configuration.h:89