| CUTLASS CUDA Templates for Linear Algebra Subroutines and Solvers

- Main Page
- Modules
- Namespaces
- Classes
- Files

- File List
- File Members

mma_tensor_op_tile_iterator_sm70.h
Go to the documentation of this file.
1 /***************************************************************************************************
2 * Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without modification, are permitted
5 * provided that the following conditions are met:
6 * * Redistributions of source code must retain the above copyright notice, this list of
7 * conditions and the following disclaimer.
8 * * Redistributions in binary form must reproduce the above copyright notice, this list of
9 * conditions and the following disclaimer in the documentation and/or other materials
10 * provided with the distribution.
11 * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
12 * to endorse or promote products derived from this software without specific prior written
13 * permission.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
17 * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
19 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
20 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
21 * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
22 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23 *
24 **************************************************************************************************/
29 #pragma once
30
31 #include "cutlass/cutlass.h"
32
33 #include "cutlass/array.h"
34 #include "cutlass/numeric_types.h"
35 #include "cutlass/tensor_ref.h"
36 #include "cutlass/matrix_shape.h"
37
38 #include "cutlass/gemm/gemm.h"
39
40 #include "cutlass/layout/matrix.h"
41 #include "cutlass/layout/pitch_linear.h"
42 #include "cutlass/layout/tensor_op_multiplicand_sm70.h"
43
44 #include "cutlass/platform/platform.h"
45
47
48 namespace cutlass {
49 namespace gemm {
50 namespace warp {
51
53
54 template <
56 typename Shape_,
58 Operand Operand,
60 typename Element_,
62 typename Layout_,
64 typename InstructionShape_,
67 int OpDelta_,
69 int Threads>
70 class MmaVoltaTensorOpMultiplicandTileIterator;
71
73
79 template <
81 typename Shape_,
83 typename Element_,
85 typename InstructionShape_,
88 int OpDelta_>
89 class MmaVoltaTensorOpMultiplicandTileIterator<
90 Shape_, Operand::kA, Element_,
91 cutlass::layout::VoltaTensorOpMultiplicandCongruous<
92 sizeof_bits<Element_>::value>,
93 InstructionShape_, OpDelta_, 32> {
94 public:
95
97 using Shape = Shape_;
98
100 static Operand const kOperand = Operand::kA;
101
103 using Element = Element_;
104
106 using Layout = cutlass::layout::VoltaTensorOpMultiplicandCongruous<sizeof_bits<Element_>::value>;
107
109 using InstructionShape = InstructionShape_;
110
112 static int const kOpDelta = OpDelta_;
113
115 static int const kThreads = 32;
116
118 using TensorRef = TensorRef<Element, Layout>;
119
121 using Index = typename TensorRef::Index;
122
124 using LongIndex = typename TensorRef::LongIndex;
125
127 using TensorCoord = typename TensorRef::TensorCoord;
128
130 struct Policy {
131 static_assert(
132 !(Shape::kContiguous % InstructionShape::kContiguous),
133 "Shape of warp-level Mma must be divisible by operator shape.");
134
135 // Shape of one individual LDS.128
136 // TODO: 32 and 4 are hardcoded, 32-by-4 is logical shape
137 using LdsShape = layout::PitchLinearShape<
138 32,
139 4
140 >;
141
142 // LdsShapes are arranged in the strided direction in SMEM
143 using LdsIterations = layout::PitchLinearShape<
144 InstructionShape::kStrided / LdsShape::kStrided,
145 Shape::kContiguous / LdsShape::kContiguous
146 >;
147 };
148
149 private:
150
152 static_assert(kOpDelta == 1,
153 "Alternative arrangements not supported at present.");
154
156 static int const kPointerCount = 2;
157
159 using AccessType = AlignedArray<Element, Layout::kElementsPerAccess>;
160
161 public:
162
163 //
164 // Derived quantities
165 //
166
168 using Fragment = Array<Element, Shape::kCount / kThreads * 2>;
169
170 private:
171
173 Index stride_;
174
176 AccessType const *pointer_[kPointerCount];
177
179 Index byte_offset_;
180
181 public:
182
184 CUTLASS_HOST_DEVICE
185 MmaVoltaTensorOpMultiplicandTileIterator(): stride_(0), byte_offset_(0) { }
186
188 CUTLASS_DEVICE
189 MmaVoltaTensorOpMultiplicandTileIterator(
190 TensorRef const &ref,
191 int lane_id
192 ):
193 stride_(ref.stride(0) / Layout::kElementsPerAccess), byte_offset_(0) {
194 // swizzle patterns for operandA LDS are
195 // 1. (tid[4] << 3) | (tid[2:0] ^ tid[4])
196 // 2. (tid[4] << 3) | (tid[2:0] ^ tid[4] ^ 0b10010)
197
198 int vec_row = (lane_id >> 4); // tid[4]
199 int vec_col = ((lane_id & 4) >> 2); // tid[2]
200
201 CUTLASS_PRAGMA_UNROLL
202 for (int i = 0; i < kPointerCount; ++i) {
203
204 if(i == 1) {
205 vec_row |= 2;
206 }
207 int access_contiguous_idx = (vec_col << 2) | ((lane_id & 3) ^ vec_row);
208 int access_contiguous = access_contiguous_idx;
209
210 int access_strided = vec_row;
211 pointer_[i] = reinterpret_cast<AccessType const *>(ref.data()) +
212 access_contiguous + access_strided * stride_;
213 }
214
215 }
216
218 CUTLASS_DEVICE
219 MmaVoltaTensorOpMultiplicandTileIterator &add_pointer_offset(LongIndex offset) {
220
221 byte_offset_ += offset * sizeof(Element);
222
223 return *this;
224 }
225
227 CUTLASS_HOST_DEVICE
228 MmaVoltaTensorOpMultiplicandTileIterator &add_tile_offset(TensorCoord const &tile_offset) {
229
230 int contiguous_offset = tile_offset.contiguous();
231 int strided_offset = tile_offset.strided();
232
233 // To support 32x32 tile size
234 if (Shape::kContiguous == Policy::LdsShape::kContiguous) {
235 if (contiguous_offset % 2) {
236 AccessType const *tmp_pointer = pointer_[0];
237 pointer_[0] = pointer_[1];
238 pointer_[1] = tmp_pointer;
239 }
240 contiguous_offset = contiguous_offset / 2;
241 }
242
243 int offset = (strided_offset * InstructionShape::kStrided) * stride_ *
244 Layout::kElementsPerAccess +
245 contiguous_offset * Shape::kContiguous;
246
247 add_pointer_offset(offset);
248
249 return *this;
250 }
251
253 CUTLASS_DEVICE
254 MmaVoltaTensorOpMultiplicandTileIterator & operator++() {
255 byte_offset_ += stride_ * InstructionShape::kStrided * sizeof(Element) *
256 Layout::kElementsPerAccess;
257
258 return *this;
259 }
260
262 CUTLASS_HOST_DEVICE
263 MmaVoltaTensorOpMultiplicandTileIterator & operator--() {
264 byte_offset_ -= stride_ * InstructionShape::kStrided * sizeof(Element) *
265 Layout::kElementsPerAccess;
266
267 return *this;
268 }
269
271 CUTLASS_DEVICE
272 MmaVoltaTensorOpMultiplicandTileIterator & operator+=(TensorCoord const &tile_offset) {
273 add_tile_offset(tile_offset);
274 return *this;
275 }
276
278 CUTLASS_DEVICE
279 MmaVoltaTensorOpMultiplicandTileIterator & operator-=(TensorCoord const &tile_offset) {
280 add_tile_offset(-tile_offset);
281 return *this;
282 }
283
285 CUTLASS_HOST_DEVICE
286 void load(Fragment &frag) const {
287
288 load_with_byte_offset(frag, 0);
289 }
290
292 CUTLASS_DEVICE
293 void load_with_byte_offset(
295 Fragment &frag,
297 Index byte_offset) const {
298
299 AccessType * fetch_ptr = reinterpret_cast<AccessType *>(&frag);
300
301 CUTLASS_PRAGMA_UNROLL
302 for (int s = 0; s < Policy::LdsIterations::kStrided; ++s) {
303
304 CUTLASS_PRAGMA_UNROLL
305 for (int c = 0; c < Policy::LdsIterations::kContiguous; ++c) {
306
307 int access_idx = c + s * Policy::LdsIterations::kContiguous;
308
309 AccessType const *source_ptr = pointer_[s & 1] +
310 Policy::LdsShape::kContiguous * c +
311 Policy::LdsShape::kStrided * (s / 2) * stride_;
312
313 char const *source_byte_ptr = reinterpret_cast<char const *>(source_ptr) + byte_offset + byte_offset_;
314 fetch_ptr[access_idx] = *(reinterpret_cast<AccessType const*> (source_byte_ptr));
315 }
316 }
317 }
318
320 CUTLASS_DEVICE
321 void load_with_pointer_offset(
323 Fragment &frag,
325 Index pointer_offset) const {
326 load_with_byte_offset(frag, pointer_offset * sizeof(Element));
327 }
328
330 CUTLASS_DEVICE
331 void load(
333 Fragment &frag,
335 TensorCoord const &tile_offset) const {
336 load_with_byte_offset(frag, tile_offset, 0);
337 }
338
340 CUTLASS_DEVICE
341 void load(
343 Fragment &frag,
345 TensorCoord const &tile_offset,
347 Index pointer_offset) const {
348 load_with_byte_offset(frag, tile_offset, pointer_offset * sizeof(Element));
349 }
350
352 CUTLASS_DEVICE
353 void load_with_byte_offset(
355 Fragment &frag,
357 TensorCoord const &tile_offset,
359 Index byte_offset) const {
360 Index pointer_offset =
361 tile_offset.contiguous() * Shape::kContiguous /
362 Layout::kElementsPerAccess +
363 tile_offset.strided() * InstructionShape::kStrided * stride_;
364
365 byte_offset += sizeof(AccessType) * pointer_offset;
366
367 load_with_byte_offset(frag, byte_offset);
368 }
369
377 CUTLASS_DEVICE
378 void set_kgroup_index(int k_group) {
379 // no operation here
380 }
381 };
382
384
390 template <
392 typename Shape_,
394 typename Element_,
396 typename InstructionShape_,
399 int OpDelta_>
400
401 class MmaVoltaTensorOpMultiplicandTileIterator<
402 Shape_, Operand::kB, Element_,
403 cutlass::layout::VoltaTensorOpMultiplicandBCongruous<
404 sizeof_bits<Element_>::value>,
405 InstructionShape_, OpDelta_, 32> {
406 public:
407
409 using Shape = Shape_;
410
412 static Operand const kOperand = Operand::kB;
413
415 using Element = Element_;
416
418 using Layout = cutlass::layout::VoltaTensorOpMultiplicandBCongruous<sizeof_bits<Element_>::value>;
419
421 using InstructionShape = InstructionShape_;
422
424 static int const kOpDelta = OpDelta_;
425
427 static int const kThreads = 32;
428
430 using TensorRef = TensorRef<Element, Layout>;
431
433 using Index = typename TensorRef::Index;
434
436 using LongIndex = typename TensorRef::LongIndex;
437
439 using TensorCoord = typename TensorRef::TensorCoord;
440
442 struct Policy {
443 static_assert(
444 !(Shape::kContiguous % InstructionShape::kContiguous),
445 "Shape of warp-level Mma must be divisible by operator shape.");
446
447 // Shape of one individual LDS
448 // TODO: remove hardcoded 32 and 4
449 using LdsShape = layout::PitchLinearShape<
450 32,
451 4
452 >;
453
454 using LdsIterations = layout::PitchLinearShape<
455 Shape::kContiguous / LdsShape::kContiguous,
456 InstructionShape::kStrided / LdsShape::kStrided
457 >;
458 };
459
460 private:
461
463 static_assert(kOpDelta == 1,
464 "Alternative arrangements not supported at present.");
465
467 using AccessType = AlignedArray<Element, Layout::kElementsPerAccess>;
468
469 public:
470
471 //
472 // Derived quantities
473 //
474
476 using Fragment = Array<Element, Shape::kCount / kThreads * 2>;
477
478 private:
479
481 Index stride_;
482
484 AccessType const *pointer_;
485
487 Index byte_offset_;
488
489 public:
490
492 CUTLASS_HOST_DEVICE
493 MmaVoltaTensorOpMultiplicandTileIterator(): stride_(0), byte_offset_(0) { }
494
496 CUTLASS_DEVICE
497 MmaVoltaTensorOpMultiplicandTileIterator(
498 TensorRef const &ref,
499 int lane_id
500 ):
501 stride_(ref.stride(0) / Layout::kElementsPerAccess), byte_offset_(0) {
502
503 // swizzle pattern is (tid & (3 << 3) | (tid[1:0] ^ tid[4:3]))
504 int access_strided = (lane_id >> 3) & 0x3;
505 int access_contiguous = ((lane_id ^ (lane_id >> 3)) & 0x3);
506
507 pointer_ = reinterpret_cast<AccessType const *>(ref.data()) +
508 access_contiguous + access_strided * stride_;
509
510 }
511
513 CUTLASS_DEVICE
514 MmaVoltaTensorOpMultiplicandTileIterator &add_pointer_offset(LongIndex offset) {
515
516 byte_offset_ += offset * sizeof(Element);
517
518 return *this;
519 }
520
522 CUTLASS_HOST_DEVICE
523 MmaVoltaTensorOpMultiplicandTileIterator &add_tile_offset(TensorCoord const &tile_offset) {
524
525 int contiguous_offset = tile_offset.contiguous();
526 int strided_offset = tile_offset.strided();
527
528 int offset = (strided_offset * InstructionShape::kStrided) * stride_ *
529 Layout::kElementsPerAccess +
530 contiguous_offset * Shape::kContiguous;
531
532 add_pointer_offset(offset);
533
534 return *this;
535 }
536
538 CUTLASS_DEVICE
539 MmaVoltaTensorOpMultiplicandTileIterator & operator++() {
540 byte_offset_ += stride_ * InstructionShape::kStrided * sizeof(Element) *
541 Layout::kElementsPerAccess;
542
543 return *this;
544 }
545
547 CUTLASS_HOST_DEVICE
548 MmaVoltaTensorOpMultiplicandTileIterator & operator--() {
549 byte_offset_ += stride_ * InstructionShape::kStrided * sizeof(Element) *
550 Layout::kElementsPerAccess;
551
552 return *this;
553 }
554
556 CUTLASS_DEVICE
557 MmaVoltaTensorOpMultiplicandTileIterator & operator+=(TensorCoord const &tile_offset) {
558 add_tile_offset(tile_offset);
559 return *this;
560 }
561
563 CUTLASS_DEVICE
564 MmaVoltaTensorOpMultiplicandTileIterator & operator-=(TensorCoord const &tile_offset) {
565 add_tile_offset(-tile_offset);
566 return *this;
567 }
568
570 CUTLASS_HOST_DEVICE
571 void load(Fragment &frag) const {
572
573 load_with_byte_offset(frag, 0);
574 }
575
577 CUTLASS_DEVICE
578 void load_with_byte_offset(
580 Fragment &frag,
582 Index byte_offset) const {
583
584 AccessType * fetch_ptr = reinterpret_cast<AccessType *>(&frag);
585
586 CUTLASS_PRAGMA_UNROLL
587 for (int s = 0; s < Policy::LdsIterations::kStrided; ++s) {
588
589 CUTLASS_PRAGMA_UNROLL
590 for (int c = 0; c < Policy::LdsIterations::kContiguous; ++c) {
591
592 int access_idx = c + s * Policy::LdsIterations::kContiguous;
593
594 AccessType const *source_ptr = pointer_ +
595 Policy::LdsShape::kContiguous / Layout::kElementsPerAccess * c +
596 Policy::LdsShape::kStrided * s * stride_;
597
598 char const *source_byte_ptr = reinterpret_cast<char const *>(source_ptr) + byte_offset + byte_offset_;
599 fetch_ptr[access_idx] = *(reinterpret_cast<AccessType const*> (source_byte_ptr));
600 }
601 }
602 }
603
605 CUTLASS_DEVICE
606 void load_with_pointer_offset(
608 Fragment &frag,
610 Index pointer_offset) const {
611 load_with_byte_offset(frag, pointer_offset * sizeof(Element));
612 }
613
615 CUTLASS_DEVICE
616 void load(
618 Fragment &frag,
620 TensorCoord const &tile_offset) const {
621 load_with_byte_offset(frag, tile_offset, 0);
622 }
623
625 CUTLASS_DEVICE
626 void load(
628 Fragment &frag,
630 TensorCoord const &tile_offset,
632 Index pointer_offset) const {
633 load_with_byte_offset(frag, tile_offset, pointer_offset * sizeof(Element));
634 }
635
637 CUTLASS_DEVICE
638 void load_with_byte_offset(
640 Fragment &frag,
642 TensorCoord const &tile_offset,
644 Index byte_offset) const {
645 Index pointer_offset =
646 tile_offset.contiguous() * Shape::kContiguous /
647 Layout::kElementsPerAccess +
648 tile_offset.strided() * InstructionShape::kStrided * stride_;
649
650 byte_offset += sizeof(AccessType) * pointer_offset;
651
652 load_with_byte_offset(frag, byte_offset);
653 }
654
662 CUTLASS_DEVICE
663 void set_kgroup_index(int k_group) {
664 // no operation here
665 }
666 };
667
669
676 template <
678 typename Shape_,
680 typename Element_,
682 typename InstructionShape_,
685 int OpDelta_>
686 class MmaVoltaTensorOpMultiplicandTileIterator<
687 Shape_, Operand::kA, Element_,
688 cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous<
689 sizeof_bits<Element_>::value>,
690 InstructionShape_, OpDelta_, 32> {
691 public:
692
694 using Shape = Shape_;
695
697 static Operand const kOperand = Operand::kA;
698
700 using Element = Element_;
701
703 using Layout = cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous<sizeof_bits<Element_>::value>;
704
706 using InstructionShape = InstructionShape_;
707
709 static int const kOpDelta = OpDelta_;
710
712 static int const kThreads = 32;
713
715 using TensorRef = TensorRef<Element, Layout>;
716
718 using Index = typename TensorRef::Index;
719
721 using LongIndex = typename TensorRef::LongIndex;
722
724 using TensorCoord = typename TensorRef::TensorCoord;
725
727 using Base = MmaVoltaTensorOpMultiplicandTileIterator<
728 layout::PitchLinearShape<Shape::kRow, Shape::kColumn>, kOperand, Element,
729 layout::VoltaTensorOpMultiplicandCongruous<sizeof_bits<Element_>::value>,
730 layout::PitchLinearShape<InstructionShape::kRow,
731 InstructionShape::kColumn>,
732 kOpDelta, kThreads>;
733
734 public:
735
736 //
737 // Derived quantities
738 //
739
741 using Fragment = Array<Element, Shape::kCount / kThreads * 2>;
742
743 private:
744
746 Base iterator_;
747
748 public:
749
751 CUTLASS_HOST_DEVICE
752 MmaVoltaTensorOpMultiplicandTileIterator() { }
753
755 CUTLASS_HOST_DEVICE
756 MmaVoltaTensorOpMultiplicandTileIterator(
757 TensorRef const &ref,
758 int lane_id
759 ): iterator_({ref.data(), ref.stride()}, lane_id) {
760 }
761
763 CUTLASS_HOST_DEVICE
764 MmaVoltaTensorOpMultiplicandTileIterator &add_pointer_offset(LongIndex offset) {
765
766 iterator_.add_pointer_offset(offset);
767
768 return *this;
769 }
770
772 CUTLASS_HOST_DEVICE
773 MmaVoltaTensorOpMultiplicandTileIterator &add_tile_offset(TensorCoord const &tile_offset) {
774
775 iterator_.add_tile_offset({tile_offset.row(), tile_offset.column()});
776
777 return *this;
778 }
779
781 CUTLASS_HOST_DEVICE
782 MmaVoltaTensorOpMultiplicandTileIterator & operator++() {
783
784 ++iterator_;
785
786 return *this;
787 }
788
790 CUTLASS_HOST_DEVICE
791 MmaVoltaTensorOpMultiplicandTileIterator & operator--() {
792
793 --iterator_;
794
795 return *this;
796 }
797
799 CUTLASS_DEVICE
800 MmaVoltaTensorOpMultiplicandTileIterator & operator+=(TensorCoord const &tile_offset) {
801 add_tile_offset(PitchLinearCoord(tile_offset.row(), tile_offset.column()));
802 return *this;
803 }
804
806 CUTLASS_DEVICE
807 MmaVoltaTensorOpMultiplicandTileIterator & operator-=(TensorCoord const &tile_offset) {
808 add_tile_offset(-PitchLinearCoord(tile_offset.row(), tile_offset.column()));
809 return *this;
810 }
811
813 CUTLASS_HOST_DEVICE
814 void load(Fragment &frag) const {
815
816 iterator_.load(frag);
817 }
818
820 CUTLASS_DEVICE
821 void load_with_pointer_offset(
823 Fragment &frag,
825 Index pointer_offset) const {
826 iterator_.load_with_pointer_offset(frag, pointer_offset);
827 }
828
830 CUTLASS_DEVICE
831 void load_with_byte_offset(
833 Fragment &frag,
835 Index byte_offset) const {
836 iterator_.load_with_byte_offset(frag, byte_offset);
837 }
838
840 CUTLASS_DEVICE
841 void load(
843 Fragment &frag,
845 TensorCoord const &tile_offset) const {
846 // TODO
847 }
848
850 CUTLASS_DEVICE
851 void load(
853 Fragment &frag,
855 TensorCoord const &tile_offset,
857 Index pointer_offset) const {
858 // TODO
859 }
860
862 CUTLASS_DEVICE
863 void load_with_byte_offset(
865 Fragment &frag,
867 TensorCoord const &tile_offset,
869 Index byte_offset) const {
870 iterator_.load_with_byte_offset(
871 frag,
872 {tile_offset.contiguous(), tile_offset.strided()},
873 byte_offset);
874 }
875
883 CUTLASS_DEVICE
884 void set_kgroup_index(int k_group) {
885 iterator_.set_kgroup_index(k_group);
886 }
887 };
888
890
897 template <
899 typename Shape_,
901 typename Element_,
903 typename InstructionShape_,
906 int OpDelta_>
907 class MmaVoltaTensorOpMultiplicandTileIterator<
908 Shape_, Operand::kB, Element_,
909 cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous<
910 sizeof_bits<Element_>::value>,
911 InstructionShape_, OpDelta_, 32> {
912 public:
913
915 using Shape = Shape_;
916
918 static Operand const kOperand = Operand::kB;
919
920 static_assert(kOperand == Operand::kA || kOperand== Operand::kB,
921 "MmaTensorOpMultiplicandIterator may only be instantiated for A or B operands to warp-level Mma.");
922
924 using Element = Element_;
925
927 using Layout = cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous<sizeof_bits<Element_>::value>;
928
930 using InstructionShape = InstructionShape_;
931
933 static int const kOpDelta = OpDelta_;
934
936 static int const kThreads = 32;
937
939 using TensorRef = TensorRef<Element, Layout>;
940
942 using Index = typename TensorRef::Index;
943
945 using LongIndex = typename TensorRef::LongIndex;
946
948 using TensorCoord = typename TensorRef::TensorCoord;
949
951 using Base = MmaVoltaTensorOpMultiplicandTileIterator<
952 layout::PitchLinearShape<Shape::kColumn, Shape::kRow>, kOperand, Element,
953 layout::VoltaTensorOpMultiplicandBCongruous<sizeof_bits<Element_>::value>,
954 layout::PitchLinearShape<InstructionShape::kColumn,
955 InstructionShape::kRow>,
956 kOpDelta, kThreads>;
957
958 public:
959
960 //
961 // Derived quantities
962 //
963
965 using Fragment = Array<Element, Shape::kCount / kThreads * 2>;
966
967 private:
968
970 Base iterator_;
971
972 public:
973
975 CUTLASS_HOST_DEVICE
976 MmaVoltaTensorOpMultiplicandTileIterator() { }
977
979 CUTLASS_HOST_DEVICE
980 MmaVoltaTensorOpMultiplicandTileIterator(
981 TensorRef const &ref,
982 int lane_id
983 ): iterator_({ref.data(), ref.stride()}, lane_id) {
984 }
985
987 CUTLASS_HOST_DEVICE
988 MmaVoltaTensorOpMultiplicandTileIterator &add_pointer_offset(LongIndex offset) {
989
990 iterator_.add_pointer_offset(offset);
991
992 return *this;
993 }
994
996 CUTLASS_HOST_DEVICE
997 MmaVoltaTensorOpMultiplicandTileIterator &add_tile_offset(TensorCoord const &tile_offset) {
998
999 iterator_.add_tile_offset({tile_offset.column(), tile_offset.row()});
1000
1001 return *this;
1002 }
1003
1005 CUTLASS_HOST_DEVICE
1006 MmaVoltaTensorOpMultiplicandTileIterator & operator++() {
1007
1008 ++iterator_;
1009
1010 return *this;
1011 }
1012
1014 CUTLASS_HOST_DEVICE
1015 MmaVoltaTensorOpMultiplicandTileIterator & operator--() {
1016
1017 --iterator_;
1018
1019 return *this;
1020 }
1021
1023 CUTLASS_DEVICE
1024 MmaVoltaTensorOpMultiplicandTileIterator & operator+=(TensorCoord const &tile_offset) {
1025 add_tile_offset(PitchLinearCoord(tile_offset.column(), tile_offset.row()));
1026 return *this;
1027 }
1028
1030 CUTLASS_DEVICE
1031 MmaVoltaTensorOpMultiplicandTileIterator & operator-=(TensorCoord const &tile_offset) {
1032 add_tile_offset(-PitchLinearCoord(tile_offset.column(), tile_offset.row()));
1033 return *this;
1034 }
1035
1037 CUTLASS_HOST_DEVICE
1038 void load(Fragment &frag) const {
1039
1040 iterator_.load(frag);
1041 }
1042
1044 CUTLASS_DEVICE
1045 void load_with_pointer_offset(
1047 Fragment &frag,
1049 Index pointer_offset) const {
1050 iterator_.load_with_pointer_offset(frag, pointer_offset);
1051 }
1052
1054 CUTLASS_DEVICE
1055 void load_with_byte_offset(
1057 Fragment &frag,
1059 Index byte_offset) const {
1060 iterator_.load_with_byte_offset(frag, byte_offset);
1061 }
1062
1064 CUTLASS_DEVICE
1065 void load(
1067 Fragment &frag,
1069 TensorCoord const &tile_offset) const {
1070 // TODO
1071 }
1072
1074 CUTLASS_DEVICE
1075 void load(
1077 Fragment &frag,
1079 TensorCoord const &tile_offset,
1081 Index pointer_offset) const {
1082 // TODO
1083 }
1084
1086 CUTLASS_DEVICE
1087 void load_with_byte_offset(
1089 Fragment &frag,
1091 TensorCoord const &tile_offset,
1093 Index byte_offset) const {
1094 iterator_.load_with_byte_offset(
1095 frag,
1096 {tile_offset.strided(), tile_offset.contiguous()},
1097 byte_offset);
1098 }
1099
1107 CUTLASS_DEVICE
1108 void set_kgroup_index(int k_group) {
1109 iterator_.set_kgroup_index(k_group);
1110 }
1111 };
1112
1114
1123 template <
1125 typename Shape_,
1127 typename Element_,
1129 typename Layout_,
1131 typename InstructionShape_,
1134 typename OpDelta_>
1135 class MmaVoltaTensorOpAccumulatorTileIterator {
1136 public:
1137
1139 using Shape = Shape_;
1140
1142 static Operand const kOperand = Operand::kC;
1143
1145 using Element = Element_;
1146
1148 using Layout = Layout_;
1149
1151 using InstructionShape = InstructionShape_;
1152
1154 using OpDelta = OpDelta_;
1155
1157 static int const kThreads = 32;
1158
1160 using TensorRef = TensorRef<Element, Layout>;
1161
1163 using Index = typename TensorRef::Index;
1164
1166 using LongIndex = typename TensorRef::LongIndex;
1167
1169 using TensorCoord = typename TensorRef::TensorCoord;
1170
1172 struct Policy {
1173
1175 using InterleavedTile = MatrixShape<32, 32>;
1176
1177 static_assert(!(Shape::kRow % InterleavedTile::kRow) && !(Shape::kColumn % InterleavedTile::kColumn),
1178 "Shape of warp-level Mma must be divisible by operator shape.");
1179
1180 static_assert(platform::is_same<TensorCoord, MatrixCoord>::value,
1181 "Layouts must be defined for logical MatrixCoord coordinate space.");
1182
1184 using TileIterations = MatrixShape<
1185 Shape::kRow / InterleavedTile::kRow,
1186 Shape::kColumn / InterleavedTile::kColumn
1187 >;
1188
1189 using MmaIterations =
1190 MatrixShape<InterleavedTile::kRow / InstructionShape::kM,
1191 InterleavedTile::kColumn / InstructionShape::kN>;
1192 };
1193
1194 private:
1195
1196 // Assume accumulator tile is multipile interleaved 32x32 tile.
1197 static int const kElementsPerPartial = 4;
1198 using EleShapePerPatial = typename platform::conditional<
1199 platform::is_same<Element, float>::value,
1200 MatrixShape<2, 2>,
1201 MatrixShape<1, 4> >::type;
1202 static int const kElementsPerMma = 8;
1203 static int const kAccumulatorPatials = 2;
1204 using QuadShapePerPatialMma = MatrixShape<4, 4>;
1205
1206 public:
1207
1208 //
1209 // Derived quantities
1210 //
1211
1213 using Fragment = Array<Element, Shape::kCount / kThreads>;
1214
1215 private:
1216
1218 TensorRef ref_;
1219
1220 public:
1221
1223 CUTLASS_HOST_DEVICE
1224 MmaVoltaTensorOpAccumulatorTileIterator() { }
1225
1227 CUTLASS_HOST_DEVICE
1228 MmaVoltaTensorOpAccumulatorTileIterator(
1229 TensorRef const &ref,
1230 int lane_id
1231 ):
1232 ref_(ref) {
1233
1234 int quad = (lane_id >> 2);
1235 int lane_in_quad = (lane_id & 3);
1236 int accum_m, accum_n;
1237
1238 if (platform::is_same<Element, float>::value) {
1239 // (quad[2],quad[0])+lane_in_quad[0]
1240 accum_m = (((quad & 0x4) >> 1) + (quad & 0x1)) * 8 + (lane_in_quad & 1);
1241 // (quad[1])+lane_in_quad[1]
1242 accum_n = ((quad >> 1) & 0x1) * kElementsPerPartial * kAccumulatorPatials +
1243 (lane_in_quad & 2);
1244 } else {
1245 accum_m = (((quad & 0x4) >> 1) + (quad & 0x1)) * 8 + lane_in_quad; // (quad[2],quad[0])
1246 accum_n = ((quad >> 1) & 0x1) * kElementsPerPartial * kAccumulatorPatials;
1247 }
1248 MatrixCoord lane_offset(accum_m, accum_n);
1249
1250 ref_.add_coord_offset(lane_offset);
1251 }
1252
1254 CUTLASS_HOST_DEVICE
1255 MmaVoltaTensorOpAccumulatorTileIterator &add_pointer_offset(LongIndex offset) {
1256 ref_.add_pointer_offset(offset);
1257 return *this;
1258 }
1259
1261 CUTLASS_HOST_DEVICE
1262 MmaVoltaTensorOpAccumulatorTileIterator &add_tile_offset(TensorCoord const &tile_offset) {
1263
1264 ref_.add_coord_offset(tile_offset * make_Coord(Shape::kRow, Shape::kColumn));
1265
1266 return *this;
1267 }
1268
1270 CUTLASS_HOST_DEVICE
1271 MmaVoltaTensorOpAccumulatorTileIterator & operator++() {
1272 // deliberate no-op
1273 return *this;
1274 }
1275
1277 CUTLASS_HOST_DEVICE
1278 MmaVoltaTensorOpAccumulatorTileIterator & operator--() {
1279 // deliberate no-op
1280 return *this;
1281 }
1282
1284 CUTLASS_DEVICE
1285 MmaVoltaTensorOpAccumulatorTileIterator & operator+=(TensorCoord const &tile_offset) {
1286 add_tile_offset(tile_offset);
1287 return *this;
1288 }
1289
1291 CUTLASS_DEVICE
1292 MmaVoltaTensorOpAccumulatorTileIterator & operator-=(TensorCoord const &tile_offset) {
1293 add_tile_offset(-tile_offset);
1294 return *this;
1295 }
1296
1298 CUTLASS_HOST_DEVICE
1299 void load(Fragment &frag) const {
1300 load_with_pointer_offset(frag, 0);
1301 }
1302
1304 CUTLASS_HOST_DEVICE
1305 void load_with_pointer_offset(
1306 Fragment &frag,
1307 Index pointer_offset) const {
1308
1309 TensorRef offset_ref(ref_);
1310 offset_ref.add_pointer_offset(pointer_offset);
1311
1312 CUTLASS_PRAGMA_UNROLL
1313 for (int tile_n = 0; tile_n < Policy::TileIterations::kColumn; ++tile_n) {
1314 CUTLASS_PRAGMA_UNROLL
1315 for (int tile_m = 0; tile_m < Policy::TileIterations::kRow; ++tile_m) {
1316 CUTLASS_PRAGMA_UNROLL
1317 for (int mma_n = 0; mma_n < Policy::MmaIterations::kColumn; ++mma_n) {
1318 CUTLASS_PRAGMA_UNROLL
1319 for (int mma_m = 0; mma_m < Policy::MmaIterations::kRow; ++mma_m) {
1320
1321 int mma_accum_start =
1322 (((tile_n * Policy::TileIterations::kRow + tile_m) *
1323 Policy::MmaIterations::kColumn + mma_n) *
1324 Policy::MmaIterations::kRow + mma_m) *
1325 kElementsPerMma;
1326
1327 CUTLASS_PRAGMA_UNROLL
1328 for (int p = 0; p < kAccumulatorPatials; ++p) {
1329 CUTLASS_PRAGMA_UNROLL
1330 for (int m = 0; m < EleShapePerPatial::kRow; ++m) {
1331 CUTLASS_PRAGMA_UNROLL
1332 for (int n = 0; n < EleShapePerPatial::kColumn; ++n) {
1333 int accum_m = tile_m * Policy::InterleavedTile::kRow +
1334 mma_m * QuadShapePerPatialMma::kRow + m * 2;
1335 int accum_n = tile_n * Policy::InterleavedTile::kColumn +
1336 mma_n * QuadShapePerPatialMma::kColumn +
1337 p * Policy::InterleavedTile::kColumn/2 + n;
1338 int idx = mma_accum_start + p * kElementsPerPartial +
1339 m * EleShapePerPatial::kColumn + n;
1340 frag[idx] = offset_ref.at({accum_m, accum_n});
1341 }
1342 }
1343 }
1344 }
1345 }
1346 }
1347 }
1348 }
1350 CUTLASS_DEVICE
1351 void load_with_byte_offset(
1352 Fragment &frag,
1353 Index byte_offset) const {
1354
1355 load_with_pointer_offset(byte_offset / sizeof(Element));
1356 }
1357
1359 CUTLASS_HOST_DEVICE
1360 void load(
1361 Fragment &frag,
1362 TensorCoord const &tile_offset) const {
1363
1364 load(frag, tile_offset, 0);
1365 }
1366
1368 CUTLASS_HOST_DEVICE
1369 void load(
1370 Fragment &frag,
1371 TensorCoord const &tile_offset,
1372 Index pointer_offset) const {
1373
1374 load_with_pointer_offset(frag, ref_.offset(tile_offset) + pointer_offset);
1375 }
1376
1378 CUTLASS_HOST_DEVICE
1379 void store(Fragment const &frag) const {
1380 store_with_pointer_offset(frag, 0);
1381 }
1382
1384 CUTLASS_HOST_DEVICE
1385 void store_with_pointer_offset(
1386 Fragment const &frag,
1387 Index pointer_offset) const {
1388
1389 TensorRef offset_ref(ref_);
1390 offset_ref.add_pointer_offset(pointer_offset);
1391
1392 CUTLASS_PRAGMA_UNROLL
1393 for (int tile_n = 0; tile_n < Policy::TileIterations::kColumn; ++tile_n) {
1394 CUTLASS_PRAGMA_UNROLL
1395 for (int tile_m = 0; tile_m < Policy::TileIterations::kRow; ++tile_m) {
1396 CUTLASS_PRAGMA_UNROLL
1397 for (int mma_n = 0; mma_n < Policy::MmaIterations::kColumn; ++mma_n) {
1398 CUTLASS_PRAGMA_UNROLL
1399 for (int mma_m = 0; mma_m < Policy::MmaIterations::kRow; ++mma_m) {
1400
1401 int mma_accum_start =
1402 (((tile_n * Policy::TileIterations::kRow + tile_m) *
1403 Policy::MmaIterations::kColumn + mma_n) *
1404 Policy::MmaIterations::kRow + mma_m) *
1405 kElementsPerMma;
1406
1407 CUTLASS_PRAGMA_UNROLL
1408 for (int p = 0; p < kAccumulatorPatials; ++p) {
1409 CUTLASS_PRAGMA_UNROLL
1410 for (int m = 0; m < EleShapePerPatial::kRow; ++m) {
1411 CUTLASS_PRAGMA_UNROLL
1412 for (int n = 0; n < EleShapePerPatial::kColumn; ++n) {
1413 int accum_m = tile_m * Policy::InterleavedTile::kRow +
1414 mma_m * QuadShapePerPatialMma::kRow + m * 2;
1415 int accum_n = tile_n * Policy::InterleavedTile::kColumn +
1416 mma_n * QuadShapePerPatialMma::kColumn +
1417 p * Policy::InterleavedTile::kColumn/2 + n;
1418 int idx = mma_accum_start + p * kElementsPerPartial +
1419 m * EleShapePerPatial::kColumn + n;
1420 offset_ref.at({accum_m, accum_n}) = frag[idx];
1421 }
1422 }
1423 }
1424 }
1425 }
1426 }
1427 }
1428 }
1429
1431 CUTLASS_HOST_DEVICE
1432 void store_with_byte_offset(
1433 Fragment const &frag,
1434 Index byte_offset) const {
1435
1436 store_with_pointer_offset(byte_offset / sizeof(Element));
1437 }
1438
1440 CUTLASS_HOST_DEVICE
1441 void store(
1442 Fragment &frag,
1443 TensorCoord const &tile_offset) const {
1444
1445 store(frag, tile_offset, 0);
1446 }
1447
1449 CUTLASS_HOST_DEVICE
1450 void store(
1452 Fragment const &frag,
1454 TensorCoord const &tile_offset,
1456 Index pointer_offset) const {
1457 store_with_pointer_offset(frag, ref_.offset(tile_offset) + pointer_offset);
1458 }
1459 };
1460
1468 template <
1470 typename Shape_,
1472 Operand Operand_,
1474 typename Element_,
1476 typename InstructionShape_,
1479 int OpDelta_,
1481 int KBlock>
1482 class MmaVoltaTensorOpMultiplicandTileIterator<
1483 Shape_, Operand_, Element_,
1484 cutlass::layout::VoltaTensorOpMultiplicandCrosswise<
1485 sizeof_bits<Element_>::value, KBlock>,
1486 InstructionShape_, OpDelta_, 32> {
1487 public:
1489 using Shape = Shape_;
1490
1492 static Operand const kOperand = Operand_;
1493
1494 static_assert(kOperand == Operand::kA || kOperand == Operand::kB,
1495 "MmaVoltaTensorOpMultiplicandIterator may only be instantiated for "
1496 "A or B operands to warp-level Mma.");
1497
1499 using Element = Element_;
1500
1502 static int const kKBlock = KBlock;
1503
1505 using Layout = cutlass::layout::VoltaTensorOpMultiplicandCrosswise<
1506 sizeof_bits<Element_>::value, kKBlock>;
1507
1509 using InstructionShape = InstructionShape_;
1510
1513 static int const kOpDelta = OpDelta_;
1514
1516 static int const kThreads = 32;
1517
1519 using TensorRef = TensorRef<Element, Layout>;
1520
1522 using Index = typename TensorRef::Index;
1523
1525 using LongIndex = typename TensorRef::LongIndex;
1526
1528 using TensorCoord = typename TensorRef::TensorCoord;
1529
1531 struct Policy {
1532
1534 using LdsShape = layout::PitchLinearShape<1, 32>;
1535
1537 using LdsIterations = layout::PitchLinearShape<1, Shape::kStrided / 32>;
1538
1540 static int const kElementsPerAccess = 8;
1541
1543 static int const kContiguousElementsPerLine = 4;
1544 };
1545
1546 private:
1548 static_assert(kOpDelta == 1,
1549 "Alternative arrangements not supported at present.");
1550
1552 using AccessType = AlignedArray<Element, Policy::kElementsPerAccess>;
1553
1554 public:
1555 //
1556 // Derived quantities
1557 //
1558
1560 using Fragment = Array<Element, Shape::kCount / kThreads * 2>;
1561
1562 private:
1563
1565 Index stride_;
1566
1568 AccessType const *pointer_;
1569
1571 Index byte_offset_;
1572
1575 Index line_size;
1576
1579 int k_group_idx_;
1580
1581 public:
1583 CUTLASS_HOST_DEVICE
1584 MmaVoltaTensorOpMultiplicandTileIterator()
1585 : pointer_(nullptr),
1586 stride_(0),
1587 line_size(0),
1588 byte_offset_(0),
1589 k_group_idx_(0) {}
1590
1592 CUTLASS_DEVICE
1593 MmaVoltaTensorOpMultiplicandTileIterator(TensorRef const &ref, int lane_id)
1594 : pointer_(reinterpret_cast<AccessType const *>(ref.data())),
1595 stride_(ref.stride(0) * Policy::kElementsPerAccess),
1596 line_size((ref.stride(0) * Policy::kContiguousElementsPerLine) /
1597 Policy::kElementsPerAccess),
1598 k_group_idx_(0),
1599 byte_offset_(0) {
1600
1601 int quad = (lane_id / 4);
1602 int lane_in_quad = (lane_id % 4);
1603 int access_contiguous;
1604
1605 if(kOperand == Operand::kA) {
1606
1607 // swizzle id: tid[4]|tid[1:0]|(tid[2]^tid[4])
1608 access_contiguous = ((quad & 0x4) << 1) + ((lane_in_quad) << 1) +
1609 ((quad & 0x1) ^ ((quad & 0x4) >> 2));
1610 } else {
1611
1612 // swizzle id: tid[4]|tid[1:0]|tid[3]
1613 access_contiguous = ((quad & 0x4) << 1) + (lane_in_quad << 1) +
1614 ((quad & 0x2) >> 1 ^ ((quad & 0x4) >> 2));
1615 }
1616
1617 byte_offset_ = access_contiguous *
1618 sizeof(Element) * Policy::kElementsPerAccess;
1619 }
1620
1622 CUTLASS_DEVICE
1623 MmaVoltaTensorOpMultiplicandTileIterator &add_pointer_offset(LongIndex offset) {
1624 byte_offset_ += offset * sizeof(Element);
1625
1626 return *this;
1627 }
1628
1631 CUTLASS_DEVICE
1632 MmaVoltaTensorOpMultiplicandTileIterator &add_tile_offset(
1633 TensorCoord const &tile_offset) {
1634
1635 int contiguous_offset = tile_offset.contiguous();
1636 int strided_offset = tile_offset.strided();
1637 k_group_idx_ = 0;
1638
1639 pointer_ += contiguous_offset *
1640 (InstructionShape::kContiguous /
1641 Policy::kContiguousElementsPerLine) *
1642 line_size +
1643 strided_offset * Shape::kStrided / 2;
1644 return *this;
1645 }
1646
1648 CUTLASS_DEVICE
1649 MmaVoltaTensorOpMultiplicandTileIterator &operator++() {
1650 k_group_idx_ = (k_group_idx_ + 1) % 8;
1651
1652 if (k_group_idx_ == 4 || k_group_idx_ == 0) {
1653 byte_offset_ ^= 1 * sizeof(Element) * Policy::kElementsPerAccess;
1654 }
1655
1656 pointer_ += line_size;
1657 return *this;
1658 }
1659
1661 CUTLASS_HOST_DEVICE
1662 MmaVoltaTensorOpMultiplicandTileIterator &operator--() { assert(0); }
1663
1666 CUTLASS_DEVICE
1667 MmaVoltaTensorOpMultiplicandTileIterator &operator+=(
1668 TensorCoord const &tile_offset) {
1669 add_tile_offset(tile_offset);
1670 return *this;
1671 }
1672
1675 CUTLASS_DEVICE
1676 MmaVoltaTensorOpMultiplicandTileIterator &operator-=(
1677 TensorCoord const &tile_offset) {
1678 add_tile_offset(-tile_offset);
1679 return *this;
1680 }
1681
1683 CUTLASS_HOST_DEVICE
1684 void load(Fragment &frag) const { load_with_byte_offset(frag, 0); }
1685
1687 CUTLASS_DEVICE
1688 void load_with_byte_offset(
1690 Fragment &frag,
1692 Index byte_offset) const {
1693
1694 AccessType * fetch_ptr = reinterpret_cast<AccessType *>(&frag);
1695
1696 CUTLASS_PRAGMA_UNROLL
1697 for (int s = 0; s < Policy::LdsIterations::kStrided; ++s) {
1698
1699 CUTLASS_PRAGMA_UNROLL
1700 for (int c = 0; c < Policy::LdsIterations::kContiguous; ++c) {
1701
1702 int access_idx = c + s * Policy::LdsIterations::kContiguous;
1703
1704 AccessType const *source_ptr = pointer_ +
1705 Policy::LdsShape::kContiguous * c * line_size +
1706 Policy::LdsShape::kStrided * s / 2;
1707
1708 char const *source_byte_ptr = reinterpret_cast<char const *>(source_ptr) + byte_offset + byte_offset_;
1709 fetch_ptr[access_idx] = *(reinterpret_cast<AccessType const*> (source_byte_ptr));
1710
1711 // swap higher 64bit and lower 64bit
1712 if (k_group_idx_ & 0x2) {
1713 uint64_t *low = reinterpret_cast<uint64_t *>(&frag) + access_idx * 2;
1714 uint64_t *high = reinterpret_cast<uint64_t *>(&frag) + access_idx * 2 + 1;
1715 uint64_t tmp = *low;
1716 *low = *high;
1717 *high = tmp;
1718 }
1719 }
1720 }
1721 }
1722
1724 CUTLASS_DEVICE
1725 void load_with_pointer_offset(
1727 Fragment &frag,
1729 Index pointer_offset) const {
1730 load_with_byte_offset(frag, pointer_offset * sizeof(Element));
1731 }
1732
1734 CUTLASS_DEVICE
1735 void load(
1737 Fragment &frag,
1739 TensorCoord const &tile_offset) const {
1740 load_with_byte_offset(frag, tile_offset, 0);
1741 }
1742
1744 CUTLASS_DEVICE
1745 void load(
1747 Fragment &frag,
1749 TensorCoord const &tile_offset,
1751 Index pointer_offset) const {
1752 load_with_byte_offset(frag, tile_offset, pointer_offset * sizeof(Element));
1753 }
1754
1756 CUTLASS_DEVICE
1757 void load_with_byte_offset(
1759 Fragment &frag,
1761 TensorCoord const &tile_offset,
1763 Index byte_offset) const {
1764 Index pointer_offset = tile_offset.contiguous() *
1765 InstructionShape::kContiguous /
1766 Policy::kElementsPerAccess +
1767 tile_offset.strided() * Shape::kStrided * stride_;
1768
1769 byte_offset += sizeof(AccessType) * pointer_offset;
1770
1771 load_with_byte_offset(frag, byte_offset);
1772 }
1773
1781 CUTLASS_DEVICE
1782 void set_kgroup_index(int k_group) {
1783 k_group_idx_ = k_group;
1784 }
1785 };
1786
1794 template <
1796 typename Shape_,
1798 Operand Operand_,
1800 typename Element_,
1802 typename InstructionShape_,
1805 int OpDelta_,
1807 int KBlock>
1808 class MmaVoltaTensorOpMultiplicandTileIterator<
1809 Shape_, Operand_, Element_,
1810 cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise<
1811 sizeof_bits<Element_>::value, KBlock>,
1812 InstructionShape_, OpDelta_, 32> {
1813 public:
1815 using Shape = Shape_;
1816
1818 static Operand const kOperand = Operand_;
1819
1820 static_assert(kOperand == Operand::kA || kOperand == Operand::kB,
1821 "MmaTensorOpMultiplicandIterator may only be instantiated for "
1822 "A or B operands to warp-level Mma.");
1823
1825 using Element = Element_;
1826
1828 static int const kKBlock = KBlock;
1829
1830
1832 using Layout = cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise<
1833 sizeof_bits<Element_>::value, kKBlock>;
1834
1836 using InstructionShape = InstructionShape_;
1837
1840 static int const kOpDelta = OpDelta_;
1841
1843 static int const kThreads = 32;
1844
1846 using TensorRef = TensorRef<Element, Layout>;
1847
1849 using Index = typename TensorRef::Index;
1850
1852 using LongIndex = typename TensorRef::LongIndex;
1853
1855 using TensorCoord = typename TensorRef::TensorCoord;
1856
1858 using Base = MmaVoltaTensorOpMultiplicandTileIterator<
1859 layout::PitchLinearShape<Shape::kRow, Shape::kColumn>, kOperand, Element,
1860 layout::VoltaTensorOpMultiplicandCrosswise<sizeof_bits<Element_>::value,
1861 kKBlock>,
1862 layout::PitchLinearShape<InstructionShape::kRow,
1863 InstructionShape::kColumn>,
1864 kOpDelta, kThreads>;
1865
1866 public:
1867 //
1868 // Derived quantities
1869 //
1870
1872 using Fragment = Array<Element, Shape::kCount / kThreads * 2>;
1873
1874 private:
1876 Base iterator_;
1877
1878 public:
1880 CUTLASS_HOST_DEVICE
1881 MmaVoltaTensorOpMultiplicandTileIterator() {}
1882
1884 CUTLASS_HOST_DEVICE
1885 MmaVoltaTensorOpMultiplicandTileIterator(TensorRef const &ref, int lane_id)
1886 : iterator_({ref.data(), ref.stride()}, lane_id) {}
1887
1889 CUTLASS_HOST_DEVICE
1890 MmaVoltaTensorOpMultiplicandTileIterator &add_pointer_offset(LongIndex offset) {
1891 iterator_.add_pointer_offset(offset);
1892
1893 return *this;
1894 }
1895
1898 CUTLASS_HOST_DEVICE
1899 MmaVoltaTensorOpMultiplicandTileIterator &add_tile_offset(
1900 TensorCoord const &tile_offset) {
1901 iterator_.add_tile_offset({tile_offset.row(), tile_offset.column()});
1902
1903 return *this;
1904 }
1905
1907 CUTLASS_HOST_DEVICE
1908 MmaVoltaTensorOpMultiplicandTileIterator &operator++() {
1909 ++iterator_;
1910
1911 return *this;
1912 }
1913
1915 CUTLASS_HOST_DEVICE
1916 MmaVoltaTensorOpMultiplicandTileIterator &operator--() {
1917 --iterator_;
1918
1919 return *this;
1920 }
1921
1924 CUTLASS_DEVICE
1925 MmaVoltaTensorOpMultiplicandTileIterator &operator+=(
1926 TensorCoord const &tile_offset) {
1927 add_tile_offset(PitchLinearCoord(tile_offset.row(), tile_offset.column()));
1928 return *this;
1929 }
1930
1933 CUTLASS_DEVICE
1934 MmaVoltaTensorOpMultiplicandTileIterator &operator-=(
1935 TensorCoord const &tile_offset) {
1936 add_tile_offset(-PitchLinearCoord(tile_offset.row(), tile_offset.column()));
1937 return *this;
1938 }
1939
1941 CUTLASS_HOST_DEVICE
1942 void load(Fragment &frag) const { iterator_.load(frag); }
1943
1945 CUTLASS_DEVICE
1946 void load_with_pointer_offset(
1948 Fragment &frag,
1950 Index pointer_offset) const {
1951 iterator_.load_with_pointer_offset(frag, pointer_offset);
1952 }
1953
1955 CUTLASS_DEVICE
1956 void load_with_byte_offset(
1958 Fragment &frag,
1960 Index byte_offset) const {
1961 iterator_.load_with_byte_offset(frag, byte_offset);
1962 }
1963
1965 CUTLASS_DEVICE
1966 void load(
1968 Fragment &frag,
1970 TensorCoord const &tile_offset) const {
1971 // TODO
1972 assert(0);
1973 }
1974
1976 CUTLASS_DEVICE
1977 void load(
1979 Fragment &frag,
1981 TensorCoord const &tile_offset,
1983 Index pointer_offset) const {
1984 // TODO
1985 assert(0);
1986 }
1987
1989 CUTLASS_DEVICE
1990 void load_with_byte_offset(
1992 Fragment &frag,
1994 TensorCoord const &tile_offset,
1996 Index byte_offset) const {
1997 iterator_.load_with_byte_offset(
1998 frag, {tile_offset.contiguous(), tile_offset.strided()}, byte_offset);
1999 }
2000
2008 CUTLASS_DEVICE
2009 void set_kgroup_index(int k_group) {
2010 iterator_.set_kgroup_index(k_group);
2011 }
2012 };
2013
2015
2023 template <
2025 typename Shape_,
2027 Operand Operand_,
2029 typename Element_,
2031 typename InstructionShape_,
2034 int OpDelta_,
2036 int KBlock>
2037 class MmaVoltaTensorOpMultiplicandTileIterator<
2038 Shape_, Operand_, Element_,
2039 cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise<
2040 sizeof_bits<Element_>::value, KBlock>,
2041 InstructionShape_, OpDelta_, 32> {
2042 public:
2044 using Shape = Shape_;
2045
2047 static Operand const kOperand = Operand_;
2048
2049 static_assert(kOperand == Operand::kA || kOperand == Operand::kB,
2050 "MmaTensorOpMultiplicandIterator may only be instantiated for "
2051 "A or B operands to warp-level Mma.");
2052
2054 using Element = Element_;
2055
2057 static int const kKBlock = KBlock;
2058
2060 using Layout = cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise<
2061 sizeof_bits<Element_>::value, kKBlock>;
2062
2064 using InstructionShape = InstructionShape_;
2065
2068 static int const kOpDelta = OpDelta_;
2069
2071 static int const kThreads = 32;
2072
2074 using TensorRef = TensorRef<Element, Layout>;
2075
2077 using Index = typename TensorRef::Index;
2078
2080 using LongIndex = typename TensorRef::LongIndex;
2081
2083 using TensorCoord = typename TensorRef::TensorCoord;
2084
2086 using Base = MmaVoltaTensorOpMultiplicandTileIterator<
2087 layout::PitchLinearShape<Shape::kColumn, Shape::kRow>, kOperand, Element,
2088 layout::VoltaTensorOpMultiplicandCrosswise<sizeof_bits<Element_>::value,
2089 kKBlock>,
2090 layout::PitchLinearShape<InstructionShape::kColumn,
2091 InstructionShape::kRow>,
2092 kOpDelta, kThreads>;
2093
2094 public:
2095 //
2096 // Derived quantities
2097 //
2098
2100 using Fragment = Array<Element, Shape::kCount / kThreads * 2>;
2101
2102 private:
2104 Base iterator_;
2105
2106 public:
2108 CUTLASS_HOST_DEVICE
2109 MmaVoltaTensorOpMultiplicandTileIterator() {}
2110
2112 CUTLASS_HOST_DEVICE
2113 MmaVoltaTensorOpMultiplicandTileIterator(TensorRef const &ref, int lane_id)
2114 : iterator_({ref.data(), ref.stride()}, lane_id) {}
2115
2117 CUTLASS_HOST_DEVICE
2118 MmaVoltaTensorOpMultiplicandTileIterator &add_pointer_offset(LongIndex offset) {
2119 iterator_.add_pointer_offset(offset);
2120
2121 return *this;
2122 }
2123
2126 CUTLASS_HOST_DEVICE
2127 MmaVoltaTensorOpMultiplicandTileIterator &add_tile_offset(
2128 TensorCoord const &tile_offset) {
2129 iterator_.add_tile_offset({tile_offset.column(), tile_offset.row()});
2130
2131 return *this;
2132 }
2133
2135 CUTLASS_HOST_DEVICE
2136 MmaVoltaTensorOpMultiplicandTileIterator &operator++() {
2137 ++iterator_;
2138
2139 return *this;
2140 }
2141
2143 CUTLASS_HOST_DEVICE
2144 MmaVoltaTensorOpMultiplicandTileIterator &operator--() {
2145 --iterator_;
2146
2147 return *this;
2148 }
2149
2152 CUTLASS_DEVICE
2153 MmaVoltaTensorOpMultiplicandTileIterator &operator+=(
2154 TensorCoord const &tile_offset) {
2155 add_tile_offset(PitchLinearCoord(tile_offset.column(), tile_offset.row()));
2156 return *this;
2157 }
2158
2161 CUTLASS_DEVICE
2162 MmaVoltaTensorOpMultiplicandTileIterator &operator-=(
2163 TensorCoord const &tile_offset) {
2164 add_tile_offset(-PitchLinearCoord(tile_offset.column(), tile_offset.row()));
2165 return *this;
2166 }
2167
2169 CUTLASS_HOST_DEVICE
2170 void load(Fragment &frag) const { iterator_.load(frag); }
2171
2173 CUTLASS_DEVICE
2174 void load_with_pointer_offset(
2176 Fragment &frag,
2178 Index pointer_offset) const {
2179 iterator_.load_with_pointer_offset(frag, pointer_offset);
2180 }
2181
2183 CUTLASS_DEVICE
2184 void load_with_byte_offset(
2186 Fragment &frag,
2188 Index byte_offset) const {
2189 iterator_.load_with_byte_offset(frag, byte_offset);
2190 }
2191
2193 CUTLASS_DEVICE
2194 void load(
2196 Fragment &frag,
2198 TensorCoord const &tile_offset) const {
2199 // TODO
2200 assert(0);
2201 }
2202
2204 CUTLASS_DEVICE
2205 void load(
2207 Fragment &frag,
2209 TensorCoord const &tile_offset,
2211 Index pointer_offset) const {
2212 // TODO
2213 assert(0);
2214 }
2215
2217 CUTLASS_DEVICE
2218 void load_with_byte_offset(
2220 Fragment &frag,
2222 TensorCoord const &tile_offset,
2224 Index byte_offset) const {
2225 iterator_.load_with_byte_offset(
2226 frag, {tile_offset.strided(), tile_offset.contiguous()}, byte_offset);
2227 }
2228
2236 CUTLASS_DEVICE
2237 void set_kgroup_index(int k_group) {
2238 iterator_.set_kgroup_index(k_group);
2239 }
2240 };
2241
2242 } // namespace warp
2243 } // namespace gemm
2244 } // namespace cutlass
2245
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::load
CUTLASS_HOST_DEVICE void load(Fragment &frag, TensorCoord const &tile_offset, Index pointer_offset) const
Loads a fragment from memory with logical offset in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:1369
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::load
CUTLASS_DEVICE void load(Fragment &frag, TensorCoord const &tile_offset) const
Loads a fragment from memory with logical offset in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:2194
cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous
Template mapping a row-major view of pitch-linear memory to VoltaTensorOpMultiplicandCongruous.
Definition: tensor_op_multiplicand_sm70.h:630
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::operator-=
CUTLASS_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator-=(TensorCoord const &tile_offset)
Definition: mma_tensor_op_tile_iterator_sm70.h:1031
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::VoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::operator++
CUTLASS_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator++()
Advances the iterator along the advance dimension.
Definition: mma_tensor_op_tile_iterator_sm70.h:254
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::VoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::load_with_byte_offset
CUTLASS_DEVICE void load_with_byte_offset(Fragment &frag, TensorCoord const &tile_offset, Index byte_offset) const
Loads a fragment from memory with logical offset in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:353
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::VoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::MmaVoltaTensorOpMultiplicandTileIterator
CUTLASS_DEVICE MmaVoltaTensorOpMultiplicandTileIterator(TensorRef const &ref, int lane_id)
Constructor from TensorRef.
Definition: mma_tensor_op_tile_iterator_sm70.h:189
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::VoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::operator++
CUTLASS_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator++()
Advances the iterator along the advance dimension.
Definition: mma_tensor_op_tile_iterator_sm70.h:539
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::VoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::TensorCoord
typename TensorRef::TensorCoord TensorCoord
Coordinate for an element in the tensor.
Definition: mma_tensor_op_tile_iterator_sm70.h:127
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::LongIndex
typename TensorRef::LongIndex LongIndex
Long Index type.
Definition: mma_tensor_op_tile_iterator_sm70.h:1166
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::Index
typename TensorRef::Index Index
Index type.
Definition: mma_tensor_op_tile_iterator_sm70.h:2077
cutlass::MatrixShape
Describes the size of a matrix tile.
Definition: matrix_shape.h:42
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::load_with_pointer_offset
CUTLASS_DEVICE void load_with_pointer_offset(Fragment &frag, Index pointer_offset) const
Loads a fragment from memory with additional logical offset.
Definition: mma_tensor_op_tile_iterator_sm70.h:821
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::InstructionShape
InstructionShape_ InstructionShape
Shape of one matrix product operation (concept: MatrixShape)
Definition: mma_tensor_op_tile_iterator_sm70.h:2064
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::InstructionShape
InstructionShape_ InstructionShape
Shape of one matrix product operation (concept: MatrixShape)
Definition: mma_tensor_op_tile_iterator_sm70.h:1151
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::Shape
Shape_ Shape
Shape of tile to load (concept: PitchLinearShape)
Definition: mma_tensor_op_tile_iterator_sm70.h:2044
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator
Definition: mma_tensor_op_tile_iterator_sm70.h:70
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::VoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::LongIndex
typename TensorRef::LongIndex LongIndex
Long Index type.
Definition: mma_tensor_op_tile_iterator_sm70.h:436
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::VoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::load
CUTLASS_DEVICE void load(Fragment &frag, TensorCoord const &tile_offset, Index pointer_offset) const
Loads a fragment from memory with logical offset in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:626
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::VoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::operator-=
CUTLASS_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator-=(TensorCoord const &tile_offset)
Definition: mma_tensor_op_tile_iterator_sm70.h:279
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::store
CUTLASS_HOST_DEVICE void store(Fragment const &frag, TensorCoord const &tile_offset, Index pointer_offset) const
Stores a fragment from memory with logical offset in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:1450
cutlass
Definition: aligned_buffer.h:35
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::load_with_pointer_offset
CUTLASS_DEVICE void load_with_pointer_offset(Fragment &frag, Index pointer_offset) const
Loads a fragment from memory with additional logical offset.
Definition: mma_tensor_op_tile_iterator_sm70.h:1045
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::Fragment
Array< Element, Shape::kCount/kThreads *2 > Fragment
Fragment object holding a thread&#39;s part of a tile.
Definition: mma_tensor_op_tile_iterator_sm70.h:965
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::LongIndex
typename TensorRef::LongIndex LongIndex
Long Index type.
Definition: mma_tensor_op_tile_iterator_sm70.h:945
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::load
CUTLASS_HOST_DEVICE void load(Fragment &frag) const
Loads a fragment from memory at the location pointed to by the iterator.
Definition: mma_tensor_op_tile_iterator_sm70.h:2170
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::operator+=
CUTLASS_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator+=(TensorCoord const &tile_offset)
Definition: mma_tensor_op_tile_iterator_sm70.h:1925
tensor_ref.h
Defines a structure containing strides, bounds, and a pointer to tensor data.
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::VoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::set_kgroup_index
CUTLASS_DEVICE void set_kgroup_index(int k_group)
Definition: mma_tensor_op_tile_iterator_sm70.h:1782
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::MmaVoltaTensorOpMultiplicandTileIterator
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator(TensorRef const &ref, int lane_id)
Constructor from TensorRef.
Definition: mma_tensor_op_tile_iterator_sm70.h:980
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::VoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::load_with_pointer_offset
CUTLASS_DEVICE void load_with_pointer_offset(Fragment &frag, Index pointer_offset) const
Loads a fragment from memory with additional logical offset.
Definition: mma_tensor_op_tile_iterator_sm70.h:606
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::OpDelta
OpDelta_ OpDelta
Delta between *MMA operations (in units of *MMA operations, concept: MatrixShape) ...
Definition: mma_tensor_op_tile_iterator_sm70.h:1154
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::VoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::Fragment
Array< Element, Shape::kCount/kThreads *2 > Fragment
Fragment object holding a thread&#39;s part of a tile.
Definition: mma_tensor_op_tile_iterator_sm70.h:1560
cutlass::TensorRef::data
CUTLASS_HOST_DEVICE Element * data() const
Returns the pointer to referenced data.
Definition: tensor_ref.h:254
cutlass::platform::is_same
std::is_same (false specialization)
Definition: platform.h:394
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::MmaVoltaTensorOpAccumulatorTileIterator
CUTLASS_HOST_DEVICE MmaVoltaTensorOpAccumulatorTileIterator()
Default ctor constructs null iterator.
Definition: mma_tensor_op_tile_iterator_sm70.h:1224
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::Shape
Shape_ Shape
Shape of tile to load (concept: PitchLinearShape)
Definition: mma_tensor_op_tile_iterator_sm70.h:694
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::VoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::Index
typename TensorRef::Index Index
Index type.
Definition: mma_tensor_op_tile_iterator_sm70.h:121
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::store_with_pointer_offset
CUTLASS_HOST_DEVICE void store_with_pointer_offset(Fragment const &frag, Index pointer_offset) const
Stores a fragment to memory with additional pointer offset.
Definition: mma_tensor_op_tile_iterator_sm70.h:1385
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::operator+=
CUTLASS_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator+=(TensorCoord const &tile_offset)
advances in units of whole tiles along the logical coordinate space of the tensor ...
Definition: mma_tensor_op_tile_iterator_sm70.h:1024
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::VoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::MmaVoltaTensorOpMultiplicandTileIterator
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator()
Default ctor constructs null iterator.
Definition: mma_tensor_op_tile_iterator_sm70.h:185
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::set_kgroup_index
CUTLASS_DEVICE void set_kgroup_index(int k_group)
Definition: mma_tensor_op_tile_iterator_sm70.h:1108
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::load
CUTLASS_HOST_DEVICE void load(Fragment &frag) const
Loads a fragment from memory at the location pointed to by the iterator.
Definition: mma_tensor_op_tile_iterator_sm70.h:1299
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::VoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::Element
Element_ Element
Element type.
Definition: mma_tensor_op_tile_iterator_sm70.h:415
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::VoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::add_pointer_offset
CUTLASS_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & add_pointer_offset(LongIndex offset)
Adds a pointer offset to internal pointer(s) to advance through memory.
Definition: mma_tensor_op_tile_iterator_sm70.h:219
cutlass::make_Coord
CUTLASS_HOST_DEVICE Coord< 1 > make_Coord(int _0)
Helper to make a 2-element coordinate.
Definition: coord.h:387
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::VoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::load_with_pointer_offset
CUTLASS_DEVICE void load_with_pointer_offset(Fragment &frag, Index pointer_offset) const
Loads a fragment from memory with additional logical offset.
Definition: mma_tensor_op_tile_iterator_sm70.h:1725
cutlass::gemm::Operand
Operand
GEMM operand enumeration: D = A * B + C.
Definition: include/cutlass/gemm/gemm.h:39
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::MmaVoltaTensorOpMultiplicandTileIterator
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator()
Default ctor constructs null iterator.
Definition: mma_tensor_op_tile_iterator_sm70.h:1881
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::VoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::Element
Element_ Element
Element type.
Definition: mma_tensor_op_tile_iterator_sm70.h:1499
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::Element
Element_ Element
Element type.
Definition: mma_tensor_op_tile_iterator_sm70.h:924
cutlass::AlignedArray
Aligned array type.
Definition: array.h:511
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::operator--
CUTLASS_HOST_DEVICE MmaVoltaTensorOpAccumulatorTileIterator & operator--()
Advances the iterator along the advance dimension.
Definition: mma_tensor_op_tile_iterator_sm70.h:1278
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::VoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::TensorCoord
typename TensorRef::TensorCoord TensorCoord
Coordinate for an element in the tensor.
Definition: mma_tensor_op_tile_iterator_sm70.h:439
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::load
CUTLASS_HOST_DEVICE void load(Fragment &frag, TensorCoord const &tile_offset) const
Loads a fragment from memory with logical offset in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:1360
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::set_kgroup_index
CUTLASS_DEVICE void set_kgroup_index(int k_group)
Definition: mma_tensor_op_tile_iterator_sm70.h:884
gemm.h
Defines common types used for all GEMM-like operators.
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::operator++
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator++()
Advances the iterator along the advance dimension.
Definition: mma_tensor_op_tile_iterator_sm70.h:782
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::TensorCoord
typename TensorRef::TensorCoord TensorCoord
Coordinate for an element in the tensor.
Definition: mma_tensor_op_tile_iterator_sm70.h:1855
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::add_tile_offset
CUTLASS_HOST_DEVICE MmaVoltaTensorOpAccumulatorTileIterator & add_tile_offset(TensorCoord const &tile_offset)
Advances an iterator along logical dimensions of matrix in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:1262
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::VoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::MmaVoltaTensorOpMultiplicandTileIterator
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator()
Default ctor constructs null iterator.
Definition: mma_tensor_op_tile_iterator_sm70.h:1584
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::VoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::Index
typename TensorRef::Index Index
Index type.
Definition: mma_tensor_op_tile_iterator_sm70.h:1522
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise
Definition: tensor_op_multiplicand_sm70.h:848
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::load_with_pointer_offset
CUTLASS_DEVICE void load_with_pointer_offset(Fragment &frag, Index pointer_offset) const
Loads a fragment from memory with additional logical offset.
Definition: mma_tensor_op_tile_iterator_sm70.h:2174
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::Element
Element_ Element
Element type.
Definition: mma_tensor_op_tile_iterator_sm70.h:1825
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::load_with_byte_offset
CUTLASS_DEVICE void load_with_byte_offset(Fragment &frag, TensorCoord const &tile_offset, Index byte_offset) const
Loads a fragment from memory with logical offset in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:1087
platform.h
C++ features that may be otherwise unimplemented for CUDA device functions.
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::MmaVoltaTensorOpAccumulatorTileIterator
CUTLASS_HOST_DEVICE MmaVoltaTensorOpAccumulatorTileIterator(TensorRef const &ref, int lane_id)
Constructor from TensorRef.
Definition: mma_tensor_op_tile_iterator_sm70.h:1228
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::load_with_byte_offset
CUTLASS_DEVICE void load_with_byte_offset(Fragment &frag, Index byte_offset) const
Loads a fragment from memory with additional logical offset.
Definition: mma_tensor_op_tile_iterator_sm70.h:1956
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::TensorCoord
typename TensorRef::TensorCoord TensorCoord
Coordinate for an element in the tensor.
Definition: mma_tensor_op_tile_iterator_sm70.h:1169
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::VoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::load_with_byte_offset
CUTLASS_DEVICE void load_with_byte_offset(Fragment &frag, Index byte_offset) const
Loads a fragment from memory with additional logical offset.
Definition: mma_tensor_op_tile_iterator_sm70.h:1688
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::operator++
CUTLASS_HOST_DEVICE MmaVoltaTensorOpAccumulatorTileIterator & operator++()
Advances the iterator along the advance dimension.
Definition: mma_tensor_op_tile_iterator_sm70.h:1271
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::Fragment
Array< Element, Shape::kCount/kThreads > Fragment
Fragment object holding a thread&#39;s part of a tile.
Definition: mma_tensor_op_tile_iterator_sm70.h:1213
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::VoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::add_tile_offset
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & add_tile_offset(TensorCoord const &tile_offset)
Advances an iterator along logical dimensions of matrix in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:523
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::Shape
Shape_ Shape
Shape of tile to load (concept: MatrixShape)
Definition: mma_tensor_op_tile_iterator_sm70.h:1139
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::VoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::load
CUTLASS_HOST_DEVICE void load(Fragment &frag) const
Loads a fragment from memory at the location pointed to by the iterator.
Definition: mma_tensor_op_tile_iterator_sm70.h:286
cutlass::TensorRef::add_coord_offset
CUTLASS_HOST_DEVICE TensorRef & add_coord_offset(TensorCoord const &coord)
Adds an offset to each pointer.
Definition: tensor_ref.h:326
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::add_pointer_offset
CUTLASS_HOST_DEVICE MmaVoltaTensorOpAccumulatorTileIterator & add_pointer_offset(LongIndex offset)
Adds a pointer offset to internal pointer(s) to advance through memory.
Definition: mma_tensor_op_tile_iterator_sm70.h:1255
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::store
CUTLASS_HOST_DEVICE void store(Fragment const &frag) const
Stores a fragment to memory.
Definition: mma_tensor_op_tile_iterator_sm70.h:1379
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::operator--
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator--()
Advances the iterator along the advance dimension.
Definition: mma_tensor_op_tile_iterator_sm70.h:791
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::operator-=
CUTLASS_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator-=(TensorCoord const &tile_offset)
Definition: mma_tensor_op_tile_iterator_sm70.h:807
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::load
CUTLASS_DEVICE void load(Fragment &frag, TensorCoord const &tile_offset) const
Loads a fragment from memory with logical offset in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:1966
cutlass::layout::PitchLinearShape
Template defining a shape used by pitch-linear operators.
Definition: pitch_linear.h:43
array.h
Statically sized array of elements that accommodates all CUTLASS-supported numeric types and is safe ...
cutlass::gemm::Operand::kC
B multiplicand.
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::VoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::Fragment
Array< Element, Shape::kCount/kThreads *2 > Fragment
Fragment object holding a thread&#39;s part of a tile, needs on more time number of registers.
Definition: mma_tensor_op_tile_iterator_sm70.h:476
CUTLASS_PRAGMA_UNROLL
#define CUTLASS_PRAGMA_UNROLL
Definition: cutlass.h:110
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::operator++
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator++()
Advances the iterator along the advance dimension.
Definition: mma_tensor_op_tile_iterator_sm70.h:1006
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::InstructionShape
InstructionShape_ InstructionShape
Shape of one matrix product operation (concept: MatrixShape)
Definition: mma_tensor_op_tile_iterator_sm70.h:1836
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::VoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::operator+=
CUTLASS_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator+=(TensorCoord const &tile_offset)
Definition: mma_tensor_op_tile_iterator_sm70.h:1667
cutlass::gemm::Operand::kA
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::Fragment
Array< Element, Shape::kCount/kThreads *2 > Fragment
Fragment object holding a thread&#39;s part of a tile.
Definition: mma_tensor_op_tile_iterator_sm70.h:741
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::Element
Element_ Element
Element type.
Definition: mma_tensor_op_tile_iterator_sm70.h:1145
cutlass::TensorRef::stride
CUTLASS_HOST_DEVICE Stride stride() const
Returns the layout object&#39;s stride vector.
Definition: tensor_ref.h:277
cutlass::TensorRef::TensorCoord
typename Layout::TensorCoord TensorCoord
Coordinate in logical tensor space.
Definition: tensor_ref.h:171
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::VoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::Fragment
Array< Element, Shape::kCount/kThreads *2 > Fragment
Fragment object holding a thread&#39;s part of a tile.
Definition: mma_tensor_op_tile_iterator_sm70.h:168
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::operator-=
CUTLASS_DEVICE MmaVoltaTensorOpAccumulatorTileIterator & operator-=(TensorCoord const &tile_offset)
Definition: mma_tensor_op_tile_iterator_sm70.h:1292
matrix_shape.h
Defines a Shape template for matrix tiles.
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::VoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::operator--
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator--()
Advances the iterator along the advance dimension.
Definition: mma_tensor_op_tile_iterator_sm70.h:263
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::load
CUTLASS_HOST_DEVICE void load(Fragment &frag) const
Loads a fragment from memory at the location pointed to by the iterator.
Definition: mma_tensor_op_tile_iterator_sm70.h:1038
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::LongIndex
typename TensorRef::LongIndex LongIndex
Long Index type.
Definition: mma_tensor_op_tile_iterator_sm70.h:1852
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::operator-=
CUTLASS_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator-=(TensorCoord const &tile_offset)
Definition: mma_tensor_op_tile_iterator_sm70.h:1934
cutlass::sizeof_bits
Defines the size of an element in bits.
Definition: numeric_types.h:42
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::load_with_byte_offset
CUTLASS_DEVICE void load_with_byte_offset(Fragment &frag, Index byte_offset) const
Loads a fragment from memory with additional logical offset.
Definition: mma_tensor_op_tile_iterator_sm70.h:831
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::Shape
Shape_ Shape
Shape of tile to load (concept: PitchLinearShape)
Definition: mma_tensor_op_tile_iterator_sm70.h:1815
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::load_with_byte_offset
CUTLASS_DEVICE void load_with_byte_offset(Fragment &frag, Index byte_offset) const
Loads a fragment from memory with additional logical offset.
Definition: mma_tensor_op_tile_iterator_sm70.h:1055
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::VoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::add_tile_offset
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & add_tile_offset(TensorCoord const &tile_offset)
Advances an iterator along logical dimensions of matrix in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:228
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::InstructionShape
InstructionShape_ InstructionShape
Shape of one matrix product operation (concept: MatrixShape)
Definition: mma_tensor_op_tile_iterator_sm70.h:706
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::VoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::operator--
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator--()
Advances the iterator along the advance dimension.
Definition: mma_tensor_op_tile_iterator_sm70.h:548
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::operator-=
CUTLASS_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator-=(TensorCoord const &tile_offset)
Definition: mma_tensor_op_tile_iterator_sm70.h:2162
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::VoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::load
CUTLASS_DEVICE void load(Fragment &frag, TensorCoord const &tile_offset) const
Loads a fragment from memory with logical offset in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:616
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::load
CUTLASS_DEVICE void load(Fragment &frag, TensorCoord const &tile_offset) const
Loads a fragment from memory with logical offset in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:841
nullptr
#define nullptr
nullptr
Definition: platform.h:144
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::VoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::load
CUTLASS_HOST_DEVICE void load(Fragment &frag) const
Loads a fragment from memory at the location pointed to by the iterator.
Definition: mma_tensor_op_tile_iterator_sm70.h:1684
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::MmaVoltaTensorOpMultiplicandTileIterator
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator(TensorRef const &ref, int lane_id)
Constructor from TensorRef.
Definition: mma_tensor_op_tile_iterator_sm70.h:756
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::TensorCoord
typename TensorRef::TensorCoord TensorCoord
Coordinate for an element in the tensor.
Definition: mma_tensor_op_tile_iterator_sm70.h:948
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::VoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::load
CUTLASS_DEVICE void load(Fragment &frag, TensorCoord const &tile_offset) const
Loads a fragment from memory with logical offset in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:331
cutlass::TensorRef< Element, Layout >
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::MmaVoltaTensorOpMultiplicandTileIterator
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator()
Default ctor constructs null iterator.
Definition: mma_tensor_op_tile_iterator_sm70.h:752
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::VoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::load_with_byte_offset
CUTLASS_DEVICE void load_with_byte_offset(Fragment &frag, Index byte_offset) const
Loads a fragment from memory with additional logical offset.
Definition: mma_tensor_op_tile_iterator_sm70.h:293
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::VoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::InstructionShape
InstructionShape_ InstructionShape
Shape of one matrix product operation (concept: GemmShape)
Definition: mma_tensor_op_tile_iterator_sm70.h:421
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::MmaVoltaTensorOpMultiplicandTileIterator
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator(TensorRef const &ref, int lane_id)
Constructor from TensorRef.
Definition: mma_tensor_op_tile_iterator_sm70.h:1885
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::VoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::Shape
Shape_ Shape
Shape of tile to load (concept: PitchLinearShape)
Definition: mma_tensor_op_tile_iterator_sm70.h:1489
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::MmaVoltaTensorOpMultiplicandTileIterator
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator()
Default ctor constructs null iterator.
Definition: mma_tensor_op_tile_iterator_sm70.h:976
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::add_pointer_offset
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & add_pointer_offset(LongIndex offset)
Adds a pointer offset to internal pointer(s) to advance through memory.
Definition: mma_tensor_op_tile_iterator_sm70.h:1890
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::load
CUTLASS_DEVICE void load(Fragment &frag, TensorCoord const &tile_offset) const
Loads a fragment from memory with logical offset in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:1065
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::VoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::Shape
Shape_ Shape
Shape of tile to load (concept: PitchLinearShape)
Definition: mma_tensor_op_tile_iterator_sm70.h:409
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::VoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::operator-=
CUTLASS_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator-=(TensorCoord const &tile_offset)
Definition: mma_tensor_op_tile_iterator_sm70.h:1676
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::operator+=
CUTLASS_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator+=(TensorCoord const &tile_offset)
advances in units of whole tiles along the logical coordinate space of the tensor ...
Definition: mma_tensor_op_tile_iterator_sm70.h:800
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::add_pointer_offset
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & add_pointer_offset(LongIndex offset)
Adds a pointer offset to internal pointer(s) to advance through memory.
Definition: mma_tensor_op_tile_iterator_sm70.h:2118
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::add_tile_offset
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & add_tile_offset(TensorCoord const &tile_offset)
Definition: mma_tensor_op_tile_iterator_sm70.h:1899
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::load_with_pointer_offset
CUTLASS_HOST_DEVICE void load_with_pointer_offset(Fragment &frag, Index pointer_offset) const
Loads a fragment from memory with additional logical offset.
Definition: mma_tensor_op_tile_iterator_sm70.h:1305
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::VoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::load_with_byte_offset
CUTLASS_DEVICE void load_with_byte_offset(Fragment &frag, TensorCoord const &tile_offset, Index byte_offset) const
Loads a fragment from memory with logical offset in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:1757
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::VoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::TensorCoord
typename TensorRef::TensorCoord TensorCoord
Coordinate for an element in the tensor.
Definition: mma_tensor_op_tile_iterator_sm70.h:1528
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::set_kgroup_index
CUTLASS_DEVICE void set_kgroup_index(int k_group)
Definition: mma_tensor_op_tile_iterator_sm70.h:2009
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::TensorCoord
typename TensorRef::TensorCoord TensorCoord
Coordinate for an element in the tensor.
Definition: mma_tensor_op_tile_iterator_sm70.h:2083
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::Index
typename TensorRef::Index Index
Index type.
Definition: mma_tensor_op_tile_iterator_sm70.h:718
cutlass::layout::VoltaTensorOpMultiplicandBCongruous
Template based on element size (in bits) - defined in terms of pitch-linear memory.
Definition: tensor_op_multiplicand_sm70.h:397
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::VoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::operator-=
CUTLASS_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator-=(TensorCoord const &tile_offset)
Definition: mma_tensor_op_tile_iterator_sm70.h:564
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::VoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::operator--
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator--()
Advances the iterator along the advance dimension.
Definition: mma_tensor_op_tile_iterator_sm70.h:1662
CUTLASS_HOST_DEVICE
#define CUTLASS_HOST_DEVICE
Definition: cutlass.h:89
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::add_tile_offset
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & add_tile_offset(TensorCoord const &tile_offset)
Advances an iterator along logical dimensions of matrix in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:997
numeric_types.h
Top-level include for all CUTLASS numeric types.
cutlass::TensorRef::offset
CUTLASS_HOST_DEVICE LongIndex offset(TensorCoord const &coord) const
Computes the offset of an index from the origin of the tensor.
Definition: tensor_ref.h:301
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::operator--
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator--()
Advances the iterator along the advance dimension.
Definition: mma_tensor_op_tile_iterator_sm70.h:1015
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::load_with_byte_offset
CUTLASS_DEVICE void load_with_byte_offset(Fragment &frag, TensorCoord const &tile_offset, Index byte_offset) const
Loads a fragment from memory with logical offset in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:863
cutlass::platform::conditional
std::conditional (true specialization)
Definition: platform.h:325
static_assert
#define static_assert(__e, __m)
Definition: platform.h:153
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::VoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::MmaVoltaTensorOpMultiplicandTileIterator
CUTLASS_DEVICE MmaVoltaTensorOpMultiplicandTileIterator(TensorRef const &ref, int lane_id)
Constructor from TensorRef.
Definition: mma_tensor_op_tile_iterator_sm70.h:497
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::add_pointer_offset
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & add_pointer_offset(LongIndex offset)
Adds a pointer offset to internal pointer(s) to advance through memory.
Definition: mma_tensor_op_tile_iterator_sm70.h:764
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::operator--
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator--()
Advances the iterator along the advance dimension.
Definition: mma_tensor_op_tile_iterator_sm70.h:2144
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::store
CUTLASS_HOST_DEVICE void store(Fragment &frag, TensorCoord const &tile_offset) const
Stores a fragment to memory with logical offset in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:1441
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::VoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::InstructionShape
InstructionShape_ InstructionShape
Shape of one matrix product operation (concept: GemmShape)
Definition: mma_tensor_op_tile_iterator_sm70.h:1509
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::Policy
Internal structure of iterator - made public to enable introspection.
Definition: mma_tensor_op_tile_iterator_sm70.h:1172
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::VoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::load_with_pointer_offset
CUTLASS_DEVICE void load_with_pointer_offset(Fragment &frag, Index pointer_offset) const
Loads a fragment from memory with additional logical offset.
Definition: mma_tensor_op_tile_iterator_sm70.h:321
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::VoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::load_with_byte_offset
CUTLASS_DEVICE void load_with_byte_offset(Fragment &frag, TensorCoord const &tile_offset, Index byte_offset) const
Loads a fragment from memory with logical offset in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:638
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::VoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::LongIndex
typename TensorRef::LongIndex LongIndex
Long Index type.
Definition: mma_tensor_op_tile_iterator_sm70.h:124
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::VoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::add_tile_offset
CUTLASS_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & add_tile_offset(TensorCoord const &tile_offset)
Definition: mma_tensor_op_tile_iterator_sm70.h:1632
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::VoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::add_pointer_offset
CUTLASS_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & add_pointer_offset(LongIndex offset)
Adds a pointer offset to internal pointer(s) to advance through memory.
Definition: mma_tensor_op_tile_iterator_sm70.h:514
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::load
CUTLASS_DEVICE void load(Fragment &frag, TensorCoord const &tile_offset, Index pointer_offset) const
Loads a fragment from memory with logical offset in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:2205
cutlass::layout::VoltaTensorOpMultiplicandCrosswise
Definition: tensor_op_multiplicand_sm70.h:733
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::add_pointer_offset
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & add_pointer_offset(LongIndex offset)
Adds a pointer offset to internal pointer(s) to advance through memory.
Definition: mma_tensor_op_tile_iterator_sm70.h:988
cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise
Definition: tensor_op_multiplicand_sm70.h:943
cutlass::TensorRef::Index
typename Layout::Index Index
Index type.
Definition: tensor_ref.h:165
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::Index
typename TensorRef::Index Index
Index type.
Definition: mma_tensor_op_tile_iterator_sm70.h:942
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::Index
typename TensorRef::Index Index
Index type.
Definition: mma_tensor_op_tile_iterator_sm70.h:1163
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::load_with_byte_offset
CUTLASS_DEVICE void load_with_byte_offset(Fragment &frag, Index byte_offset) const
Loads a fragment from memory with additional logical offset.
Definition: mma_tensor_op_tile_iterator_sm70.h:1351
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::VoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::load
CUTLASS_DEVICE void load(Fragment &frag, TensorCoord const &tile_offset, Index pointer_offset) const
Loads a fragment from memory with logical offset in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:341
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::load
CUTLASS_DEVICE void load(Fragment &frag, TensorCoord const &tile_offset, Index pointer_offset) const
Loads a fragment from memory with logical offset in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:851
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::VoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::load
CUTLASS_HOST_DEVICE void load(Fragment &frag) const
Loads a fragment from memory at the location pointed to by the iterator.
Definition: mma_tensor_op_tile_iterator_sm70.h:571
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::Fragment
Array< Element, Shape::kCount/kThreads *2 > Fragment
Fragment object holding a thread&#39;s part of a tile.
Definition: mma_tensor_op_tile_iterator_sm70.h:1872
cutlass::TensorRef::at
CUTLASS_HOST_DEVICE Reference at(TensorCoord const &coord) const
Returns a reference to the element at a given Coord.
Definition: tensor_ref.h:307
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::load_with_byte_offset
CUTLASS_DEVICE void load_with_byte_offset(Fragment &frag, Index byte_offset) const
Loads a fragment from memory with additional logical offset.
Definition: mma_tensor_op_tile_iterator_sm70.h:2184
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator
Definition: mma_tensor_op_tile_iterator_sm70.h:1135
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::VoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::load_with_byte_offset
CUTLASS_DEVICE void load_with_byte_offset(Fragment &frag, Index byte_offset) const
Loads a fragment from memory with additional logical offset.
Definition: mma_tensor_op_tile_iterator_sm70.h:578
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::VoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::add_pointer_offset
CUTLASS_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & add_pointer_offset(LongIndex offset)
Adds a pointer offset to internal pointer(s) to advance through memory.
Definition: mma_tensor_op_tile_iterator_sm70.h:1623
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::VoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::load
CUTLASS_DEVICE void load(Fragment &frag, TensorCoord const &tile_offset) const
Loads a fragment from memory with logical offset in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:1735
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::Shape
Shape_ Shape
Shape of tile to load (concept: PitchLinearShape)
Definition: mma_tensor_op_tile_iterator_sm70.h:915
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::load
CUTLASS_DEVICE void load(Fragment &frag, TensorCoord const &tile_offset, Index pointer_offset) const
Loads a fragment from memory with logical offset in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:1977
matrix.h
Defines layout functions used by TensorRef and derived classes.
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::Fragment
Array< Element, Shape::kCount/kThreads *2 > Fragment
Fragment object holding a thread&#39;s part of a tile.
Definition: mma_tensor_op_tile_iterator_sm70.h:2100
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::store_with_byte_offset
CUTLASS_HOST_DEVICE void store_with_byte_offset(Fragment const &frag, Index byte_offset) const
Stores a fragment to memory with additional pointer offset.
Definition: mma_tensor_op_tile_iterator_sm70.h:1432
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::Element
Element_ Element
Element type.
Definition: mma_tensor_op_tile_iterator_sm70.h:2054
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::MmaVoltaTensorOpMultiplicandTileIterator
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator()
Default ctor constructs null iterator.
Definition: mma_tensor_op_tile_iterator_sm70.h:2109
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::VoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::load
CUTLASS_DEVICE void load(Fragment &frag, TensorCoord const &tile_offset, Index pointer_offset) const
Loads a fragment from memory with logical offset in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:1745
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::VoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::Shape
Shape_ Shape
Shape of tile to load (concept: PitchLinearShape)
Definition: mma_tensor_op_tile_iterator_sm70.h:97
pitch_linear.h
Defines layout functions used by TensorRef and derived classes for pitch-linear memory.
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::load
CUTLASS_HOST_DEVICE void load(Fragment &frag) const
Loads a fragment from memory at the location pointed to by the iterator.
Definition: mma_tensor_op_tile_iterator_sm70.h:814
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::add_tile_offset
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & add_tile_offset(TensorCoord const &tile_offset)
Advances an iterator along logical dimensions of matrix in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:773
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::operator+=
CUTLASS_DEVICE MmaVoltaTensorOpAccumulatorTileIterator & operator+=(TensorCoord const &tile_offset)
advances in units of whole tiles along the logical coordinate space of the tensor ...
Definition: mma_tensor_op_tile_iterator_sm70.h:1285
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::load
CUTLASS_HOST_DEVICE void load(Fragment &frag) const
Loads a fragment from memory at the location pointed to by the iterator.
Definition: mma_tensor_op_tile_iterator_sm70.h:1942
cutlass::TensorRef::add_pointer_offset
CUTLASS_HOST_DEVICE TensorRef & add_pointer_offset(LongIndex offset_)
Adds an offset to each pointer.
Definition: tensor_ref.h:319
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::VoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::operator+=
CUTLASS_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator+=(TensorCoord const &tile_offset)
advances in units of whole tiles along the logical coordinate space of the tensor ...
Definition: mma_tensor_op_tile_iterator_sm70.h:272
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::Index
typename TensorRef::Index Index
Index type.
Definition: mma_tensor_op_tile_iterator_sm70.h:1849
cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous
Template mapping a column-major view of pitch-linear memory to VoltaTensorOpMultiplicandCongruous.
Definition: tensor_op_multiplicand_sm70.h:191
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::load_with_byte_offset
CUTLASS_DEVICE void load_with_byte_offset(Fragment &frag, TensorCoord const &tile_offset, Index byte_offset) const
Loads a fragment from memory with logical offset in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:1990
tensor_op_multiplicand_sm70.h
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::load_with_pointer_offset
CUTLASS_DEVICE void load_with_pointer_offset(Fragment &frag, Index pointer_offset) const
Loads a fragment from memory with additional logical offset.
Definition: mma_tensor_op_tile_iterator_sm70.h:1946
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::VoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::set_kgroup_index
CUTLASS_DEVICE void set_kgroup_index(int k_group)
Definition: mma_tensor_op_tile_iterator_sm70.h:663
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::MmaVoltaTensorOpMultiplicandTileIterator
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator(TensorRef const &ref, int lane_id)
Constructor from TensorRef.
Definition: mma_tensor_op_tile_iterator_sm70.h:2113
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::set_kgroup_index
CUTLASS_DEVICE void set_kgroup_index(int k_group)
Definition: mma_tensor_op_tile_iterator_sm70.h:2237
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::VoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::Element
Element_ Element
Element type.
Definition: mma_tensor_op_tile_iterator_sm70.h:103
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::VoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::operator+=
CUTLASS_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator+=(TensorCoord const &tile_offset)
advances in units of whole tiles along the logical coordinate space of the tensor ...
Definition: mma_tensor_op_tile_iterator_sm70.h:557
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::InstructionShape
InstructionShape_ InstructionShape
Shape of one matrix product operation (concept: MatrixShape)
Definition: mma_tensor_op_tile_iterator_sm70.h:930
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::operator++
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator++()
Advances the iterator along the advance dimension.
Definition: mma_tensor_op_tile_iterator_sm70.h:2136
cutlass::gemm::Operand::kB
A multiplicand.
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::VoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::operator++
CUTLASS_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator++()
Advances the iterator along the advance dimension.
Definition: mma_tensor_op_tile_iterator_sm70.h:1649
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::load
CUTLASS_DEVICE void load(Fragment &frag, TensorCoord const &tile_offset, Index pointer_offset) const
Loads a fragment from memory with logical offset in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:1075
cutlass::gemm::warp::MmaVoltaTensorOpAccumulatorTileIterator::Layout
Layout_ Layout
Layout of source tile.
Definition: mma_tensor_op_tile_iterator_sm70.h:1148
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::operator--
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator--()
Advances the iterator along the advance dimension.
Definition: mma_tensor_op_tile_iterator_sm70.h:1916
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::operator+=
CUTLASS_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator+=(TensorCoord const &tile_offset)
Definition: mma_tensor_op_tile_iterator_sm70.h:2153
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::VoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::TensorRef
TensorRef< Element, Layout > TensorRef
TensorRef type for loading element from a tensor.
Definition: mma_tensor_op_tile_iterator_sm70.h:1519
cutlass::layout::VoltaTensorOpMultiplicandCongruous
Template based on element size (in bits) - defined in terms of pitch-linear memory.
Definition: tensor_op_multiplicand_sm70.h:60
cutlass.h
Basic include for CUTLASS.
cutlass::MatrixCoord
Definition: matrix_coord.h:39
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::TensorCoord
typename TensorRef::TensorCoord TensorCoord
Coordinate for an element in the tensor.
Definition: mma_tensor_op_tile_iterator_sm70.h:724
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::operator++
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & operator++()
Advances the iterator along the advance dimension.
Definition: mma_tensor_op_tile_iterator_sm70.h:1908
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::LongIndex
typename TensorRef::LongIndex LongIndex
Long Index type.
Definition: mma_tensor_op_tile_iterator_sm70.h:721
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::Element
Element_ Element
Element type.
Definition: mma_tensor_op_tile_iterator_sm70.h:700
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::load_with_byte_offset
CUTLASS_DEVICE void load_with_byte_offset(Fragment &frag, TensorCoord const &tile_offset, Index byte_offset) const
Loads a fragment from memory with logical offset in units of whole tiles.
Definition: mma_tensor_op_tile_iterator_sm70.h:2218
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::add_tile_offset
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator & add_tile_offset(TensorCoord const &tile_offset)
Definition: mma_tensor_op_tile_iterator_sm70.h:2127
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::VoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::MmaVoltaTensorOpMultiplicandTileIterator
CUTLASS_HOST_DEVICE MmaVoltaTensorOpMultiplicandTileIterator()
Default ctor constructs null iterator.
Definition: mma_tensor_op_tile_iterator_sm70.h:493
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::VoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::set_kgroup_index
CUTLASS_DEVICE void set_kgroup_index(int k_group)
Definition: mma_tensor_op_tile_iterator_sm70.h:378
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kB, Element_, cutlass::layout::VoltaTensorOpMultiplicandBCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::Index
typename TensorRef::Index Index
Index type.
Definition: mma_tensor_op_tile_iterator_sm70.h:433
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand::kA, Element_, cutlass::layout::VoltaTensorOpMultiplicandCongruous< sizeof_bits< Element_ >::value >, InstructionShape_, OpDelta_, 32 >::InstructionShape
InstructionShape_ InstructionShape
Shape of one matrix product operation (concept: GemmShape)
Definition: mma_tensor_op_tile_iterator_sm70.h:109
cutlass::TensorRef::LongIndex
typename Layout::LongIndex LongIndex
Long index used for pointer offsets.
Definition: tensor_ref.h:168
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::VoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::MmaVoltaTensorOpMultiplicandTileIterator
CUTLASS_DEVICE MmaVoltaTensorOpMultiplicandTileIterator(TensorRef const &ref, int lane_id)
Constructor from TensorRef.
Definition: mma_tensor_op_tile_iterator_sm70.h:1593
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::VoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::LongIndex
typename TensorRef::LongIndex LongIndex
Long Index type.
Definition: mma_tensor_op_tile_iterator_sm70.h:1525
cutlass::gemm::warp::MmaVoltaTensorOpMultiplicandTileIterator< Shape_, Operand_, Element_, cutlass::layout::RowMajorVoltaTensorOpMultiplicandCrosswise< sizeof_bits< Element_ >::value, KBlock >, InstructionShape_, OpDelta_, 32 >::LongIndex
typename TensorRef::LongIndex LongIndex
Long Index type.
Definition: mma_tensor_op_tile_iterator_sm70.h:2080