| CUTLASS CUDA Templates for Linear Algebra Subroutines and Solvers

- Main Page
- Modules
- Namespaces
- Classes
- Files

- File List
- File Members

transform/threadblock/predicated_tile_iterator.h
Go to the documentation of this file.
1 /***************************************************************************************************
2 * Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without modification, are permitted
5 * provided that the following conditions are met:
6 * * Redistributions of source code must retain the above copyright notice, this list of
7 * conditions and the following disclaimer.
8 * * Redistributions in binary form must reproduce the above copyright notice, this list of
9 * conditions and the following disclaimer in the documentation and/or other materials
10 * provided with the distribution.
11 * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
12 * to endorse or promote products derived from this software without specific prior written
13 * permission.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
17 * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
19 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
20 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
21 * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
22 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23 *
24 **************************************************************************************************/
35 #pragma once
36
37 #include "cutlass/arch/memory.h"
38 #include "cutlass/transform/threadblock/predicated_tile_access_iterator.h"
39
41
42 namespace cutlass {
43 namespace transform {
44 namespace threadblock {
45
47
84 // template <typename Iterator>
85 // __global__ void kernel(
86 // typename Iterator::Params params,
87 // typename Iterator::Element *ptr,
88 // TensorCoord extent) {
89 //
90 // typename Iterator::Fragment fragment;
91 //
92 // TensorCoord threadblock_offset(0, 0);
93 //
94 // Iterator iter(params, ptr, extent, threadIdx.x, threadblock_offsets);
95 //
96 //
97 // fragment = *iter; // load "residue" tile first
98 // ++iter; // advance to first "steady state" tile and update internal masks
99 //
100 //
101 // #pragma unroll
102 // for (int i = Remaining - 1; i >= 0; --i) {
103 //
104 // f(fragment);
105 //
106 // if (!i) {
107 // iter.clear_mask(); // light-weight operation to clear masks - subsequent loads become NO-OPs.
108 // }
109 //
110 // fragment = *iter; // load tile during "steady state" phase
111 // ++iter; // advance to next tile - lightweight due to steady-state masks
112 // }
113 // }
114 //
115 // void host(TensorView<Element, 2, layout::PitchLinear> view) {
116 //
117 // using Iterator = transform::threadblock::PredicatedTileIterator;
118 //
119 // typename Iterator::Params params(view.layout());
120 //
121 // kernel<Iterator>(params, view.data());
122 // }
125 template <
126 typename Shape,
127 typename Element,
128 typename Layout,
129 int AdvanceRank,
130 typename ThreadMap,
131 int AccessSize = ThreadMap::kElementsPerAccess
132 >
133 class PredicatedTileIterator;
134
136
144 template <typename Shape_, typename Element_, int AdvanceRank,
145 typename ThreadMap_, int AccessSize>
146 class PredicatedTileIterator<Shape_, Element_, layout::PitchLinear, AdvanceRank,
147 ThreadMap_, AccessSize> {
148 public:
149 static_assert(
150 AdvanceRank == 0 || AdvanceRank == 1,
151 "Specialization for pitch-linear iterator may along advance along the "
152 "contiguous(rank=0) or strided(rank=1) dimension.");
153
154 using Shape = Shape_;
155 using Element = Element_;
156 using Layout = layout::PitchLinear;
157 static int const kAdvanceRank = AdvanceRank;
158 using ThreadMap = ThreadMap_;
159
160 using Index = typename Layout::Index;
161 using LongIndex = typename Layout::LongIndex;
162
163 using TensorRef = TensorRef<Element, Layout>;
164 using TensorView = TensorView<Element, Layout>;
165 using TensorCoord = typename Layout::TensorCoord;
166
167 using Pointer = Element *;
168 using NonConstPointer = typename platform::remove_const<Element>::type *;
169
171 using AccessType = AlignedArray<Element, AccessSize, (AccessSize * sizeof_bits<Element>::value / 8)>;
172
174 using TileAccessIterator =
175 PredicatedTileAccessIterator<Shape, Element, Layout, kAdvanceRank,
176 ThreadMap, AccessType>;
177
178 static int const kAccessesPerVector = TileAccessIterator::kAccessesPerVector;
179
181 using Fragment = cutlass::Array<Element, ThreadMap::Iterations::kCount *
182 ThreadMap::kElementsPerAccess>;
183
185 using Mask = typename TileAccessIterator::Mask;
186
188 class Params {
189 public:
190 friend PredicatedTileIterator;
191
192 private:
194 typename TileAccessIterator::Params params_;
195
196 public:
198 CUTLASS_HOST_DEVICE
199 Params(Layout const &layout) : params_(layout) { }
200
201 CUTLASS_HOST_DEVICE
202 Params() { }
203 };
204
205 private:
207 using BytePointer = char *;
208
209 private:
210 //
211 // Data members
212 //
213
215 TileAccessIterator address_iterator_;
216
217 public:
220 CUTLASS_HOST_DEVICE
221 PredicatedTileIterator(
223 Params const &params,
225 Pointer pointer,
227 TensorCoord extent,
229 int thread_id,
231 TensorCoord const &threadblock_offset)
232 : address_iterator_(params.params_, pointer, extent, thread_id,
233 threadblock_offset) {}
234
236 CUTLASS_HOST_DEVICE
237 PredicatedTileIterator(
238 Params const &params,
239 Pointer pointer,
240 TensorCoord extent,
241 int thread_id
242 )
243 : PredicatedTileIterator(params, pointer, extent, thread_id,
244 make_Coord(0, 0)) {}
245
247 CUTLASS_HOST_DEVICE
248 void add_pointer_offset(LongIndex pointer_offset) {
249 address_iterator_.add_pointer_offset(pointer_offset);
250 }
251
258 CUTLASS_HOST_DEVICE
259 PredicatedTileIterator &operator++() {
260 if (kAdvanceRank)
261 address_iterator_.add_tile_offset({0, 1});
262 else
263 address_iterator_.add_tile_offset({1, 0});
264
265 return *this;
266 }
267
274 CUTLASS_HOST_DEVICE
275 PredicatedTileIterator operator++(int) {
276 PredicatedTileIterator self(*this);
277 operator++();
278 return self;
279 }
280
282 CUTLASS_HOST_DEVICE
283 void clear_mask() { address_iterator_.clear_mask(); }
284
286 CUTLASS_HOST_DEVICE
287 void enable_mask() { address_iterator_.enable_mask(); }
288
290 CUTLASS_HOST_DEVICE
291 void set_mask(Mask const &mask) { address_iterator_.set_mask(mask); }
292
294 CUTLASS_HOST_DEVICE
295 void get_mask(Mask &mask) { address_iterator_.get_mask(mask); }
296
297 CUTLASS_DEVICE
298 void load_with_pointer_offset(Fragment &frag, Index pointer_offset) {
299
300 AccessType *frag_ptr = reinterpret_cast<AccessType *>(&frag);
301
302 CUTLASS_PRAGMA_UNROLL
303 for (int s = 0; s < ThreadMap::Iterations::kStrided; ++s) {
304 CUTLASS_PRAGMA_UNROLL
305 for (int c = 0; c < ThreadMap::Iterations::kContiguous; ++c) {
306
307 CUTLASS_PRAGMA_UNROLL
308 for (int v = 0; v < kAccessesPerVector; ++v) {
309
310 int idx = v + kAccessesPerVector * (c + s * ThreadMap::Iterations::kContiguous);
311
312 address_iterator_.set_iteration_index(idx);
313 auto ptr = (address_iterator_.get() + pointer_offset);
314
315 if (address_iterator_.valid()) {
316 frag_ptr[idx] = *ptr;
317 }
318 ++address_iterator_;
319 }
320 }
321 }
322 }
323
325 CUTLASS_DEVICE
326 void load(Fragment &frag) { load_with_pointer_offset(frag, 0); }
327
329 CUTLASS_DEVICE
330 void store_with_pointer_offset(Fragment const &frag, Index pointer_offset) {
331 address_iterator_.set_iteration_index(0);
332 AccessType const *frag_ptr = reinterpret_cast<AccessType const *>(&frag);
333
334 CUTLASS_PRAGMA_UNROLL
335 for (int s = 0; s < ThreadMap::Iterations::kStrided; ++s) {
336 CUTLASS_PRAGMA_UNROLL
337 for (int c = 0; c < ThreadMap::Iterations::kContiguous; ++c) {
338 CUTLASS_PRAGMA_UNROLL
339 for (int v = 0; v < kAccessesPerVector; ++v) {
340
341 int idx = v + kAccessesPerVector * (c + s * ThreadMap::Iterations::kContiguous);
342
343 if (address_iterator_.valid()) {
344 *(address_iterator_.get() + pointer_offset) = frag_ptr[idx];
345 }
346 ++address_iterator_;
347 }
348 }
349 }
350 }
351
353 CUTLASS_DEVICE
354 void store(Fragment const &frag) { store_with_pointer_offset(frag, 0); }
355 };
356
358
366 template <
367 typename Shape_,
368 typename Element_,
369 int AdvanceRank,
370 typename ThreadMap_,
371 int AccessSize
372 >
373 class PredicatedTileIterator<Shape_, Element_, layout::ColumnMajor, AdvanceRank, ThreadMap_, AccessSize> {
374 public:
375
376 static_assert(AdvanceRank == 0 || AdvanceRank == 1,
377 "Specialization for pitch-linear iterator may along advance along the "
378 "contiguous(rank=0) or strided(rank=1) dimension.");
379
380 using Shape = Shape_;
381 using Element = Element_;
382 using Layout = layout::ColumnMajor;
383 static int const kAdvanceRank = AdvanceRank;
384 using ThreadMap = ThreadMap_;
385
386 using Index = typename Layout::Index;
387 using LongIndex = typename Layout::LongIndex;
388
389 using TensorRef = TensorRef<Element, Layout>;
390 using TensorView = TensorView<Element, Layout>;
391 using TensorCoord = typename Layout::TensorCoord;
392
393 using Pointer = Element *;
394 using NonConstPointer = typename platform::remove_const<Element>::type *;
395
396 using UnderlyingIterator = PredicatedTileIterator<
397 layout::PitchLinearShape<Shape::kRow, Shape::kColumn>,
398 Element,
399 layout::PitchLinear,
400 (kAdvanceRank == 0 ? 0 : 1),
401 ThreadMap,
402 AccessSize
403 >;
404
405 using AccessType = typename UnderlyingIterator::AccessType;
406
408 using Fragment = cutlass::Array<Element, ThreadMap::Iterations::kCount * ThreadMap::kElementsPerAccess>;
409
411 using Mask = typename UnderlyingIterator::Mask;
412
414 class Params {
415 private:
416
417 friend PredicatedTileIterator;
418
420 typename UnderlyingIterator::Params params_;
421
422 public:
423
424 CUTLASS_HOST_DEVICE
425 Params() { }
426
428 CUTLASS_HOST_DEVICE
429 Params(Layout const &layout): params_(layout::PitchLinear(layout.stride(0))) {
430
431 }
432 };
433
434
435 private:
436
437 //
438 // Data members
439 //
440
442 UnderlyingIterator iterator_;
443
444 public:
445
447 CUTLASS_HOST_DEVICE
448 PredicatedTileIterator(
449 Params const &params,
450 Pointer pointer,
451 TensorCoord extent,
452 int thread_id,
453 TensorCoord const &threadblock_offset
454 ):
455 iterator_(
456 params.params_,
457 pointer,
458 layout::PitchLinearCoord(extent.row(), extent.column()),
459 thread_id,
460 layout::PitchLinearCoord(threadblock_offset.row(), threadblock_offset.column())
461 ) { }
462
464 CUTLASS_HOST_DEVICE
465 PredicatedTileIterator(
466 Params const &params,
467 Pointer pointer,
468 TensorCoord extent,
469 int thread_id
470 ): PredicatedTileIterator(params, pointer, extent, thread_id, make_Coord(0, 0)) { }
471
473 CUTLASS_HOST_DEVICE
474 void add_pointer_offset(LongIndex pointer_offset) {
475 iterator_.add_pointer_offset(pointer_offset);
476 }
477
483 CUTLASS_HOST_DEVICE
484 PredicatedTileIterator &operator++() {
485 ++iterator_;
486 return *this;
487 }
488
494 CUTLASS_HOST_DEVICE
495 PredicatedTileIterator operator++(int) {
496 PredicatedTileIterator self(*this);
497 operator++();
498 return self;
499 }
500
502 CUTLASS_HOST_DEVICE
503 void clear_mask() {
504 iterator_.clear_mask();
505 }
506
508 CUTLASS_HOST_DEVICE
509 void enable_mask() {
510 iterator_.enable_mask();
511 }
512
514 CUTLASS_HOST_DEVICE
515 void set_mask(Mask const &mask) {
516 iterator_.set_mask(mask);
517 }
518
520 CUTLASS_HOST_DEVICE
521 void get_mask(Mask &mask) {
522 iterator_.get_mask(mask);
523 }
524
526 CUTLASS_DEVICE
527 void load_with_pointer_offset(Fragment &frag, Index pointer_offset) {
528 iterator_.load_with_pointer_offset(frag, pointer_offset);
529 }
530
532 CUTLASS_DEVICE
533 void load(Fragment &frag) {
534 load_with_pointer_offset(frag, 0);
535 }
536
538 CUTLASS_DEVICE
539 void store_with_pointer_offset(Fragment const &frag, Index pointer_offset) {
540 iterator_.store_with_pointer_offset(frag, pointer_offset);
541 }
542
544 CUTLASS_DEVICE
545 void store(Fragment const &frag) {
546 store_with_pointer_offset(frag, 0);
547 }
548 };
549
551
559 template <
560 typename Shape_,
561 typename Element_,
562 int AdvanceRank,
563 typename ThreadMap_,
564 int AccessSize
565 >
566 class PredicatedTileIterator<Shape_, Element_, layout::RowMajor, AdvanceRank, ThreadMap_, AccessSize> {
567 public:
568
569 static_assert(AdvanceRank == 0 || AdvanceRank == 1,
570 "Specialization for pitch-linear iterator may along advance along the "
571 "contiguous(rank=0) or strided(rank=1) dimension.");
572
573 using Shape = Shape_;
574 using Element = Element_;
575 using Layout = layout::RowMajor;
576 static int const kAdvanceRank = AdvanceRank;
577 using ThreadMap = ThreadMap_;
578
579 using Index = typename Layout::Index;
580 using LongIndex = typename Layout::LongIndex;
581
582 using TensorRef = TensorRef<Element, Layout>;
583 using TensorView = TensorView<Element, Layout>;
584 using TensorCoord = typename Layout::TensorCoord;
585
586 using Pointer = Element *;
587 using NonConstPointer = typename platform::remove_const<Element>::type *;
588
589 using UnderlyingIterator = PredicatedTileIterator<
590 layout::PitchLinearShape<Shape::kColumn, Shape::kRow>,
591 Element,
592 layout::PitchLinear,
593 (kAdvanceRank == 0 ? 1 : 0),
594 ThreadMap,
595 AccessSize
596 >;
597
598 using AccessType = typename UnderlyingIterator::AccessType;
599
601 using Fragment = cutlass::Array<Element, ThreadMap::Iterations::kCount * ThreadMap::kElementsPerAccess>;
602
604 using Mask = typename UnderlyingIterator::Mask;
605
607 class Params {
608 private:
609
610 friend PredicatedTileIterator;
611
613 typename UnderlyingIterator::Params params_;
614
615 public:
616
617 CUTLASS_HOST_DEVICE
618 Params() { }
619
621 CUTLASS_HOST_DEVICE
622 Params(Layout const &layout): params_(layout::PitchLinear(layout.stride(0))) {
623
624 };
625 };
626
627
628 private:
629
630 //
631 // Data members
632 //
633
635 UnderlyingIterator iterator_;
636
637 public:
638
640 CUTLASS_HOST_DEVICE
641 PredicatedTileIterator(
642 Params const &params,
643 Pointer pointer,
644 TensorCoord extent,
645 int thread_id,
646 TensorCoord const &threadblock_offset
647 ):
648 iterator_(
649 params.params_,
650 pointer,
651 layout::PitchLinearCoord(extent.column(), extent.row()),
652 thread_id,
653 layout::PitchLinearCoord(threadblock_offset.column(), threadblock_offset.row())
654 ) { }
655
657 CUTLASS_HOST_DEVICE
658 PredicatedTileIterator(
659 Params const &params,
660 Pointer pointer,
661 TensorCoord extent,
662 int thread_id
663 ): PredicatedTileIterator(params, pointer, extent, thread_id, make_Coord(0, 0)) { }
664
666 CUTLASS_HOST_DEVICE
667 void add_pointer_offset(LongIndex pointer_offset) {
668 iterator_.add_pointer_offset(pointer_offset);
669 }
670
676 CUTLASS_HOST_DEVICE
677 PredicatedTileIterator &operator++() {
678 ++iterator_;
679 return *this;
680 }
681
687 CUTLASS_HOST_DEVICE
688 PredicatedTileIterator operator++(int) {
689 PredicatedTileIterator self(*this);
690 operator++();
691 return self;
692 }
693
695 CUTLASS_HOST_DEVICE
696 void clear_mask() {
697 iterator_.clear_mask();
698 }
699
701 CUTLASS_HOST_DEVICE
702 void enable_mask() {
703 iterator_.enable_mask();
704 }
705
707 CUTLASS_HOST_DEVICE
708 void set_mask(Mask const &mask) {
709 iterator_.set_mask(mask);
710 }
711
713 CUTLASS_HOST_DEVICE
714 void get_mask(Mask &mask) {
715 iterator_.get_mask(mask);
716 }
717
719 CUTLASS_DEVICE
720 void load_with_pointer_offset(Fragment &frag, Index pointer_offset) {
721 iterator_.load_with_pointer_offset(frag, pointer_offset);
722 }
723
725 CUTLASS_DEVICE
726 void load(Fragment &frag) {
727 load_with_pointer_offset(frag, 0);
728 }
729
731 CUTLASS_DEVICE
732 void store_with_pointer_offset(Fragment const &frag, Index pointer_offset) {
733 iterator_.store_with_pointer_offset(frag, pointer_offset);
734 }
735
737 CUTLASS_DEVICE
738 void store(Fragment const &frag) {
739 store_with_pointer_offset(frag, 0);
740 }
741 };
742
744
753
754 template <typename Shape_, typename Element_, int AdvanceRank,
755 typename ThreadMap_, int AccessSize, int InterleavedK>
756 class PredicatedTileIterator<Shape_, Element_,
757 layout::ColumnMajorInterleaved<InterleavedK>,
758 AdvanceRank, ThreadMap_, AccessSize> {
759 public:
760 static_assert(
761 AdvanceRank == 0 || AdvanceRank == 1,
762 "Specialization for pitch-linear iterator may along advance along the "
763 "contiguous(rank=0) or strided(rank=1) dimension.");
764
765 using Shape = Shape_;
766 using Element = Element_;
767 static int const kInterleavedK = InterleavedK;
768 using Layout = layout::ColumnMajorInterleaved<kInterleavedK>;
769 static int const kAdvanceRank = AdvanceRank;
770 using ThreadMap = ThreadMap_;
771
772 using Index = typename Layout::Index;
773 using LongIndex = typename Layout::LongIndex;
774
775 using TensorRef = TensorRef<Element, Layout>;
776 using TensorView = TensorView<Element, Layout>;
777 using TensorCoord = typename Layout::TensorCoord;
778
779 using Pointer = Element *;
780 using NonConstPointer = typename platform::remove_const<Element>::type *;
781
782 using UnderlyingIterator = PredicatedTileIterator<
783 layout::PitchLinearShape<Shape::kRow * kInterleavedK,
784 Shape::kColumn / kInterleavedK>,
785 Element, layout::PitchLinear, (kAdvanceRank == 0 ? 0 : 1), ThreadMap, AccessSize>;
786
787
788 using AccessType = typename UnderlyingIterator::AccessType;
789
791 using Fragment = cutlass::Array<Element, ThreadMap::Iterations::kCount *
792 ThreadMap::kElementsPerAccess>;
793
795 using Mask = typename UnderlyingIterator::Mask;
796
798 class Params {
799 private:
800 friend PredicatedTileIterator;
801
803 typename UnderlyingIterator::Params params_;
804
805 public:
806 CUTLASS_HOST_DEVICE
807 Params() {}
808
810 CUTLASS_HOST_DEVICE
811 Params(Layout const &layout)
812 : params_(layout::PitchLinear(layout.stride(0))) {}
813 };
814
815 private:
816 //
817 // Data members
818 //
819
821 UnderlyingIterator iterator_;
822
823 public:
826 CUTLASS_HOST_DEVICE
827 PredicatedTileIterator(
829 Params const &params,
831 Pointer pointer,
833 TensorCoord extent,
835 int thread_id,
837 TensorCoord const &threadblock_offset)
838 : iterator_(params.params_, pointer,
839 layout::PitchLinearCoord(extent.row() * kInterleavedK,
840 extent.column() / kInterleavedK),
841 thread_id,
842 layout::PitchLinearCoord(
843 threadblock_offset.row() * kInterleavedK,
844 threadblock_offset.column() / kInterleavedK)) {}
845
847 CUTLASS_HOST_DEVICE
848 PredicatedTileIterator(
849 Params const &params,
850 Pointer pointer,
851 TensorCoord extent,
852 int thread_id
853 )
854 : PredicatedTileIterator(params, pointer, extent, thread_id,
855 make_Coord(0, 0)) {}
856
858 CUTLASS_HOST_DEVICE
859 void add_pointer_offset(LongIndex pointer_offset) {
860 iterator_.add_pointer_offset(pointer_offset);
861 }
862
869 CUTLASS_HOST_DEVICE
870 PredicatedTileIterator &operator++() {
871 ++iterator_;
872 return *this;
873 }
874
881 CUTLASS_HOST_DEVICE
882 PredicatedTileIterator operator++(int) {
883 PredicatedTileIterator self(*this);
884 operator++();
885 return self;
886 }
887
889 CUTLASS_HOST_DEVICE
890 void clear_mask() { iterator_.clear_mask(); }
891
893 CUTLASS_HOST_DEVICE
894 void enable_mask() { iterator_.enable_mask(); }
895
897 CUTLASS_HOST_DEVICE
898 void set_mask(Mask const &mask) { iterator_.set_mask(mask); }
899
901 CUTLASS_HOST_DEVICE
902 void get_mask(Mask &mask) { iterator_.get_mask(mask); }
903
905 CUTLASS_DEVICE
906 void load_with_pointer_offset(Fragment &frag, Index pointer_offset) {
907 iterator_.load_with_pointer_offset(frag, pointer_offset);
908 }
909
911 CUTLASS_DEVICE
912 void load(Fragment &frag) { load_with_pointer_offset(frag, 0); }
913
915 CUTLASS_DEVICE
916 void store_with_pointer_offset(Fragment const &frag, Index pointer_offset) {
917 iterator_.store_with_pointer_offset(frag, pointer_offset);
918 }
919
921 CUTLASS_DEVICE
922 void store(Fragment const &frag) { store_with_pointer_offset(frag, 0); }
923 };
924
926
935 template <typename Shape_, typename Element_, int AdvanceRank,
936 typename ThreadMap_, int AccessSize, int InterleavedK>
937 class PredicatedTileIterator<Shape_, Element_,
938 layout::RowMajorInterleaved<InterleavedK>,
939 AdvanceRank, ThreadMap_, AccessSize> {
940 public:
941 static_assert(
942 AdvanceRank == 0 || AdvanceRank == 1,
943 "Specialization for pitch-linear iterator may along advance along the "
944 "contiguous(rank=0) or strided(rank=1) dimension.");
945
946 using Shape = Shape_;
947 using Element = Element_;
948 static int const kInterleavedK = InterleavedK;
949 using Layout = layout::RowMajorInterleaved<kInterleavedK>;
950 static int const kAdvanceRank = AdvanceRank;
951 using ThreadMap = ThreadMap_;
952
953 using Index = typename Layout::Index;
954 using LongIndex = typename Layout::LongIndex;
955
956 using TensorRef = TensorRef<Element, Layout>;
957 using TensorView = TensorView<Element, Layout>;
958 using TensorCoord = typename Layout::TensorCoord;
959
960 using Pointer = Element *;
961 using NonConstPointer = typename platform::remove_const<Element>::type *;
962
963 using UnderlyingIterator = PredicatedTileIterator<
964 layout::PitchLinearShape<Shape::kColumn * kInterleavedK,
965 Shape::kRow / kInterleavedK>,
966 Element, layout::PitchLinear, (kAdvanceRank == 0 ? 1 : 0), ThreadMap, AccessSize>;
967
968
969 using AccessType = typename UnderlyingIterator::AccessType;
970
972 using Fragment = cutlass::Array<Element, ThreadMap::Iterations::kCount *
973 ThreadMap::kElementsPerAccess>;
974
976 using Mask = typename UnderlyingIterator::Mask;
977
979 class Params {
980 private:
981 friend PredicatedTileIterator;
982
984 typename UnderlyingIterator::Params params_;
985
986 public:
987 CUTLASS_HOST_DEVICE
988 Params() {}
989
991 CUTLASS_HOST_DEVICE
992 Params(Layout const &layout)
993 : params_(layout::PitchLinear(layout.stride(0))) {}
994 };
995
996 private:
997 //
998 // Data members
999 //
1000
1002 UnderlyingIterator iterator_;
1003
1004 public:
1007 CUTLASS_HOST_DEVICE
1008 PredicatedTileIterator(
1010 Params const &params,
1012 Pointer pointer,
1014 TensorCoord extent,
1016 int thread_id,
1018 TensorCoord const &threadblock_offset)
1019 : iterator_(params.params_, pointer,
1020 layout::PitchLinearCoord(extent.column() * kInterleavedK,
1021 extent.row() / kInterleavedK),
1022 thread_id,
1023 layout::PitchLinearCoord(
1024 threadblock_offset.column() * kInterleavedK,
1025 threadblock_offset.row() / kInterleavedK)) {}
1026
1028 CUTLASS_HOST_DEVICE
1029 PredicatedTileIterator(
1030 Params const &params,
1031 Pointer pointer,
1032 TensorCoord extent,
1033 int thread_id
1034 )
1035 : PredicatedTileIterator(params, pointer, extent, thread_id,
1036 make_Coord(0, 0)) {}
1037
1039 CUTLASS_HOST_DEVICE
1040 void add_pointer_offset(LongIndex pointer_offset) {
1041 iterator_.add_pointer_offset(pointer_offset);
1042 }
1043
1050 CUTLASS_HOST_DEVICE
1051 PredicatedTileIterator &operator++() {
1052 ++iterator_;
1053 return *this;
1054 }
1055
1062 CUTLASS_HOST_DEVICE
1063 PredicatedTileIterator operator++(int) {
1064 PredicatedTileIterator self(*this);
1065 operator++();
1066 return self;
1067 }
1068
1070 CUTLASS_HOST_DEVICE
1071 void clear_mask() { iterator_.clear_mask(); }
1072
1074 CUTLASS_HOST_DEVICE
1075 void enable_mask() { iterator_.enable_mask(); }
1076
1078 CUTLASS_HOST_DEVICE
1079 void set_mask(Mask const &mask) { iterator_.set_mask(mask); }
1080
1082 CUTLASS_HOST_DEVICE
1083 void get_mask(Mask &mask) { iterator_.get_mask(mask); }
1084
1086 CUTLASS_DEVICE
1087 void load_with_pointer_offset(Fragment &frag, Index pointer_offset) {
1088 iterator_.load_with_pointer_offset(frag, pointer_offset);
1089 }
1090
1092 CUTLASS_DEVICE
1093 void load(Fragment &frag) { load_with_pointer_offset(frag, 0); }
1094
1096 CUTLASS_DEVICE
1097 void store_with_pointer_offset(Fragment const &frag, Index pointer_offset) {
1098 iterator_.store_with_pointer_offset(frag, pointer_offset);
1099 }
1100
1102 CUTLASS_DEVICE
1103 void store(Fragment const &frag) { store_with_pointer_offset(frag, 0); }
1104 };
1105
1107
1108 } // namespace threadblock
1109 } // namespace transform
1110 } // namespace cutlass
1111
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajor, AdvanceRank, ThreadMap_, AccessSize >::AccessType
typename UnderlyingIterator::AccessType AccessType
Definition: transform/threadblock/predicated_tile_iterator.h:405
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajor, AdvanceRank, ThreadMap_, AccessSize >::Shape
Shape_ Shape
Definition: transform/threadblock/predicated_tile_iterator.h:380
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajor, AdvanceRank, ThreadMap_, AccessSize >::clear_mask
CUTLASS_HOST_DEVICE void clear_mask()
Clears the predicate set efficiently.
Definition: transform/threadblock/predicated_tile_iterator.h:503
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::LongIndex
typename Layout::LongIndex LongIndex
Definition: transform/threadblock/predicated_tile_iterator.h:954
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajor, AdvanceRank, ThreadMap_, AccessSize >::load
CUTLASS_DEVICE void load(Fragment &frag)
Loads a fragment from memory.
Definition: transform/threadblock/predicated_tile_iterator.h:533
cutlass::layout::RowMajor::LongIndex
int64_t LongIndex
Long index type used for offsets.
Definition: layout/matrix.h:62
cutlass::layout::ColumnMajorInterleaved::LongIndex
int64_t LongIndex
Long index type used for offsets.
Definition: layout/matrix.h:355
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::PitchLinear, AdvanceRank, ThreadMap_, AccessSize >::store
CUTLASS_DEVICE void store(Fragment const &frag)
Store a fragment to memory.
Definition: transform/threadblock/predicated_tile_iterator.h:354
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajor, AdvanceRank, ThreadMap_, AccessSize >::Pointer
Element * Pointer
Definition: transform/threadblock/predicated_tile_iterator.h:586
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::add_pointer_offset
CUTLASS_HOST_DEVICE void add_pointer_offset(LongIndex pointer_offset)
Adds a pointer offset in units of Element.
Definition: transform/threadblock/predicated_tile_iterator.h:1040
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::PitchLinear, AdvanceRank, ThreadMap_, AccessSize >::Shape
Shape_ Shape
Definition: transform/threadblock/predicated_tile_iterator.h:154
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::PitchLinear, AdvanceRank, ThreadMap_, AccessSize >::Mask
typename TileAccessIterator::Mask Mask
Predicate vector stores mask to guard accesses.
Definition: transform/threadblock/predicated_tile_iterator.h:185
cutlass
Definition: aligned_buffer.h:35
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::operator++
CUTLASS_HOST_DEVICE PredicatedTileIterator & operator++()
Definition: transform/threadblock/predicated_tile_iterator.h:870
cutlass::layout::PitchLinearCoord
Coordinate in pitch-linear space.
Definition: pitch_linear.h:52
memory.h
Architecture-specific operators on memory.
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajor, AdvanceRank, ThreadMap_, AccessSize >::operator++
CUTLASS_HOST_DEVICE PredicatedTileIterator operator++(int)
Definition: transform/threadblock/predicated_tile_iterator.h:495
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::Index
typename Layout::Index Index
Definition: transform/threadblock/predicated_tile_iterator.h:953
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajor, AdvanceRank, ThreadMap_, AccessSize >::set_mask
CUTLASS_HOST_DEVICE void set_mask(Mask const &mask)
Sets the predicate mask, overriding value stored in predicate iterator.
Definition: transform/threadblock/predicated_tile_iterator.h:515
cutlass::platform::remove_const::type
T type
Definition: platform.h:351
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajor, AdvanceRank, ThreadMap_, AccessSize >::PredicatedTileIterator
CUTLASS_HOST_DEVICE PredicatedTileIterator(Params const &params, Pointer pointer, TensorCoord extent, int thread_id)
Construct a PredicatedTileIterator with zero threadblock offset.
Definition: transform/threadblock/predicated_tile_iterator.h:465
cutlass::layout::RowMajorInterleaved::LongIndex
int64_t LongIndex
Long index type used for offsets.
Definition: layout/matrix.h:249
cutlass::layout::PitchLinear
Mapping function for pitch-linear memory.
Definition: pitch_linear.h:163
cutlass::layout::ColumnMajorInterleaved::Index
int32_t Index
Index type used for coordinates.
Definition: layout/matrix.h:352
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::TensorCoord
typename Layout::TensorCoord TensorCoord
Definition: transform/threadblock/predicated_tile_iterator.h:958
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajor, AdvanceRank, ThreadMap_, AccessSize >::Pointer
Element * Pointer
Definition: transform/threadblock/predicated_tile_iterator.h:393
cutlass::make_Coord
CUTLASS_HOST_DEVICE Coord< 1 > make_Coord(int _0)
Helper to make a 2-element coordinate.
Definition: coord.h:387
cutlass::layout::ColumnMajor::LongIndex
int64_t LongIndex
Long index type used for offsets.
Definition: layout/matrix.h:154
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajor, AdvanceRank, ThreadMap_, AccessSize >::clear_mask
CUTLASS_HOST_DEVICE void clear_mask()
Clears the predicate set efficiently.
Definition: transform/threadblock/predicated_tile_iterator.h:696
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajor, AdvanceRank, ThreadMap_, AccessSize >::Index
typename Layout::Index Index
Definition: transform/threadblock/predicated_tile_iterator.h:579
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajor, AdvanceRank, ThreadMap_, AccessSize >::TensorCoord
typename Layout::TensorCoord TensorCoord
Definition: transform/threadblock/predicated_tile_iterator.h:584
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajor, AdvanceRank, ThreadMap_, AccessSize >::LongIndex
typename Layout::LongIndex LongIndex
Definition: transform/threadblock/predicated_tile_iterator.h:580
cutlass::AlignedArray
Aligned array type.
Definition: array.h:511
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::store_with_pointer_offset
CUTLASS_DEVICE void store_with_pointer_offset(Fragment const &frag, Index pointer_offset)
Store a fragment to memory.
Definition: transform/threadblock/predicated_tile_iterator.h:916
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::Shape
Shape_ Shape
Definition: transform/threadblock/predicated_tile_iterator.h:765
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajor, AdvanceRank, ThreadMap_, AccessSize >::NonConstPointer
typename platform::remove_const< Element >::type * NonConstPointer
Definition: transform/threadblock/predicated_tile_iterator.h:394
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::PredicatedTileIterator
CUTLASS_HOST_DEVICE PredicatedTileIterator(Params const &params, Pointer pointer, TensorCoord extent, int thread_id)
Construct a PredicatedTileIterator with zero threadblock offset.
Definition: transform/threadblock/predicated_tile_iterator.h:1029
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajor, AdvanceRank, ThreadMap_, AccessSize >::store_with_pointer_offset
CUTLASS_DEVICE void store_with_pointer_offset(Fragment const &frag, Index pointer_offset)
Store a fragment to memory.
Definition: transform/threadblock/predicated_tile_iterator.h:732
cutlass::layout::RowMajorInterleaved::Index
int32_t Index
Index type used for coordinates.
Definition: layout/matrix.h:246
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajor, AdvanceRank, ThreadMap_, AccessSize >::PredicatedTileIterator
CUTLASS_HOST_DEVICE PredicatedTileIterator(Params const &params, Pointer pointer, TensorCoord extent, int thread_id, TensorCoord const &threadblock_offset)
Constructs a TileIterator from its precomputed state, threadblock offset, and thread ID...
Definition: transform/threadblock/predicated_tile_iterator.h:641
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::add_pointer_offset
CUTLASS_HOST_DEVICE void add_pointer_offset(LongIndex pointer_offset)
Adds a pointer offset in units of Element.
Definition: transform/threadblock/predicated_tile_iterator.h:859
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::PitchLinear, AdvanceRank, ThreadMap_, AccessSize >::load_with_pointer_offset
CUTLASS_DEVICE void load_with_pointer_offset(Fragment &frag, Index pointer_offset)
Definition: transform/threadblock/predicated_tile_iterator.h:298
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::PredicatedTileIterator
CUTLASS_HOST_DEVICE PredicatedTileIterator(Params const &params, Pointer pointer, TensorCoord extent, int thread_id, TensorCoord const &threadblock_offset)
Definition: transform/threadblock/predicated_tile_iterator.h:1008
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::load
CUTLASS_DEVICE void load(Fragment &frag)
Loads a fragment from memory.
Definition: transform/threadblock/predicated_tile_iterator.h:912
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::LongIndex
typename Layout::LongIndex LongIndex
Definition: transform/threadblock/predicated_tile_iterator.h:773
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::clear_mask
CUTLASS_HOST_DEVICE void clear_mask()
Clears the predicate set efficiently.
Definition: transform/threadblock/predicated_tile_iterator.h:1071
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajor, AdvanceRank, ThreadMap_, AccessSize >::operator++
CUTLASS_HOST_DEVICE PredicatedTileIterator & operator++()
Definition: transform/threadblock/predicated_tile_iterator.h:484
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::PredicatedTileIterator
CUTLASS_HOST_DEVICE PredicatedTileIterator(Params const &params, Pointer pointer, TensorCoord extent, int thread_id)
Construct a PredicatedTileIterator with zero threadblock offset.
Definition: transform/threadblock/predicated_tile_iterator.h:848
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::store_with_pointer_offset
CUTLASS_DEVICE void store_with_pointer_offset(Fragment const &frag, Index pointer_offset)
Store a fragment to memory.
Definition: transform/threadblock/predicated_tile_iterator.h:1097
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::PitchLinear, AdvanceRank, ThreadMap_, AccessSize >::PredicatedTileIterator
CUTLASS_HOST_DEVICE PredicatedTileIterator(Params const &params, Pointer pointer, TensorCoord extent, int thread_id, TensorCoord const &threadblock_offset)
Definition: transform/threadblock/predicated_tile_iterator.h:221
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::set_mask
CUTLASS_HOST_DEVICE void set_mask(Mask const &mask)
Sets the predicate mask, overriding value stored in predicate iterator.
Definition: transform/threadblock/predicated_tile_iterator.h:1079
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajor, AdvanceRank, ThreadMap_, AccessSize >::Fragment
cutlass::Array< Element, ThreadMap::Iterations::kCount *ThreadMap::kElementsPerAccess > Fragment
Fragment object to be loaded or stored.
Definition: transform/threadblock/predicated_tile_iterator.h:408
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::Params::Params
CUTLASS_HOST_DEVICE Params(Layout const &layout)
Construct the Params object given a pitch-linear tensor&#39;s layout.
Definition: transform/threadblock/predicated_tile_iterator.h:811
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajor, AdvanceRank, ThreadMap_, AccessSize >::get_mask
CUTLASS_HOST_DEVICE void get_mask(Mask &mask)
Gets the mask.
Definition: transform/threadblock/predicated_tile_iterator.h:714
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::Fragment
cutlass::Array< Element, ThreadMap::Iterations::kCount *ThreadMap::kElementsPerAccess > Fragment
Fragment object to be loaded or stored.
Definition: transform/threadblock/predicated_tile_iterator.h:792
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::PitchLinear, AdvanceRank, ThreadMap_, AccessSize >::add_pointer_offset
CUTLASS_HOST_DEVICE void add_pointer_offset(LongIndex pointer_offset)
Adds a pointer offset in units of Element.
Definition: transform/threadblock/predicated_tile_iterator.h:248
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajor, AdvanceRank, ThreadMap_, AccessSize >::Mask
typename UnderlyingIterator::Mask Mask
Predicate vector stores mask to guard accesses.
Definition: transform/threadblock/predicated_tile_iterator.h:411
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::PredicatedTileIterator
CUTLASS_HOST_DEVICE PredicatedTileIterator(Params const &params, Pointer pointer, TensorCoord extent, int thread_id, TensorCoord const &threadblock_offset)
Definition: transform/threadblock/predicated_tile_iterator.h:827
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajor, AdvanceRank, ThreadMap_, AccessSize >::get_mask
CUTLASS_HOST_DEVICE void get_mask(Mask &mask)
Gets the mask.
Definition: transform/threadblock/predicated_tile_iterator.h:521
cutlass::layout::ColumnMajor
Mapping function for column-major matrices.
Definition: layout/matrix.h:142
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::load
CUTLASS_DEVICE void load(Fragment &frag)
Loads a fragment from memory.
Definition: transform/threadblock/predicated_tile_iterator.h:1093
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajor, AdvanceRank, ThreadMap_, AccessSize >::Params::Params
CUTLASS_HOST_DEVICE Params(Layout const &layout)
Construct the Params object given a pitch-linear tensor&#39;s layout.
Definition: transform/threadblock/predicated_tile_iterator.h:429
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajor, AdvanceRank, ThreadMap_, AccessSize >::Params::Params
CUTLASS_HOST_DEVICE Params()
Definition: transform/threadblock/predicated_tile_iterator.h:618
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::PitchLinear, AdvanceRank, ThreadMap_, AccessSize >::get_mask
CUTLASS_HOST_DEVICE void get_mask(Mask &mask)
Gets the mask.
Definition: transform/threadblock/predicated_tile_iterator.h:295
cutlass::layout::PitchLinearShape
Template defining a shape used by pitch-linear operators.
Definition: pitch_linear.h:43
CUTLASS_PRAGMA_UNROLL
#define CUTLASS_PRAGMA_UNROLL
Definition: cutlass.h:110
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajor, AdvanceRank, ThreadMap_, AccessSize >::Mask
typename UnderlyingIterator::Mask Mask
Predicate vector stores mask to guard accesses.
Definition: transform/threadblock/predicated_tile_iterator.h:604
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajor, AdvanceRank, ThreadMap_, AccessSize >::operator++
CUTLASS_HOST_DEVICE PredicatedTileIterator operator++(int)
Definition: transform/threadblock/predicated_tile_iterator.h:688
cutlass::layout::RowMajor::Index
int32_t Index
Index type used for coordinates.
Definition: layout/matrix.h:59
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::PitchLinear, AdvanceRank, ThreadMap_, AccessSize >::Params::Params
CUTLASS_HOST_DEVICE Params(Layout const &layout)
Construct the Params object given a pitch-linear tensor&#39;s layout.
Definition: transform/threadblock/predicated_tile_iterator.h:199
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::Params::Params
CUTLASS_HOST_DEVICE Params()
Definition: transform/threadblock/predicated_tile_iterator.h:807
cutlass::operator++
CUTLASS_HOST_DEVICE half_t & operator++(half_t &lhs)
Definition: half.h:694
cutlass::layout::PitchLinear::LongIndex
int64_t LongIndex
Long index type used for offsets.
Definition: pitch_linear.h:175
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajor, AdvanceRank, ThreadMap_, AccessSize >::set_mask
CUTLASS_HOST_DEVICE void set_mask(Mask const &mask)
Sets the predicate mask, overriding value stored in predicate iterator.
Definition: transform/threadblock/predicated_tile_iterator.h:708
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::get_mask
CUTLASS_HOST_DEVICE void get_mask(Mask &mask)
Gets the mask.
Definition: transform/threadblock/predicated_tile_iterator.h:1083
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajor, AdvanceRank, ThreadMap_, AccessSize >::PredicatedTileIterator
CUTLASS_HOST_DEVICE PredicatedTileIterator(Params const &params, Pointer pointer, TensorCoord extent, int thread_id, TensorCoord const &threadblock_offset)
Constructs a TileIterator from its precomputed state, threadblock offset, and thread ID...
Definition: transform/threadblock/predicated_tile_iterator.h:448
cutlass::TensorView< Element, Layout >
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::AccessType
typename UnderlyingIterator::AccessType AccessType
Definition: transform/threadblock/predicated_tile_iterator.h:788
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::PitchLinear, AdvanceRank, ThreadMap_, AccessSize >::Fragment
cutlass::Array< Element, ThreadMap::Iterations::kCount *ThreadMap::kElementsPerAccess > Fragment
Fragment object to be loaded or stored.
Definition: transform/threadblock/predicated_tile_iterator.h:182
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajor, AdvanceRank, ThreadMap_, AccessSize >::AccessType
typename UnderlyingIterator::AccessType AccessType
Definition: transform/threadblock/predicated_tile_iterator.h:598
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajor, AdvanceRank, ThreadMap_, AccessSize >::add_pointer_offset
CUTLASS_HOST_DEVICE void add_pointer_offset(LongIndex pointer_offset)
Adds a pointer offset in units of Element.
Definition: transform/threadblock/predicated_tile_iterator.h:667
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::Params::Params
CUTLASS_HOST_DEVICE Params()
Definition: transform/threadblock/predicated_tile_iterator.h:988
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::operator++
CUTLASS_HOST_DEVICE PredicatedTileIterator operator++(int)
Definition: transform/threadblock/predicated_tile_iterator.h:882
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajor, AdvanceRank, ThreadMap_, AccessSize >::operator++
CUTLASS_HOST_DEVICE PredicatedTileIterator & operator++()
Definition: transform/threadblock/predicated_tile_iterator.h:677
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajor, AdvanceRank, ThreadMap_, AccessSize >::enable_mask
CUTLASS_HOST_DEVICE void enable_mask()
Clears the predicate set efficiently.
Definition: transform/threadblock/predicated_tile_iterator.h:509
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajor, AdvanceRank, ThreadMap_, AccessSize >::PredicatedTileIterator
CUTLASS_HOST_DEVICE PredicatedTileIterator(Params const &params, Pointer pointer, TensorCoord extent, int thread_id)
Construct a PredicatedTileIterator with zero threadblock offset.
Definition: transform/threadblock/predicated_tile_iterator.h:658
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::PitchLinear, AdvanceRank, ThreadMap_, AccessSize >::load
CUTLASS_DEVICE void load(Fragment &frag)
Loads a fragment from memory.
Definition: transform/threadblock/predicated_tile_iterator.h:326
cutlass::TensorRef< Element, Layout >
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajor, AdvanceRank, ThreadMap_, AccessSize >::store
CUTLASS_DEVICE void store(Fragment const &frag)
Store a fragment to memory.
Definition: transform/threadblock/predicated_tile_iterator.h:738
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::PitchLinear, AdvanceRank, ThreadMap_, AccessSize >::LongIndex
typename Layout::LongIndex LongIndex
Definition: transform/threadblock/predicated_tile_iterator.h:161
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajor, AdvanceRank, ThreadMap_, AccessSize >::TensorCoord
typename Layout::TensorCoord TensorCoord
Definition: transform/threadblock/predicated_tile_iterator.h:391
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::Fragment
cutlass::Array< Element, ThreadMap::Iterations::kCount *ThreadMap::kElementsPerAccess > Fragment
Fragment object to be loaded or stored.
Definition: transform/threadblock/predicated_tile_iterator.h:973
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::operator++
CUTLASS_HOST_DEVICE PredicatedTileIterator & operator++()
Definition: transform/threadblock/predicated_tile_iterator.h:1051
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::enable_mask
CUTLASS_HOST_DEVICE void enable_mask()
Clears the predicate set efficiently.
Definition: transform/threadblock/predicated_tile_iterator.h:1075
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::Pointer
Element * Pointer
Definition: transform/threadblock/predicated_tile_iterator.h:960
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::Index
typename Layout::Index Index
Definition: transform/threadblock/predicated_tile_iterator.h:772
predicated_tile_access_iterator.h
Templates calculating the address and predicates to the load of tiles from pitch-linear rank=2 tensor...
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::PitchLinear, AdvanceRank, ThreadMap_, AccessSize >::operator++
CUTLASS_HOST_DEVICE PredicatedTileIterator & operator++()
Definition: transform/threadblock/predicated_tile_iterator.h:259
CUTLASS_HOST_DEVICE
#define CUTLASS_HOST_DEVICE
Definition: cutlass.h:89
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajor, AdvanceRank, ThreadMap_, AccessSize >::Shape
Shape_ Shape
Definition: transform/threadblock/predicated_tile_iterator.h:573
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajor, AdvanceRank, ThreadMap_, AccessSize >::Params::Params
CUTLASS_HOST_DEVICE Params()
Definition: transform/threadblock/predicated_tile_iterator.h:425
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::PitchLinear, AdvanceRank, ThreadMap_, AccessSize >::clear_mask
CUTLASS_HOST_DEVICE void clear_mask()
Clears the predicate set efficiently.
Definition: transform/threadblock/predicated_tile_iterator.h:283
static_assert
#define static_assert(__e, __m)
Definition: platform.h:153
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajor, AdvanceRank, ThreadMap_, AccessSize >::enable_mask
CUTLASS_HOST_DEVICE void enable_mask()
Clears the predicate set efficiently.
Definition: transform/threadblock/predicated_tile_iterator.h:702
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::PitchLinear, AdvanceRank, ThreadMap_, AccessSize >::Pointer
Element * Pointer
Definition: transform/threadblock/predicated_tile_iterator.h:167
cutlass::layout::PitchLinear::Index
int32_t Index
Index type used for coordinates.
Definition: pitch_linear.h:172
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::operator++
CUTLASS_HOST_DEVICE PredicatedTileIterator operator++(int)
Definition: transform/threadblock/predicated_tile_iterator.h:1063
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::TensorCoord
typename Layout::TensorCoord TensorCoord
Definition: transform/threadblock/predicated_tile_iterator.h:777
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::get_mask
CUTLASS_HOST_DEVICE void get_mask(Mask &mask)
Gets the mask.
Definition: transform/threadblock/predicated_tile_iterator.h:902
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajor, AdvanceRank, ThreadMap_, AccessSize >::Fragment
cutlass::Array< Element, ThreadMap::Iterations::kCount *ThreadMap::kElementsPerAccess > Fragment
Fragment object to be loaded or stored.
Definition: transform/threadblock/predicated_tile_iterator.h:601
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::NonConstPointer
typename platform::remove_const< Element >::type * NonConstPointer
Definition: transform/threadblock/predicated_tile_iterator.h:780
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajor, AdvanceRank, ThreadMap_, AccessSize >::store
CUTLASS_DEVICE void store(Fragment const &frag)
Store a fragment to memory.
Definition: transform/threadblock/predicated_tile_iterator.h:545
cutlass::transform::threadblock::PredicatedTileIterator
Definition: transform/threadblock/predicated_tile_iterator.h:133
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajor, AdvanceRank, ThreadMap_, AccessSize >::Params::Params
CUTLASS_HOST_DEVICE Params(Layout const &layout)
Construct the Params object given a pitch-linear tensor&#39;s layout.
Definition: transform/threadblock/predicated_tile_iterator.h:622
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::clear_mask
CUTLASS_HOST_DEVICE void clear_mask()
Clears the predicate set efficiently.
Definition: transform/threadblock/predicated_tile_iterator.h:890
cutlass::layout::RowMajor
Mapping function for row-major matrices.
Definition: layout/matrix.h:50
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::PitchLinear, AdvanceRank, ThreadMap_, AccessSize >::Index
typename Layout::Index Index
Definition: transform/threadblock/predicated_tile_iterator.h:160
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::Shape
Shape_ Shape
Definition: transform/threadblock/predicated_tile_iterator.h:946
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::store
CUTLASS_DEVICE void store(Fragment const &frag)
Store a fragment to memory.
Definition: transform/threadblock/predicated_tile_iterator.h:922
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::PitchLinear, AdvanceRank, ThreadMap_, AccessSize >::Params::PredicatedTileIterator
friend PredicatedTileIterator
Definition: transform/threadblock/predicated_tile_iterator.h:190
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajor, AdvanceRank, ThreadMap_, AccessSize >::load_with_pointer_offset
CUTLASS_DEVICE void load_with_pointer_offset(Fragment &frag, Index pointer_offset)
Loads a fragment from memory.
Definition: transform/threadblock/predicated_tile_iterator.h:720
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::set_mask
CUTLASS_HOST_DEVICE void set_mask(Mask const &mask)
Sets the predicate mask, overriding value stored in predicate iterator.
Definition: transform/threadblock/predicated_tile_iterator.h:898
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::load_with_pointer_offset
CUTLASS_DEVICE void load_with_pointer_offset(Fragment &frag, Index pointer_offset)
Loads a fragment from memory.
Definition: transform/threadblock/predicated_tile_iterator.h:1087
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::Pointer
Element * Pointer
Definition: transform/threadblock/predicated_tile_iterator.h:779
cutlass::layout::ColumnMajorInterleaved
Definition: layout/matrix.h:343
cutlass::layout::ColumnMajor::Index
int32_t Index
Index type used for coordinates.
Definition: layout/matrix.h:151
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::NonConstPointer
typename platform::remove_const< Element >::type * NonConstPointer
Definition: transform/threadblock/predicated_tile_iterator.h:961
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajor, AdvanceRank, ThreadMap_, AccessSize >::LongIndex
typename Layout::LongIndex LongIndex
Definition: transform/threadblock/predicated_tile_iterator.h:387
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::PitchLinear, AdvanceRank, ThreadMap_, AccessSize >::operator++
CUTLASS_HOST_DEVICE PredicatedTileIterator operator++(int)
Definition: transform/threadblock/predicated_tile_iterator.h:275
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajor, AdvanceRank, ThreadMap_, AccessSize >::load
CUTLASS_DEVICE void load(Fragment &frag)
Loads a fragment from memory.
Definition: transform/threadblock/predicated_tile_iterator.h:726
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::Mask
typename UnderlyingIterator::Mask Mask
Predicate vector stores mask to guard accesses.
Definition: transform/threadblock/predicated_tile_iterator.h:795
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::PitchLinear, AdvanceRank, ThreadMap_, AccessSize >::store_with_pointer_offset
CUTLASS_DEVICE void store_with_pointer_offset(Fragment const &frag, Index pointer_offset)
Store a fragment to memory.
Definition: transform/threadblock/predicated_tile_iterator.h:330
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::Params::Params
CUTLASS_HOST_DEVICE Params(Layout const &layout)
Construct the Params object given a pitch-linear tensor&#39;s layout.
Definition: transform/threadblock/predicated_tile_iterator.h:992
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::load_with_pointer_offset
CUTLASS_DEVICE void load_with_pointer_offset(Fragment &frag, Index pointer_offset)
Loads a fragment from memory.
Definition: transform/threadblock/predicated_tile_iterator.h:906
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::PitchLinear, AdvanceRank, ThreadMap_, AccessSize >::NonConstPointer
typename platform::remove_const< Element >::type * NonConstPointer
Definition: transform/threadblock/predicated_tile_iterator.h:168
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajor, AdvanceRank, ThreadMap_, AccessSize >::add_pointer_offset
CUTLASS_HOST_DEVICE void add_pointer_offset(LongIndex pointer_offset)
Adds a pointer offset in units of Element.
Definition: transform/threadblock/predicated_tile_iterator.h:474
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::PitchLinear, AdvanceRank, ThreadMap_, AccessSize >::enable_mask
CUTLASS_HOST_DEVICE void enable_mask()
Clears the predicate set efficiently.
Definition: transform/threadblock/predicated_tile_iterator.h:287
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::AccessType
typename UnderlyingIterator::AccessType AccessType
Definition: transform/threadblock/predicated_tile_iterator.h:969
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::PitchLinear, AdvanceRank, ThreadMap_, AccessSize >::PredicatedTileIterator
CUTLASS_HOST_DEVICE PredicatedTileIterator(Params const &params, Pointer pointer, TensorCoord extent, int thread_id)
Construct a PredicatedTileIterator with zero threadblock offset.
Definition: transform/threadblock/predicated_tile_iterator.h:237
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::PitchLinear, AdvanceRank, ThreadMap_, AccessSize >::set_mask
CUTLASS_HOST_DEVICE void set_mask(Mask const &mask)
Sets the predicate mask, overriding value stored in predicate iterator.
Definition: transform/threadblock/predicated_tile_iterator.h:291
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::store
CUTLASS_DEVICE void store(Fragment const &frag)
Store a fragment to memory.
Definition: transform/threadblock/predicated_tile_iterator.h:1103
cutlass::MatrixCoord
Definition: matrix_coord.h:39
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajor, AdvanceRank, ThreadMap_, AccessSize >::Index
typename Layout::Index Index
Definition: transform/threadblock/predicated_tile_iterator.h:386
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::PitchLinear, AdvanceRank, ThreadMap_, AccessSize >::Params::Params
CUTLASS_HOST_DEVICE Params()
Definition: transform/threadblock/predicated_tile_iterator.h:202
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::PitchLinear, AdvanceRank, ThreadMap_, AccessSize >::TensorCoord
typename Layout::TensorCoord TensorCoord
Definition: transform/threadblock/predicated_tile_iterator.h:165
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::Mask
typename UnderlyingIterator::Mask Mask
Predicate vector stores mask to guard accesses.
Definition: transform/threadblock/predicated_tile_iterator.h:976
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajor, AdvanceRank, ThreadMap_, AccessSize >::store_with_pointer_offset
CUTLASS_DEVICE void store_with_pointer_offset(Fragment const &frag, Index pointer_offset)
Store a fragment to memory.
Definition: transform/threadblock/predicated_tile_iterator.h:539
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajorInterleaved< InterleavedK >, AdvanceRank, ThreadMap_, AccessSize >::enable_mask
CUTLASS_HOST_DEVICE void enable_mask()
Clears the predicate set efficiently.
Definition: transform/threadblock/predicated_tile_iterator.h:894
cutlass::transform::threadblock::PredicatedTileAccessIterator< Shape, Element, Layout, kAdvanceRank, ThreadMap, AccessType >
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::ColumnMajor, AdvanceRank, ThreadMap_, AccessSize >::load_with_pointer_offset
CUTLASS_DEVICE void load_with_pointer_offset(Fragment &frag, Index pointer_offset)
Loads a fragment from memory.
Definition: transform/threadblock/predicated_tile_iterator.h:527
cutlass::transform::threadblock::PredicatedTileIterator< Shape_, Element_, layout::RowMajor, AdvanceRank, ThreadMap_, AccessSize >::NonConstPointer
typename platform::remove_const< Element >::type * NonConstPointer
Definition: transform/threadblock/predicated_tile_iterator.h:587
cutlass::layout::RowMajorInterleaved
Definition: layout/matrix.h:237