| CUTLASS CUDA Templates for Linear Algebra Subroutines and Solvers

- Main Page
- Modules
- Namespaces
- Classes
- Files

- File List
- File Members

tools/util/include/cutlass/util/reference/device/kernel/gemm.h
Go to the documentation of this file.
1 /***************************************************************************************************
2 * Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without modification, are permitted
5 * provided that the following conditions are met:
6 * * Redistributions of source code must retain the above copyright notice, this list of
7 * conditions and the following disclaimer.
8 * * Redistributions in binary form must reproduce the above copyright notice, this list of
9 * conditions and the following disclaimer in the documentation and/or other materials
10 * provided with the distribution.
11 * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
12 * to endorse or promote products derived from this software without specific prior written
13 * permission.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
17 * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
19 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
20 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
21 * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
22 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23 *
24 **************************************************************************************************/
29 #pragma once
30
31 #include "cutlass/coord.h"
32 #include "cutlass/matrix_traits.h"
33 #include "cutlass/tensor_view.h"
34 #include "cutlass/gemm/gemm.h"
35
36 #include "cutlass/util/reference/device/thread/gemm.h"
37
38 namespace cutlass {
39 namespace reference {
40 namespace device {
41 namespace kernel {
42
44
47 template <
48 typename TensorRefA,
49 typename TensorRefB,
50 typename TensorRefC,
51 typename ScalarType,
52 typename AccumulatorType,
53 typename OutputTile,
54 typename InnerProductOp,
55 typename ConvertOp
56 >
57 __global__ void Gemm(
58 gemm::GemmCoord problem_size,
59 ScalarType alpha,
60 TensorRefA tensor_a,
61 TensorRefB tensor_b,
62 ScalarType beta,
63 TensorRefC tensor_c,
64 TensorRefC tensor_d,
65 AccumulatorType initial_accum) {
66
67 // Map each thread to a unique tile of the output matrix
68 MatrixCoord output_coord(
69 (threadIdx.x + blockIdx.x * blockDim.x) * OutputTile::kRow,
70 (threadIdx.y + blockIdx.y * blockDim.y) * OutputTile::kColumn
71 );
72
73 // Compute the general matrix product
74 thread::Gemm<
75 TensorRefA,
76 TensorRefB,
77 TensorRefC,
78 ScalarType,
79 AccumulatorType,
80 OutputTile,
81 InnerProductOp,
82 ConvertOp
83 > gemm(initial_accum);
84
85 gemm.multiply_add(
86 problem_size,
87 tensor_a,
88 tensor_b,
89 output_coord);
90
91 gemm.epilogue(problem_size, alpha, beta, tensor_c, tensor_d, output_coord);
92 }
93
95
98 template <
99 typename TensorRefCollectionA,
100 typename TensorRefCollectionB,
101 typename TensorRefCollectionC,
102 typename ScalarType,
103 typename AccumulatorType,
104 typename OutputTile,
105 typename InnerProductOp,
106 typename ConvertOp
107 >
108 __global__ void BatchedGemm(
109 gemm::GemmCoord problem_size,
110 ScalarType alpha,
111 TensorRefCollectionA tensor_collection_a,
112 TensorRefCollectionB tensor_collection_b,
113 ScalarType beta,
114 TensorRefCollectionC tensor_collection_c,
115 AccumulatorType initial_accum) {
116
117 // Obtain batch ID
118 int batch_id = blockIdx.z;
119
120 // Dereference based on batch_id
121 typename TensorRefCollectionA::TensorRef tensor_a = tensor_collection_a.at(batch_id);
122 typename TensorRefCollectionB::TensorRef tensor_b = tensor_collection_b.at(batch_id);
123 typename TensorRefCollectionC::TensorRef tensor_c = tensor_collection_c.at(batch_id);
124
125 // Map each thread to a unique tile of the output matrix
126 MatrixCoord output_coord(
127 (threadIdx.x + blockIdx.x * blockDim.x) * OutputTile::kColumn,
128 (threadIdx.y + blockIdx.y * blockDim.y) * OutputTile::kRow
129 );
130
131 // Compute the general matrix product
132 thread::Gemm<
133 typename TensorRefCollectionA::TensorRef,
134 typename TensorRefCollectionB::TensorRef,
135 typename TensorRefCollectionC::TensorRef,
136 ScalarType,
137 AccumulatorType,
138 OutputTile,
139 InnerProductOp,
140 ConvertOp
141 > gemm(initial_accum);
142
143 gemm.multiply_add(
144 problem_size,
145 tensor_a,
146 tensor_b,
147 output_coord);
148
149 gemm.epilogue(problem_size, alpha, beta, tensor_c, output_coord);
150 }
151
153
154 } // namespace kernel
155 } // namespace device
156 } // namespace reference
157 } // namespace cutlass
cutlass::reference::device::thread::Gemm
Thread-level blocked general matrix product.
Definition: tools/util/include/cutlass/util/reference/device/thread/gemm.h:57
cutlass
Definition: aligned_buffer.h:35
coord.h
A Coord is a coordinate of arbitrary rank into a tensor or matrix.
cutlass::gemm::GemmCoord
Definition: include/cutlass/gemm/gemm.h:94
gemm.h
Defines common types used for all GEMM-like operators.
tensor_view.h
Defines a structure containing strides and a pointer to tensor data.
cutlass::reference::device::kernel::BatchedGemm
__global__ void BatchedGemm(gemm::GemmCoord problem_size, ScalarType alpha, TensorRefCollectionA tensor_collection_a, TensorRefCollectionB tensor_collection_b, ScalarType beta, TensorRefCollectionC tensor_collection_c, AccumulatorType initial_accum)
Definition: tools/util/include/cutlass/util/reference/device/kernel/gemm.h:108
gemm.h
Reference implementation for GEMM in host-side code.
matrix_traits.h
Defines properties of matrices used to denote layout and operands to GEMM kernels.
cutlass::reference::device::kernel::Gemm
__global__ void Gemm(gemm::GemmCoord problem_size, ScalarType alpha, TensorRefA tensor_a, TensorRefB tensor_b, ScalarType beta, TensorRefC tensor_c, TensorRefC tensor_d, AccumulatorType initial_accum)
Definition: tools/util/include/cutlass/util/reference/device/kernel/gemm.h:57
cutlass::MatrixCoord
Definition: matrix_coord.h:39