Robert Lopez
/
CMSIS5
Important changes to repositories hosted on mbed.com
Mbed hosted mercurial repositories are deprecated and are due to be permanently deleted in July 2026.
To keep a copy of this software download the repository Zip archive or clone locally using Mercurial.
It is also possible to export all your personal repositories from the account settings page.
Embed:
(wiki syntax)
Show/hide line numbers
arm_convolve_HWC_q7_fast.c
00001 /* 00002 * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. 00003 * 00004 * SPDX-License-Identifier: Apache-2.0 00005 * 00006 * Licensed under the Apache License, Version 2.0 (the License); you may 00007 * not use this file except in compliance with the License. 00008 * You may obtain a copy of the License at 00009 * 00010 * www.apache.org/licenses/LICENSE-2.0 00011 * 00012 * Unless required by applicable law or agreed to in writing, software 00013 * distributed under the License is distributed on an AS IS BASIS, WITHOUT 00014 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 00015 * See the License for the specific language governing permissions and 00016 * limitations under the License. 00017 */ 00018 00019 /* ---------------------------------------------------------------------- 00020 * Project: CMSIS NN Library 00021 * Title: arm_convolve_HWC_q7_fast.c 00022 * Description: Fast Q7 version of convolution 00023 * 00024 * $Date: 17. January 2018 00025 * $Revision: V.1.0.0 00026 * 00027 * Target Processor: Cortex-M cores 00028 * 00029 * -------------------------------------------------------------------- */ 00030 00031 #include "arm_math.h" 00032 #include "arm_nnfunctions.h" 00033 00034 /** 00035 * @ingroup groupNN 00036 */ 00037 00038 /** 00039 * @addtogroup NNConv 00040 * @{ 00041 */ 00042 00043 /** 00044 * @brief Fast Q7 convolution function 00045 * @param[in] Im_in pointer to input tensor 00046 * @param[in] dim_im_in input tensor dimention 00047 * @param[in] ch_im_in number of input tensor channels 00048 * @param[in] wt pointer to kernel weights 00049 * @param[in] ch_im_out number of filters, i.e., output tensor channels 00050 * @param[in] dim_kernel filter kernel size 00051 * @param[in] padding padding sizes 00052 * @param[in] stride convolution stride 00053 * @param[in] bias pointer to bias 00054 * @param[in] bias_shift amount of left-shift for bias 00055 * @param[in] out_shift amount of right-shift for output 00056 * @param[in,out] Im_out pointer to output tensor 00057 * @param[in] dim_im_out output tensor dimension 00058 * @param[in,out] bufferA pointer to buffer space for input 00059 * @param[in,out] bufferB pointer to buffer space for output 00060 * @return The function returns either 00061 * <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking. 00062 * 00063 * @details 00064 * 00065 * <b>Buffer size:</b> 00066 * 00067 * bufferA size: 2*ch_im_in*dim_kernel*dim_kernel 00068 * 00069 * bufferB size: 0 00070 * 00071 * <b>Input dimension constraints:</b> 00072 * 00073 * ch_im_in is multiple of 4 ( because of the SIMD32 read and swap ) 00074 * 00075 * ch_im_out is multipe of 2 ( bacause 2x2 mat_mult kernel ) 00076 * 00077 * The im2col converts the Q7 tensor input into Q15 column, which is stored in 00078 * bufferA. There is reordering happenning during this im2col process with 00079 * arm_q7_to_q15_reordered_no_shift. For every four elements, the second and 00080 * third elements are swapped. 00081 * 00082 * The computation kernel arm_nn_mat_mult_kernel_q7_q15_reordered does the 00083 * GEMM computation with the reordered columns. 00084 * 00085 * To speed-up the determination of the padding condition, we split the 00086 * computation into 3x3 parts, i.e., {top, mid, bottom} X {left, mid, right}. 00087 * This reduces the total number of boundary condition checks and improves 00088 * the data copying performance. 00089 */ 00090 00091 arm_status 00092 arm_convolve_HWC_q7_fast(const q7_t * Im_in, 00093 const uint16_t dim_im_in, 00094 const uint16_t ch_im_in, 00095 const q7_t * wt, 00096 const uint16_t ch_im_out, 00097 const uint16_t dim_kernel, 00098 const uint16_t padding, 00099 const uint16_t stride, 00100 const q7_t * bias, 00101 const uint16_t bias_shift, 00102 const uint16_t out_shift, 00103 q7_t * Im_out, 00104 const uint16_t dim_im_out, 00105 q15_t * bufferA, 00106 q7_t * bufferB) 00107 { 00108 00109 #if defined (ARM_MATH_DSP) 00110 /* Run the following code for Cortex-M4 and Cortex-M7 */ 00111 00112 int16_t i_out_y, i_out_x, i_ker_y, i_ker_x; 00113 00114 /* 00115 * Here we use bufferA as q15_t internally as computation are done with q15_t level 00116 * im2col are done to output in q15_t format from q7_t input 00117 */ 00118 00119 q15_t *pBuffer = bufferA; 00120 q7_t *pOut = Im_out; 00121 00122 if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0) 00123 { 00124 /* check if the input dimension meets the constraints */ 00125 return ARM_MATH_SIZE_MISMATCH; 00126 } 00127 00128 /* 00129 * Here we split the entire matrix into three regions depending on the padding situation 00130 * Top: i_out_y from 0 to padding - 1 00131 * Middle: i_out_y from padding to dim_im_out-padding-1 00132 * Bottom: i_out_y from dim_im_out-padding to dim_im_out-1 00133 */ 00134 00135 /* top part */ 00136 for (i_out_y = 0; i_out_y < padding; i_out_y++) 00137 { 00138 for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++) 00139 { 00140 /* This part implements the im2col function */ 00141 for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++) 00142 { 00143 for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++) 00144 { 00145 if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in) 00146 { 00147 /* arm_fill_q15(0, pBuffer, ch_im_in); */ 00148 memset(pBuffer, 0, sizeof(q15_t)*ch_im_in); 00149 } else 00150 { 00151 arm_q7_to_q15_reordered_no_shift 00152 ((q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); 00153 } 00154 pBuffer += ch_im_in; 00155 } 00156 } 00157 00158 if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel) 00159 { 00160 pOut = 00161 arm_nn_mat_mult_kernel_q7_q15_reordered(wt, 00162 bufferA, 00163 ch_im_out, 00164 ch_im_in 00165 * 00166 dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut); 00167 /* counter reset */ 00168 pBuffer = bufferA; 00169 } 00170 } 00171 } 00172 00173 /* middle part, here we also divide the x into left, mid and right */ 00174 for (; i_out_y < dim_im_out - padding; i_out_y++) 00175 { 00176 00177 /* left part */ 00178 for (i_out_x = 0; i_out_x < padding; i_out_x++) 00179 { 00180 /* This part implements the im2col function */ 00181 for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++) 00182 { 00183 for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++) 00184 { 00185 if (i_ker_x < 0 || i_ker_x >= dim_im_in) 00186 { 00187 /* arm_fill_q15(0, pBuffer, ch_im_in); */ 00188 memset(pBuffer, 0, sizeof(q15_t)*ch_im_in); 00189 } else 00190 { 00191 arm_q7_to_q15_reordered_no_shift 00192 ((q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); 00193 } 00194 pBuffer += ch_im_in; 00195 } 00196 } 00197 00198 if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel) 00199 { 00200 pOut = 00201 arm_nn_mat_mult_kernel_q7_q15_reordered(wt, 00202 bufferA, 00203 ch_im_out, 00204 ch_im_in 00205 * 00206 dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut); 00207 /* counter reset */ 00208 pBuffer = bufferA; 00209 } 00210 } 00211 00212 /* mid part */ 00213 for (; i_out_x < dim_im_out - padding; i_out_x++) 00214 { 00215 /* This part implements the im2col function */ 00216 for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++) 00217 { 00218 arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in 00219 + 00220 (i_ker_y * 00221 dim_im_in + 00222 i_out_x * 00223 stride - padding) * ch_im_in, pBuffer, ch_im_in * dim_kernel); 00224 pBuffer += ch_im_in * dim_kernel; 00225 } 00226 00227 if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel) 00228 { 00229 pOut = 00230 arm_nn_mat_mult_kernel_q7_q15_reordered(wt, 00231 bufferA, 00232 ch_im_out, 00233 ch_im_in 00234 * 00235 dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut); 00236 /* counter reset */ 00237 pBuffer = bufferA; 00238 } 00239 } 00240 00241 /* right part */ 00242 for (; i_out_x < dim_im_out; i_out_x++) 00243 { 00244 /* This part implements the im2col function */ 00245 for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++) 00246 { 00247 for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++) 00248 { 00249 if (i_ker_x < 0 || i_ker_x >= dim_im_in) 00250 { 00251 /* arm_fill_q15(0, pBuffer, ch_im_in); */ 00252 memset(pBuffer, 0, sizeof(q15_t)*ch_im_in); 00253 } else 00254 { 00255 arm_q7_to_q15_reordered_no_shift 00256 ((q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); 00257 } 00258 pBuffer += ch_im_in; 00259 } 00260 } 00261 00262 if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel) 00263 { 00264 pOut = 00265 arm_nn_mat_mult_kernel_q7_q15_reordered(wt, 00266 bufferA, 00267 ch_im_out, 00268 ch_im_in 00269 * 00270 dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut); 00271 /* counter reset */ 00272 pBuffer = bufferA; 00273 } 00274 } 00275 } 00276 00277 for (; i_out_y < dim_im_out; i_out_y++) 00278 { 00279 for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++) 00280 { 00281 /* This part implements the im2col function */ 00282 for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++) 00283 { 00284 for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++) 00285 { 00286 if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in) 00287 { 00288 /* arm_fill_q15(0, pBuffer, ch_im_in); */ 00289 memset(pBuffer, 0, sizeof(q15_t)*ch_im_in); 00290 } else 00291 { 00292 arm_q7_to_q15_reordered_no_shift 00293 ((q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); 00294 } 00295 pBuffer += ch_im_in; 00296 } 00297 } 00298 00299 if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel) 00300 { 00301 pOut = 00302 arm_nn_mat_mult_kernel_q7_q15_reordered(wt, 00303 bufferA, 00304 ch_im_out, 00305 ch_im_in 00306 * 00307 dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut); 00308 /* counter reset */ 00309 pBuffer = bufferA; 00310 } 00311 } 00312 } 00313 00314 /* check if there is left-over for compute */ 00315 if (pBuffer != bufferA) 00316 { 00317 const q7_t *pA = wt; 00318 int i; 00319 00320 for (i = 0; i < ch_im_out; i++) 00321 { 00322 q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); 00323 q15_t *pB = bufferA; 00324 /* each time it process 4 entries */ 00325 uint16_t colCnt = ch_im_in * dim_kernel * dim_kernel >> 2; 00326 00327 while (colCnt) 00328 { 00329 00330 q31_t inA1, inA2; 00331 q31_t inB1, inB2; 00332 00333 pA = (q7_t *) read_and_pad_reordered((void *)pA, &inA1, &inA2); 00334 00335 inB1 = *__SIMD32(pB)++; 00336 sum = __SMLAD(inA1, inB1, sum); 00337 inB2 = *__SIMD32(pB)++; 00338 sum = __SMLAD(inA2, inB2, sum); 00339 00340 colCnt--; 00341 } 00342 colCnt = ch_im_in * dim_kernel * dim_kernel & 0x3; 00343 while (colCnt) 00344 { 00345 q7_t inA1 = *pA++; 00346 q15_t inB1 = *pB++; 00347 sum += inA1 * inB1; 00348 colCnt--; 00349 } 00350 *pOut = (q7_t) __SSAT((sum >> out_shift), 8); 00351 pOut++; 00352 00353 } 00354 00355 } 00356 #else 00357 /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ 00358 00359 uint16_t i, j, k, l, m, n; 00360 int conv_out; 00361 signed char in_row, in_col; 00362 00363 if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0) 00364 { 00365 /* check if the input dimension meets the constraints */ 00366 return ARM_MATH_SIZE_MISMATCH; 00367 } 00368 00369 for (i = 0; i < ch_im_out; i++) 00370 { 00371 for (j = 0; j < dim_im_out; j++) 00372 { 00373 for (k = 0; k < dim_im_out; k++) 00374 { 00375 conv_out = (bias[i] << bias_shift) + NN_ROUND(out_shift); 00376 for (m = 0; m < dim_kernel; m++) 00377 { 00378 for (n = 0; n < dim_kernel; n++) 00379 { 00380 // if-for implementation 00381 in_row = stride * j + m - padding; 00382 in_col = stride * k + n - padding; 00383 if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in) 00384 { 00385 for (l = 0; l < ch_im_in; l++) 00386 { 00387 conv_out += 00388 Im_in[(in_row * dim_im_in + in_col) * ch_im_in + 00389 l] * wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel + 00390 n) * ch_im_in + l]; 00391 } 00392 } 00393 } 00394 } 00395 Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q7_t) __SSAT((conv_out >> out_shift), 8); 00396 } 00397 } 00398 } 00399 00400 #endif /* ARM_MATH_DSP */ 00401 00402 /* Return to application */ 00403 return ARM_MATH_SUCCESS; 00404 } 00405 00406 /** 00407 * @} end of NNConv group 00408 */ 00409
Generated on Tue Jul 12 2022 16:46:23 by
1.7.2