-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  --------------------------------------------------------------------------------  ------------  
                                                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg     Self CUDA   Self CUDA %    CUDA total  CUDA time avg       CPU Mem  Self CPU Mem      CUDA Mem  Self CUDA Mem    # of Calls                                                                      Input Shapes   Total FLOPs  
-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  --------------------------------------------------------------------------------  ------------  
                                          ProfilerStep*         0.00%       0.000us         0.00%       0.000us       0.000us     594.965ms       168.05%     594.965ms       9.916ms           0 B           0 B           0 B           0 B            60                                                                                []            --  
                                          ProfilerStep*        50.16%      96.213ms        68.97%     132.292ms      13.229ms       0.000us         0.00%     354.049ms      35.405ms           0 B           0 B           0 B     -11.20 GB            10                                                                                []            --  
                                       aten::_cudnn_rnn         5.96%      11.428ms        21.17%      40.613ms       2.031ms     106.581ms        30.10%     232.131ms      11.607ms           0 B           0 B       4.80 GB     -15.94 GB            20  [[8192, 68, 200], [], [], [588672], [2, 8192, 224], [], [], [], [], [], [], [],             --  
                                         aten::_to_copy         0.02%      46.740us         0.26%     495.815us      24.791us       0.000us         0.00%     195.770ms       9.788ms           0 B           0 B       6.24 GB           0 B            20                                         [[8192, 68, 200], [], [], [], [], [], []]            --  
                                            aten::copy_         0.03%      51.194us         0.07%     137.787us       6.889us     195.770ms        55.29%     195.770ms       9.788ms           0 B           0 B           0 B           0 B            20                                            [[8192, 68, 200], [8192, 68, 200], []]            --  
                         Memcpy HtoD (Pinned -> Device)         0.00%       0.000us         0.00%       0.000us       0.000us     188.566ms        53.26%     188.566ms       6.286ms           0 B           0 B           0 B           0 B            30                                                                                []            --  
                                               aten::to         0.01%      10.520us         0.22%     412.535us      20.627us       0.000us         0.00%     188.501ms       9.425ms           0 B           0 B       4.16 GB           0 B            20                                     [[8192, 68, 200], [], [], [], [], [], [], []]            --  
                                              aten::gru         0.13%     258.493us        11.69%      22.430ms       2.243ms       0.000us         0.00%     119.827ms      11.983ms           0 B           0 B       2.39 GB     -11.23 MB            10                     [[8192, 68, 200], [2, 8192, 224], [], [], [], [], [], [], []]            --  
ampere_fp16_s1688gemm_fp16_128x128_ldg8_f2f_stages_3...         0.00%       0.000us         0.00%       0.000us       0.000us      38.408ms        10.85%      38.408ms      28.241us           0 B           0 B           0 B           0 B          1360                                                                                []            --  
void elemWiseRNNcell<__half, __half, float, (cudnnRN...         0.00%       0.000us         0.00%       0.000us       0.000us      35.513ms        10.03%      35.513ms      26.113us           0 B           0 B           0 B           0 B          1360                                                                                []            --  
ampere_fp16_s1688gemm_fp16_256x64_ldg8_f2f_stages_32...         0.00%       0.000us         0.00%       0.000us       0.000us      32.660ms         9.22%      32.660ms      96.058us           0 B           0 B           0 B           0 B           340                                                                                []            --  
                                             aten::sub_         0.12%     233.591us         0.15%     283.926us      28.393us       9.764ms         2.76%       9.764ms     976.354us           0 B           0 B           0 B           0 B            10                                                [[8192, 68, 200], [1, 1, 200], []]            --  
void at::native::elementwise_kernel<128, 2, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       9.764ms         2.76%       9.764ms     976.354us           0 B           0 B           0 B           0 B            10                                                                                []            --  
                                             aten::mul_         0.07%     136.628us         0.09%     177.167us      17.717us       9.701ms         2.74%       9.701ms     970.149us           0 B           0 B           0 B           0 B            10                                                    [[8192, 68, 200], [1, 1, 200]]            --  
void at::native::elementwise_kernel<128, 2, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       9.701ms         2.74%       9.701ms     970.149us           0 B           0 B           0 B           0 B            10                                                                                []            --  
void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       9.579ms         2.71%       9.579ms     239.481us           0 B           0 B           0 B           0 B            40                                                                                []            --  
                                      aten::nan_to_num_         0.01%      27.971us         0.10%     195.733us      19.573us       0.000us         0.00%       9.545ms     954.534us           0 B           0 B           0 B           0 B            10                                                     [[8192, 68, 200], [], [], []]            --  
                                       aten::nan_to_num         0.03%      57.296us         0.09%     167.762us      16.776us       9.545ms         2.70%       9.545ms     954.534us           0 B           0 B           0 B           0 B            10                                    [[8192, 68, 200], [], [], [], [8192, 68, 200]]            --  
                                              aten::mul         0.08%     159.349us         0.10%     200.371us      20.037us       9.376ms         2.65%       9.376ms     937.586us           0 B           0 B       4.16 GB       4.16 GB            10                                                 [[8192, 68, 200], [8192, 1, 200]]  1114112000.000  
void at::native::elementwise_kernel<128, 4, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       9.376ms         2.65%       9.376ms     937.586us           0 B           0 B           0 B           0 B            10                                                                                []            --  
void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       7.508ms         2.12%       7.508ms      75.080us           0 B           0 B           0 B           0 B           100                                                                                []            --  
                                               aten::to         0.01%      11.088us         0.05%     104.888us       5.244us       0.000us         0.00%       7.269ms     363.426us           0 B           0 B       2.08 GB           0 B            20                                                 [[8192, 68, 200], [], [], [], []]            --  
                                       aten::contiguous         0.01%      12.716us         0.08%     160.118us      16.012us       0.000us         0.00%       5.722ms     572.243us           0 B           0 B       2.08 GB           0 B            10                                                             [[68, 8192, 200], []]            --  
                                            aten::clone         0.01%      23.476us         0.08%     147.402us      14.740us       0.000us         0.00%       5.722ms     572.243us           0 B           0 B       2.08 GB           0 B            10                                                             [[68, 8192, 200], []]            --  
                                            aten::copy_         0.02%      39.609us         0.04%      75.522us       7.552us       5.722ms         1.62%       5.722ms     572.243us           0 B           0 B           0 B           0 B            10                                            [[68, 8192, 200], [68, 8192, 200], []]            --  
void at::native::elementwise_kernel<128, 4, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       5.722ms         1.62%       5.722ms     572.243us           0 B           0 B           0 B           0 B            10                                                                                []            --  
                                             aten::mean         0.17%     334.764us         0.21%     400.880us      40.088us       5.381ms         1.52%       5.381ms     538.148us           0 B           0 B      62.50 MB      62.50 MB            10                                                     [[8192, 68, 200], [], [], []]            --  
void at::native::reduce_kernel<128, 4, at::native::R...         0.00%       0.000us         0.00%       0.000us       0.000us       5.381ms         1.52%       5.381ms     538.148us           0 B           0 B           0 B           0 B            10                                                                                []            --  
                                              aten::sum         0.54%       1.038ms         0.81%       1.563ms      10.420us     385.845us         0.11%     385.845us       2.572us           0 B           0 B      75.00 KB      75.00 KB           150                                                                      [[8192], []]            --  
void at::native::reduce_kernel<512, 1, at::native::R...         0.00%       0.000us         0.00%       0.000us       0.000us     385.845us         0.11%     385.845us       2.572us           0 B           0 B           0 B           0 B           150                                                                                []            --  
                                       aten::layer_norm         0.03%      56.856us         0.39%     756.153us      37.808us       0.000us         0.00%     328.699us      16.435us           0 B           0 B     161.06 MB     -49.62 MB            20                                           [[8192, 224], [], [224], [224], [], []]            --  
void at::native::(anonymous namespace)::vectorized_l...         0.00%       0.000us         0.00%       0.000us       0.000us     273.695us         0.08%     273.695us      13.685us           0 B           0 B           0 B           0 B            20                                                                                []            --  
                                         aten::_to_copy         0.23%     444.069us         0.78%       1.503ms      21.465us       0.000us         0.00%     245.278us       3.504us           0 B           0 B       2.81 MB           0 B            70                                                  [[8192], [], [], [], [], [], []]            --  
                                            aten::copy_         0.18%     351.281us         0.43%     820.059us      11.715us     245.278us         0.07%     245.278us       3.504us           0 B           0 B           0 B           0 B            70                                                              [[8192], [8192], []]            --  
                                           aten::linear         0.06%     110.810us         0.64%       1.225ms      61.238us       0.000us         0.00%     207.774us      10.389us           0 B           0 B      55.02 MB     -28.00 MB            20                                                  [[8192, 224], [112, 224], [112]]            --  
                                               aten::to         0.14%     274.983us         0.73%       1.403ms      23.386us       0.000us         0.00%     184.699us       3.078us           0 B           0 B       1.88 MB           0 B            60                                              [[8192], [], [], [], [], [], [], []]            --  
                                              aten::mul         0.32%     605.178us         0.59%       1.128ms       7.051us     176.926us         0.05%     176.926us       1.106us           0 B           0 B       4.53 MB       4.53 MB           160                                                                  [[8192], [8192]]   1310720.000  
void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us     172.574us         0.05%     172.574us       1.726us           0 B           0 B           0 B           0 B           100                                                                                []            --  
void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us     155.646us         0.04%     155.646us       1.112us           0 B           0 B           0 B           0 B           140                                                                                []            --  
                                           aten::linear         0.10%     189.789us         0.79%       1.515ms      75.772us       0.000us         0.00%     145.466us       7.273us           0 B           0 B      16.38 MB     -25.00 MB            20                                                    [[8192, 200], [32, 200], [32]]            --  
-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  --------------------------------------------------------------------------------  ------------  
Self CPU time total: 191.807ms
Self CUDA time total: 354.049ms
