-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  --------------------------------------------------------------------------------  ------------  
                                                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg     Self CUDA   Self CUDA %    CUDA total  CUDA time avg       CPU Mem  Self CPU Mem      CUDA Mem  Self CUDA Mem    # of Calls                                                                      Input Shapes   Total FLOPs  
-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  --------------------------------------------------------------------------------  ------------  
                                          ProfilerStep*         0.00%       0.000us         0.00%       0.000us       0.000us     597.908ms       167.06%     597.908ms       9.965ms           0 B           0 B           0 B           0 B            60                                                                                []            --  
                                          ProfilerStep*        35.72%      69.752ms        56.97%     111.259ms      11.126ms       0.000us         0.00%     357.890ms      35.789ms           0 B           0 B           0 B     -11.18 GB            10                                                                                []            --  
                                       aten::_cudnn_rnn         6.69%      13.071ms        23.79%      46.453ms       2.323ms     106.233ms        29.68%     231.366ms      11.568ms           0 B           0 B       4.80 GB     -15.94 GB            20  [[8192, 68, 200], [], [], [588672], [2, 8192, 224], [], [], [], [], [], [], [],             --  
                                         aten::_to_copy         0.03%      62.236us         1.21%       2.359ms     117.969us       0.000us         0.00%     200.022ms      10.001ms           0 B           0 B       6.24 GB           0 B            20                                         [[8192, 68, 200], [], [], [], [], [], []]            --  
                                            aten::copy_         0.03%      58.576us         0.09%     167.247us       8.362us     200.022ms        55.89%     200.022ms      10.001ms           0 B           0 B           0 B           0 B            20                                            [[8192, 68, 200], [8192, 68, 200], []]            --  
                         Memcpy HtoD (Pinned -> Device)         0.00%       0.000us         0.00%       0.000us       0.000us     192.807ms        53.87%     192.807ms       6.427ms           0 B           0 B           0 B           0 B            30                                                                                []            --  
                                               aten::to         0.01%      12.418us         1.15%       2.252ms     112.586us       0.000us         0.00%     192.745ms       9.637ms           0 B           0 B       4.16 GB           0 B            20                                     [[8192, 68, 200], [], [], [], [], [], [], []]            --  
                                              aten::gru         0.15%     295.151us        13.14%      25.672ms       2.567ms       0.000us         0.00%     119.449ms      11.945ms           0 B           0 B       2.39 GB     -11.23 MB            10                     [[8192, 68, 200], [2, 8192, 224], [], [], [], [], [], [], []]            --  
ampere_fp16_s1688gemm_fp16_128x128_ldg8_f2f_stages_3...         0.00%       0.000us         0.00%       0.000us       0.000us      38.526ms        10.76%      38.526ms      28.328us           0 B           0 B           0 B           0 B          1360                                                                                []            --  
void elemWiseRNNcell<__half, __half, float, (cudnnRN...         0.00%       0.000us         0.00%       0.000us       0.000us      35.084ms         9.80%      35.084ms      25.797us           0 B           0 B           0 B           0 B          1360                                                                                []            --  
ampere_fp16_s1688gemm_fp16_256x64_ldg8_f2f_stages_32...         0.00%       0.000us         0.00%       0.000us       0.000us      32.623ms         9.12%      32.623ms      95.949us           0 B           0 B           0 B           0 B           340                                                                                []            --  
                                             aten::sub_         0.05%      96.914us         0.08%     153.571us      15.357us       9.754ms         2.73%       9.754ms     975.374us           0 B           0 B           0 B           0 B            10                                                [[8192, 68, 200], [1, 1, 200], []]            --  
void at::native::elementwise_kernel<128, 2, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       9.754ms         2.73%       9.754ms     975.374us           0 B           0 B           0 B           0 B            10                                                                                []            --  
                                             aten::mul_         0.03%      61.694us         0.06%     109.125us      10.912us       9.703ms         2.71%       9.703ms     970.296us           0 B           0 B           0 B           0 B            10                                                    [[8192, 68, 200], [1, 1, 200]]            --  
void at::native::elementwise_kernel<128, 2, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       9.703ms         2.71%       9.703ms     970.296us           0 B           0 B           0 B           0 B            10                                                                                []            --  
void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       9.581ms         2.68%       9.581ms     239.537us           0 B           0 B           0 B           0 B            40                                                                                []            --  
                                      aten::nan_to_num_         0.02%      30.756us         0.12%     229.685us      22.968us       0.000us         0.00%       9.548ms     954.786us           0 B           0 B           0 B           0 B            10                                                     [[8192, 68, 200], [], [], []]            --  
                                       aten::nan_to_num         0.04%      68.856us         0.10%     198.929us      19.893us       9.548ms         2.67%       9.548ms     954.786us           0 B           0 B           0 B           0 B            10                                    [[8192, 68, 200], [], [], [], [8192, 68, 200]]            --  
                                              aten::mul         0.05%      91.327us         0.07%     134.321us      13.432us       9.372ms         2.62%       9.372ms     937.243us           0 B           0 B       4.16 GB       4.16 GB            10                                                 [[8192, 68, 200], [8192, 1, 200]]  1114112000.000  
void at::native::elementwise_kernel<128, 4, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       9.372ms         2.62%       9.372ms     937.243us           0 B           0 B           0 B           0 B            10                                                                                []            --  
void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       7.516ms         2.10%       7.516ms      75.157us           0 B           0 B           0 B           0 B           100                                                                                []            --  
                                               aten::to         0.01%      22.151us         0.07%     142.230us       7.111us       0.000us         0.00%       7.277ms     363.859us           0 B           0 B       2.08 GB           0 B            20                                                 [[8192, 68, 200], [], [], [], []]            --  
                                       aten::contiguous         0.01%      14.231us         0.10%     189.885us      18.988us       0.000us         0.00%       5.684ms     568.437us           0 B           0 B       2.08 GB           0 B            10                                                             [[68, 8192, 200], []]            --  
                                            aten::clone         0.01%      28.924us         0.09%     175.654us      17.565us       0.000us         0.00%       5.684ms     568.437us           0 B           0 B       2.08 GB           0 B            10                                                             [[68, 8192, 200], []]            --  
                                            aten::copy_         0.02%      46.039us         0.05%      88.342us       8.834us       5.684ms         1.59%       5.684ms     568.437us           0 B           0 B           0 B           0 B            10                                            [[68, 8192, 200], [68, 8192, 200], []]            --  
void at::native::elementwise_kernel<128, 4, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       5.684ms         1.59%       5.684ms     568.437us           0 B           0 B           0 B           0 B            10                                                                                []            --  
                                             aten::mean         0.11%     210.964us         0.15%     289.143us      28.914us       5.369ms         1.50%       5.369ms     536.943us           0 B           0 B      62.50 MB      62.50 MB            10                                                     [[8192, 68, 200], [], [], []]            --  
void at::native::reduce_kernel<128, 4, at::native::R...         0.00%       0.000us         0.00%       0.000us       0.000us       5.369ms         1.50%       5.369ms     536.943us           0 B           0 B           0 B           0 B            10                                                                                []            --  
                                              aten::sum         0.62%       1.213ms         0.91%       1.781ms      11.871us     385.695us         0.11%     385.695us       2.571us           0 B           0 B      75.00 KB      75.00 KB           150                                                                      [[8192], []]            --  
void at::native::reduce_kernel<512, 1, at::native::R...         0.00%       0.000us         0.00%       0.000us       0.000us     385.695us         0.11%     385.695us       2.571us           0 B           0 B           0 B           0 B           150                                                                                []            --  
                                       aten::layer_norm         0.03%      60.070us         0.42%     816.969us      40.848us       0.000us         0.00%     326.878us      16.344us           0 B           0 B     147.09 MB     -63.62 MB            20                                           [[8192, 224], [], [224], [224], [], []]            --  
void at::native::(anonymous namespace)::vectorized_l...         0.00%       0.000us         0.00%       0.000us       0.000us     272.861us         0.08%     272.861us      13.643us           0 B           0 B           0 B           0 B            20                                                                                []            --  
                                         aten::_to_copy         0.16%     308.021us         0.74%       1.451ms      20.730us       0.000us         0.00%     238.143us       3.402us           0 B           0 B       2.81 MB           0 B            70                                                  [[8192], [], [], [], [], [], []]            --  
                                            aten::copy_         0.13%     260.812us         0.45%     872.310us      12.462us     238.143us         0.07%     238.143us       3.402us           0 B           0 B           0 B           0 B            70                                                              [[8192], [8192], []]            --  
                                           aten::linear         0.06%     119.129us         0.64%       1.246ms      62.300us       0.000us         0.00%     210.468us      10.523us           0 B           0 B      58.52 MB     -24.50 MB            20                                                  [[8192, 224], [112, 224], [112]]            --  
                                              aten::mul         0.35%     679.820us         0.64%       1.241ms       7.756us     177.947us         0.05%     177.947us       1.112us           0 B           0 B       4.53 MB       4.53 MB           160                                                                  [[8192], [8192]]   1310720.000  
                                               aten::to         0.06%     122.955us         0.61%       1.191ms      19.852us       0.000us         0.00%     177.600us       2.960us           0 B           0 B       1.88 MB           0 B            60                                              [[8192], [], [], [], [], [], [], []]            --  
void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us     172.901us         0.05%     172.901us       1.729us           0 B           0 B           0 B           0 B           100                                                                                []            --  
void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us     156.636us         0.04%     156.636us       1.119us           0 B           0 B           0 B           0 B           140                                                                                []            --  
                                           aten::linear         0.11%     212.006us         0.90%       1.752ms      87.590us       0.000us         0.00%     147.581us       7.379us           0 B           0 B      13.25 MB     -28.12 MB            20                                                    [[8192, 200], [32, 200], [32]]            --  
-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  --------------------------------------------------------------------------------  ------------  
Self CPU time total: 195.299ms
Self CUDA time total: 357.890ms
