-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  --------------------------------------------------------------------------------  ------------  
                                                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg     Self CUDA   Self CUDA %    CUDA total  CUDA time avg       CPU Mem  Self CPU Mem      CUDA Mem  Self CUDA Mem    # of Calls                                                                      Input Shapes   Total FLOPs  
-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  --------------------------------------------------------------------------------  ------------  
                                          ProfilerStep*         0.00%       0.000us         0.00%       0.000us       0.000us     669.000ms        64.33%     669.000ms      33.450ms           0 B           0 B           0 B           0 B            20                                                                                []            --  
autograd::engine::evaluate_function: CudnnRnnBackwar...         0.02%     179.739us         3.57%      30.881ms       3.088ms       0.000us         0.00%     603.345ms      60.334ms           0 B           0 B     -35.08 GB     -37.17 GB            10                                                                                []            --  
                                      CudnnRnnBackward0         0.05%     435.615us         3.55%      30.701ms       3.070ms       0.000us         0.00%     603.345ms      60.334ms           0 B           0 B       2.09 GB     -70.00 MB            10                                                         [[8192, 68, 224], [], []]            --  
                              aten::_cudnn_rnn_backward         2.03%      17.537ms         3.50%      30.265ms       3.027ms     586.166ms        56.37%     603.345ms      60.334ms           0 B           0 B       2.15 GB     -34.64 GB            10  [[8192, 68, 200], [], [], [588672], [2, 8192, 224], [], [8192, 68, 224], [8192,             --  
                        DistributedDataParallel.forward         0.00%       0.000us         0.00%       0.000us       0.000us     491.004ms        47.22%     491.004ms       9.820ms           0 B           0 B           0 B           0 B            50                                                                                []            --  
                                          ProfilerStep*       -27.79%  -240477.710us        89.13%     771.257ms      77.126ms       0.000us         0.00%     382.331ms      38.233ms           0 B           0 B     747.00 KB     -39.25 GB            10                                                                                []            --  
                                       aten::_cudnn_rnn         1.48%      12.816ms         5.05%      43.655ms       2.183ms     149.038ms        14.33%     316.577ms      15.829ms           0 B           0 B      67.66 GB     -16.12 GB            20  [[8192, 68, 200], [], [], [588672], [2, 8192, 224], [], [], [], [], [], [], [],             --  
ampere_fp16_s16816gemm_fp16_128x64_ldg8_f2f_stages_6...         0.00%       0.000us         0.00%       0.000us       0.000us     261.473ms        25.14%     261.473ms       4.358ms           0 B           0 B           0 B           0 B            60                                                                                []            --  
                                         aten::_to_copy         0.01%     122.155us         0.25%       2.181ms      72.704us       0.000us         0.00%     188.142ms       6.271ms           0 B           0 B      10.40 GB           0 B            30                                         [[8192, 68, 200], [], [], [], [], [], []]            --  
                                            aten::copy_         0.02%     177.325us         0.04%     363.575us      12.119us     188.142ms        18.09%     188.142ms       6.271ms           0 B           0 B           0 B           0 B            30                                            [[8192, 68, 200], [8192, 68, 200], []]            --  
                                               aten::to         0.00%      24.781us         0.24%       2.089ms      69.638us       0.000us         0.00%     181.102ms       6.037ms           0 B           0 B       8.32 GB           0 B            30                                     [[8192, 68, 200], [], [], [], [], [], [], []]            --  
                        DistributedDataParallel.forward         0.76%       6.557ms        42.88%     371.053ms      37.105ms       0.000us         0.00%     177.203ms      17.720ms           0 B         -92 B      35.10 GB      -4.55 GB            10                                                                                []            --  
                         Memcpy HtoD (Pinned -> Device)         0.00%       0.000us         0.00%       0.000us       0.000us     174.288ms        16.76%     174.288ms       5.810ms           0 B           0 B           0 B           0 B            30                                                                                []            --  
                                              aten::gru         0.19%       1.650ms         2.95%      25.494ms       2.549ms       0.000us         0.00%     161.930ms      16.193ms           0 B           0 B      34.91 GB           0 B            10                     [[8192, 68, 200], [2, 8192, 224], [], [], [], [], [], [], []]            --  
void GRU_elementWise_bp2<__half, __half, float, 3, (...         0.00%       0.000us         0.00%       0.000us       0.000us     128.662ms        12.37%     128.662ms       6.433ms           0 B           0 B           0 B           0 B            20                                                                                []            --  
ampere_fp16_s1688gemm_fp16_128x128_ldg8_f2f_stages_3...         0.00%       0.000us         0.00%       0.000us       0.000us     108.190ms        10.40%     108.190ms      53.034us           0 B           0 B           0 B           0 B          2040                                                                                []            --  
void GRU_elementWise_bp1<__half, __half, float>(int,...         0.00%       0.000us         0.00%       0.000us       0.000us      72.949ms         7.02%      72.949ms      53.639us           0 B           0 B           0 B           0 B          1360                                                                                []            --  
void elemWiseRNNcell<__half, __half, float, (cudnnRN...         0.00%       0.000us         0.00%       0.000us       0.000us      66.531ms         6.40%      66.531ms      48.920us           0 B           0 B           0 B           0 B          1360                                                                                []            --  
ampere_fp16_s1688gemm_fp16_128x128_ldg8_f2f_stages_3...         0.00%       0.000us         0.00%       0.000us       0.000us      39.801ms         3.83%      39.801ms      29.265us           0 B           0 B           0 B           0 B          1360                                                                                []            --  
ampere_fp16_s1688gemm_fp16_256x64_ldg8_f2f_stages_32...         0.00%       0.000us         0.00%       0.000us       0.000us      32.698ms         3.14%      32.698ms      96.171us           0 B           0 B           0 B           0 B           340                                                                                []            --  
void at::native::elementwise_kernel<128, 2, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us      24.170ms         2.32%      24.170ms     805.660us           0 B           0 B           0 B           0 B            30                                                                                []            --  
void at::native::elementwise_kernel<128, 4, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us      22.910ms         2.20%      22.910ms     458.208us           0 B           0 B           0 B           0 B            50                                                                                []            --  
autograd::engine::evaluate_function: torch::autograd...         0.18%       1.537ms         0.84%       7.293ms      36.466us       0.000us         0.00%      21.483ms     107.415us           0 B          16 B           0 B           0 B           200                                                                                []            --  
                                     record_param_comms         0.13%       1.088ms         0.17%       1.479ms      67.238us      20.924ms         2.01%      20.924ms     951.069us           0 B           0 B           0 B           0 B            22                                          [[], [], [], [], [], [], [], [], [], []]            --  
void cudnn::dropout_apply_mask<__half, float, 4>(flo...         0.00%       0.000us         0.00%       0.000us       0.000us      20.915ms         2.01%      20.915ms      40.222us           0 B           0 B           0 B           0 B           520                                                                                []            --  
                                       c10d::allreduce_         0.05%     420.893us         0.18%       1.585ms      79.235us       0.000us         0.00%      20.906ms       1.045ms           0 B           0 B           0 B           0 B            20                                                          [[], [], [], [], [], []]            --  
ncclDevKernel_AllReduce_Sum_f32_RING_LL(ncclDevKerne...         0.00%       0.000us         0.00%       0.000us       0.000us      20.906ms         2.01%      20.906ms       1.045ms           0 B           0 B           0 B           0 B            20                                                                                []            --  
                                        nccl:all_reduce         0.00%       0.000us         0.00%       0.000us       0.000us      20.906ms         2.01%      20.906ms       1.045ms           0 B           0 B           0 B           0 B            20                                                                                []            --  
      autograd::engine::evaluate_function: MulBackward0         0.02%     148.959us         0.10%     873.761us      43.688us       0.000us         0.00%      20.029ms       1.001ms           0 B           0 B      -4.13 GB      -8.37 GB            20                                                                                []            --  
                                       aten::contiguous         0.01%      65.966us         0.06%     558.127us      18.604us       0.000us         0.00%      16.843ms     561.431us           0 B           0 B       6.23 GB           0 B            30                                                             [[68, 8192, 200], []]            --  
                                            aten::clone         0.01%      66.756us         0.06%     492.161us      16.405us       0.000us         0.00%      16.843ms     561.431us           0 B           0 B       6.23 GB           0 B            30                                                             [[68, 8192, 200], []]            --  
                                            aten::copy_         0.02%     154.385us         0.03%     286.713us       9.557us      16.843ms         1.62%      16.843ms     561.431us           0 B           0 B           0 B           0 B            30                                            [[68, 8192, 200], [68, 8192, 200], []]            --  
                                           MulBackward0         0.00%      28.224us         0.02%     136.276us      13.628us       0.000us         0.00%      14.705ms       1.471ms           0 B           0 B       4.15 GB           0 B            10                                                                 [[8192, 68, 200]]            --  
                                              aten::mul         0.01%      74.742us         0.01%     108.052us      10.805us      14.705ms         1.41%      14.705ms       1.471ms           0 B           0 B       4.15 GB       4.15 GB            10                                                [[8192, 68, 200], [8192, 68, 200]]  1114112000.000  
                                             aten::mul_         0.01%      79.771us         0.02%     134.862us      13.486us       9.451ms         0.91%       9.451ms     945.071us           0 B           0 B           0 B           0 B            10                                                    [[8192, 68, 200], [1, 1, 200]]            --  
                                             aten::sub_         0.02%     142.695us         0.03%     219.871us      21.987us       9.442ms         0.91%       9.442ms     944.198us           0 B           0 B           0 B           0 B            10                                                [[8192, 68, 200], [1, 1, 200], []]            --  
void at::native::elementwise_kernel<128, 2, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       9.442ms         0.91%       9.442ms     944.198us           0 B           0 B           0 B           0 B            10                                                                                []            --  
                                      aten::nan_to_num_         0.00%      39.535us         0.03%     249.492us      24.949us       0.000us         0.00%       9.193ms     919.280us           0 B           0 B           0 B           0 B            10                                                     [[8192, 68, 200], [], [], []]            --  
                                       aten::nan_to_num         0.01%      81.684us         0.02%     209.957us      20.996us       9.193ms         0.88%       9.193ms     919.280us           0 B           0 B           0 B           0 B            10                                    [[8192, 68, 200], [], [], [], [8192, 68, 200]]            --  
void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       9.193ms         0.88%       9.193ms     919.280us           0 B           0 B           0 B           0 B            10                                                                                []            --  
-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  --------------------------------------------------------------------------------  ------------  
Self CPU time total: 865.270ms
Self CUDA time total: 1.040s
