-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  --------------------------------------------------------------------------------  ------------  
                                                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg     Self CUDA   Self CUDA %    CUDA total  CUDA time avg       CPU Mem  Self CPU Mem      CUDA Mem  Self CUDA Mem    # of Calls                                                                      Input Shapes   Total FLOPs  
-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  --------------------------------------------------------------------------------  ------------  
                                          ProfilerStep*         0.00%       0.000us         0.00%       0.000us       0.000us     210.129ms        64.97%     210.129ms      35.021ms           0 B           0 B           0 B           0 B             6                                                                                []            --  
autograd::engine::evaluate_function: CudnnRnnBackwar...         0.01%      52.471us         2.45%      10.095ms       3.365ms       0.000us         0.00%     191.622ms      63.874ms           0 B           0 B     -10.52 GB     -11.15 GB             3                                                                                []            --  
                                      CudnnRnnBackward0         0.08%     331.306us         2.44%      10.042ms       3.347ms       0.000us         0.00%     191.622ms      63.874ms           0 B           0 B     640.87 MB     -21.00 MB             3                                                         [[8192, 68, 224], [], []]            --  
                              aten::_cudnn_rnn_backward         1.29%       5.318ms         2.36%       9.711ms       3.237ms     186.462ms        57.65%     191.622ms      63.874ms           0 B           0 B     661.87 MB     -10.39 GB             3  [[8192, 68, 200], [], [], [588672], [2, 8192, 224], [], [8192, 68, 224], [8192,             --  
                        DistributedDataParallel.forward         0.00%       0.000us         0.00%       0.000us       0.000us     154.232ms        47.68%     154.232ms      10.282ms           0 B           0 B           0 B           0 B            15                                                                                []            --  
                                          ProfilerStep*       -49.61%  -204421.911us        86.11%     354.867ms     118.289ms       0.000us         0.00%     115.664ms      38.555ms           0 B           0 B     747.00 KB     -11.77 GB             3                                                                                []            --  
                                       aten::_cudnn_rnn         0.95%       3.904ms         3.35%      13.804ms       2.301ms      46.052ms        14.24%      97.667ms      16.278ms           0 B           0 B      20.30 GB      -4.84 GB             6  [[8192, 68, 200], [], [], [588672], [2, 8192, 224], [], [], [], [], [], [], [],             --  
ampere_fp16_s16816gemm_fp16_128x64_ldg8_f2f_stages_6...         0.00%       0.000us         0.00%       0.000us       0.000us      85.512ms        26.44%      85.512ms       4.751ms           0 B           0 B           0 B           0 B            18                                                                                []            --  
                                         aten::_to_copy         0.03%     130.588us         0.16%     669.083us      74.343us       0.000us         0.00%      55.862ms       6.207ms           0 B           0 B       3.12 GB           0 B             9                                         [[8192, 68, 200], [], [], [], [], [], []]            --  
                                            aten::copy_         0.01%      37.898us         0.02%      96.715us      10.746us      55.862ms        17.27%      55.862ms       6.207ms           0 B           0 B           0 B           0 B             9                                            [[8192, 68, 200], [8192, 68, 200], []]            --  
                        DistributedDataParallel.forward         0.95%       3.905ms        21.50%      88.584ms      29.528ms       0.000us         0.00%      54.576ms      18.192ms           0 B         -92 B      10.53 GB      -1.37 GB             3                                                                                []            --  
                                               aten::to         0.00%      17.487us         0.16%     655.875us      72.875us       0.000us         0.00%      53.739ms       5.971ms           0 B           0 B       2.50 GB           0 B             9                                     [[8192, 68, 200], [], [], [], [], [], [], []]            --  
                         Memcpy HtoD (Pinned -> Device)         0.00%       0.000us         0.00%       0.000us       0.000us      51.705ms        15.99%      51.705ms       5.745ms           0 B           0 B           0 B           0 B             9                                                                                []            --  
                                              aten::gru         0.24%     991.811us         2.11%       8.712ms       2.904ms       0.000us         0.00%      49.931ms      16.644ms           0 B           0 B      10.47 GB           0 B             3                     [[8192, 68, 200], [2, 8192, 224], [], [], [], [], [], [], []]            --  
void GRU_elementWise_bp2<__half, __half, float, 3, (...         0.00%       0.000us         0.00%       0.000us       0.000us      40.296ms        12.46%      40.296ms       6.716ms           0 B           0 B           0 B           0 B             6                                                                                []            --  
ampere_fp16_s1688gemm_fp16_128x128_ldg8_f2f_stages_3...         0.00%       0.000us         0.00%       0.000us       0.000us      34.458ms        10.65%      34.458ms      56.304us           0 B           0 B           0 B           0 B           612                                                                                []            --  
void GRU_elementWise_bp1<__half, __half, float>(int,...         0.00%       0.000us         0.00%       0.000us       0.000us      21.781ms         6.73%      21.781ms      53.384us           0 B           0 B           0 B           0 B           408                                                                                []            --  
void elemWiseRNNcell<__half, __half, float, (cudnnRN...         0.00%       0.000us         0.00%       0.000us       0.000us      20.101ms         6.21%      20.101ms      49.268us           0 B           0 B           0 B           0 B           408                                                                                []            --  
ampere_fp16_s1688gemm_fp16_128x128_ldg8_f2f_stages_3...         0.00%       0.000us         0.00%       0.000us       0.000us      12.676ms         3.92%      12.676ms      31.068us           0 B           0 B           0 B           0 B           408                                                                                []            --  
ampere_fp16_s1688gemm_fp16_256x64_ldg8_f2f_stages_32...         0.00%       0.000us         0.00%       0.000us       0.000us      10.212ms         3.16%      10.212ms     100.117us           0 B           0 B           0 B           0 B           102                                                                                []            --  
void at::native::elementwise_kernel<128, 2, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       7.255ms         2.24%       7.255ms     806.085us           0 B           0 B           0 B           0 B             9                                                                                []            --  
void at::native::elementwise_kernel<128, 4, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       6.880ms         2.13%       6.880ms     458.653us           0 B           0 B           0 B           0 B            15                                                                                []            --  
autograd::engine::evaluate_function: torch::autograd...         0.11%     451.831us         0.45%       1.840ms      30.673us       0.000us         0.00%       6.439ms     107.319us           0 B           8 B           0 B           0 B            60                                                                                []            --  
                                     record_param_comms         0.09%     388.763us         0.13%     519.268us      64.909us       6.285ms         1.94%       6.285ms     785.583us           0 B           0 B           0 B           0 B             8                                          [[], [], [], [], [], [], [], [], [], []]            --  
                                       c10d::allreduce_         0.03%     116.907us         0.12%     480.722us      80.120us       0.000us         0.00%       6.267ms       1.045ms           0 B           0 B           0 B           0 B             6                                                          [[], [], [], [], [], []]            --  
ncclDevKernel_AllReduce_Sum_f32_RING_LL(ncclDevKerne...         0.00%       0.000us         0.00%       0.000us       0.000us       6.267ms         1.94%       6.267ms       1.045ms           0 B           0 B           0 B           0 B             6                                                                                []            --  
                                        nccl:all_reduce         0.00%       0.000us         0.00%       0.000us       0.000us       6.267ms         1.94%       6.267ms       1.045ms           0 B           0 B           0 B           0 B             6                                                                                []            --  
void cudnn::dropout_apply_mask<__half, float, 4>(flo...         0.00%       0.000us         0.00%       0.000us       0.000us       6.241ms         1.93%       6.241ms      40.004us           0 B           0 B           0 B           0 B           156                                                                                []            --  
      autograd::engine::evaluate_function: MulBackward0         0.01%      46.459us         0.07%     268.532us      44.755us       0.000us         0.00%       5.953ms     992.101us           0 B           0 B      -1.24 GB      -2.51 GB             6                                                                                []            --  
                                       aten::contiguous         0.00%      13.491us         0.04%     173.250us      19.250us       0.000us         0.00%       5.058ms     561.960us           0 B           0 B       1.87 GB           0 B             9                                                             [[68, 8192, 200], []]            --  
                                            aten::clone         0.01%      24.959us         0.04%     159.759us      17.751us       0.000us         0.00%       5.058ms     561.960us           0 B           0 B       1.87 GB           0 B             9                                                             [[68, 8192, 200], []]            --  
                                            aten::copy_         0.01%      51.275us         0.02%      91.917us      10.213us       5.058ms         1.56%       5.058ms     561.960us           0 B           0 B           0 B           0 B             9                                            [[68, 8192, 200], [68, 8192, 200], []]            --  
                                           MulBackward0         0.00%       8.653us         0.01%      43.265us      14.422us       0.000us         0.00%       4.389ms       1.463ms           0 B           0 B       1.25 GB           0 B             3                                                                 [[8192, 68, 200]]            --  
                                              aten::mul         0.01%      23.875us         0.01%      34.612us      11.537us       4.389ms         1.36%       4.389ms       1.463ms           0 B           0 B       1.25 GB       1.25 GB             3                                                [[8192, 68, 200], [8192, 68, 200]]  334233600.000  
                                             aten::mul_         0.01%      56.856us         0.02%      76.515us      25.505us       2.861ms         0.88%       2.861ms     953.728us           0 B           0 B           0 B           0 B             3                                                    [[8192, 68, 200], [1, 1, 200]]            --  
                                             aten::sub_         0.01%      52.187us         0.02%      80.740us      26.913us       2.844ms         0.88%       2.844ms     948.117us           0 B           0 B           0 B           0 B             3                                                [[8192, 68, 200], [1, 1, 200], []]            --  
void at::native::elementwise_kernel<128, 2, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       2.844ms         0.88%       2.844ms     948.117us           0 B           0 B           0 B           0 B             3                                                                                []            --  
                                      aten::nan_to_num_         0.00%      12.748us         0.02%      98.428us      32.809us       0.000us         0.00%       2.779ms     926.465us           0 B           0 B           0 B           0 B             3                                                     [[8192, 68, 200], [], [], []]            --  
                                       aten::nan_to_num         0.01%      31.800us         0.02%      85.680us      28.560us       2.779ms         0.86%       2.779ms     926.465us           0 B           0 B           0 B           0 B             3                                    [[8192, 68, 200], [], [], [], [8192, 68, 200]]            --  
void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       2.779ms         0.86%       2.779ms     926.465us           0 B           0 B           0 B           0 B             3                                                                                []            --  
-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  --------------------------------------------------------------------------------  ------------  
Self CPU time total: 412.097ms
Self CUDA time total: 323.441ms
