-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  --------------------------------------------------------------------------------  ------------  
                                                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg     Self CUDA   Self CUDA %    CUDA total  CUDA time avg       CPU Mem  Self CPU Mem      CUDA Mem  Self CUDA Mem    # of Calls                                                                      Input Shapes   Total FLOPs  
-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  --------------------------------------------------------------------------------  ------------  
                                          ProfilerStep*         0.00%       0.000us         0.00%       0.000us       0.000us     679.418ms        64.25%     679.418ms      33.971ms           0 B           0 B           0 B           0 B            20                                                                                []            --  
autograd::engine::evaluate_function: CudnnRnnBackwar...         0.04%     200.921us         5.44%      30.654ms       3.065ms       0.000us         0.00%     604.278ms      60.428ms           0 B           0 B     -35.08 GB     -37.17 GB            10                                                                                []            --  
                                      CudnnRnnBackward0         0.02%     131.669us         5.41%      30.453ms       3.045ms       0.000us         0.00%     604.278ms      60.428ms           0 B           0 B       2.09 GB     -70.00 MB            10                                                         [[8192, 68, 224], [], []]            --  
                              aten::_cudnn_rnn_backward         3.13%      17.612ms         5.38%      30.321ms       3.032ms     587.027ms        55.52%     604.278ms      60.428ms           0 B           0 B       2.15 GB     -34.64 GB            10  [[8192, 68, 200], [], [], [588672], [2, 8192, 224], [], [8192, 68, 224], [8192,             --  
                        DistributedDataParallel.forward         0.00%       0.000us         0.00%       0.000us       0.000us     489.103ms        46.26%     489.103ms       9.782ms           0 B           0 B           0 B           0 B            50                                                                                []            --  
                                          ProfilerStep*        17.11%      96.397ms        29.78%     167.716ms      16.772ms       0.000us         0.00%     419.025ms      41.903ms           0 B           0 B           0 B     -39.24 GB            10                                                                                []            --  
                                       aten::_cudnn_rnn         2.52%      14.212ms         8.98%      50.605ms       2.530ms     150.175ms        14.20%     319.003ms      15.950ms           0 B           0 B      67.66 GB     -16.12 GB            20  [[8192, 68, 200], [], [], [588672], [2, 8192, 224], [], [], [], [], [], [], [],             --  
ampere_fp16_s16816gemm_fp16_128x64_ldg8_f2f_stages_6...         0.00%       0.000us         0.00%       0.000us       0.000us     262.847ms        24.86%     262.847ms       4.381ms           0 B           0 B           0 B           0 B            60                                                                                []            --  
                                         aten::_to_copy         0.02%     111.916us         0.18%       1.017ms      33.907us       0.000us         0.00%     208.220ms       6.941ms           0 B           0 B      10.40 GB           0 B            30                                         [[8192, 68, 200], [], [], [], [], [], []]            --  
                                            aten::copy_         0.02%     116.591us         0.05%     309.591us      10.320us     208.220ms        19.69%     208.220ms       6.941ms           0 B           0 B           0 B           0 B            30                                            [[8192, 68, 200], [8192, 68, 200], []]            --  
                                               aten::to         0.00%      25.483us         0.16%     924.162us      30.805us       0.000us         0.00%     201.130ms       6.704ms           0 B           0 B       8.32 GB           0 B            30                                     [[8192, 68, 200], [], [], [], [], [], [], []]            --  
                         Memcpy HtoD (Pinned -> Device)         0.00%       0.000us         0.00%       0.000us       0.000us     194.304ms        18.38%     194.304ms       6.477ms           0 B           0 B           0 B           0 B            30                                                                                []            --  
                        DistributedDataParallel.forward         0.66%       3.726ms         7.00%      39.419ms       3.942ms       0.000us         0.00%     193.317ms      19.332ms           0 B           0 B      35.10 GB      -4.54 GB            10                                                                                []            --  
                                              aten::gru         0.08%     467.149us         4.99%      28.135ms       2.814ms       0.000us         0.00%     163.169ms      16.317ms           0 B           0 B      34.91 GB           0 B            10                     [[8192, 68, 200], [2, 8192, 224], [], [], [], [], [], [], []]            --  
void GRU_elementWise_bp2<__half, __half, float, 3, (...         0.00%       0.000us         0.00%       0.000us       0.000us     129.394ms        12.24%     129.394ms       6.470ms           0 B           0 B           0 B           0 B            20                                                                                []            --  
ampere_fp16_s1688gemm_fp16_128x128_ldg8_f2f_stages_3...         0.00%       0.000us         0.00%       0.000us       0.000us     106.806ms        10.10%     106.806ms      52.356us           0 B           0 B           0 B           0 B          2040                                                                                []            --  
void GRU_elementWise_bp1<__half, __half, float>(int,...         0.00%       0.000us         0.00%       0.000us       0.000us      72.318ms         6.84%      72.318ms      53.175us           0 B           0 B           0 B           0 B          1360                                                                                []            --  
void elemWiseRNNcell<__half, __half, float, (cudnnRN...         0.00%       0.000us         0.00%       0.000us       0.000us      66.832ms         6.32%      66.832ms      49.141us           0 B           0 B           0 B           0 B          1360                                                                                []            --  
ampere_fp16_s1688gemm_fp16_128x128_ldg8_f2f_stages_3...         0.00%       0.000us         0.00%       0.000us       0.000us      40.181ms         3.80%      40.181ms      29.545us           0 B           0 B           0 B           0 B          1360                                                                                []            --  
ampere_fp16_s1688gemm_fp16_256x64_ldg8_f2f_stages_32...         0.00%       0.000us         0.00%       0.000us       0.000us      33.038ms         3.12%      33.038ms      97.170us           0 B           0 B           0 B           0 B           340                                                                                []            --  
void at::native::elementwise_kernel<128, 2, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us      24.267ms         2.29%      24.267ms     808.886us           0 B           0 B           0 B           0 B            30                                                                                []            --  
void at::native::elementwise_kernel<128, 4, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us      23.032ms         2.18%      23.032ms     460.636us           0 B           0 B           0 B           0 B            50                                                                                []            --  
void cudnn::dropout_apply_mask<__half, float, 4>(flo...         0.00%       0.000us         0.00%       0.000us       0.000us      21.744ms         2.06%      21.744ms      41.815us           0 B           0 B           0 B           0 B           520                                                                                []            --  
      autograd::engine::evaluate_function: MulBackward0         0.03%     171.860us         0.15%     819.027us      40.951us       0.000us         0.00%      20.065ms       1.003ms           0 B           0 B      -4.13 GB      -8.37 GB            20                                                                                []            --  
                                       aten::contiguous         0.01%      51.080us         0.11%     592.570us      19.752us       0.000us         0.00%      16.944ms     564.817us           0 B           0 B       6.23 GB           0 B            30                                                             [[68, 8192, 200], []]            --  
                                            aten::clone         0.01%      73.133us         0.10%     541.490us      18.050us       0.000us         0.00%      16.944ms     564.817us           0 B           0 B       6.23 GB           0 B            30                                                             [[68, 8192, 200], []]            --  
                                            aten::copy_         0.03%     166.841us         0.06%     322.115us      10.737us      16.944ms         1.60%      16.944ms     564.817us           0 B           0 B           0 B           0 B            30                                            [[68, 8192, 200], [68, 8192, 200], []]            --  
autograd::engine::evaluate_function: torch::autograd...         0.24%       1.326ms         1.06%       5.950ms      29.748us       0.000us         0.00%      15.868ms      79.340us           0 B          32 B           0 B           0 B           200                                                                                []            --  
                                       c10d::allreduce_         0.06%     325.646us         0.24%       1.346ms      67.280us       0.000us         0.00%      15.344ms     767.195us           0 B           0 B           0 B           0 B            20                                                          [[], [], [], [], [], []]            --  
                                     record_param_comms         0.13%     745.753us         0.18%       1.020ms      50.998us      15.344ms         1.45%      15.344ms     767.195us           0 B           0 B           0 B           0 B            20                                          [[], [], [], [], [], [], [], [], [], []]            --  
ncclDevKernel_AllReduce_Sum_f32_RING_LL(ncclDevKerne...         0.00%       0.000us         0.00%       0.000us       0.000us      15.344ms         1.45%      15.344ms     767.195us           0 B           0 B           0 B           0 B            20                                                                                []            --  
                                        nccl:all_reduce         0.00%       0.000us         0.00%       0.000us       0.000us      15.344ms         1.45%      15.344ms     767.195us           0 B           0 B           0 B           0 B            20                                                                                []            --  
                                           Buffer Flush         0.03%     152.868us         0.03%     170.596us     170.596us      14.834ms         1.40%      14.834ms      14.834ms           0 B           0 B      -7.38 MB      -7.38 MB             1                                                                                []            --  
                                           MulBackward0         0.01%      29.421us         0.02%     138.177us      13.818us       0.000us         0.00%      14.750ms       1.475ms           0 B           0 B       4.15 GB           0 B            10                                                                 [[8192, 68, 200]]            --  
                                              aten::mul         0.01%      72.633us         0.02%     108.756us      10.876us      14.750ms         1.39%      14.750ms       1.475ms           0 B           0 B       4.15 GB       4.15 GB            10                                                [[8192, 68, 200], [8192, 68, 200]]  1114112000.000  
                                             aten::sub_         0.02%     135.951us         0.04%     209.103us      20.910us       9.695ms         0.92%       9.695ms     969.464us           0 B           0 B           0 B           0 B            10                                                [[8192, 68, 200], [1, 1, 200], []]            --  
void at::native::elementwise_kernel<128, 2, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       9.695ms         0.92%       9.695ms     969.464us           0 B           0 B           0 B           0 B            10                                                                                []            --  
                                             aten::mul_         0.02%      88.092us         0.03%     150.736us      15.074us       9.503ms         0.90%       9.503ms     950.283us           0 B           0 B           0 B           0 B            10                                                    [[8192, 68, 200], [1, 1, 200]]            --  
                                      aten::nan_to_num_         0.01%      71.868us         0.05%     305.938us      30.594us       0.000us         0.00%       9.442ms     944.242us           0 B           0 B           0 B           0 B            10                                                     [[8192, 68, 200], [], [], []]            --  
                                       aten::nan_to_num         0.02%     106.450us         0.04%     234.070us      23.407us       9.442ms         0.89%       9.442ms     944.242us           0 B           0 B           0 B           0 B            10                                    [[8192, 68, 200], [], [], [], [8192, 68, 200]]            --  
-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  --------------------------------------------------------------------------------  ------------  
Self CPU time total: 563.272ms
Self CUDA time total: 1.057s
