-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  --------------------------------------------------------------------------------  ------------  
                                                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg     Self CUDA   Self CUDA %    CUDA total  CUDA time avg       CPU Mem  Self CPU Mem      CUDA Mem  Self CUDA Mem    # of Calls                                                                      Input Shapes   Total FLOPs  
-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  --------------------------------------------------------------------------------  ------------  
                                          ProfilerStep*         0.00%       0.000us         0.00%       0.000us       0.000us     588.031ms       166.76%     588.031ms       9.801ms           0 B           0 B           0 B           0 B            60                                                                                []            --  
                                          ProfilerStep*        47.87%      92.506ms        69.00%     133.328ms      13.333ms       0.000us         0.00%     352.612ms      35.261ms           0 B           0 B           0 B     -11.19 GB            10                                                                                []            --  
                                       aten::_cudnn_rnn         6.74%      13.032ms        22.60%      43.662ms       2.183ms     104.630ms        29.67%     228.193ms      11.410ms           0 B           0 B       4.80 GB     -15.94 GB            20  [[8192, 68, 200], [], [], [588672], [2, 8192, 224], [], [], [], [], [], [], [],             --  
                                         aten::_to_copy         0.05%     105.888us         1.29%       2.491ms     124.541us       0.000us         0.00%     196.436ms       9.822ms           0 B           0 B       6.24 GB           0 B            20                                         [[8192, 68, 200], [], [], [], [], [], []]            --  
                                            aten::copy_         0.03%      59.241us         0.08%     159.432us       7.972us     196.436ms        55.71%     196.436ms       9.822ms           0 B           0 B           0 B           0 B            20                                            [[8192, 68, 200], [8192, 68, 200], []]            --  
                         Memcpy HtoD (Pinned -> Device)         0.00%       0.000us         0.00%       0.000us       0.000us     189.228ms        53.66%     189.228ms       6.308ms           0 B           0 B           0 B           0 B            30                                                                                []            --  
                                               aten::to         0.01%      12.128us         1.23%       2.372ms     118.582us       0.000us         0.00%     189.163ms       9.458ms           0 B           0 B       4.16 GB           0 B            20                                     [[8192, 68, 200], [], [], [], [], [], [], []]            --  
                                              aten::gru         0.16%     305.028us        12.70%      24.544ms       2.454ms       0.000us         0.00%     117.854ms      11.785ms           0 B           0 B       2.39 GB     -11.23 MB            10                     [[8192, 68, 200], [2, 8192, 224], [], [], [], [], [], [], []]            --  
ampere_fp16_s1688gemm_fp16_128x128_ldg8_f2f_stages_3...         0.00%       0.000us         0.00%       0.000us       0.000us      37.595ms        10.66%      37.595ms      27.644us           0 B           0 B           0 B           0 B          1360                                                                                []            --  
void elemWiseRNNcell<__half, __half, float, (cudnnRN...         0.00%       0.000us         0.00%       0.000us       0.000us      34.829ms         9.88%      34.829ms      25.609us           0 B           0 B           0 B           0 B          1360                                                                                []            --  
ampere_fp16_s1688gemm_fp16_256x64_ldg8_f2f_stages_32...         0.00%       0.000us         0.00%       0.000us       0.000us      32.206ms         9.13%      32.206ms      94.724us           0 B           0 B           0 B           0 B           340                                                                                []            --  
                                             aten::sub_         0.05%     104.106us         0.09%     165.386us      16.539us       9.760ms         2.77%       9.760ms     975.952us           0 B           0 B           0 B           0 B            10                                                [[8192, 68, 200], [1, 1, 200], []]            --  
void at::native::elementwise_kernel<128, 2, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       9.760ms         2.77%       9.760ms     975.952us           0 B           0 B           0 B           0 B            10                                                                                []            --  
                                             aten::mul_         0.03%      66.391us         0.06%     113.821us      11.382us       9.695ms         2.75%       9.695ms     969.508us           0 B           0 B           0 B           0 B            10                                                    [[8192, 68, 200], [1, 1, 200]]            --  
void at::native::elementwise_kernel<128, 2, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       9.695ms         2.75%       9.695ms     969.508us           0 B           0 B           0 B           0 B            10                                                                                []            --  
void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       9.561ms         2.71%       9.561ms     239.016us           0 B           0 B           0 B           0 B            40                                                                                []            --  
                                      aten::nan_to_num_         0.02%      30.184us         0.12%     229.664us      22.966us       0.000us         0.00%       9.529ms     952.852us           0 B           0 B           0 B           0 B            10                                                     [[8192, 68, 200], [], [], []]            --  
                                       aten::nan_to_num         0.04%      71.128us         0.10%     199.480us      19.948us       9.529ms         2.70%       9.529ms     952.852us           0 B           0 B           0 B           0 B            10                                    [[8192, 68, 200], [], [], [], [8192, 68, 200]]            --  
                                              aten::mul         0.05%     105.958us         0.08%     153.450us      15.345us       9.364ms         2.66%       9.364ms     936.382us           0 B           0 B       4.16 GB       4.16 GB            10                                                 [[8192, 68, 200], [8192, 1, 200]]  1114112000.000  
void at::native::elementwise_kernel<128, 4, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       9.364ms         2.66%       9.364ms     936.382us           0 B           0 B           0 B           0 B            10                                                                                []            --  
void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       7.500ms         2.13%       7.500ms      75.000us           0 B           0 B           0 B           0 B           100                                                                                []            --  
                                               aten::to         0.01%      16.066us         0.08%     147.383us       7.369us       0.000us         0.00%       7.273ms     363.625us           0 B           0 B       2.08 GB           0 B            20                                                 [[8192, 68, 200], [], [], [], []]            --  
                                       aten::contiguous         0.01%      14.579us         0.10%     197.335us      19.734us       0.000us         0.00%       5.709ms     570.875us           0 B           0 B       2.08 GB           0 B            10                                                             [[68, 8192, 200], []]            --  
                                            aten::clone         0.02%      34.195us         0.09%     182.756us      18.276us       0.000us         0.00%       5.709ms     570.875us           0 B           0 B       2.08 GB           0 B            10                                                             [[68, 8192, 200], []]            --  
                                            aten::copy_         0.03%      50.524us         0.05%      94.499us       9.450us       5.709ms         1.62%       5.709ms     570.875us           0 B           0 B           0 B           0 B            10                                            [[68, 8192, 200], [68, 8192, 200], []]            --  
void at::native::elementwise_kernel<128, 4, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       5.709ms         1.62%       5.709ms     570.875us           0 B           0 B           0 B           0 B            10                                                                                []            --  
                                             aten::mean         0.14%     261.361us         0.18%     344.424us      34.442us       5.369ms         1.52%       5.369ms     536.881us           0 B           0 B      62.50 MB      62.50 MB            10                                                     [[8192, 68, 200], [], [], []]            --  
void at::native::reduce_kernel<128, 4, at::native::R...         0.00%       0.000us         0.00%       0.000us       0.000us       5.369ms         1.52%       5.369ms     536.881us           0 B           0 B           0 B           0 B            10                                                                                []            --  
                                              aten::sum         0.59%       1.138ms         0.87%       1.675ms      11.168us     367.746us         0.10%     367.746us       2.452us           0 B           0 B      75.00 KB      75.00 KB           150                                                                      [[8192], []]            --  
void at::native::reduce_kernel<512, 1, at::native::R...         0.00%       0.000us         0.00%       0.000us       0.000us     367.746us         0.10%     367.746us       2.452us           0 B           0 B           0 B           0 B           150                                                                                []            --  
                                       aten::layer_norm         0.03%      61.424us         0.41%     791.379us      39.569us       0.000us         0.00%     312.351us      15.618us           0 B           0 B     154.09 MB     -56.62 MB            20                                           [[8192, 224], [], [224], [224], [], []]            --  
void at::native::(anonymous namespace)::vectorized_l...         0.00%       0.000us         0.00%       0.000us       0.000us     261.531us         0.07%     261.531us      13.077us           0 B           0 B           0 B           0 B            20                                                                                []            --  
                                         aten::_to_copy         0.19%     375.813us         0.76%       1.472ms      21.023us       0.000us         0.00%     246.487us       3.521us           0 B           0 B       2.81 MB           0 B            70                                                  [[8192], [], [], [], [], [], []]            --  
                                            aten::copy_         0.15%     297.969us         0.43%     826.937us      11.813us     246.487us         0.07%     246.487us       3.521us           0 B           0 B           0 B           0 B            70                                                              [[8192], [8192], []]            --  
                                           aten::linear         0.06%     120.012us         0.64%       1.238ms      61.903us       0.000us         0.00%     197.502us       9.875us           0 B           0 B      58.52 MB     -24.50 MB            20                                                  [[8192, 224], [112, 224], [112]]            --  
                                               aten::to         0.24%     468.114us         0.81%       1.570ms      26.166us       0.000us         0.00%     189.050us       3.151us           0 B           0 B       1.88 MB           0 B            60                                              [[8192], [], [], [], [], [], [], []]            --  
                                              aten::mul         0.34%     657.247us         0.59%       1.148ms       7.176us     169.465us         0.05%     169.465us       1.059us           0 B           0 B       4.53 MB       4.53 MB           160                                                                  [[8192], [8192]]   1310720.000  
void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us     163.711us         0.05%     163.711us       1.637us           0 B           0 B           0 B           0 B           100                                                                                []            --  
void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us     149.178us         0.04%     149.178us       1.066us           0 B           0 B           0 B           0 B           140                                                                                []            --  
                                           aten::linear         0.10%     200.882us         0.95%       1.828ms      91.423us       0.000us         0.00%     139.426us       6.971us           0 B           0 B      13.25 MB     -28.12 MB            20                                                    [[8192, 200], [32, 200], [32]]            --  
-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  --------------------------------------------------------------------------------  ------------  
Self CPU time total: 193.229ms
Self CUDA time total: 352.612ms
