Transformer.Config(
                channels_in=4,
                channels_out=4,
                num_layers=1,
                proj_in=Embedding.Config(
                        channels_in=2,
                        channels_out=4,
                        padding_idx=None,
                        device=None,
                        dtype=None,
                        shard='vocab',
                        depth_index=(),
                        init_weight=functools.partial(<function priml.model.init.truncated_normal at 0xdefacedeface>, std=0.02)
                ),
                block=[
                │       TransformerBlock.Config(
                │               channels_in=4,
                │               channels_out=4,
                │               attn=Attention.Config(
                │                       channels_in=4,
                │                       channels_out=4,
                │                       num_heads=2,
                │                       channels_head=2,
                │                       num_heads_kv=2,
                │                       bias=False,
                │                       dropout=0.0,
                │                       causal=True,
                │                       rope=None,
                │                       norm_qk=None,
                │                       share_qk_norm=True,
                │                       norm_out=None,
                │                       split_qkv_projection=False,
                │                       init_weight=<function priml.model.init.kaiming_uniform at 0xdefacedeface>,
                │                       depth_index=((0, 1),),
                │                       attn_kernel=SdpaFused.Config()
                │               ),
                │               ffn=SwiGLU.Config(
                │                       channels_in=4,
                │                       channels_out=4,
                │                       channels_hidden=2,
                │                       expansion=2.6666666666666665,
                │                       round_to=1,
                │                       gate=True,
                │                       bias=False,
                │                       split_gate_projection=False,
                │                       norm=None,
                │                       act=<function priml.model.swiglu.silu at 0xdefacedeface>,
                │                       depth_index=((0, 1),),
                │                       init_weight=<function priml.model.init.unit_fan_in_uniform at 0xdefacedeface>,
                │                       init_weight_out=<function torch.nn.init.zeros_ at 0xdefacedeface>,
                │                       shard='colwise'
                │               ),
                │               norm1=RMSNorm.Config(
                │                       channels_in=4,
                │                       channels_out=4,
                │                       eps=1e-06,
                │                       elementwise_affine=False,
                │                       device=None,
                │                       dtype=None
                │               ),
                │               norm2=RMSNorm.Config(
                │                       channels_in=4,
                │                       channels_out=4,
                │                       eps=1e-06,
                │                       elementwise_affine=False,
                │                       device=None,
                │                       dtype=None
                │               ),
                │               prenorm=True,
                │               checkpoint=False,
                │               depth_index=((0, 1),)
                │       )
                ],
                proj_out=Sequential.Config(
                        channels_in=4,
                        channels_out=4,
                        elements=[
                                RMSNorm.Config(
                                        channels_in=4,
                                        channels_out=4,
                                        eps=1e-06,
                                        elementwise_affine=False,
                                        device=None,
                                        dtype=None
                                ),
                                Linear.Config(
                                        channels_in=4,
                                        channels_out=4,
                                        bias=False,
                                        depth_index=(),
                                        device=None,
                                        dtype=None,
                                        init_weight=<function priml.model.init.kaiming_uniform at 0xdefacedeface>,
                                        init_bias=<function torch.nn.init.zeros_ at 0xdefacedeface>,
                                        shard='vocab'
                                )
                        ],
                        repeat=1,
                        depth_index=()
                )
        )
