MoE.Config(
                channels_in=4,
                channels_out=4,
                router=SoftmaxRouter.Config(
                        channels_in=4,
                        num_experts=2,
                        top_k=1,
                        norm_topk_prob=False,
                        jitter_noise=0.0
                ),
                expert=SwiGLU.Config(
                        channels_in=4,
                        channels_out=4,
                        channels_hidden=256,
                        expansion=2.6666666666666665,
                        round_to=256,
                        gate=True,
                        bias=False,
                        split_gate_projection=False,
                        norm=None,
                        act=<function priml.model.swiglu.silu at 0xdefacedeface>,
                        depth_index=(),
                        init_weight=<function priml.model.init.unit_fan_in_uniform at 0xdefacedeface>,
                        init_weight_out=<function torch.nn.init.zeros_ at 0xdefacedeface>,
                        shard='colwise'
                ),
                num_shared_experts=0,
                shared_expert=SwiGLU.Config(
                        channels_in=4,
                        channels_out=4,
                        channels_hidden=256,
                        expansion=2.6666666666666665,
                        round_to=256,
                        gate=True,
                        bias=False,
                        split_gate_projection=False,
                        norm=None,
                        act=<function priml.model.swiglu.silu at 0xdefacedeface>,
                        depth_index=(),
                        init_weight=<function priml.model.init.unit_fan_in_uniform at 0xdefacedeface>,
                        init_weight_out=<function torch.nn.init.zeros_ at 0xdefacedeface>,
                        shard='colwise'
                ),
                aux_loss_weight=0.01,
                depth_index=()
        )
