MoE.Config(
        channels_in=4,
        channels_out=4,
        router=SoftmaxRouter.Config(
            channels_in=4,
            num_experts=2,
            top_k=1,
            norm_topk_prob=False,
            jitter_noise=0.0
        ),
        expert=SwiGLU.Config(
            channels_in=4,
            channels_out=4,
            channels_hidden=256,
            expansion=2.6666666666666665,
            round_to=256,
            gate=True,
            bias=False,
            split_gate_projection=False,
            norm=None,
            act=priml.model.swiglu.silu,
            depth_index=(),
            init_weight=priml.model.init.unit_fan_in_uniform,
            init_weight_out=torch.nn.init.zeros_,
            up_proj=None,
            middle=None,
            down_proj=None,
            shard='colwise'
        ),
        num_shared_experts=0,
        shared_expert=SwiGLU.Config(
            channels_in=4,
            channels_out=4,
            channels_hidden=256,
            expansion=2.6666666666666665,
            round_to=256,
            gate=True,
            bias=False,
            split_gate_projection=False,
            norm=None,
            act=priml.model.swiglu.silu,
            depth_index=(),
            init_weight=priml.model.init.unit_fan_in_uniform,
            init_weight_out=torch.nn.init.zeros_,
            up_proj=None,
            middle=None,
            down_proj=None,
            shard='colwise'
        ),
        aux_loss_weight=0.01,
        depth_index=()
    )
