SwiGLU.Config(
        channels_in=4,
        channels_out=4,
        channels_hidden=4,
        expansion=2.6666666666666665,
        round_to=256,
        gate=True,
        bias=False,
        split_gate_projection=False,
        norm=None,
        act=priml.model.swiglu.silu,
        depth_index=(),
        init_weight=priml.model.init.unit_fan_in_uniform,
        init_weight_out=torch.nn.init.zeros_,
        up_proj=None,
        middle=None,
        down_proj=None,
        shard=None
    )
