{
  "model": "llama-3.1-8b",
  "context": 32768,
  "concurrency": 1,
  "placements": [
    {
      "profile": "l4",
      "name": "NVIDIA L4",
      "feasible": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec": 18,
      "hourly_usd": 0.28,
      "usd_per_mtok": 4.44,
      "moe_batch_optimism": null,
      "reason": null
    },
    {
      "profile": "rtx-4090",
      "name": "NVIDIA RTX 4090",
      "feasible": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec": 59,
      "hourly_usd": 0.35,
      "usd_per_mtok": 1.65,
      "moe_batch_optimism": null,
      "reason": null
    },
    {
      "profile": "rtx-5090",
      "name": "NVIDIA RTX 5090",
      "feasible": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec": 105,
      "hourly_usd": 0.55,
      "usd_per_mtok": 1.46,
      "moe_batch_optimism": null,
      "reason": null
    },
    {
      "profile": "l40s",
      "name": "NVIDIA L40S",
      "feasible": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec": 50,
      "hourly_usd": 0.8,
      "usd_per_mtok": 4.4,
      "moe_batch_optimism": null,
      "reason": null
    },
    {
      "profile": "a100-40",
      "name": "NVIDIA A100 40 GB",
      "feasible": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec": 91,
      "hourly_usd": 1.1,
      "usd_per_mtok": 3.36,
      "moe_batch_optimism": null,
      "reason": null
    },
    {
      "profile": "a100-80",
      "name": "NVIDIA A100 80 GB",
      "feasible": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec": 119,
      "hourly_usd": 1.8,
      "usd_per_mtok": 4.2,
      "moe_batch_optimism": null,
      "reason": null
    },
    {
      "profile": "mi300x",
      "name": "AMD MI300X 192 GB",
      "feasible": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec": 310,
      "hourly_usd": 2.5,
      "usd_per_mtok": 2.24,
      "moe_batch_optimism": null,
      "reason": null
    },
    {
      "profile": "h100",
      "name": "NVIDIA H100 80 GB SXM",
      "feasible": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec": 196,
      "hourly_usd": 2.9,
      "usd_per_mtok": 4.12,
      "moe_batch_optimism": null,
      "reason": null
    },
    {
      "profile": "h200",
      "name": "NVIDIA H200 141 GB",
      "feasible": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec": 280,
      "hourly_usd": 3.6,
      "usd_per_mtok": 3.57,
      "moe_batch_optimism": null,
      "reason": null
    },
    {
      "profile": "b200",
      "name": "NVIDIA B200 192 GB",
      "feasible": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec": 467,
      "hourly_usd": 5.5,
      "usd_per_mtok": 3.27,
      "moe_batch_optimism": null,
      "reason": null
    },
    {
      "profile": "h100-x2",
      "name": "2\u00d7 NVIDIA H100",
      "feasible": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec": 362,
      "hourly_usd": 5.8,
      "usd_per_mtok": 4.45,
      "moe_batch_optimism": null,
      "reason": null
    },
    {
      "profile": "tpu-v5e-8",
      "name": "TPU v5e (8 chips)",
      "feasible": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec": 349,
      "hourly_usd": 9.6,
      "usd_per_mtok": 7.65,
      "moe_batch_optimism": null,
      "reason": null
    },
    {
      "profile": "h100-x4",
      "name": "4\u00d7 NVIDIA H100",
      "feasible": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec": 695,
      "hourly_usd": 11.6,
      "usd_per_mtok": 4.64,
      "moe_batch_optimism": null,
      "reason": null
    },
    {
      "profile": "tpu-v5p-4",
      "name": "TPU v5p (4 chips)",
      "feasible": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec": 573,
      "hourly_usd": 16.8,
      "usd_per_mtok": 8.14,
      "moe_batch_optimism": null,
      "reason": null
    },
    {
      "profile": "tpu-v6e-8",
      "name": "TPU v6e Trillium (8 chips)",
      "feasible": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec": 665,
      "hourly_usd": 21.6,
      "usd_per_mtok": 9.02,
      "moe_batch_optimism": null,
      "reason": null
    },
    {
      "profile": "h200-x8",
      "name": "8\u00d7 NVIDIA H200",
      "feasible": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec": 1949,
      "hourly_usd": 28.8,
      "usd_per_mtok": 4.11,
      "moe_batch_optimism": null,
      "reason": null
    },
    {
      "profile": "m3-ultra-512",
      "name": "Apple M3 Ultra 512 GB",
      "feasible": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec": 48,
      "hourly_usd": null,
      "usd_per_mtok": null,
      "moe_batch_optimism": null,
      "reason": null
    },
    {
      "profile": "m4-max-128",
      "name": "Apple M4 Max 128 GB",
      "feasible": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec": 32,
      "hourly_usd": null,
      "usd_per_mtok": null,
      "moe_batch_optimism": null,
      "reason": null
    },
    {
      "profile": "m4-pro-48",
      "name": "Apple M4 Pro 48 GB",
      "feasible": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec": 16,
      "hourly_usd": null,
      "usd_per_mtok": null,
      "moe_batch_optimism": null,
      "reason": null
    },
    {
      "profile": "cpu-64",
      "name": "CPU only (64 GB DDR5)",
      "feasible": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec": 4,
      "hourly_usd": null,
      "usd_per_mtok": null,
      "moe_batch_optimism": null,
      "reason": null
    }
  ]
}
