{
  "model": "llama-3.1-8b",
  "context": 32768,
  "concurrency": 1,
  "estimates_are_roofline_not_measured": true,
  "placements": [
    {
      "profile": "l4",
      "name": "NVIDIA L4",
      "fits": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec_estimate": 18,
      "hourly_usd": 0.28,
      "reason": ""
    },
    {
      "profile": "rtx-4090",
      "name": "NVIDIA RTX 4090",
      "fits": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec_estimate": 59,
      "hourly_usd": 0.35,
      "reason": ""
    },
    {
      "profile": "rtx-5090",
      "name": "NVIDIA RTX 5090",
      "fits": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec_estimate": 105,
      "hourly_usd": 0.55,
      "reason": ""
    },
    {
      "profile": "l40s",
      "name": "NVIDIA L40S",
      "fits": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec_estimate": 50,
      "hourly_usd": 0.8,
      "reason": ""
    },
    {
      "profile": "a100-40",
      "name": "NVIDIA A100 40 GB",
      "fits": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec_estimate": 91,
      "hourly_usd": 1.1,
      "reason": ""
    },
    {
      "profile": "a100-80",
      "name": "NVIDIA A100 80 GB",
      "fits": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec_estimate": 119,
      "hourly_usd": 1.8,
      "reason": ""
    },
    {
      "profile": "mi300x",
      "name": "AMD MI300X 192 GB",
      "fits": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec_estimate": 310,
      "hourly_usd": 2.5,
      "reason": ""
    },
    {
      "profile": "h100",
      "name": "NVIDIA H100 80 GB SXM",
      "fits": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec_estimate": 196,
      "hourly_usd": 2.9,
      "reason": ""
    },
    {
      "profile": "h200",
      "name": "NVIDIA H200 141 GB",
      "fits": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec_estimate": 280,
      "hourly_usd": 3.6,
      "reason": ""
    },
    {
      "profile": "b200",
      "name": "NVIDIA B200 192 GB",
      "fits": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec_estimate": 467,
      "hourly_usd": 5.5,
      "reason": ""
    },
    {
      "profile": "h100-x2",
      "name": "2\u00d7 NVIDIA H100",
      "fits": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec_estimate": 362,
      "hourly_usd": 5.8,
      "reason": ""
    },
    {
      "profile": "tpu-v5e-8",
      "name": "TPU v5e (8 chips)",
      "fits": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec_estimate": 349,
      "hourly_usd": 9.6,
      "reason": ""
    },
    {
      "profile": "h100-x4",
      "name": "4\u00d7 NVIDIA H100",
      "fits": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec_estimate": 695,
      "hourly_usd": 11.6,
      "reason": ""
    },
    {
      "profile": "tpu-v5p-4",
      "name": "TPU v5p (4 chips)",
      "fits": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec_estimate": 573,
      "hourly_usd": 16.8,
      "reason": ""
    },
    {
      "profile": "tpu-v6e-8",
      "name": "TPU v6e Trillium (8 chips)",
      "fits": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec_estimate": 665,
      "hourly_usd": 21.6,
      "reason": ""
    },
    {
      "profile": "h200-x8",
      "name": "8\u00d7 NVIDIA H200",
      "fits": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec_estimate": 1949,
      "hourly_usd": 28.8,
      "reason": ""
    },
    {
      "profile": "m3-ultra-512",
      "name": "Apple M3 Ultra 512 GB",
      "fits": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec_estimate": 48,
      "hourly_usd": null,
      "reason": ""
    },
    {
      "profile": "m4-max-128",
      "name": "Apple M4 Max 128 GB",
      "fits": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec_estimate": 32,
      "hourly_usd": null,
      "reason": ""
    },
    {
      "profile": "m4-pro-48",
      "name": "Apple M4 Pro 48 GB",
      "fits": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec_estimate": 16,
      "hourly_usd": null,
      "reason": ""
    },
    {
      "profile": "cpu-64",
      "name": "CPU only (64 GB DDR5)",
      "fits": true,
      "quant": "q8",
      "total_gb": 13.6,
      "tokens_per_sec_estimate": 4,
      "hourly_usd": null,
      "reason": ""
    }
  ]
}
