{
  "version": "1.4",
  "generated_at": "2026-10-26T12:13:38.524400",
  "schema_version": "2.4",
  "source_thread": "https://forum.level1techs.com/t/256443",
  "total_submissions": 50,
  "skipped_templates": 3,
  "submissions": [
    {
      "id": "post_5_digitalscream_0",
      "source": {
        "post_number": 5,
        "post_id": 4119255,
        "username": "digitalscream",
        "submitted_at": "2026-09-18T14:00:04.030Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "48-64GB",
        "hardware": "2 x R9700",
        "backend": "vLLM",
        "model_name": "d0xin/Swift-Qwen3.8-27B-Uncensored-FP8",
        "quant_package": "FP8",
        "download_url": "https://huggingface.co/d0xin/Swift-Qwen3.8-27B-Uncensored-FP8",
        "primary_use_case": "Coding, Research (OpenCode)",
        "stable_context_tested": "~240k",
        "kv_cache_setting": "FP16",
        "throughput_tps": "120t/s code, 70t/s chat",
        "setup_notes": "magiccodingman/vllm-radiance, with adaptive MTP = 8"
      },
      "tier": "t5",
      "hardware": {
        "description": "2 x R9700",
        "vendor": "AMD"
      },
      "backend": "vLLM",
      "os_driver": "",
      "model": {
        "name": "d0xin/Swift-Qwen3.8-27B-Uncensored-FP8",
        "url": "https://huggingface.co/d0xin/Swift-Qwen3.8-27B-Uncensored-FP8"
      },
      "quant": "FP8",
      "weight_precision": "",
      "use_cases": "Coding, Research (OpenCode)",
      "context": {
        "stable": "~240k",
        "kv_cache": "FP16"
      },
      "performance": {
        "decode_tps": "120t/s code, 70t/s chat",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "magiccodingman/vllm-radiance, with adaptive MTP = 8 In truth, I could\u2019ve used any of the Swift options, I just went for uncensored because I occasionally have to track down DoS attacks on my forum and that sort of thing makes standard Qwen-based models nervy. Swift is the only fine-tune I\u2019ve encountered that\u2019s a straight upgrade on the original model with no catches or caveats. It just works, and it does what it says on the tin.",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_6_burhan_0",
      "source": {
        "post_number": 6,
        "post_id": 4119262,
        "username": "Burhan",
        "submitted_at": "2026-09-18T14:30:23.937Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "16-24GB",
        "hardware": "1x RTX-4000 Ada SFF (20 GB)",
        "backend": "Ollama",
        "model_name": "Qwen3.6-27b-q4_KM",
        "quant_package": "GGUF Q4_K_M",
        "download_url": "https://ollama.com/library/qwen3.6:27b-q4_K_M",
        "primary_use_case": "Document Q&A, Deep Research, Brainstorming, Coding",
        "stable_context_tested": "52000",
        "kv_cache_setting": "q8_0",
        "engine_mode": "Hybrid (VRAM+RAM)",
        "throughput_tps": "~20-30 t/s",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_draw": "75 W",
        "setup_notes": ""
      },
      "tier": "t2",
      "hardware": {
        "description": "1x RTX-4000 Ada SFF (20 GB)",
        "vendor": "NVIDIA"
      },
      "backend": "Ollama",
      "os_driver": "",
      "model": {
        "name": "Qwen3.6-27b-q4_KM",
        "url": "https://ollama.com/library/qwen3.6:27b-q4_K_M"
      },
      "quant": "GGUF Q4_K_M",
      "weight_precision": "",
      "use_cases": "Document Q&A, Deep Research, Brainstorming, Coding",
      "context": {
        "stable": "52000",
        "kv_cache": "q8_0"
      },
      "performance": {
        "decode_tps": "~20-30 t/s",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": "75 W"
      },
      "engine_mode": "Hybrid (VRAM+RAM)",
      "notes": "Updated to list each model separately",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_6_burhan_1",
      "source": {
        "post_number": 6,
        "post_id": 4119262,
        "username": "Burhan",
        "submitted_at": "2026-09-18T14:30:23.937Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "16-24GB",
        "hardware": "1x RTX-4000 Ada SFF (20 GB)",
        "backend": "Ollama",
        "model_name": "LFM2.5-thinking:1.2b-q8_0",
        "quant_package": "GGUF Q8_0",
        "download_url": "https://ollama.com/library/lfm2.5-thinking:1.2b-q8_0",
        "primary_use_case": "Q&A, Brainstorming, Complex Tasks",
        "stable_context_tested": "64000",
        "kv_cache_setting": "q8_0",
        "engine_mode": "Hybrid (VRAM+RAM)",
        "throughput_tps": "~35 t/s",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_draw": "75 W",
        "setup_notes": ""
      },
      "tier": "t2",
      "hardware": {
        "description": "1x RTX-4000 Ada SFF (20 GB)",
        "vendor": "NVIDIA"
      },
      "backend": "Ollama",
      "os_driver": "",
      "model": {
        "name": "LFM2.5-thinking:1.2b-q8_0",
        "url": "https://ollama.com/library/lfm2.5-thinking:1.2b-q8_0"
      },
      "quant": "GGUF Q8_0",
      "weight_precision": "",
      "use_cases": "Q&A, Brainstorming, Complex Tasks",
      "context": {
        "stable": "64000",
        "kv_cache": "q8_0"
      },
      "performance": {
        "decode_tps": "~35 t/s",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": "75 W"
      },
      "engine_mode": "Hybrid (VRAM+RAM)",
      "notes": "",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_6_burhan_2",
      "source": {
        "post_number": 6,
        "post_id": 4119262,
        "username": "Burhan",
        "submitted_at": "2026-09-18T14:30:23.937Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "16-24GB",
        "hardware": "1x RTX-4000 Ada SFF (20 GB)",
        "backend": "Ollama",
        "model_name": "Gemma4:26b-a4b-it-mtp-q4_K_M",
        "quant_package": "GGUF Q4_K_M",
        "download_url": "https://ollama.com/library/gemma4:26b-a4b-it-mtp-q4_K_M",
        "primary_use_case": "Document Q&A, Deep Research, Creative Writing",
        "stable_context_tested": "64000",
        "kv_cache_setting": "q8_0",
        "engine_mode": "Hybrid (VRAM+RAM)",
        "throughput_tps": "~25-30 t/s",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_draw": "75 W",
        "setup_notes": ""
      },
      "tier": "t2",
      "hardware": {
        "description": "1x RTX-4000 Ada SFF (20 GB)",
        "vendor": "NVIDIA"
      },
      "backend": "Ollama",
      "os_driver": "",
      "model": {
        "name": "Gemma4:26b-a4b-it-mtp-q4_K_M",
        "url": "https://ollama.com/library/gemma4:26b-a4b-it-mtp-q4_K_M"
      },
      "quant": "GGUF Q4_K_M",
      "weight_precision": "",
      "use_cases": "Document Q&A, Deep Research, Creative Writing",
      "context": {
        "stable": "64000",
        "kv_cache": "q8_0"
      },
      "performance": {
        "decode_tps": "~25-30 t/s",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": "75 W"
      },
      "engine_mode": "Hybrid (VRAM+RAM)",
      "notes": "",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_6_burhan_3",
      "source": {
        "post_number": 6,
        "post_id": 4119262,
        "username": "Burhan",
        "submitted_at": "2026-09-18T14:30:23.937Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "16-24GB",
        "hardware": "1x RTX-4000 Ada SFF (20 GB)",
        "backend": "Ollama",
        "model_name": "Qwen3.5:9b-q8_0",
        "quant_package": "GGUF Q8_0",
        "download_url": "https://ollama.com/library/qwen3.5:9b-q8_0",
        "primary_use_case": "Document Q&A, Deep Research, Creative Writing, Brainstorming, Planning",
        "stable_context_tested": "128000",
        "kv_cache_setting": "q8_0",
        "engine_mode": "Hybrid (VRAM+RAM)",
        "throughput_tps": "~30 t/s",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_draw": "75 W",
        "setup_notes": ""
      },
      "tier": "t2",
      "hardware": {
        "description": "1x RTX-4000 Ada SFF (20 GB)",
        "vendor": "NVIDIA"
      },
      "backend": "Ollama",
      "os_driver": "",
      "model": {
        "name": "Qwen3.5:9b-q8_0",
        "url": "https://ollama.com/library/qwen3.5:9b-q8_0"
      },
      "quant": "GGUF Q8_0",
      "weight_precision": "",
      "use_cases": "Document Q&A, Deep Research, Creative Writing, Brainstorming, Planning",
      "context": {
        "stable": "128000",
        "kv_cache": "q8_0"
      },
      "performance": {
        "decode_tps": "~30 t/s",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": "75 W"
      },
      "engine_mode": "Hybrid (VRAM+RAM)",
      "notes": "I have Ollama + Open WebUI + SearXNG configured for general purpose usage. I\u2019ve had great results with just having Qwen3.6-27b (low thinking mode) doing web research for me, even the Qwen3.5-9b has done surprisingly well. The KV cache quantization has helped a lot for getting bigger models and context windows running with limited VRAM, and the 75W max on the small GPU has been very kind to my electric bill. Most of my usage here is \u2018throwing tasks over the fence.\u2019 So speed isn\u2019t critical, it\u2019s just getting the job done as best as possible. Generally I\u2019ll send a message for a research task and then come back several minutes or even hours later to review the results. I do have another system for coding that I\u2019ll share separately later on.",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_8_s1ugh34d_0",
      "source": {
        "post_number": 8,
        "post_id": 4119449,
        "username": "s1ugh34d",
        "submitted_at": "2026-09-18T20:53:48.887Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "\u226416GB",
        "hardware": "Radeon VII 16GB HBM2 (gfx906), 32-core EPYC host, 67GB RAM",
        "backend": "llama-swap (llama.cpp subprocess, ROCm/ROCm backends)",
        "model_name": "samuelcardillo/Carnice-Qwen3.6-MoE-35B-A3B (I-Nano)",
        "quant_package": "GGUF I-Nano",
        "download_url": "samuelcardillo/Carnice-Qwen3.6-MoE-35B-A3B \u00b7 Hugging Face",
        "primary_use_case": "LLM assited coding / technical documentation",
        "stable_context_tested": "65536 tokens native (NIAH hit YES at 65k)",
        "kv_cache_setting": "quantized KV (observed near-flat VRAM growth w/ context)",
        "throughput_tps": "54.4 tps decode / 215 tok/s prefill (short ctx)",
        "setup_notes": "MoE A3B \u2014 fits 12.7GB incl. KV headroom. 9B active params \u2248 like a 9B at 9B-compute. Fast cold load (19s). I-Nano = very low quant; verify quality for your task, but pipeline ran coherent through 65k.",
        "power_draw": "~150 W peak, 99% GPU util",
        "ttft_cold": "19.3 s"
      },
      "tier": "t2",
      "hardware": {
        "description": "Radeon VII 16GB HBM2 (gfx906), 32-core EPYC host, 67GB RAM",
        "vendor": "AMD"
      },
      "backend": "llama-swap",
      "os_driver": "",
      "model": {
        "name": "samuelcardillo/Carnice-Qwen3.6-MoE-35B-A3B (I-Nano)",
        "url": "samuelcardillo/Carnice-Qwen3.6-MoE-35B-A3B \u00b7 Hugging Face"
      },
      "quant": "GGUF I-Nano",
      "weight_precision": "",
      "use_cases": "LLM assited coding / technical documentation",
      "context": {
        "stable": "65536 tokens native (NIAH hit YES at 65k)",
        "kv_cache": "quantized KV (observed near-flat VRAM growth w/ context)"
      },
      "performance": {
        "decode_tps": "54.4",
        "prefill_tps": "215",
        "ttft_cold": "19.3 s",
        "power_w": "~150 W peak, 99% GPU util"
      },
      "engine_mode": "",
      "notes": "MoE A3B \u2014 fits 12.7GB incl. KV headroom. 9B active params \u2248 like a 9B at 9B-compute. Fast cold load (19s). I-Nano = very low quant; verify quality for your task, but pipeline ran coherent through 65k. | llama.cpp subprocess, ROCm/ROCm backends",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_8_s1ugh34d_1",
      "source": {
        "post_number": 8,
        "post_id": 4119449,
        "username": "s1ugh34d",
        "submitted_at": "2026-09-18T20:53:48.887Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "\u226416GB",
        "hardware": "Radeon VII 16GB HBM2 (gfx906)",
        "backend": "llama-swap (llama.cpp subprocess)",
        "model_name": "deadbydawn101/gemma-4-E4B-Agentic-Opus-Reasoning-GeminiCLI-GGUF",
        "quant_package": "GGUF Q4_K_M",
        "download_url": "deadbydawn101/gemma-4-E4B-Agentic-Opus-Reasoning-GeminiCLI-GGUF \u00b7 Hugging Face",
        "primary_use_case": "LLM assited coding / technical documentation / large repo generation",
        "stable_context_tested": "49152 tokens (NIAH hit YES at 49k; 65k not probed)",
        "kv_cache_setting": "quantized KV (VRAM ~flat: 4618MB @49k vs 4619MB @4k)",
        "throughput_tps": "86.7 tps decode / 408 tok/s prefill",
        "setup_notes": "Thinking model \u2014 emits reasoning_content; budget 512+ tokens or it burns max_tokens in reasoning and returns finish_reason=length with empty content. Answered verbal-math correctly (final answer 4). ~5GB VRAM footprint.",
        "coherence": "holds through 49k NIAH.",
        "power_draw": "~214 W peak, 99% GPU util"
      },
      "tier": "t2",
      "hardware": {
        "description": "Radeon VII 16GB HBM2 (gfx906)",
        "vendor": "AMD"
      },
      "backend": "llama-swap",
      "os_driver": "",
      "model": {
        "name": "deadbydawn101/gemma-4-E4B-Agentic-Opus-Reasoning-GeminiCLI-GGUF",
        "url": "deadbydawn101/gemma-4-E4B-Agentic-Opus-Reasoning-GeminiCLI-GGUF \u00b7 Hugging Face"
      },
      "quant": "GGUF Q4_K_M",
      "weight_precision": "",
      "use_cases": "LLM assited coding / technical documentation / large repo generation",
      "context": {
        "stable": "49152 tokens (NIAH hit YES at 49k; 65k not probed)",
        "kv_cache": "quantized KV (VRAM ~flat: 4618MB @49k vs 4619MB @4k)"
      },
      "performance": {
        "decode_tps": "86.7",
        "prefill_tps": "408",
        "ttft_cold": "",
        "power_w": "~214 W peak, 99% GPU util"
      },
      "engine_mode": "",
      "notes": "Thinking model \u2014 emits reasoning_content; budget 512+ tokens or it burns max_tokens in reasoning and returns finish_reason=length with empty content. Answered verbal-math correctly (final answer 4). ~5GB VRAM footprint. | llama.cpp subprocess coherence: holds through 49k NIAH.",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_8_s1ugh34d_2",
      "source": {
        "post_number": 8,
        "post_id": 4119449,
        "username": "s1ugh34d",
        "submitted_at": "2026-09-18T20:53:48.887Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "\u226416GB",
        "hardware": "Radeon VII 16GB HBM2 (gfx906)",
        "backend": "llama-swap (llama.cpp subprocess)",
        "model_name": "unsloth/Qwen3.8-27B-GGUF (UD-Q2_K_XL)",
        "quant_package": "GGUF UD-Q2_K_XL",
        "download_url": "unsloth/Qwen3.8-27B-GGUF \u00b7 Hugging Face",
        "primary_use_case": "LLM assited coding / technical documentation",
        "stable_context_tested": "49152 tokens (NIAH hit YES at 49k)",
        "kv_cache_setting": "quantized KV (VRAM 13770\u219213798MB across 8k\u219232k, +28MB)",
        "throughput_tps": "13.9 tps decode / 79 tok/s prefill",
        "setup_notes": "13.8GB VRAM = nearly full weight residency on 16GB; decode speed collapses to 14 tps (big dense model). Slow prefill: 41s TTFT @8k, 245s @49k.",
        "coherence": "holds through full 49k NIAH. Good value if you need 30k+ ctx on a dense 27B and can live with 14 tps.",
        "power_draw": "~233 W peak, 100% GPU util"
      },
      "tier": "t2",
      "hardware": {
        "description": "Radeon VII 16GB HBM2 (gfx906)",
        "vendor": "AMD"
      },
      "backend": "llama-swap",
      "os_driver": "",
      "model": {
        "name": "unsloth/Qwen3.8-27B-GGUF (UD-Q2_K_XL)",
        "url": "unsloth/Qwen3.8-27B-GGUF \u00b7 Hugging Face"
      },
      "quant": "GGUF UD-Q2_K_XL",
      "weight_precision": "",
      "use_cases": "LLM assited coding / technical documentation",
      "context": {
        "stable": "49152 tokens (NIAH hit YES at 49k)",
        "kv_cache": "quantized KV (VRAM 13770\u219213798MB across 8k\u219232k, +28MB)"
      },
      "performance": {
        "decode_tps": "13.9",
        "prefill_tps": "79",
        "ttft_cold": "",
        "power_w": "~233 W peak, 100% GPU util"
      },
      "engine_mode": "",
      "notes": "13.8GB VRAM = nearly full weight residency on 16GB; decode speed collapses to 14 tps (big dense model). Slow prefill: 41s TTFT @8k, 245s @49k. | llama.cpp subprocess coherence: holds through full 49k NIAH. Good value if you need 30k+ ctx on a dense 27B and can live with 14 tps.",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_8_s1ugh34d_3",
      "source": {
        "post_number": 8,
        "post_id": 4119449,
        "username": "s1ugh34d",
        "submitted_at": "2026-09-18T20:53:48.887Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "\u226416GB",
        "hardware": "Radeon VII 16GB HBM2 (gfx906)",
        "backend": "llama-swap (llama.cpp subprocess)",
        "model_name": "MaziyarPanahi/DeepSeek-R1-0528-Qwen3-8B-GGUF",
        "quant_package": "GGUF Q4_K_M",
        "download_url": "MaziyarPanahi/DeepSeek-R1-0528-Qwen3-8B-GGUF \u00b7 Hugging Face",
        "primary_use_case": "LLM assited coding / technical documentation",
        "stable_context_tested": "16384 tokens (native server cap)",
        "kv_cache_setting": "quantized KV (7327\u21927299 MB, flat)",
        "throughput_tps": "84.4 tps decode / 490 tok/s prefill",
        "setup_notes": "Reasoning model (reasoning_content stream). NIAH hit at native 16k limit. Cheap 7.4GB. Run with generous max_tokens; single-request math budget 512 burned in reasoning without emitting final answer (content overflow quirk). Strong ttft + throughput for an 8B R1 distill.",
        "power_draw": "~268 W peak, 93% GPU util"
      },
      "tier": "t2",
      "hardware": {
        "description": "Radeon VII 16GB HBM2 (gfx906)",
        "vendor": "AMD"
      },
      "backend": "llama-swap",
      "os_driver": "",
      "model": {
        "name": "MaziyarPanahi/DeepSeek-R1-0528-Qwen3-8B-GGUF",
        "url": "MaziyarPanahi/DeepSeek-R1-0528-Qwen3-8B-GGUF \u00b7 Hugging Face"
      },
      "quant": "GGUF Q4_K_M",
      "weight_precision": "",
      "use_cases": "LLM assited coding / technical documentation",
      "context": {
        "stable": "16384 tokens (native server cap)",
        "kv_cache": "quantized KV (7327\u21927299 MB, flat)"
      },
      "performance": {
        "decode_tps": "84.4",
        "prefill_tps": "490",
        "ttft_cold": "",
        "power_w": "~268 W peak, 93% GPU util"
      },
      "engine_mode": "",
      "notes": "Reasoning model (reasoning_content stream). NIAH hit at native 16k limit. Cheap 7.4GB. Run with generous max_tokens; single-request math budget 512 burned in reasoning without emitting final answer (content overflow quirk). Strong ttft + throughput for an 8B R1 distill. | llama.cpp subprocess",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_8_s1ugh34d_4",
      "source": {
        "post_number": 8,
        "post_id": 4119449,
        "username": "s1ugh34d",
        "submitted_at": "2026-09-18T20:53:48.887Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "\u226416GB",
        "hardware": "Radeon VII 16GB HBM2 (gfx906)",
        "backend": "llama-swap (llama.cpp subprocess)",
        "model_name": "bartowski/Qwen_Qwen3.5-4B-GGUF",
        "quant_package": "GGUF Q4_K_M",
        "download_url": "bartowski/Qwen_Qwen3.5-4B-GGUF \u00b7 Hugging Face",
        "primary_use_case": "LLM assited coding / technical documentation",
        "stable_context_tested": "65536 tokens (native cap; NIAH hit YES; 96k rejected, n_ctx=65536 enforced)",
        "kv_cache_setting": "quantized KV (VRAM 5312MB @4k \u2192 5312MB @65k, +0)",
        "throughput_tps": "96.4 tps decode / 468 tok/s prefill",
        "setup_notes": "Fastest model tested; everything fits with massive KV headroom. Full native 65k context usable. Verified arithmetic (final answer 4). Budget max_tokens \u2014 it reasons too. Best \u201cleave always loaded\u201d generalist.",
        "power_draw": "~225 W peak, 99% GPU util"
      },
      "tier": "t2",
      "hardware": {
        "description": "Radeon VII 16GB HBM2 (gfx906)",
        "vendor": "AMD"
      },
      "backend": "llama-swap",
      "os_driver": "",
      "model": {
        "name": "bartowski/Qwen_Qwen3.5-4B-GGUF",
        "url": "bartowski/Qwen_Qwen3.5-4B-GGUF \u00b7 Hugging Face"
      },
      "quant": "GGUF Q4_K_M",
      "weight_precision": "",
      "use_cases": "LLM assited coding / technical documentation",
      "context": {
        "stable": "65536 tokens (native cap; NIAH hit YES; 96k rejected, n_ctx=65536 enforced)",
        "kv_cache": "quantized KV (VRAM 5312MB @4k \u2192 5312MB @65k, +0)"
      },
      "performance": {
        "decode_tps": "96.4",
        "prefill_tps": "468",
        "ttft_cold": "",
        "power_w": "~225 W peak, 99% GPU util"
      },
      "engine_mode": "",
      "notes": "Fastest model tested; everything fits with massive KV headroom. Full native 65k context usable. Verified arithmetic (final answer 4). Budget max_tokens \u2014 it reasons too. Best \u201cleave always loaded\u201d generalist. | llama.cpp subprocess",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_12_manoreken2_0",
      "source": {
        "post_number": 12,
        "post_id": 4119505,
        "username": "manoreken2",
        "submitted_at": "2026-09-19T00:47:52.759Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "[192-256GB+]",
        "hardware": "[RTX 5090 + IceLake Xeon]",
        "backend": "[llama.cpp]",
        "model_name": "[DeepSeek-R1:671b-0528_q8_0]",
        "quant_package": "[GGUF Q8_0]",
        "download_url": "[ unsloth/DeepSeek-R1-GGUF-UD at main ]",
        "stable_context_tested": "[8192]",
        "kv_cache_setting": "[default]",
        "throughput_tps": "[1 tokens/sec]",
        "setup_notes": "[llama-server -m C:/work/hf/DeepSeek-R1.gguf -c 8192 -t 10 --parallel 1]"
      },
      "tier": "t6",
      "hardware": {
        "description": "RTX 5090 + IceLake Xeon",
        "vendor": "NVIDIA"
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "DeepSeek-R1:671b-0528_q8_0",
        "url": "unsloth/DeepSeek-R1-GGUF-UD at main"
      },
      "quant": "GGUF Q8_0",
      "weight_precision": "",
      "use_cases": "",
      "context": {
        "stable": "8192",
        "kv_cache": "default"
      },
      "performance": {
        "decode_tps": "1 tokens/sec",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "llama-server -m C:/work/hf/DeepSeek-R1.gguf -c 8192 -t 10 --parallel 1 My task is niche: 2500\uff5e1000 years ago Chinese historical documents to modern Japanese translation, it is for my recreational reading. Slow token throughput is no problem for this task, because it is unattended batch translation. Model parameters are spilled out to main memory (1024GB). Following is my subjective output text quality evaluation. This is one of the most suitable model for this task. This is output example: \u53f2\u8a18 \u65e5\u672c\u8a9e\u8a33 DeepSeek-R1:671b-0528-q8_0",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_12_manoreken2_1",
      "source": {
        "post_number": 12,
        "post_id": 4119505,
        "username": "manoreken2",
        "submitted_at": "2026-09-19T00:47:52.759Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "[192-256GB+]",
        "model_name": "[DeepSeek-V3.1 Terminus:671b_q8_0]",
        "download_url": "[ unsloth/DeepSeek-V3.1-Terminus-GGUF \u00b7 Hugging Face ]",
        "throughput_tps": "[1 tokens/sec]"
      },
      "tier": "t6",
      "hardware": {
        "description": "RTX 5090 + IceLake Xeon",
        "vendor": "NVIDIA"
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "DeepSeek-V3.1 Terminus:671b_q8_0",
        "url": "unsloth/DeepSeek-V3.1-Terminus-GGUF \u00b7 Hugging Face"
      },
      "quant": "GGUF Q8_0",
      "weight_precision": "",
      "use_cases": "",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "1 tokens/sec",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "This is also good.",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": [],
      "inherited_from": {
        "hardware": "block_0",
        "backend": "block_0",
        "quant": "block_0"
      }
    },
    {
      "id": "post_12_manoreken2_2",
      "source": {
        "post_number": 12,
        "post_id": 4119505,
        "username": "manoreken2",
        "submitted_at": "2026-09-19T00:47:52.759Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "[192-256GB+]",
        "model_name": "[DeepSeek-V3.2:671b_UD-Q8_K_XL]",
        "download_url": "[ unsloth/DeepSeek-V3.2-GGUF \u00b7 Hugging Face ]",
        "throughput_tps": "[1 tokens/sec]"
      },
      "tier": "t6",
      "hardware": {
        "description": "RTX 5090 + IceLake Xeon",
        "vendor": "NVIDIA"
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "DeepSeek-V3.2:671b_UD-Q8_K_XL",
        "url": "unsloth/DeepSeek-V3.2-GGUF \u00b7 Hugging Face"
      },
      "quant": "",
      "weight_precision": "",
      "use_cases": "",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "1 tokens/sec",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "Outputs literal translation style text and difficult to understand.",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": [],
      "inherited_from": {
        "hardware": "block_0",
        "backend": "block_0"
      }
    },
    {
      "id": "post_12_manoreken2_3",
      "source": {
        "post_number": 12,
        "post_id": 4119505,
        "username": "manoreken2",
        "submitted_at": "2026-09-19T00:47:52.759Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "[192-256GB+]",
        "model_name": "[GLM-5.2-UD-Q6_K_XL]",
        "download_url": "[ unsloth/GLM-5.2-GGUF \u00b7 Hugging Face ]",
        "throughput_tps": "[1 tokens/sec]"
      },
      "tier": "t6",
      "hardware": {
        "description": "RTX 5090 + IceLake Xeon",
        "vendor": "NVIDIA"
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "GLM-5.2-UD-Q6_K_XL",
        "url": "unsloth/GLM-5.2-GGUF \u00b7 Hugging Face"
      },
      "quant": "",
      "weight_precision": "",
      "use_cases": "",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "1 tokens/sec",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "Outputs literal translation style text. not suitable for this task.",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": [],
      "inherited_from": {
        "hardware": "block_0",
        "backend": "block_0"
      }
    },
    {
      "id": "post_12_manoreken2_4",
      "source": {
        "post_number": 12,
        "post_id": 4119505,
        "username": "manoreken2",
        "submitted_at": "2026-09-19T00:47:52.759Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "[192-256GB+]",
        "model_name": "[Kimi-K2.6-UD-Q8_K_XL]",
        "download_url": "[ unsloth/Kimi-K2.6 \u00b7 Hugging Face ]",
        "throughput_tps": "[1 tokens/sec]"
      },
      "tier": "t6",
      "hardware": {
        "description": "RTX 5090 + IceLake Xeon",
        "vendor": "NVIDIA"
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "Kimi-K2.6-UD-Q8_K_XL",
        "url": "unsloth/Kimi-K2.6 \u00b7 Hugging Face"
      },
      "quant": "",
      "weight_precision": "",
      "use_cases": "",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "1 tokens/sec",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "Outputs youth slang-like text. not suitable for history textbook translation.",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": [],
      "inherited_from": {
        "hardware": "block_0",
        "backend": "block_0"
      }
    },
    {
      "id": "post_12_manoreken2_5",
      "source": {
        "post_number": 12,
        "post_id": 4119505,
        "username": "manoreken2",
        "submitted_at": "2026-09-19T00:47:52.759Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "[128-196GB]",
        "model_name": "[Qwen3:235b-a22b-thinking-2507-q4]",
        "download_url": "[ Qwen/Qwen3-235B-A22B-Thinking-2507 \u00b7 Hugging Face ]"
      },
      "tier": "t6",
      "hardware": {
        "description": "RTX 5090 + IceLake Xeon",
        "vendor": "NVIDIA"
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "Qwen3:235b-a22b-thinking-2507-q4",
        "url": "Qwen/Qwen3-235B-A22B-Thinking-2507 \u00b7 Hugging Face"
      },
      "quant": "",
      "weight_precision": "",
      "use_cases": "",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "Quite sloppy, outputs many incorrect things.",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": [],
      "inherited_from": {
        "hardware": "block_0",
        "backend": "block_0"
      }
    },
    {
      "id": "post_12_manoreken2_6",
      "source": {
        "post_number": 12,
        "post_id": 4119505,
        "username": "manoreken2",
        "submitted_at": "2026-09-19T00:47:52.759Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "[48-64GB]",
        "model_name": "[nemotron-3-super:120b]"
      },
      "tier": "t5",
      "hardware": {
        "description": "RTX 5090 + IceLake Xeon",
        "vendor": "NVIDIA"
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "nemotron-3-super:120b",
        "url": ""
      },
      "quant": "",
      "weight_precision": "",
      "use_cases": "",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "Unusable, do not know Asian history, refuse to translate cruel text, cannot output proper Japanese text.",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": [],
      "inherited_from": {
        "hardware": "block_0",
        "backend": "block_0"
      }
    },
    {
      "id": "post_12_manoreken2_7",
      "source": {
        "post_number": 12,
        "post_id": 4119505,
        "username": "manoreken2",
        "submitted_at": "2026-09-19T00:47:52.759Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "[48-64GB]",
        "model_name": "[GPT-OSS:120b]",
        "download_url": "[ unsloth/gpt-oss-120b-GGUF \u00b7 Hugging Face ]"
      },
      "tier": "t5",
      "hardware": {
        "description": "RTX 5090 + IceLake Xeon",
        "vendor": "NVIDIA"
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "GPT-OSS:120b",
        "url": "unsloth/gpt-oss-120b-GGUF \u00b7 Hugging Face"
      },
      "quant": "",
      "weight_precision": "",
      "use_cases": "",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "This model does not know basic culture and history of Asia. Also it refuse to translate some cruel texts.",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": [],
      "inherited_from": {
        "hardware": "block_0",
        "backend": "block_0"
      }
    },
    {
      "id": "post_17_manoreken2_0",
      "source": {
        "post_number": 17,
        "post_id": 4120037,
        "username": "manoreken2",
        "submitted_at": "2026-09-20T22:37:48.204Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "196 GB \u2013 256 GB+",
        "hardware": "1x RTX5090 (32 GB) + IceLake Xeon CPU (1024 GB)",
        "backend": "[llama-cpp]",
        "model_name": "[DeepSeek-V4-Pro-0813-UD_K_XL.gguf]",
        "download_url": "[ unsloth/DeepSeek-V4-Pro-0813-GGUF \u00b7 Hugging Face ]",
        "setup_notes": "[llama-server --model C:/hf/DeepSeek-V4-Pro-0813-UD-Q8_K_XL.gguf --ctx-size 16384 --flash-attn on --parallel 1 --no-cont-batching --load-mode none --batch-size 4096 --ubatch-size 4096 --cache-type-k q8_0 --cache-type-v q8_0 --ctx-checkpoints 0 --threads 20 --threads-batch 20 --jinja --log-verbosity 4 --cache-reuse 256 --cache-ram -1 --swa-full --timeout 3600 --host 0.0.0.0 --port 8888]",
        "primary_use_case": "[translation of government journal from 481 BCE China]",
        "stable_context_tested": "[16384]"
      },
      "tier": "t6",
      "hardware": {
        "description": "1x RTX5090 (32 GB) + IceLake Xeon CPU (1024 GB)",
        "vendor": "NVIDIA"
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "DeepSeek-V4-Pro-0813-UD_K_XL.gguf",
        "url": "unsloth/DeepSeek-V4-Pro-0813-GGUF \u00b7 Hugging Face"
      },
      "quant": "",
      "weight_precision": "",
      "use_cases": "translation of government journal from 481 BCE China",
      "context": {
        "stable": "16384",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "llama-server --model C:/hf/DeepSeek-V4-Pro-0813-UD-Q8_K_XL.gguf --ctx-size 16384 --flash-attn on --parallel 1 --no-cont-batching --load-mode none --batch-size 4096 --ubatch-size 4096 --cache-type-k q8_0 --cache-type-v q8_0 --ctx-checkpoints 0 --threads 20 --threads-batch 20 --jinja --log-verbosity 4 --cache-reuse 256 --cache-ram -1 --swa-full --timeout 3600 --host 0.0.0.0 --port 8888 Output example: \u73fe\u4ee3\u65e5\u672c\u8a9e ../document/\u6625\u79cb\u5de6\u6c0f\u50b3 unsloth/DeepSeek-V4-Pro-0813-UD-Q8_K_XL This is pretty good so far, focusing faithful translation of original text and less clutter than DeepSeek-R1:671b_Q8, which sometimes outputs nonsense such as: a warning against SNS-driven society, or it is the model ruler aligning with SDGs goal 16",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_18_manoreken2_0",
      "source": {
        "post_number": 18,
        "post_id": 4121718,
        "username": "manoreken2",
        "submitted_at": "2026-09-25T09:03:49.180Z",
        "marked": false
      },
      "type": "full",
      "raw": {
        "backend": "[colibri]",
        "model_name": "[DeepSeek-V4.1-Flash]",
        "download_url": "[ deepseek-ai/DeepSeek-V4.1-Flash \u00b7 Hugging Face ]",
        "setup_notes": "download exe from https://github.com/JustVugg/colibri/releases/download/v1.12.1/colibri-v1.12.1-windows-x86_64.zip, hf download deepseek-ai/DeepSeek-V4.1-Flash --local-dir C:\\hf\\dfv41f, set COLI_ALLOW_INSECURE_BIND=1, coli serve --model C:\\hf\\dsv41f --host 0.0.0.0 --port 8888",
        "primary_use_case": "[translation of government journal from 481 BCE China]",
        "stable_context_tested": "[12288]"
      },
      "tier": null,
      "hardware": {
        "description": "",
        "vendor": ""
      },
      "backend": "Colibri",
      "os_driver": "",
      "model": {
        "name": "DeepSeek-V4.1-Flash",
        "url": "deepseek-ai/DeepSeek-V4.1-Flash \u00b7 Hugging Face"
      },
      "quant": "",
      "weight_precision": "",
      "use_cases": "translation of government journal from 481 BCE China",
      "context": {
        "stable": "12288",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "download exe from https://github.com/JustVugg/colibri/releases/download/v1.12.1/colibri-v1.12.1-windows-x86_64.zip, hf download deepseek-ai/DeepSeek-V4.1-Flash --local-dir C:\\hf\\dfv41f, set COLI_ALLOW_INSECURE_BIND=1, coli serve --model C:\\hf\\dsv41f --host 0.0.0.0 --port 8888 Output text example : \u73fe\u4ee3\u65e5\u672c\u8a9e\u8a33 \u6625\u79cb\u5de6\u6c0f\u50b3 deepseek-v4.1-flash-colibri CPU-only inference using colibri. output text is antiquated and literal word-to-word style, difficult to read.",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_21_pawingunplug_0",
      "source": {
        "post_number": 21,
        "post_id": 4122536,
        "username": "PawingUnplug",
        "submitted_at": "2026-09-28T07:14:44.464Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "48-64GB",
        "hardware": "2 x R9700",
        "backend": "llamacpp",
        "model_name": "bartowski/Kwaipilot_KAT-Coder-V2.5-Dev-Q5_K_M.gguf, Qwen3-Embedding-8B-Q8_0.gguf, Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-MTP-Q8_0.gguf",
        "download_url": "https://huggingface.co/bartowski/Kwaipilot_KAT-Coder-V2.5-Dev-GGUF, https://huggingface.co/Qwen/Qwen3-Embedding-8B-GGUF, https://huggingface.co/DavidAU/Qwen3.5-9B-The-Defiant-Fable-Uncensored-Heretic-NEO-IMATRIX-MAX-MTP-GGUF",
        "primary_use_case": "Coding Judge, Research (OpenCode and own harness), Embeddings",
        "stable_context_tested": "~262k",
        "kv_cache_setting": "FP16",
        "power_draw": "~600w",
        "throughput_tps": "~50t/s chat"
      },
      "tier": "t5",
      "hardware": {
        "description": "2 x R9700",
        "vendor": "AMD"
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "bartowski/Kwaipilot_KAT-Coder-V2.5-Dev-Q5_K_M.gguf, Qwen3-Embedding-8B-Q8_0.gguf, Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-MTP-Q8_0.gguf",
        "url": "https://huggingface.co/bartowski/Kwaipilot_KAT-Coder-V2.5-Dev-GGUF, https://huggingface.co/Qwen/Qwen3-Embedding-8B-GGUF, https://huggingface.co/DavidAU/Qwen3.5-9B-The-Defiant-Fable-Uncensored-Heretic-NEO-IMATRIX-MAX-MTP-GGUF"
      },
      "quant": "",
      "weight_precision": "",
      "use_cases": "Coding Judge, Research (OpenCode and own harness), Embeddings",
      "context": {
        "stable": "~262k",
        "kv_cache": "FP16"
      },
      "performance": {
        "decode_tps": "~50t/s chat",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": "~600w"
      },
      "engine_mode": "",
      "notes": "",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_21_pawingunplug_1",
      "source": {
        "post_number": 21,
        "post_id": 4122536,
        "username": "PawingUnplug",
        "submitted_at": "2026-09-28T07:14:44.464Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "48-96GB",
        "hardware": "2 x w7900",
        "backend": "llamacpp",
        "model_name": "Qwen3.8-Flash-Next-Q4_K_M",
        "download_url": "https://huggingface.co/bartowski/Qwen3.8-Flash-Next-GGUF",
        "primary_use_case": "Coding",
        "stable_context_tested": "~262k",
        "kv_cache_setting": "FP16",
        "throughput_tps": "~35t/s chat",
        "power_draw": "~750w"
      },
      "tier": "t5",
      "hardware": {
        "description": "2 x w7900",
        "vendor": "AMD"
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "Qwen3.8-Flash-Next-Q4_K_M",
        "url": "https://huggingface.co/bartowski/Qwen3.8-Flash-Next-GGUF"
      },
      "quant": "",
      "weight_precision": "",
      "use_cases": "Coding",
      "context": {
        "stable": "~262k",
        "kv_cache": "FP16"
      },
      "performance": {
        "decode_tps": "~35t/s chat",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": "~750w"
      },
      "engine_mode": "",
      "notes": "",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_22_sgtawesomesauce_0",
      "source": {
        "post_number": 22,
        "post_id": 4122551,
        "username": "SgtAwesomesauce",
        "submitted_at": "2026-09-28T08:00:27.341Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "96-128GB",
        "hardware": "4x AMD v620",
        "backend": "vLLM (custom framework, see below)",
        "os_driver": "Linux/ROCm",
        "model_name": "qwen3.8-flash-next",
        "quant_package": "W4A16",
        "weight_precision": "W4A16",
        "download_url": "https://huggingface.co/wtdcode/Qwen3.8-Flash-Next-AWQ-W4A16",
        "primary_use_case": "Agentic coding, chatbot tomfoolery",
        "stable_context_tested": "245K",
        "kv_cache_setting": "FP16/BF16",
        "throughput_tps": "68-92",
        "prefill_tps": "1200@0ctx",
        "ttft_cold": "3.5s",
        "power_draw": "~900w, full system power from wall",
        "setup_notes": "Following https://github.com/leapdragon/vllm-rdna2-recipe, I got the flash-next fork working. Tweaks from that guide: enable vision, `max_pixels=1638400`, `DENSE_INT8=1`, `DENSE_INT8_ONLY=1`, `MTP=0` (collapses under multiple inference threads), `GPUUTIL=0.98` (needs fully headless system), `MAXLEN=262144`, PLE table `ples_fp8`, ~50GB of system memory required for it. Total KV capacity ~840k tokens Quantized PLE doesn't seem to have any meaningful impact on quality."
      },
      "tier": "t6",
      "hardware": {
        "description": "4x AMD v620",
        "vendor": "AMD"
      },
      "backend": "vLLM",
      "os_driver": "Linux/ROCm",
      "model": {
        "name": "qwen3.8-flash-next",
        "url": "https://huggingface.co/wtdcode/Qwen3.8-Flash-Next-AWQ-W4A16"
      },
      "quant": "W4A16",
      "weight_precision": "W4A16",
      "use_cases": "Agentic coding, chatbot tomfoolery",
      "context": {
        "stable": "245K",
        "kv_cache": "FP16/BF16"
      },
      "performance": {
        "decode_tps": "68-92",
        "prefill_tps": "1200@0ctx",
        "ttft_cold": "3.5s",
        "power_w": "~900w, full system power from wall"
      },
      "engine_mode": "",
      "notes": "Following https://github.com/leapdragon/vllm-rdna2-recipe, I got the flash-next fork working. Tweaks from that guide: enable vision, `max_pixels=1638400`, `DENSE_INT8=1`, `DENSE_INT8_ONLY=1`, `MTP=0` (collapses under multiple inference threads), `GPUUTIL=0.98` (needs fully headless system), `MAXLEN=262144`, PLE table `ples_fp8`, ~50GB of system memory required for it. Total KV capacity ~840k tokens Quantized PLE doesn't seem to have any meaningful impact on quality. | custom framework, see below",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_26_kif_0",
      "source": {
        "post_number": 26,
        "post_id": 4128355,
        "username": "Kif",
        "submitted_at": "2026-10-12T09:32:10.190Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "24-32GB",
        "hardware": "1x RTX 4090 24GB, Ryzen 9 5900X, 128GB DDR4-3600 CL18 (2x CMK64GX4M2D3600C18, 4x32GB double-rank, holding 3600 MT/s XMP), Gigabyte Aorus X570 Master, Samsung 990 Pro NVMe (PCIe 4.0)",
        "backend": "Strata 0.1.42 (dedicated Qwen3.8-Flash-Next engine, OpenAI/Anthropic API)",
        "os_driver": "Windows 11 Pro + WSL2/Ubuntu 24.04 (96 GB allocated via .wslconfig); NVIDIA 610.43.02, KMD 610.47, CUDA UMD 13.3",
        "model_name": "Qwen/Qwen3.8-Flash-Next (GGUF IQ3_S)",
        "quant_package": "GGUF IQ3_S (ISTA-DASLab GSQ-RCO)",
        "weight_precision": "IQ3_S",
        "download_url": "https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF",
        "primary_use_case": "Agentic coding / agent harness via OpenAI-compatible API",
        "stable_context_tested": "131072 configured; >=106,493 tokens observed in live agent sessions, no OOM, no coherence collapse",
        "kv_cache_setting": "int8 (engine-native quantized KV)",
        "throughput_tps": "99-128 decode (median 108, on 50k+ token prompts)",
        "prefill_tps": "825-3600 on uncached tokens (median ~2800, prompt cache ~90% hits)",
        "ttft_cold": "",
        "power_draw": "450W stock cap, 12W idle at P8; peak under load not measured",
        "setup_notes": "125B MoE hybrid on a single 24GB card: 6892 experts (~13.4 GB) in VRAM, rest in RAM, lookup table on NVMe. VRAM permanently occupied 23471/24564 MiB (processes /strata + /strata-vision). Speculative decoding MTP=4 + lookup=3, pcie_frac 0.35, image input enabled, one request at a time. WSL2 is not an officially documented path: RAM must be allocated via .wslconfig (96 GB here) or the engine can't load the 35-55 GB of experts. 4x32GB double-rank DDR4 holding 3600 CL18 (unusual on Ryzen 5000 - most boards drop to 3200 with this many ranks), so the RAM-resident experts get ~57 GB/s of dual-channel bandwidth."
      },
      "tier": "t3",
      "hardware": {
        "description": "1x RTX 4090 24GB, Ryzen 9 5900X, 128GB DDR4-3600 CL18 (2x CMK64GX4M2D3600C18, 4x32GB double-rank, holding 3600 MT/s XMP), Gigabyte Aorus X570 Master, Samsung 990 Pro NVMe (PCIe 4.0)",
        "vendor": "NVIDIA"
      },
      "backend": "Strata",
      "os_driver": "Windows 11 Pro + WSL2/Ubuntu 24.04 (96 GB allocated via .wslconfig); NVIDIA 610.43.02, KMD 610.47, CUDA UMD 13.3",
      "model": {
        "name": "Qwen/Qwen3.8-Flash-Next (GGUF IQ3_S)",
        "url": "https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF"
      },
      "quant": "GGUF IQ3_S (ISTA-DASLab GSQ-RCO)",
      "weight_precision": "IQ3_S",
      "use_cases": "Agentic coding / agent harness via OpenAI-compatible API",
      "context": {
        "stable": "131072 configured; >=106,493 tokens observed in live agent sessions, no OOM, no coherence collapse",
        "kv_cache": "int8 (engine-native quantized KV)"
      },
      "performance": {
        "decode_tps": "128",
        "prefill_tps": "825-3600 on uncached tokens (median ~2800, prompt cache ~90% hits)",
        "ttft_cold": "",
        "power_w": "450W stock cap, 12W idle at P8; peak under load not measured"
      },
      "engine_mode": "",
      "notes": "125B MoE hybrid on a single 24GB card: 6892 experts (~13.4 GB) in VRAM, rest in RAM, lookup table on NVMe. VRAM permanently occupied 23471/24564 MiB (processes /strata + /strata-vision). Speculative decoding MTP=4 + lookup=3, pcie_frac 0.35, image input enabled, one request at a time. WSL2 is not an officially documented path: RAM must be allocated via .wslconfig (96 GB here) or the engine can't load the 35-55 GB of experts. 4x32GB double-rank DDR4 holding 3600 CL18 (unusual on Ryzen 5000 - most boards drop to 3200 with this many ranks), so the RAM-resident experts get ~57 GB/s of dual-channel bandwidth. | dedicated Qwen3.8-Flash-Next engine, OpenAI/Anthropic API Hi, Please note that big parts of this post have been written with the help of the hereafter described model using OpenWork harness as I was afraid to miss relevant infos Strata on a 4090 + 128 GB, under WSL2 \u2014 agentic coding Direct answer to @lhl (post #25): yes, Strata changes the \u201csmall GPU\u201d equation. A Qwen3.8-Flash-Next 125B in IQ3_S runs on a single 4090 here, at ~108 tok/s decode in real agent use. Beyond raw speed, what sold me is the prompt cache: ~90% hit rate, so agent follow-ups on ~50k-token contexts resume in 0.3-0.8 s instead of re-prefilling. One note for anyone eyeing WSL2: the docs target native Windows or native Linux. Under WSL2 you must allocate RAM explicitly (.wslconfig, 96 GB here - WSL2 defaults to 50% of host), otherwise the engine doesn\u2019t have enough RAM to load the 35-55 GB of experts. I came from llama.cpp (Qwen3.8-27B-UD-IQ4_X@256k, KV q4_0), where I was already doing very good agentic coding with Claude Code or Copilot CLI (both via VSCode terminal). No formal comparison, but Strata at IQ3_S/131k doesn\u2019t make me miss the 27B at 256k - I do miss some context depth, and I\u2019m planning to test 256k under Strata. On the tier brackets: with hybrid offload engines (Strata), the VRAM tier alone no longer describes what actually fits - this 125B IQ3_S run is RAM+SSD-bound, not VRAM-bound. A RAM column or an \u201chybrid/offload\u201d flag would be more honest than a VRAM bracket. Cheers !",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": true,
      "tags": [
        "breakthrough",
        "hybrid",
        "consumer"
      ],
      "highlight_reason": "First proof of 125B MoE on single consumer GPU via hybrid offload. Changes the VRAM tier model."
    },
    {
      "id": "post_28_digitalscream_0",
      "source": {
        "post_number": 28,
        "post_id": 4128376,
        "username": "digitalscream",
        "submitted_at": "2026-10-12T11:17:02.223Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "32-64GB",
        "hardware": "2 x R9700 32GB, Threadripper 3960X, 128GB DDR4",
        "backend": "radiance 1.3.0",
        "os_driver": "amdgpu",
        "model_name": "Swift 1.5 Flash Next",
        "quant_package": "IQ4 weights, FP8 PLE",
        "weight_precision": "4-bit/8-bit",
        "download_url": "https://huggingface.co/StillDeadcode/swift1.5-qwen3.8-next-flash-fp8-iq4r-moe",
        "primary_use_case": "General purpose / agentic coding / agent harness via OpenAI-compatible API",
        "stable_context_tested": "262,144 configured; >=200,000 tokens observed in live agent sessions, no OOM, no coherence collapse",
        "kv_cache_setting": "FP8",
        "throughput_tps": "200t/s+ decode average",
        "prefill_tps": "5200-6200t/s",
        "ttft_cold": "",
        "power_draw": "500-600W during inference, GPUs at 330W cap each",
        "setup_notes": "Everything running in VRAM and system RAM, no disk tier"
      },
      "tier": "t5",
      "hardware": {
        "description": "2 x R9700 32GB, Threadripper 3960X, 128GB DDR4",
        "vendor": "AMD"
      },
      "backend": "Radiance",
      "os_driver": "amdgpu",
      "model": {
        "name": "Swift 1.5 Flash Next",
        "url": "https://huggingface.co/StillDeadcode/swift1.5-qwen3.8-next-flash-fp8-iq4r-moe"
      },
      "quant": "IQ4 weights, FP8 PLE",
      "weight_precision": "4-bit/8-bit",
      "use_cases": "General purpose / agentic coding / agent harness via OpenAI-compatible API",
      "context": {
        "stable": "262,144 configured; >=200,000 tokens observed in live agent sessions, no OOM, no coherence collapse",
        "kv_cache": "FP8"
      },
      "performance": {
        "decode_tps": "200",
        "prefill_tps": "5200-6200t/s",
        "ttft_cold": "",
        "power_w": "500-600W during inference, GPUs at 330W cap each"
      },
      "engine_mode": "",
      "notes": "Everything running in VRAM and system RAM, no disk tier OK, new submission now that I\u2019ve got Swift 1.5 Flash Next running accurately on my R9700s. Honestly, this thing is nuts. It sometimes takes longer to get to the solution than 27B, but it\u2019s better at finding the right solution and much better at understanding vaguely-defined requirements. The fact that it\u2019s hitting 200t/s+ with MTP, rather than DFlash2, is incredible in itself.",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": true,
      "tags": [
        "amd",
        "high-performance",
        "mtp"
      ],
      "highlight_reason": "200t/s+ on AMD with MTP. Radiance is the AMD answer to vLLM."
    },
    {
      "id": "post_33_manoreken2_0",
      "source": {
        "post_number": 33,
        "post_id": 4128818,
        "username": "manoreken2",
        "submitted_at": "2026-10-13T12:45:58.600Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "hardware": "Intel Arc B60 24GB x4",
        "backend": "llama.cpp vulkan with no performance tuning",
        "tier": "96-128",
        "model_name": "gemma-4-31B-it-BF16.gguf",
        "quant_package": "BF16",
        "primary_use_case": "CtoJ translation of Chinese history \u8cc7\u6cbb\u901a\u9451",
        "setup_notes": "It outputs clean Japanese. The model knows kings of \u897f\u79e6 therefore it knows China history more than GPT-OSS-120B but less than DeepSeek-R1-0528-671b-q8_0 : Gemma4 31B does not know the older sexagenary cycle year names such as \u65c3\u8499\u5927\u8352\u843d, \u67d4\u5146\u56f0\u6566 or \u662d\u967d\u5927\u6df5\u737b and those words trigger pretty sloppy quasi-history. Denial of translation of cruel historical text is somewhat relaxed than DeepSeek-V4-Pro-0813.",
        "throughput_tps": "PP 120t/s, TG 2.5 t/s"
      },
      "tier": "t6",
      "hardware": {
        "description": "Intel Arc B60 24GB x4",
        "vendor": "Intel"
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "gemma-4-31B-it-BF16.gguf",
        "url": ""
      },
      "quant": "BF16",
      "weight_precision": "",
      "use_cases": "CtoJ translation of Chinese history \u8cc7\u6cbb\u901a\u9451",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "2.5",
        "prefill_tps": "120",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "It outputs clean Japanese. The model knows kings of \u897f\u79e6 therefore it knows China history more than GPT-OSS-120B but less than DeepSeek-R1-0528-671b-q8_0 : Gemma4 31B does not know the older sexagenary cycle year names such as \u65c3\u8499\u5927\u8352\u843d, \u67d4\u5146\u56f0\u6566 or \u662d\u967d\u5927\u6df5\u737b and those words trigger pretty sloppy quasi-history. Denial of translation of cruel historical text is somewhat relaxed than DeepSeek-V4-Pro-0813. | vulkan with no performance tuning @lhl Thanks for your suggestion!",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_35_novasty_0",
      "source": {
        "post_number": 35,
        "post_id": 4131388,
        "username": "Novasty",
        "submitted_at": "2026-10-20T14:23:09.762Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "t1",
        "hardware": "RTX 3060 12GB",
        "backend": "Other",
        "os_driver": "Linux/CUDA",
        "model_name": "HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive",
        "quant_package": "GGUF Q4_K_M",
        "download_url": "https://huggingface.co/HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive",
        "primary_use_case": "RAG",
        "stable_context_tested": "196608",
        "kv_cache_setting": "Default",
        "engine_mode": "Hybrid (VRAM+RAM)",
        "throughput_tps": "14.8",
        "power_draw": "170w"
      },
      "tier": "t1",
      "hardware": {
        "description": "RTX 3060 12GB",
        "vendor": "NVIDIA"
      },
      "backend": "Other",
      "os_driver": "Linux/CUDA",
      "model": {
        "name": "HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive",
        "url": "https://huggingface.co/HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive"
      },
      "quant": "GGUF Q4_K_M",
      "weight_precision": "",
      "use_cases": "RAG",
      "context": {
        "stable": "196608",
        "kv_cache": "Default"
      },
      "performance": {
        "decode_tps": "14.8",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": "170w"
      },
      "engine_mode": "Hybrid (VRAM+RAM)",
      "notes": "My server is running DDR4 RAM, so there is definitely loss there.",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_36_yehya_karout_0",
      "source": {
        "post_number": 36,
        "post_id": 4132067,
        "username": "Yehya_Karout",
        "submitted_at": "2026-10-22T03:29:22.581Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "hardware": "1x RTX 5080 16GB + 64GB DDR5-6400, Core Ultra 9 285K",
        "backend": "Strata v0.2.2 (engine 0.1.47), Fedora 44",
        "tier": "16GB \u2014 \u226416 (hybrid, experts live in RAM)",
        "model_name": "Qwen3.8-Flash-Next (GSQ-RCO GGUF)",
        "quant_package": "IQ3_XXS, int8 KV",
        "primary_use_case": "agentic coding (opencode, pi), full 262K context",
        "setup_notes": "",
        "throughput_tps": "~170 tok/s tg / ~3,000 pp at 4K, ~150 tg / ~3,950 pp at 180K, 152 tg / 3,800 pp at 250K. Real opencode agent session: ~165-170 tok/s avg"
      },
      "tier": "t2",
      "hardware": {
        "description": "1x RTX 5080 16GB + 64GB DDR5-6400, Core Ultra 9 285K",
        "vendor": "NVIDIA"
      },
      "backend": "Strata",
      "os_driver": "",
      "model": {
        "name": "Qwen3.8-Flash-Next (GSQ-RCO GGUF)",
        "url": ""
      },
      "quant": "IQ3_XXS, int8 KV",
      "weight_precision": "",
      "use_cases": "agentic coding (opencode, pi), full 262K context",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "~170 tok/s tg / ~3,000 pp at 4K, ~150 tg / ~3,950 pp at 180K, 152 tg / 3,800 pp at 250K. Real opencode agent session: ~165-170 tok/s avg",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "v0.2.2 engine 0.1.47), Fedora 44 Running Flash-Next on a 16GB card with Strata, figured this belongs in the hybrid bucket. Numbers are from my own fresh-prompt sweeps + a real opencode agent run, not the built-in bench.",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_37_kif_0",
      "source": {
        "post_number": 37,
        "post_id": 4133535,
        "username": "Kif",
        "submitted_at": "2026-10-25T13:34:15.525Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "24-32GB",
        "hardware": "1x RTX 4090 24GB, Ryzen 9 5900X, 128GB DDR4-3600 CL18 (4x32GB double-rank), Gigabyte Aorus X570 Master, Samsung 990 Pro NVMe",
        "backend": "Strata 0.1.49.2",
        "os_driver": "Windows 11 Pro + WSL2/Ubuntu 24.04 (96 GB via .wslconfig); NVIDIA 615.78.05, KMD 617.42, CUDA UMD 13.4",
        "model_name": "Qwen/Qwen3.8-Flash-Next (GGUF IQ3_S)",
        "quant_package": "GGUF IQ3_S (ISTA-DASLab GSQ-RCO)",
        "weight_precision": "IQ3_S",
        "download_url": "https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF",
        "primary_use_case": "Agentic coding / agent harness via OpenAI-compatible API (+ vision via mmproj)",
        "stable_context_tested": "262144 configured; 228,553 tokens observed in live sessions (65 s cold prefill), no OOM, no truncation, no coherence collapse",
        "kv_cache_setting": "int8",
        "throughput_tps": "95-107 decode (p25-p75 at 262k single stream, median 101; 131k entry in my Oct 12 post was median 108)",
        "prefill_tps": "3460 median (requests with >=8k fresh tokens)",
        "ttft_cold": "~65 s at full 228k cold; under 1 s on cached follow-ups",
        "power_draw": "450W stock cap, 12W idle at P8",
        "setup_notes": "chat profile, 1 slot, 5781 expert slots / 11 GB VRAM; ~88% of all input tokens served from KV reuse across the 3291-request log, ~99% per turn inside a growing live session; vision .mmproj unchanged (1024 image tokens); supersedes the 131k part of my Oct 12 entry"
      },
      "tier": "t3",
      "hardware": {
        "description": "1x RTX 4090 24GB, Ryzen 9 5900X, 128GB DDR4-3600 CL18 (4x32GB double-rank), Gigabyte Aorus X570 Master, Samsung 990 Pro NVMe",
        "vendor": "NVIDIA"
      },
      "backend": "Strata",
      "os_driver": "Windows 11 Pro + WSL2/Ubuntu 24.04 (96 GB via .wslconfig); NVIDIA 615.78.05, KMD 617.42, CUDA UMD 13.4",
      "model": {
        "name": "Qwen/Qwen3.8-Flash-Next (GGUF IQ3_S)",
        "url": "https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF"
      },
      "quant": "GGUF IQ3_S (ISTA-DASLab GSQ-RCO)",
      "weight_precision": "IQ3_S",
      "use_cases": "Agentic coding / agent harness via OpenAI-compatible API (+ vision via mmproj)",
      "context": {
        "stable": "262144 configured; 228,553 tokens observed in live sessions (65 s cold prefill), no OOM, no truncation, no coherence collapse",
        "kv_cache": "int8"
      },
      "performance": {
        "decode_tps": "107",
        "prefill_tps": "3460 median (requests with >=8k fresh tokens)",
        "ttft_cold": "~65 s at full 228k cold; under 1 s on cached follow-ups",
        "power_w": "450W stock cap, 12W idle at P8"
      },
      "engine_mode": "",
      "notes": "chat profile, 1 slot, 5781 expert slots / 11 GB VRAM; ~88% of all input tokens served from KV reuse across the 3291-request log, ~99% per turn inside a growing live session; vision .mmproj unchanged (1024 image tokens); supersedes the 131k part of my Oct 12 entry Follow-up: 256k context on the same 4090 rig \u2014 and why I now run two launch profiles Hi again \u2014 my Oct 12 post said I was planning to try 256k under Strata. I did, and TL;DR: decode barely moved, prefill latency is the real cost, and 262k forced me into two launch profiles. (Still hybrid \u2014 glad to see the matrix grew a flag for it.) The numbers, from the engine log: 3,291 requests, 95 boots, Oct 7 \u2192 25. Decode filtered to requests generating \u2265128 tokens: decode p25 / median / p75 \u2014 131k: 92 / 103 / 113 \u00b7 262k one stream: 95 / 101 / 107 \u00b7 262k two streams: 52 / 55 / 59 prefill median (>=8k fresh tokens) \u2014 ~3480 tok/s at 131k \u00b7 ~3460 single-stream \u00b7 ~1720 two streams expert cache in VRAM \u2014 6400-7700 slots (12-14.6 GB) at 131k \u00b7 5781 (11.0 GB) single-stream \u00b7 1786 (3.4 GB) two streams Longest live session: 228,553 tokens \u2014 65 s cold prefill (~3.5k tok/s), then 101 tok/s decode. Zero OOM, zero context-full errors, zero coherence collapse across the whole log. Vision still fine at 262k (same .mmproj, stress-tested at 2x context). The real cost is prefill latency, not throughput. Per-token prefill speed didn\u2019t change at all; the cold-cache wall-clock roughly doubled: ~65 s for a 228k prompt vs ~35 s at 131k. That delay on cache misses is what pushed me into prompt caching. Why two launch profiles: at 262k the int8 KV eats ~3.6 GB of VRAM per slot, so 2 streams starve the expert cache (5781 \u2192 1786 slots \u2248 70 % of experts pushed to RAM) and decode drops to ~55 tok/s. So I pick chat (1 stream, full expert cache \u2014 my default) or agentic (2 streams, roughly half TG). Claude Code on the agentic profile kept KV reuse at 86 % and sessions at 201k tokens across both slots \u2014 it works, but the fan-out didn\u2019t offset the TG drop. Back-to-131k A/B matched my gut almost exactly \u2014 TG identical, 262k only slower on cold prefill. Subjective bonus at 256k: fewer auto-compactions in Claude Code, and long OpenWork conversations stay coherent over more turns (impression, not a metric). KV variant benches (same day, one fixed prompt: ~10k fresh tokens forced cold with a unique salt, then 900 decoded): single-stream chat, k8v4 changes nothing (5636 \u2192 6011 experts, ~99 tok/s either way). Under agentic load it does: experts 1620 \u2192 2878, decode per stream 28.8/24.1 \u2192 34.1/28.5 tok/s, prefill 1560 \u2192 1925 \u2014 reproduced on an A/B reversal. Caveats: those decode figures are fresh conversations (adapted sessions reach the ~55 above), and speculation served only one of the two streams in every pass, all three variants. (Sidenote) Forcing KV streaming under WSL2. The 3.6 GB-per-slot KV made me try --kv-resident 32768 \u2014 keep the hot KV window in VRAM (32,768 positions per layer), the rest in host RAM. Strata normally refuses that under WSL2; injected via user_args it ran: 255k-token request, 3602 prefill / 104.7 decode tok/s \u2014 no collapse over WSL\u2019s ~26 GB/s PCIe \u2014 with 7257 experts in the freed VRAM. The log explains why: pinning works (cudaHostRegister PORTABLE ok) and 97 %+ of KV block reads still hit VRAM, locality keeping the working set inside the window. Even two-stream agentic benefits: 6343 experts, 40-52 tok/s decode. Daily driver still int8; native Win11 stays on the bench. \"Chat\" profile \u2014 IQ3_S, 262k single stream (matrix submission \u2014 click to expand) \"Agentic\" profile \u2014 2 streams + KV streaming (matrix submission \u2014 click to expand)",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_37_kif_1",
      "source": {
        "post_number": 37,
        "post_id": 4133535,
        "username": "Kif",
        "submitted_at": "2026-10-25T13:34:15.525Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "tier": "24-32GB",
        "hardware": "1x RTX 4090 24GB, Ryzen 9 5900X, 128GB DDR4-3600 CL18 (4x32GB double-rank), Gigabyte Aorus X570 Master, Samsung 990 Pro NVMe",
        "backend": "Strata 0.1.49.2",
        "os_driver": "Windows 11 Pro + WSL2/Ubuntu 24.04 (96 GB via .wslconfig); NVIDIA 615.78.05, KMD 617.42, CUDA UMD 13.4",
        "model_name": "Qwen/Qwen3.8-Flash-Next (GGUF IQ3_S)",
        "quant_package": "GGUF IQ3_S (ISTA-DASLab GSQ-RCO)",
        "weight_precision": "IQ3_S",
        "download_url": "https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF",
        "primary_use_case": "Agentic coding, 2 parallel streams via OpenAI-compatible API",
        "stable_context_tested": "262144 per slot x 2 slots; 2x41k-token parallel fresh streams measured with KV in host RAM, plus a single 255k-token request (74 s cold prefill)",
        "kv_cache_setting": "k8v4 + --kv-resident 32768 (32,768 positions per layer in VRAM, rest in host RAM; injected via config user_args since the engine declines it under WSL2)",
        "throughput_tps": "50.6/39.9 decode per stream on two fresh 10k conversations; 52.2/29.9 at 41k each",
        "prefill_tps": "1953-2088 two streams, 10k fresh tokens each",
        "ttft_cold": "11-15 s for a fresh 41k prompt per stream",
        "power_draw": "450W stock cap, 12W idle at P8",
        "setup_notes": "agentic profile, 2 slots, 6343 expert slots / 12.0 GB VRAM thanks to KV streaming (vs 1786 slots int8 all-VRAM); engine log: cudaHostRegister PORTABLE ok under WSL2, 97% of KV block reads still hit VRAM with both streams sharing the 32k window; caveat: MTP speculation served only one of the two streams in every measured pass"
      },
      "tier": "t3",
      "hardware": {
        "description": "1x RTX 4090 24GB, Ryzen 9 5900X, 128GB DDR4-3600 CL18 (4x32GB double-rank), Gigabyte Aorus X570 Master, Samsung 990 Pro NVMe",
        "vendor": "NVIDIA"
      },
      "backend": "Strata",
      "os_driver": "Windows 11 Pro + WSL2/Ubuntu 24.04 (96 GB via .wslconfig); NVIDIA 615.78.05, KMD 617.42, CUDA UMD 13.4",
      "model": {
        "name": "Qwen/Qwen3.8-Flash-Next (GGUF IQ3_S)",
        "url": "https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF"
      },
      "quant": "GGUF IQ3_S (ISTA-DASLab GSQ-RCO)",
      "weight_precision": "IQ3_S",
      "use_cases": "Agentic coding, 2 parallel streams via OpenAI-compatible API",
      "context": {
        "stable": "262144 per slot x 2 slots; 2x41k-token parallel fresh streams measured with KV in host RAM, plus a single 255k-token request (74 s cold prefill)",
        "kv_cache": "k8v4 + --kv-resident 32768 (32,768 positions per layer in VRAM, rest in host RAM; injected via config user_args since the engine declines it under WSL2)"
      },
      "performance": {
        "decode_tps": "39.9",
        "prefill_tps": "1953-2088 two streams, 10k fresh tokens each",
        "ttft_cold": "11-15 s for a fresh 41k prompt per stream",
        "power_w": "450W stock cap, 12W idle at P8"
      },
      "engine_mode": "",
      "notes": "agentic profile, 2 slots, 6343 expert slots / 12.0 GB VRAM thanks to KV streaming (vs 1786 slots int8 all-VRAM); engine log: cudaHostRegister PORTABLE ok under WSL2, 97% of KV block reads still hit VRAM with both streams sharing the 32k window; caveat: MTP speculation served only one of the two streams in every measured pass Cheers !",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_42_f41n_0",
      "source": {
        "post_number": 42,
        "post_id": 4133610,
        "username": "f41n",
        "submitted_at": "2026-10-25T18:26:47.415Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "hardware": "1xRX 6800 XT 16GB + Ryzen 7 7700X + 32GB DDR5-5200",
        "backend": "llama.cpp (ROCm, build b10066)",
        "tier": "\u226416",
        "model_name": "gpt-oss 20B",
        "quant_package": "Q8_0",
        "primary_use_case": "coding, agents",
        "setup_notes": "",
        "throughput_tps": "131 t/s generation, 2762 t/s prompt"
      },
      "tier": "t2",
      "hardware": {
        "description": "1xRX 6800 XT 16GB + Ryzen 7 7700X + 32GB DDR5-5200",
        "vendor": ""
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "gpt-oss 20B",
        "url": ""
      },
      "quant": "Q8_0",
      "weight_precision": "",
      "use_cases": "coding, agents",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "131 t/s generation, 2762 t/s prompt",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "ROCm, build b10066 Sorry for the absolute brick that\u2019s going to ensue, i run a lot of models in my system for everything. Hope it\u2019s all formatted well, let me know if i need to edit anything. How measured: llama-bench pp512 / tg128, 3 reps, all layers on GPU. MTP and Flash-Next can\u2019t run in llama-bench, so those used llama-server: the same ~30-token coding prompt sent 3+ times, warm runs reported. As for all the data and testing I\u2019ve done, ill push that to my repo (eventually) even tho i know no one runs mi25\u2019s, but maybe helpful somehow. ~my desktop~",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_42_f41n_1",
      "source": {
        "post_number": 42,
        "post_id": 4133610,
        "username": "f41n",
        "submitted_at": "2026-10-25T18:26:47.415Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "hardware": "1x RX 6800 XT 16GB + Ryzen 7 7700X + 32GB DDR5-5200",
        "backend": "llama.cpp (ROCm, build b10066)",
        "tier": "\u226416",
        "model_name": "Nemotron3 Nano 4B",
        "quant_package": "Q4_K_M",
        "primary_use_case": "quick tasks, small helper model",
        "setup_notes": "",
        "throughput_tps": "123 t/s generation, 2412 t/s prompt"
      },
      "tier": "t2",
      "hardware": {
        "description": "1x RX 6800 XT 16GB + Ryzen 7 7700X + 32GB DDR5-5200",
        "vendor": ""
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "Nemotron3 Nano 4B",
        "url": ""
      },
      "quant": "Q4_K_M",
      "weight_precision": "",
      "use_cases": "quick tasks, small helper model",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "123 t/s generation, 2412 t/s prompt",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "ROCm, build b10066 [/COMMUNITY-SUBMISSION] hardware: 1x RX 6800 XT 16GB + Ryzen 7 7700X + 32GB DDR5-5200 vram: \u226416 model: Qwen3.6 35B-A3B backend: llama.cpp (ROCm, build b10066) quant: IQ3_S (Unsloth UD) speed: 84 t/s generation, 1722 t/s prompt use_case: coding, agents notes: -ngl 99 -fa 1 -ctk q8_0 -ctv q4_0 -c 32768. 12.7 GB. [/COMMUNITY-SUBMISSION]",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_42_f41n_2",
      "source": {
        "post_number": 42,
        "post_id": 4133610,
        "username": "f41n",
        "submitted_at": "2026-10-25T18:26:47.415Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "hardware": "1x RX 6800 XT 16GB + Ryzen 7 7700X + 32GB DDR5-5200",
        "backend": "llama.cpp (ROCm, build b10066)",
        "tier": "\u226416",
        "model_name": "Gemma 4 26B-A4B",
        "quant_package": "UD-IQ4_XS",
        "primary_use_case": "daily chat, general",
        "setup_notes": "",
        "throughput_tps": "73 t/s generation, 2278 t/s prompt"
      },
      "tier": "t2",
      "hardware": {
        "description": "1x RX 6800 XT 16GB + Ryzen 7 7700X + 32GB DDR5-5200",
        "vendor": ""
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "Gemma 4 26B-A4B",
        "url": ""
      },
      "quant": "UD-IQ4_XS",
      "weight_precision": "",
      "use_cases": "daily chat, general",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "73 t/s generation, 2278 t/s prompt",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "ROCm, build b10066",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_42_f41n_3",
      "source": {
        "post_number": 42,
        "post_id": 4133610,
        "username": "f41n",
        "submitted_at": "2026-10-25T18:26:47.415Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "hardware": "1x RX 6800 XT 16GB + Ryzen 7 7700X + 32GB DDR5-5200",
        "backend": "llama.cpp (ROCm, build b10066)",
        "tier": "\u226416",
        "model_name": "Qwen3.8-27B",
        "quant_package": "3-bit GSQ-RCO (mixed IQ3_S etc.) + MTP head",
        "primary_use_case": "coding, agents",
        "setup_notes": "",
        "throughput_tps": "\\~51 t/s generation"
      },
      "tier": "t2",
      "hardware": {
        "description": "1x RX 6800 XT 16GB + Ryzen 7 7700X + 32GB DDR5-5200",
        "vendor": ""
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "Qwen3.8-27B",
        "url": ""
      },
      "quant": "3-bit GSQ-RCO (mixed IQ3_S etc.) + MTP head",
      "weight_precision": "",
      "use_cases": "coding, agents",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "\\~51 t/s generation",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "ROCm, build b10066",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_42_f41n_4",
      "source": {
        "post_number": 42,
        "post_id": 4133610,
        "username": "f41n",
        "submitted_at": "2026-10-25T18:26:47.415Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "hardware": "1x RX 6800 XT 16GB + Ryzen 7 7700X + 32GB DDR5-5200",
        "backend": "llama.cpp (ROCm, build b10066)",
        "tier": "\u226416",
        "model_name": "Qwen3.8-27B",
        "quant_package": "3-bit GSQ-RCO (mixed IQ3_S etc.)",
        "primary_use_case": "coding, agents",
        "setup_notes": "",
        "throughput_tps": "27 t/s generation, 441 t/s prompt"
      },
      "tier": "t2",
      "hardware": {
        "description": "1x RX 6800 XT 16GB + Ryzen 7 7700X + 32GB DDR5-5200",
        "vendor": ""
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "Qwen3.8-27B",
        "url": ""
      },
      "quant": "3-bit GSQ-RCO (mixed IQ3_S etc.)",
      "weight_precision": "",
      "use_cases": "coding, agents",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "27 t/s generation, 441 t/s prompt",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "ROCm, build b10066",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_42_f41n_5",
      "source": {
        "post_number": 42,
        "post_id": 4133610,
        "username": "f41n",
        "submitted_at": "2026-10-25T18:26:47.415Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "hardware": "1x RX 6800 XT 16GB + Ryzen 7 7700X + 32GB DDR5-5200",
        "backend": "llama.cpp (Vulkan, upstream build with --moe-cache-mib)",
        "tier": "\u226416",
        "model_name": "Qwen3.8-Flash-Next (125B MoE)",
        "quant_package": "GSQ-RCO Q2_0 (37.6 GB shard 1 + 29 GB n-gram table shard 2)",
        "primary_use_case": "coding, agents",
        "setup_notes": "",
        "throughput_tps": "\\~12 t/s generation (warm), \\~12 t/s prompt, \\~5 t/s first cold prompt"
      },
      "tier": "t2",
      "hardware": {
        "description": "1x RX 6800 XT 16GB + Ryzen 7 7700X + 32GB DDR5-5200",
        "vendor": ""
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "Qwen3.8-Flash-Next (125B MoE)",
        "url": ""
      },
      "quant": "GSQ-RCO Q2_0 (37.6 GB shard 1 + 29 GB n-gram table shard 2)",
      "weight_precision": "",
      "use_cases": "coding, agents",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "\\~12 t/s generation (warm), \\~12 t/s prompt, \\~5 t/s first cold prompt",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "Vulkan, upstream build with --moe-cache-mib ~my inference node~",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_42_f41n_6",
      "source": {
        "post_number": 42,
        "post_id": 4133610,
        "username": "f41n",
        "submitted_at": "2026-10-25T18:26:47.415Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "hardware": "1x Radeon Instinct MI25 16GB HBM2 @ 170W (WX 9100 BIOS) + Ryzen 5 3600 + 32GB DDR4-3200",
        "backend": "llama.cpp (Vulkan/RADV, build 11829)",
        "tier": "\u226416",
        "model_name": "gpt-oss 20B",
        "quant_package": "Q8_0",
        "primary_use_case": "coding, agents",
        "setup_notes": "",
        "throughput_tps": "89 t/s generation, 858 t/s prompt"
      },
      "tier": "t2",
      "hardware": {
        "description": "1x Radeon Instinct MI25 16GB HBM2 @ 170W (WX 9100 BIOS) + Ryzen 5 3600 + 32GB DDR4-3200",
        "vendor": "AMD"
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "gpt-oss 20B",
        "url": ""
      },
      "quant": "Q8_0",
      "weight_precision": "",
      "use_cases": "coding, agents",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "89 t/s generation, 858 t/s prompt",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "Vulkan/RADV, build 11829",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_42_f41n_7",
      "source": {
        "post_number": 42,
        "post_id": 4133610,
        "username": "f41n",
        "submitted_at": "2026-10-25T18:26:47.415Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "hardware": "1x Radeon Instinct MI25 16GB HBM2 @ 170W (WX 9100 BIOS) + Ryzen 5 3600 + 32GB DDR4-3200",
        "backend": "llama.cpp (Vulkan/RADV, build 11829)",
        "tier": "\u226416",
        "model_name": "Nemotron3 Nano 4B",
        "quant_package": "Q4_K_M",
        "primary_use_case": "quick tasks, small helper model",
        "setup_notes": "",
        "throughput_tps": "77 t/s generation, 783 t/s prompt"
      },
      "tier": "t2",
      "hardware": {
        "description": "1x Radeon Instinct MI25 16GB HBM2 @ 170W (WX 9100 BIOS) + Ryzen 5 3600 + 32GB DDR4-3200",
        "vendor": "AMD"
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "Nemotron3 Nano 4B",
        "url": ""
      },
      "quant": "Q4_K_M",
      "weight_precision": "",
      "use_cases": "quick tasks, small helper model",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "77 t/s generation, 783 t/s prompt",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "Vulkan/RADV, build 11829",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_42_f41n_8",
      "source": {
        "post_number": 42,
        "post_id": 4133610,
        "username": "f41n",
        "submitted_at": "2026-10-25T18:26:47.415Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "hardware": "1x Radeon Instinct MI25 16GB HBM2 @ 170W (WX 9100 BIOS) + Ryzen 5 3600 + 32GB DDR4-3200",
        "backend": "llama.cpp (Vulkan/RADV, build 11829)",
        "tier": "\u226416",
        "model_name": "Qwen3.6 35B-A3B",
        "quant_package": "IQ3_S (Unsloth UD)",
        "primary_use_case": "coding, agents",
        "setup_notes": "",
        "throughput_tps": "68 t/s generation, 586 t/s prompt"
      },
      "tier": "t2",
      "hardware": {
        "description": "1x Radeon Instinct MI25 16GB HBM2 @ 170W (WX 9100 BIOS) + Ryzen 5 3600 + 32GB DDR4-3200",
        "vendor": "AMD"
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "Qwen3.6 35B-A3B",
        "url": ""
      },
      "quant": "IQ3_S (Unsloth UD)",
      "weight_precision": "",
      "use_cases": "coding, agents",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "68 t/s generation, 586 t/s prompt",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "Vulkan/RADV, build 11829",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_42_f41n_9",
      "source": {
        "post_number": 42,
        "post_id": 4133610,
        "username": "f41n",
        "submitted_at": "2026-10-25T18:26:47.415Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "hardware": "1x Radeon Instinct MI25 16GB HBM2 @ 170W (WX 9100 BIOS) + Ryzen 5 3600 + 32GB DDR4-3200",
        "backend": "llama.cpp (Vulkan/RADV, build 11829)",
        "tier": "\u226416",
        "model_name": "Gemma 4 26B-A4B",
        "quant_package": "UD-IQ4_XS",
        "primary_use_case": "daily chat, general",
        "setup_notes": "",
        "throughput_tps": "57 t/s generation, 678 t/s prompt"
      },
      "tier": "t2",
      "hardware": {
        "description": "1x Radeon Instinct MI25 16GB HBM2 @ 170W (WX 9100 BIOS) + Ryzen 5 3600 + 32GB DDR4-3200",
        "vendor": "AMD"
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "Gemma 4 26B-A4B",
        "url": ""
      },
      "quant": "UD-IQ4_XS",
      "weight_precision": "",
      "use_cases": "daily chat, general",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "57 t/s generation, 678 t/s prompt",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "Vulkan/RADV, build 11829",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_42_f41n_10",
      "source": {
        "post_number": 42,
        "post_id": 4133610,
        "username": "f41n",
        "submitted_at": "2026-10-25T18:26:47.415Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "hardware": "1x Radeon Instinct MI25 16GB HBM2 @ 170W (WX 9100 BIOS) + Ryzen 5 3600 + 32GB DDR4-3200",
        "backend": "llama.cpp (Vulkan/RADV, build 11829)",
        "tier": "\u226416",
        "model_name": "Qwen3.8-27B",
        "quant_package": "3-bit GSQ-RCO (mixed IQ3_S etc.) + MTP head",
        "primary_use_case": "coding, agents",
        "setup_notes": "",
        "throughput_tps": "\\~23 t/s generation"
      },
      "tier": "t2",
      "hardware": {
        "description": "1x Radeon Instinct MI25 16GB HBM2 @ 170W (WX 9100 BIOS) + Ryzen 5 3600 + 32GB DDR4-3200",
        "vendor": "AMD"
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "Qwen3.8-27B",
        "url": ""
      },
      "quant": "3-bit GSQ-RCO (mixed IQ3_S etc.) + MTP head",
      "weight_precision": "",
      "use_cases": "coding, agents",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "\\~23 t/s generation",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "Vulkan/RADV, build 11829",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_42_f41n_11",
      "source": {
        "post_number": 42,
        "post_id": 4133610,
        "username": "f41n",
        "submitted_at": "2026-10-25T18:26:47.415Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "hardware": "1x Radeon Instinct MI25 16GB HBM2 @ 170W (WX 9100 BIOS) + Ryzen 5 3600 + 32GB DDR4-3200",
        "backend": "llama.cpp (Vulkan/RADV, build 11829)",
        "tier": "\u226416",
        "model_name": "Qwen3.8-27B",
        "quant_package": "3-bit GSQ-RCO (mixed IQ3_S etc.)",
        "primary_use_case": "coding, agents",
        "setup_notes": "",
        "throughput_tps": "17 t/s generation, 134 t/s prompt"
      },
      "tier": "t2",
      "hardware": {
        "description": "1x Radeon Instinct MI25 16GB HBM2 @ 170W (WX 9100 BIOS) + Ryzen 5 3600 + 32GB DDR4-3200",
        "vendor": "AMD"
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "Qwen3.8-27B",
        "url": ""
      },
      "quant": "3-bit GSQ-RCO (mixed IQ3_S etc.)",
      "weight_precision": "",
      "use_cases": "coding, agents",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "17 t/s generation, 134 t/s prompt",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "Vulkan/RADV, build 11829",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_42_f41n_12",
      "source": {
        "post_number": 42,
        "post_id": 4133610,
        "username": "f41n",
        "submitted_at": "2026-10-25T18:26:47.415Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "hardware": "1x Radeon Instinct MI25 16GB HBM2 @ 170W (WX 9100 BIOS) + Ryzen 5 3600 + 32GB DDR4-3200",
        "backend": "llama.cpp (Vulkan/RADV, build 11829)",
        "tier": "\u226416",
        "model_name": "Qwen3.8-Flash-Next (125B MoE)",
        "quant_package": "GSQ-RCO Q2_0 (37.6 GB shard 1 + 29 GB n-gram table shard 2)",
        "primary_use_case": "coding, agents",
        "setup_notes": "",
        "throughput_tps": "\\~17 t/s generation (warm, 15.5-18), \\~14 t/s prompt, \\~10 t/s first cold prompt"
      },
      "tier": "t2",
      "hardware": {
        "description": "1x Radeon Instinct MI25 16GB HBM2 @ 170W (WX 9100 BIOS) + Ryzen 5 3600 + 32GB DDR4-3200",
        "vendor": "AMD"
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "Qwen3.8-Flash-Next (125B MoE)",
        "url": ""
      },
      "quant": "GSQ-RCO Q2_0 (37.6 GB shard 1 + 29 GB n-gram table shard 2)",
      "weight_precision": "",
      "use_cases": "coding, agents",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "\\~17 t/s generation (warm, 15.5-18), \\~14 t/s prompt, \\~10 t/s first cold prompt",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "Vulkan/RADV, build 11829",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_46_mattlach_0",
      "source": {
        "post_number": 46,
        "post_id": 4133763,
        "username": "mattlach",
        "submitted_at": "2026-10-26T02:05:35.793Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "hardware": "1x AMD Instinct MI210 passed through to VM w. 32GB of RAM",
        "backend": "llama.cpp",
        "tier": "64GB total. 31.5GB used of which 27.2GB is model, rest is mostly context.",
        "model_name": "Gemma4:26B",
        "quant_package": "Q8_0",
        "primary_use_case": "Daily chat, basic scripting, summarization",
        "setup_notes": "Tends to lose reasoning capability and enter reasoning loops as context gets large. Needs frequent context compaction as a result",
        "throughput_tps": "Pre-fill: ~2,200 tok/s. Eval ~75-80 tok/s"
      },
      "tier": "t5",
      "hardware": {
        "description": "1x AMD Instinct MI210 passed through to VM w. 32GB of RAM",
        "vendor": "AMD"
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "Gemma4:26B",
        "url": ""
      },
      "quant": "Q8_0",
      "weight_precision": "",
      "use_cases": "Daily chat, basic scripting, summarization",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "Pre-fill: ~2,200 tok/s. Eval ~75-80 tok/s",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "Tends to lose reasoning capability and enter reasoning loops as context gets large. Needs frequent context compaction as a result For all of the submissions below the system is as follows: Supermicro H12SSL-NT motherboard AMD Epyc 7543 (Milan, 32C/64T) 512GB Registered DDR4-3200 ECC RAM Proxmox Hypervisor All of these models are running at the same time, with the four models on the GPU sharing its RAM.",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_46_mattlach_1",
      "source": {
        "post_number": 46,
        "post_id": 4133763,
        "username": "mattlach",
        "submitted_at": "2026-10-26T02:05:35.793Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "hardware": "1x AMD Instinct MI210 passed through to VM w. 32GB of RAM",
        "backend": "llama.cpp",
        "tier": "64GB total. 13.5GB used of which 12GB is model, rest is mostly context.",
        "model_name": "Gemma4:12B",
        "quant_package": "Q8_0",
        "primary_use_case": "Model dedicated to context compaction for other models.",
        "setup_notes": "",
        "throughput_tps": "Pre-fill: ~1,300 tok/s. Eval ~40-45 tok/s"
      },
      "tier": "t5",
      "hardware": {
        "description": "1x AMD Instinct MI210 passed through to VM w. 32GB of RAM",
        "vendor": "AMD"
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "Gemma4:12B",
        "url": ""
      },
      "quant": "Q8_0",
      "weight_precision": "",
      "use_cases": "Model dedicated to context compaction for other models.",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "Pre-fill: ~1,300 tok/s. Eval ~40-45 tok/s",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_46_mattlach_2",
      "source": {
        "post_number": 46,
        "post_id": 4133763,
        "username": "mattlach",
        "submitted_at": "2026-10-26T02:05:35.793Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "hardware": "1x AMD Instinct MI210 passed through to VM w. 32GB of RAM",
        "backend": "llama.cpp",
        "tier": "64GB total. 4.3GB used, but the confusing part is that the model file is 5GB. Not sure what is going on here.",
        "model_name": "Gemma4:E4B",
        "quant_package": "Q4_K_M",
        "primary_use_case": "Model dedicated to small front end tasks (title generation, tags, etc.)",
        "setup_notes": "Numbers are probably low due to prompts being very short, so it doesn't have enough tokens to really get going. Very weird that VRAM used for both weights and context is smaller that weights file... I can't explain that.",
        "throughput_tps": "Pre-fill: ~2,500 - 3,000 tok/s. Eval ~85-90 tok/s"
      },
      "tier": "t5",
      "hardware": {
        "description": "1x AMD Instinct MI210 passed through to VM w. 32GB of RAM",
        "vendor": "AMD"
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "Gemma4:E4B",
        "url": ""
      },
      "quant": "Q4_K_M",
      "weight_precision": "",
      "use_cases": "Model dedicated to small front end tasks (title generation, tags, etc.)",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "Pre-fill: ~2,500 - 3,000 tok/s. Eval ~85-90 tok/s",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "Numbers are probably low due to prompts being very short, so it doesn't have enough tokens to really get going. Very weird that VRAM used for both weights and context is smaller that weights file... I can't explain that.",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_46_mattlach_3",
      "source": {
        "post_number": 46,
        "post_id": 4133763,
        "username": "mattlach",
        "submitted_at": "2026-10-26T02:05:35.793Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "hardware": "1x AMD Instinct MI210 passed through to VM w. 32GB of RAM",
        "backend": "llama.cpp",
        "tier": "64GB total. 1.1GB used",
        "model_name": "Jina Embeddings V3",
        "quant_package": "F16",
        "primary_use_case": "Model dedicated for embeddings",
        "throughput_tps": ""
      },
      "tier": "t5",
      "hardware": {
        "description": "1x AMD Instinct MI210 passed through to VM w. 32GB of RAM",
        "vendor": "AMD"
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "Jina Embeddings V3",
        "url": ""
      },
      "quant": "F16",
      "weight_precision": "",
      "use_cases": "Model dedicated for embeddings",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_46_mattlach_4",
      "source": {
        "post_number": 46,
        "post_id": 4133763,
        "username": "mattlach",
        "submitted_at": "2026-10-26T02:05:35.793Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "hardware": "AMD Epyc 7543 (Milan, 32C/64T), LXC container with 128GB of RAM (no GPU)",
        "backend": "Ollama",
        "tier": "",
        "model_name": "Poolside Laguna S2.1 (118B)",
        "quant_package": "Q4_K_M",
        "primary_use_case": "Coding, and long form reasoning beyond what Gemma4:26B is capable of",
        "setup_notes": "I tend to refine my coding prompts using Gemma4:26B, and then copy and paste the plan for the coding into Laguna and have it run in the background or overnight. Needed to tweak some settings to make the model run better on the CPU. Setting \"PARAMETER num_batch 512\" significantly improved performance. Pre-fill will always run very slowly on CPU's, which is what makes this more of an offline prompt model. The only reason this is usable at all for eval is because this is an enterprise server with 8x memory channels of DDR4-3200 each. On consumer hardware with only two memory channels this would be slow as molasses.",
        "throughput_tps": "Pre-fill 60-62 tok/s. Evaluation 12-14 tok/s"
      },
      "tier": null,
      "hardware": {
        "description": "AMD Epyc 7543 (Milan, 32C/64T), LXC container with 128GB of RAM (no GPU)",
        "vendor": "AMD"
      },
      "backend": "Ollama",
      "os_driver": "",
      "model": {
        "name": "Poolside Laguna S2.1 (118B)",
        "url": ""
      },
      "quant": "Q4_K_M",
      "weight_precision": "",
      "use_cases": "Coding, and long form reasoning beyond what Gemma4:26B is capable of",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "Pre-fill 60-62 tok/s. Evaluation 12-14 tok/s",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "I tend to refine my coding prompts using Gemma4:26B, and then copy and paste the plan for the coding into Laguna and have it run in the background or overnight. Needed to tweak some settings to make the model run better on the CPU. Setting \"PARAMETER num_batch 512\" significantly improved performance. Pre-fill will always run very slowly on CPU's, which is what makes this more of an offline prompt model. The only reason this is usable at all for eval is because this is an enterprise server with 8x memory channels of DDR4-3200 each. On consumer hardware with only two memory channels this would be slow as molasses.",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_47_f41n_0",
      "source": {
        "post_number": 47,
        "post_id": 4133788,
        "username": "f41n",
        "submitted_at": "2026-10-26T05:00:36.518Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "hardware": "1x RX 6800 XT 16GB + Ryzen 7 7700X + 32GB DDR5-5200",
        "backend": "llama.cpp (ROCm, build b11889)",
        "tier": "\u226416",
        "model_name": "LFM2.5 8B-A1B",
        "quant_package": "Q4_K_M",
        "primary_use_case": "quick tasks, small helper model",
        "setup_notes": "",
        "throughput_tps": "221 t/s generation, 4718 t/s prompt"
      },
      "tier": "t2",
      "hardware": {
        "description": "1x RX 6800 XT 16GB + Ryzen 7 7700X + 32GB DDR5-5200",
        "vendor": ""
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "LFM2.5 8B-A1B",
        "url": ""
      },
      "quant": "Q4_K_M",
      "weight_precision": "",
      "use_cases": "quick tasks, small helper model",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "221 t/s generation, 4718 t/s prompt",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "ROCm, build b11889 more model testing~",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    },
    {
      "id": "post_47_f41n_1",
      "source": {
        "post_number": 47,
        "post_id": 4133788,
        "username": "f41n",
        "submitted_at": "2026-10-26T05:00:36.518Z",
        "marked": true
      },
      "type": "full",
      "raw": {
        "hardware": "1x RX 6800 XT 16GB + Ryzen 7 7700X + 32GB DDR5-5200",
        "backend": "llama.cpp (ROCm, build b11889)",
        "tier": "\u226416",
        "model_name": "LFM2.5 8B-A1B",
        "quant_package": "Q4_K_M + DSpark draft (Q4_K_M)",
        "primary_use_case": "quick tasks, small helper model",
        "setup_notes": "",
        "throughput_tps": "240 t/s generation"
      },
      "tier": "t2",
      "hardware": {
        "description": "1x RX 6800 XT 16GB + Ryzen 7 7700X + 32GB DDR5-5200",
        "vendor": ""
      },
      "backend": "llama.cpp",
      "os_driver": "",
      "model": {
        "name": "LFM2.5 8B-A1B",
        "url": ""
      },
      "quant": "Q4_K_M + DSpark draft (Q4_K_M)",
      "weight_precision": "",
      "use_cases": "quick tasks, small helper model",
      "context": {
        "stable": "",
        "kv_cache": ""
      },
      "performance": {
        "decode_tps": "240 t/s generation",
        "prefill_tps": "",
        "ttft_cold": "",
        "power_w": ""
      },
      "engine_mode": "",
      "notes": "ROCm, build b11889",
      "confidence": "community-reported",
      "lifecycle": "current",
      "highlight": false,
      "tags": []
    }
  ]
}