Bonsai 2 27B local benchmark notes
Test date: 2026-10-03
Prepared: 2026-10-23
Hardware: RTX 4070 Ti 12GB
The structured measurements and request examples follow. This is a UTF-8 text report.

{
  "test_date": "2026-10-03",
  "prepared_for_article": "2026-10-23",
  "gpu": "RTX 4070 Ti 12GB",
  "model": "Bonsai 2 27B CRACK PQ2 (derived from Qwen3.8-27B)",
  "engine_build": "NInfer KVMem 20261002 / sm89",
  "method": "Single-request engine decode throughput. Same prompts, greedy sampling, thinking off. Preset comparison, not single-variable A/B: GDN state precision, chunk and draft backend differ. Generated lengths may differ. Chinese answer capped at 512 tokens for speed measurement.",
  "short_comparison": [
    {
      "task": "chinese",
      "mtp_tok_s": 106.64380468364564,
      "dflash2_tok_s": 146.84540578342728,
      "mtp_tokens": 512,
      "dflash2_tokens": 512
    },
    {
      "task": "code",
      "mtp_tok_s": 123.3022798130307,
      "dflash2_tok_s": 229.17395928677,
      "mtp_tokens": 402,
      "dflash2_tokens": 362
    },
    {
      "task": "count",
      "mtp_tok_s": 154.3618816568758,
      "dflash2_tok_s": 465.13650335386154,
      "mtp_tokens": 492,
      "dflash2_tokens": 492
    }
  ],
  "long_tests": [
    {
      "prompt_tokens": 44461,
      "prefill_seconds": 38.4385452,
      "prefill_tok_s": 1156.6774904894164,
      "decode_tok_s": 83.61582365521163,
      "three_needles_correct": true
    },
    {
      "prompt_tokens": 128461,
      "prefill_seconds": 232.7752528,
      "prefill_tok_s": 551.8670840425352,
      "decode_tok_s": 59.880619179691884,
      "three_needles_correct": true
    }
  ],
  "dflash_runtime": {
    "backend": "dflash2",
    "draft_tokens": 12,
    "kv_dtype": "k8v4",
    "kv_capacity": 17152,
    "kv_window": 16384,
    "kv_retrieve": 8192,
    "prefill_chunk": 256,
    "gdn_state": "FP16",
    "device_state_slots": 0,
    "max_concurrency": 1,
    "ngram_draft_tokens": "engine default 15"
  },
  "limits": [
    "256K logical context initialized; full 256K input not tested",
    "128461-token three-position synthetic retrieval passed once; not a general long-context accuracy guarantee",
    "12K resident-window variant missed the middle needle; not exposed in launcher",
    "Roughly 300-430 MiB GPU memory free during tests; other GPU use may prevent startup",
    "GDN FP16 state changes runtime numerical precision; no broad accuracy evaluation",
    "Short arithmetic, synthetic needles and tool-call parsing are limited functional tests"
  ],
  "short_requests": {
    "mtp3": [
      {
        "task": "chinese",
        "request": {
          "model": "qwen3.8-27b",
          "messages": [
            {
              "role": "user",
              "content": "用中文向初学者解释 Python 字典与列表的区别。给出各自的适用场景、两个简短例子和常见错误。请直接回答。"
            }
          ],
          "max_tokens": 512,
          "temperature": 0,
          "reasoning_effort": "none"
        },
        "prompt_tokens": 39,
        "output_tokens": 512,
        "decode_ms": 4791.652,
        "decode_tok_s": 106.64380468364564,
        "prompt_ms": 150.7641,
        "wall_s": 4.953000000008615
      },
      {
        "task": "code",
        "request": {
          "model": "qwen3.8-27b",
          "messages": [
            {
              "role": "user",
              "content": "Write a Python function merge_intervals(intervals) that merges overlapping intervals. Include the function, three assert examples, and a concise explanation of complexity."
            }
          ],
          "max_tokens": 512,
          "temperature": 0,
          "reasoning_effort": "none"
        },
        "prompt_tokens": 41,
        "output_tokens": 402,
        "decode_ms": 3252.1702,
        "decode_tok_s": 123.3022798130307,
        "prompt_ms": 90.9686,
        "wall_s": 3.375
      },
      {
        "task": "count",
        "request": {
          "model": "qwen3.8-27b",
          "messages": [
            {
              "role": "user",
              "content": "Output the integers from 1 to 150 in ascending order, separated by commas. No other text."
            }
          ],
          "max_tokens": 512,
          "temperature": 0,
          "reasoning_effort": "none"
        },
        "prompt_tokens": 35,
        "output_tokens": 492,
        "decode_ms": 3180.8371,
        "decode_tok_s": 154.3618816568758,
        "prompt_ms": 82.5531,
        "wall_s": 3.2820000000065193
      }
    ],
    "dflash2": [
      {
        "task": "chinese",
        "request": {
          "model": "qwen3.8-27b",
          "messages": [
            {
              "role": "user",
              "content": "用中文向初学者解释 Python 字典与列表的区别。给出各自的适用场景、两个简短例子和常见错误。请直接回答。"
            }
          ],
          "max_tokens": 512,
          "temperature": 0,
          "reasoning_effort": "none"
        },
        "prompt_tokens": 39,
        "output_tokens": 512,
        "decode_ms": 3479.8501,
        "decode_tok_s": 146.84540578342728,
        "prompt_ms": 115.3819,
        "wall_s": 3.60999999998603
      },
      {
        "task": "code",
        "request": {
          "model": "qwen3.8-27b",
          "messages": [
            {
              "role": "user",
              "content": "Write a Python function merge_intervals(intervals) that merges overlapping intervals. Include the function, three assert examples, and a concise explanation of complexity."
            }
          ],
          "max_tokens": 512,
          "temperature": 0,
          "reasoning_effort": "none"
        },
        "prompt_tokens": 41,
        "output_tokens": 362,
        "decode_ms": 1575.2226,
        "decode_tok_s": 229.17395928677,
        "prompt_ms": 107.3082,
        "wall_s": 1.7030000000086147
      },
      {
        "task": "count",
        "request": {
          "model": "qwen3.8-27b",
          "messages": [
            {
              "role": "user",
              "content": "Output the integers from 1 to 150 in ascending order, separated by commas. No other text."
            }
          ],
          "max_tokens": 512,
          "temperature": 0,
          "reasoning_effort": "none"
        },
        "prompt_tokens": 35,
        "output_tokens": 492,
        "decode_ms": 1055.6041,
        "decode_tok_s": 465.13650335386154,
        "prompt_ms": 73.3607,
        "wall_s": 1.139999999984866
      }
    ]
  }
}