From 7a0ff55eb5dcdc8e386e2aea7864cba2581093db324df6bcb3da17b12cba1e2a Mon Sep 17 00:00:00 2001 From: tlg Date: Sun, 5 Apr 2026 23:35:05 +0200 Subject: [PATCH] fix: remove unsupported KV cache quantization in llama-cpp backend GGML_TYPE_Q8_0 for type_k/type_v not supported in this llama-cpp-python version. Keep reduced n_ctx=4096 for VRAM savings. Co-Authored-By: Claude Opus 4.6 (1M context) --- kischdle/llmux/llmux/backends/llamacpp.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/kischdle/llmux/llmux/backends/llamacpp.py b/kischdle/llmux/llmux/backends/llamacpp.py index f2da464..aae0991 100644 --- a/kischdle/llmux/llmux/backends/llamacpp.py +++ b/kischdle/llmux/llmux/backends/llamacpp.py @@ -38,13 +38,10 @@ class LlamaCppBackend(BaseBackend): logger.info(f"Loading GGUF model {model_path} with n_gpu_layers={n_gpu_layers}") def _load(): - from llama_cpp import GGML_TYPE_Q8_0 kwargs = { "model_path": model_path, "n_gpu_layers": n_gpu_layers, "n_ctx": 4096, - "type_k": GGML_TYPE_Q8_0, - "type_v": GGML_TYPE_Q8_0, "verbose": False, } if physical.mmproj_file: