{
  "schema_version": 1,
  "sample_id": "kv-cache-survey",
  "title": "KV Cache Optimization in Large Language Models",
  "artifact_disclosure": {
    "kind": "generated_demonstration_draft",
    "research_mode": "survey",
    "omniwise_conducted_experiments": false,
    "peer_reviewed": false,
    "original_full_latex_available": false,
    "latex_download_kind": "companion_excerpt"
  },
  "sources": [
    {
      "key": "kwon2023pagedattention",
      "title": "Efficient Memory Management for Large Language Model Serving with PagedAttention",
      "authors": "Woosuk Kwon, Zhuohan Li, Siyuan Zhuang, Ying Sheng, Lianmin Zheng, Cody Hao Yu, Joseph E. Gonzalez, Hao Zhang, and Ion Stoica",
      "year": 2023,
      "venue": "SOSP 2023",
      "official_url": "https://dl.acm.org/doi/10.1145/3600006.3613165",
      "doi": "10.1145/3600006.3613165"
    },
    {
      "key": "zhang2023h2o",
      "title": "H2O: Heavy-Hitter Oracle for Efficient Generative Inference of Large Language Models",
      "authors": "Zhenyu Zhang, Ying Sheng, Tianyi Zhou, Tianlong Chen, Lianmin Zheng, Ruisi Cai, Zhao Song, Yuandong Tian, Christopher Re, Clark Barrett, Zhangyang Wang, and Beidi Chen",
      "year": 2023,
      "venue": "NeurIPS 2023",
      "official_url": "https://proceedings.neurips.cc/paper_files/paper/2023/hash/6ceefa7b15572587b78ecfcebb2827f8-Abstract-Conference.html"
    },
    {
      "key": "li2024snapkv",
      "title": "SnapKV: LLM Knows What You are Looking for Before Generation",
      "authors": "Yuhong Li, Yingbing Huang, Bowen Yang, Bharat Venkitesh, Acyr Locatelli, Hanchen Ye, Tianle Cai, Patrick Lewis, and Deming Chen",
      "year": 2024,
      "venue": "NeurIPS 2024",
      "official_url": "https://proceedings.neurips.cc/paper_files/paper/2024/hash/28ab418242603e0f7323e54185d19bde-Abstract-Conference.html",
      "doi": "10.52202/079017-0722"
    },
    {
      "key": "zhao2025semsharekv",
      "title": "SemShareKV: Efficient KVCache Sharing for Semantically Similar Prompts via Token-Level LSH Matching",
      "authors": "Xinye Zhao and Spyridon Mastorakis",
      "year": 2025,
      "venue": "Findings of IJCNLP-AACL 2025",
      "official_url": "https://aclanthology.org/2025.findings-ijcnlp.25/",
      "doi": "10.18653/v1/2025.findings-ijcnlp.25"
    }
  ],
  "claims": [
    {
      "id": "pagedattention-serving-memory",
      "source_key": "kwon2023pagedattention",
      "source_reported_result": "The paper reports that vLLM improves throughput by 2-4x at a comparable latency level against FasterTransformer and Orca in its evaluations.",
      "source_reported_result_zh": "论文报告称，在其评测条件下，vLLM 相比 FasterTransformer 和 Orca 在延迟相当时将吞吐量提高了 2-4 倍。",
      "applies_to": "Serving-time KV-cache memory management and sharing under the models and workloads evaluated by the paper.",
      "applies_to_zh": "适用于论文所评测模型与工作负载下的服务期 KV 缓存内存管理和共享。",
      "caveat": "This is a source-reported comparison, not a universal speedup guarantee or evidence of semantic cache equivalence.",
      "caveat_zh": "这是原论文报告的对比结果，不是普适加速承诺，也不能证明语义级缓存等价。",
      "source_locator": "ACM Digital Library abstract"
    },
    {
      "id": "h2o-heavy-hitter-eviction",
      "source_key": "zhang2023h2o",
      "source_reported_result": "With a 20% heavy-hitter setting, the paper reports throughput gains up to 29x, 29x, and 3x against three named inference systems, plus latency reduction up to 1.9x at the same batch size.",
      "source_reported_result_zh": "在保留 20% 重 hitter 的设置下，论文报告相对三种指定推理系统的吞吐提升上限分别为 29 倍、29 倍和 3 倍，同批量下延迟最多降低 1.9 倍。",
      "applies_to": "KV-cache eviction on the OPT-6.7B and OPT-30B experiments and baselines described by the paper.",
      "applies_to_zh": "适用于论文描述的 OPT-6.7B、OPT-30B 实验和对应基线中的 KV 缓存淘汰。",
      "caveat": "The maxima depend on the selected models, 20% policy setting, workloads, and comparison systems; they are not directly comparable with other papers' numbers.",
      "caveat_zh": "这些上限依赖模型、20% 策略、工作负载和对比系统，不能与其他论文数字直接横向比较。",
      "source_locator": "NeurIPS 2023 official abstract"
    },
    {
      "id": "snapkv-observation-window",
      "source_key": "li2024snapkv",
      "source_reported_result": "For 16K-token inputs, the paper reports 3.6x higher generation speed and 8.2x memory efficiency while evaluating 16 long-sequence datasets.",
      "source_reported_result_zh": "针对 16K token 输入，论文报告生成速度提高 3.6 倍、内存效率提高 8.2 倍，并在 16 个长序列数据集上进行了评测。",
      "applies_to": "Fine-tuning-free selection of important KV positions using the prompt-end observation window studied by SnapKV.",
      "applies_to_zh": "适用于 SnapKV 所研究的、利用提示末端观察窗口选择重要 KV 位置的免微调方法。",
      "caveat": "The reported values are tied to the paper's hardware, baselines, input length, and task suite; the page does not generalize them to every deployment.",
      "caveat_zh": "报告数值受论文硬件、基线、输入长度和任务集约束，本页面不将其外推到所有部署。",
      "source_locator": "NeurIPS 2024 official abstract"
    },
    {
      "id": "semsharekv-fuzzy-token-reuse",
      "source_key": "zhao2025semsharekv",
      "source_reported_result": "On the paper's summarization experiments with 5K-token inputs, SemShareKV reports up to 6.25x speedup and 42% lower GPU memory use.",
      "source_reported_result_zh": "在论文针对 5K token 输入的摘要实验中，SemShareKV 报告最高 6.25 倍加速和 42% 的 GPU 内存下降。",
      "applies_to": "Fuzzy token matching and selective KV reuse between semantically similar prompts in the evaluated summarization setting.",
      "applies_to_zh": "适用于所评测摘要场景中，语义相似提示之间的模糊 token 匹配与选择性 KV 复用。",
      "caveat": "The evidence is task- and configuration-specific and does not establish production-wide cross-model cache sharing.",
      "caveat_zh": "证据受任务和配置约束，不能据此认定已实现生产范围的跨模型缓存共享。",
      "source_locator": "ACL Anthology abstract and metadata"
    }
  ]
}