NVIDIA Dory 4B llama.cpp部署与Loop性能实测
How to Run NVIDIA Dory 4B with llama.cpp: Loop by Loop Guide
手把手教你跑通 NVIDIA 新出的 Dory 4B，包含精确的 commit 锁定和 Loop 性能对比，做端侧推理优化的同学可以直接抄作业。
In this tutorial, we work with Dory 1.0 4B, a small reasoning model from NVIDIA’s Nemotron Labs whose middle layers form a shared block that can be applied once, twice, three, or four times for every token. Each extra pass, or loop, spends more computation per token without adding a single parameter, so one checkpoint can run at four different compute budgets with no retraining. We build the llama.cpp branch that NVIDIA’s model card pins, download the 3.7 GB GGUF, and read the architecture straight from the file to predict what each loop should cost in parameters and cache memory. We then check those predictions against the running server, measure decode speed at every loop count, test whether the cheaper last-loop cache changes the model’s predictions, and run a paired GSM8K evaluation to see how accuracy, answer length, and time per solved question move as the loops increase.
在本教程中，我们使用 Dory 1.0 4B，这是 NVIDIA Nemotron Labs 推出的一个小型推理模型，其中间层构成一个共享块，每个 token 可应用一次、两次、三次或四次。每次额外的遍历（或循环）在不增加任何参数的情况下为每个 token 消耗更多计算资源，因此单个检查点可以在四种不同的计算预算下运行，无需重新训练。我们构建 NVIDIA 模型卡所固定的 llama.cpp 分支，下载 3.7 GB 的 GGUF 文件，并直接从文件中读取架构信息，以预测每次循环在参数和缓存内存方面的成本。随后，我们将这些预测与运行中的服务器进行比对，测量每种循环次数下的解码速度，测试较便宜的末次循环缓存是否改变模型的预测结果，并运行配对 GSM8K 评估，观察随着循环次数增加，准确率、答案长度以及每道已解答题目的耗时如何变化。
import os
import re
import sys
import json
import time
import math
import random
import signal
import shutil
import hashlib
import platform
import statistics
import traceback
import subprocess
import collections
import urllib.request
from pathlib import Path
RESULTS = {} # step -> one-line result, for the summary table
CLAIMS = [] # (step, claim, held)
CURRENT = ["1"]
def banner(title):
print("\n" + "=" * 78)
print(title)
print("=" * 78)
def section(name):
def wrap(fn):
def run(*a, **kw):
CURRENT[0] = name.split(".")[0]
banner(name)
t0 = time.time()
try:
out = fn(*a, **kw)
RESULTS[name] = out if isinstance(out, str) else "ok"
return out
except Exception as e:
RESULTS[name] = f"SKIPPED / FAILED -> {type(e).__name__}: {e}"
print(f"\n[!] {name} did not complete: {type(e).__name__}: {e}")
traceback.print_exc(limit=3)
return None
finally:
print(f"\n[{time.time() - t0:.1f} s]")
return run
return wrap
def claim(text, held):
CLAIMS.append((CURRENT[0], text, bool(held)))
return held
banner("1. Build the pinned Dory llama.cpp fork and record the environment")
FORK = "https://github.com/brianchmielNV/llama.cpp.git"
FORK_COMMIT = "98a74c97ea09ac72f6ceda1559808fc0a2f447ce" # the commit NVIDIA's model card pins; ggml-org PR #30306 is still a draft
GGUF_REPO = "nvidia/NVIDIA-Nemotron-Labs-Dory1.0-4B-GGUF"
GGUF_REV = "df53755cb8741843af8e019e088819fbef17e29b"
GGUF_FILE = "NVIDIA-Nemotron-Labs-Dory1.0-4B-Q5_K_M.gguf" # the card's commands call it dory-q5_k_m.gguf
GGUF_SIZE = 3_673_288_864
GGUF_SHA256 = "96080be49ffd67b916018b7bdef8c484762576fb52780e10ab98ac4978a660e0"
GSM8K_REV = "740312add88f781978c0658806c59bc2815b9866"
WORK = Path(os.environ.get("DORY_WORK") or ("/content/dory" if os.path.isdir("/content") else "dory_work")).resolve()
WORK.mkdir(parents=True, exist_ok=True)
(WORK / "logs").mkdir(exist_ok=True)
subprocess.run([sys.executable, "-m", "pip", "install", "-q", "gguf==0.19.0", "huggingface_hub", "pandas", "pyarrow"], check=True)
SYSTEM = platform.system()
GPU = ""
if shutil.which("nvidia-smi"):
q = subprocess.run(["nvidia-smi", "--query-gpu=name,memory.total,compute_cap", "--format=csv,noheader"],
capture_output=True, text=True)
GPU = q.stdout.strip().splitlines()[0] if q.returncode == 0 and q.stdout.strip() else ""
BACKEND = "metal" if SYSTEM == "Darwin" else ("cuda" if GPU else "cpu")
SRC = WORK / "llama.cpp-dory"
SERVER_BIN = SRC / "build" / "bin" / "llama-server"
free_gb = shutil.disk_usage(WORK).free / 1e9
print(f"{SYSTEM} {platform.machine()}, Python {platform.python_version()}, {os.cpu_count()} CPU cores, "
f"backend {BACKEND}{' (' + GPU + ')' if GPU else ''}, {free_gb:.1f} GB free disk")
if not SERVER_BIN.exists():
t0 = time.time()
if not (SRC / ".git").exists():
subprocess.run(["git", "init", "-q", str(SRC)], check=True)
subprocess.run(["git", "-C", str(SRC), "remote", "add", "origin", FORK], check=True)
subprocess.run(["git", "-C", str(SRC), "fetch", "-q", "--depth", "1", "origin", FORK_COMMIT], check=True)
subprocess.run(["git", "-C", str(SRC), "checkout", "-q", "--detach", "FETCH_HEAD"], check=True)
flags = ["-DCMAKE_BUILD_TYPE=Release", "-DLLAMA_BUILD_SERVER=ON", "-DLLAMA_BUILD_TESTS=OFF",
"-DLLAMA_BUILD_EXAMPLES=OFF", "-DLLAMA_BUILD_APP=OFF", "-DLLAMA_BUILD_UI=OFF",
"-DLLAMA_USE_PREBUILT_UI=ON", "-DLLAMA_OPENSSL=OFF"]
if BACKEND == "metal": # the card's Apple-silicon flags
flags += ["-DCMAKE_OSX_ARCHITECTURES=arm64", "-DGGML_METAL=ON", "-DGGML_METAL_EMBED_LIBRARY=ON",
"-DGGML_BLAS=ON", "-DGGML_BLAS_VENDOR=Apple"]
elif BACKEND == "cuda": # kernels for this GPU only (a T4 is 7.5); flash-attention kernels are unused with --flash-attn off
cap = GPU.split(",")[-1].strip().replace(".", "")
flags += ["-DGGML_CUDA=ON", f"-DCMAKE_CUDA_ARCHITECTURES={cap}", "-DGGML_CUDA_FA=OFF", "-DGGML_NATIVE=OFF"]
env = dict(os.environ, HF_UI_VERSION="b11551")
with open(WORK / "build.log", "w") as log:
subprocess.run(["cmake", "-S", str(SRC), "-B", str(SRC / "build")] + flags, check=True, env=env, stdout=log, stderr=log)
subprocess.run(["cmake", "--build", str(SRC / "build"), "--parallel", str(os.cpu_count()), "--target", "llama-server"],
check=True, env=env, stdout=log, stderr=log)
print(f"built llama-server in {time.time() - t0:.0f} s")
head = subprocess.run(["git", "-C", str(SRC), "rev-parse", "HEAD"], capture_output=True, text=True).stdout.strip()
assert head == FORK_COMMIT, f"checked out {head}, expected {FORK_COMMIT}"
version = subprocess.run([str(SERVER_BIN), "--version"], capture_output=True, text=True)
print(f"fork commit {head[:12]}; {(version.stdout + version.stderr).strip().splitlines()[-1]}")
RESULTS["1. Build the pinned Dory llama.cpp fork and record the environment"] = f"fork {head[:12]}, backend {BACKEND}"We build llama-server from the exact commit of the Dory branch that NVIDIA’s model card pins, because Dory support in llama.cpp is still an open draft pull request and no released binary can load the model. The cell detects the platform and passes the card’s Metal flags on a Mac, or compiles CUDA kernels for the attached GPU alone on Colab, skipping the flash-attention kernels that the card’s –flash-attn off never uses, which keeps a T4 build manageable. It then checks that the checked-out commit is the pinned one, so every later number traces to one known version of the code.
我们从 NVIDIA 模型卡所固定的 Dory 分支的确切提交版本构建 llama-server，因为 llama.cpp 中对 Dory 的支持仍处于开放草稿拉取请求阶段，且没有任何发布版二进制文件能够加载该模型。该单元格会检测平台，并在 Mac 上传递显卡的 Metal 标志，或在 Colab 上仅为附加 GPU 编译 CUDA 内核，跳过显卡通过 –flash-attn off 禁用的 flash-attention 内核，从而使 T4 上的构建保持可控。接着，它会验证检出的提交是否为固定版本，确保后续所有数据均追溯至同一已知代码版本。
@section("2. Download the Q5_K_M GGUF at a pinned revision and check its SHA-256")
def download_step():
global GGUF
from huggingface_hub import hf_hub_download
t0 = time.time()
GGUF = Path(hf_hub_download(GGUF_REPO, GGUF_FILE, revision=GGUF_REV, local_dir=str(WORK)))
h = hashlib.sha256()
with open(GGUF, "rb") as f:
for block in iter(lambda: f.read(1 << 24), b""):
h.update(block)
size, digest = GGUF.stat().st_size, h.hexdigest()
print(f"{GGUF.name}: {size:,} bytes, sha256 {digest[:16]}... ({time.time() - t0:.0f} s)")
assert size == GGUF_SIZE and digest == GGUF_SHA256, "the file does not match the card's size and checksum"
claim("the GGUF matches the size and SHA-256 published on the card", True)
return f"{size / 1e9:.2f} GB, checksum matches the card"
download_step()更进一步：量化金融体系
看懂新闻只是起点——沿量化金融路径，把它变成能交付的工程能力