从计算到湿实验：评估AI蛋白设计性能
From In-Silico to Wet-Lab: Evaluating AI Protein Design Performance
In this tutorial, we use Anthropic’s claude-protein-binder-design dataset, which contains 1,440 AI-designed miniprotein binders tested against 16 targets. Because the release includes both computational predictions and real wet-lab results from two independent labs, we can go beyond simply studying the designs. We evaluate how well structure predictors identify successful binders, whether combining predictions improves performance, how rankings translate into practical testing budgets, and how much disagreement comes from the assays themselves. Also, we train a target-aware classifier to test whether these signals can reliably predict experimental success.
在本教程中，我们使用Anthropic的claude-protein-binder-design数据集，该数据集包含针对16个靶标测试的1,440个AI设计的微型蛋白结合剂。由于该发布包含来自两个独立实验室的计算预测和真实湿实验（wet-lab）结果，我们不仅能研究设计本身，还能评估结构预测器识别成功结合剂的能力、组合预测是否提升性能、排名如何转化为实际测试预算，以及检测方法本身带来的分歧程度。此外，我们训练一个靶标感知的分类器，以测试这些信号能否可靠预测实验成功。
import subprocess, sys, warnings, itertools, math
warnings.filterwarnings("ignore")
import importlib.util
_needed = {"huggingface_hub": "huggingface_hub>=0.24", "pyarrow": "pyarrow",
"pandas": "pandas", "sklearn": "scikit-learn",
"matplotlib": "matplotlib", "scipy": "scipy"}
_missing = [pkg for mod, pkg in _needed.items() if importlib.util.find_spec(mod) is None]
if _missing:
print("installing:", ", ".join(_missing))
subprocess.run([sys.executable, "-m", "pip", "install", "-q", *_missing], check=False)
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy import stats
from huggingface_hub import HfApi, hf_hub_download
from sklearn.metrics import roc_auc_score, cohen_kappa_score, average_precision_score
from sklearn.model_selection import GroupKFold, StratifiedKFold
from sklearn.ensemble import HistGradientBoostingClassifier
from sklearn.inspection import permutation_importance
SEED = 0
rng_global = np.random.default_rng(SEED)
pd.set_option("display.width", 200)
pd.set_option("display.max_columns", 100)
plt.rcParams.update({"figure.dpi": 110, "font.size": 9, "axes.grid": True,
"grid.alpha": 0.25, "axes.spines.top": False, "axes.spines.right": False})
REPO = "Anthropic/claude-protein-binder-design"
BAR = "=" * 78
def head(n, title):
prefix = f"{n}. " if str(n) else ""
print(f"\n{BAR}\n {prefix}{title}\n{BAR}")
head(1, "TABLE DISCOVERY")
api = HfApi()
repo_files = api.list_repo_files(REPO, repo_type="dataset")
TABLES = {}
for f in repo_files:
if f.startswith("data/tables/") and f.endswith(".parquet"):
key = f[len("data/tables/"): -len(".parquet")].replace("/", "_")
TABLES[key] = f
print(f"Found {len(TABLES)} Parquet tables:")
for k in sorted(TABLES):
print(f" - {k:38s} {TABLES[k]}")
def load_table(name: str) -> pd.DataFrame:
"""Load a subset by its viewer name, with a datasets-library fallback."""
if name in TABLES:
return pd.read_parquet(hf_hub_download(REPO, TABLES[name], repo_type="dataset"))
from datasets import load_dataset
return load_dataset(REPO, name, split="full").to_pandas()
ds = load_table("design_summary")
print(f"\ndesign_summary: {ds.shape[0]:,} rows x {ds.shape[1]} columns")import subprocess, sys, warnings, itertools, math
warnings.filterwarnings("ignore")
import importlib.util
_needed = {"huggingface_hub": "huggingface_hub>=0.24", "pyarrow": "pyarrow",
"pandas": "pandas", "sklearn": "scikit-learn",
"matplotlib": "matplotlib", "scipy": "scipy"}
_missing = [pkg for mod, pkg in _needed.items() if importlib.util.find_spec(mod) is None]
if _missing:
print("installing:", ", ".join(_missing))
subprocess.run([sys.executable, "-m", "pip", "install", "-q", *_missing], check=False)
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy import stats
from huggingface_hub import HfApi, hf_hub_download
from sklearn.metrics import roc_auc_score, cohen_kappa_score, average_precision_score
from sklearn.model_selection import GroupKFold, StratifiedKFold
from sklearn.ensemble import HistGradientBoostingClassifier
from sklearn.inspection import permutation_importance
SEED = 0
rng_global = np.random.default_rng(SEED)
pd.set_option("display.width", 200)
pd.set_option("display.max_columns", 100)
plt.rcParams.update({"figure.dpi": 110, "font.size": 9, "axes.grid": True,
"grid.alpha": 0.25, "axes.spines.top": False, "axes.spines.right": False})
REPO = "Anthropic/claude-protein-binder-design"
BAR = "=" * 78
def head(n, title):
prefix = f"{n}. " if str(n) else ""
print(f"\n{BAR}\n {prefix}{title}\n{BAR}")
head(1, "TABLE DISCOVERY")
api = HfApi()
repo_files = api.list_repo_files(REPO, repo_type="dataset")
TABLES = {}
for f in repo_files:
if f.startswith("data/tables/") and f.endswith(".parquet"):
key = f[len("data/tables/"): -len(".parquet")].replace("/", "_")
TABLES[key] = f
print(f"Found {len(TABLES)} Parquet tables:")
for k in sorted(TABLES):
print(f" - {k:38s} {TABLES[k]}")
def load_table(name: str) -> pd.DataFrame:
"""Load a subset by its viewer name, with a datasets-library fallback."""
if name in TABLES:
return pd.read_parquet(hf_hub_download(REPO, TABLES[name], repo_type="dataset"))
from datasets import load_dataset
return load_dataset(REPO, name, split="full").to_pandas()
ds = load_table("design_summary")
print(f"\ndesign_summary: {ds.shape[0]:,} rows x {ds.shape[1]} columns")We start by installing only what the runtime is actually missing, then enumerate the repository once and build a {subset to path} map instead of hard-coding file locations. This matters because the naming is not uniform; the subset wetlab_summary lives at data/tables/wetlab/summary.parquet, and a guessed path would fail silently. With the map in place we pull design_summary, one row per design, 1,440 rows wide enough to carry every join we need downstream.
我们首先仅安装运行时实际缺失的组件，然后一次性枚举仓库，并构建一个{子集到路径}的映射，而不是硬编码文件位置。这很重要，因为命名并不统一；子集wetlab_summary位于data/tables/wetlab/summary.parquet，而猜测的路径会静默失败。有了映射后，我们提取design_summary，每个设计一行，共1,440行，宽度足以承载我们下游所需的每一次连接。
head(2, "SCHEMA + EVALUABLE SET")
CALLS = {"binder", "non_binder"}
tested = ds["adaptyv_binding"].isin(CALLS) | ds["twist_binding"].isin(CALLS)
ev = ds[tested].copy()
ev["y"] = ev["binder_final"].astype(int)
print(f"All designs : {len(ds):,}")
print(f"Evaluable (>=1 vendor call): {len(ev):,}")
print(f"Confirmed binders : {int(ev['y'].sum()):,} "
f"({100 * ev['y'].mean():.1f}% base rate)")
print(f"Never measured : {len(ds) - len(ev):,}")
print("\nCategorical levels:")
for c in ["design_model", "campaign", "generator", "sequence_design_method", "vendor_agreement"]:
vals = ds[c].astype(str).value_counts()
print(f" {c:24s} ({len(vals)}): {', '.join(vals.index[:6])}"
+ (" ..." if len(vals) > 6 else ""))
print(f"\nTargets ({ds['target'].nunique()}): {', '.join(sorted(ds['target'].unique()))}")
print(f"Binder length: {ds.binder_length.min()}-{ds.binder_length.max()} aa "
f"(median {ds.binder_length.median():.0f})")
head(3, "HIT-RATE LANDSCAPE")
def wilson(k, n, z=1.96):
if n == 0:
return (np.nan, np.nan, np.nan)
p = k / n
d = 1 + z**2 / n
c = (p + z**2 / (2 * n)) / d
h = z * math.sqrt(p * (1 - p) / n + z**2 / (4 * n**2)) / d
return p, max(0.0, c - h), min(1.0, c + h)
def rate_table(df, by):
rows = []
for key, g in df.groupby(by, dropna=False):
p, lo, hi = wilson(int(g.y.sum()), len(g))
rows.append({by: key, "n": len(g), "hits": int(g.y.sum()),
"rate": p, "lo": lo, "hi": hi})
return pd.DataFrame(rows).sort_values("rate", ascending=False).reset_index(drop=True)
for dim in ["design_model", "campaign", "generator", "sequence_design_method"]:
t = rate_table(ev, dim)
print(f"\n--- hit rate by {dim} ---")
print(t.to_string(index=False,
formatters={"rate": "{:.3f}".format, "lo": "{:.3f}".format, "hi": "{:.3f}".format}))
tt = rate_table(ev, "target")
fig, ax = plt.subplots(figsize=(9, 4.2))
ax.bar(tt.target, tt.rate, color="#4C72B0")
ax.errorbar(tt.target, tt.rate,
yerr=[(tt.rate - tt.lo).clip(lower=0), (tt.hi - tt.rate).clip(lower=0)],
fmt="none", ecolor="0.25", capsize=3, lw=1)
ax.axhline(ev.y.mean(), ls="--", c="crimson", lw=1, label=f"pooled {ev.y.mean():.2f}")
ax.set_ylabel("experimental hit rate"); ax.set_title("Hit rate by target (Wilson 95% CI)")
ax.tick_params(axis="x", rotation=55); ax.legend(); plt.tight_layout(); plt.show()
print("\nRead this plot as the dominant effect size in the dataset: target choice "
"swamps generator choice. Any model comparison that does not stratify by "
"target is mostly measuring which targets that model was pointed at.")head(2, "SCHEMA + EVALUABLE SET")
CALLS = {"binder", "non_binder"}
tested = ds["adaptyv_binding"].isin(CALLS) | ds["twist_binding"].isin(CALLS)
ev = ds[tested].copy()
ev["y"] = ev["binder_final"].astype(int)
print(f"All designs : {len(ds):,}")
print(f"Evaluable (>=1 vendor call): {len(ev):,}")
print(f"Confirmed binders : {int(ev['y'].sum()):,} "
f"({100 * ev['y'].mean():.1f}% base rate)")
print(f"Never measured : {len(ds) - len(ev):,}")
print("\nCategorical levels:")
for c in ["design_model", "campaign", "generator", "sequence_design_method", "vendor_agreement"]:
vals = ds[c].astype(str).value_counts()
print(f" {c:24s} ({len(vals)}): {', '.join(vals.index[:6])}"
+ (" ..." if len(vals) > 6 else ""))
print(f"\nTargets ({ds['target'].nunique()}): {', '.join(sorted(ds['target'].unique()))}")
print(f"Binder length: {ds.binder_length.min()}-{ds.binder_length.max()} aa "
f"(median {ds.binder_length.median():.0f})")
head(3, "HIT-RATE LANDSCAPE")
def wilson(k, n, z=1.96):
if n == 0:
return (np.nan, np.nan, np.nan)
p = k / n
d = 1 + z**2 / n
c = (p + z**2 / (2 * n)) / d
h = z * math.sqrt(p * (1 - p) / n + z**2 / (4 * n**2)) / d
return p, max(0.0, c - h), min(1.0, c + h)
def rate_table(df, by):
rows = []
for key, g in df.groupby(by, dropna=False):
p, lo, hi = wilson(int(g.y.sum()), len(g))
rows.append({by: key, "n": len(g), "hits": int(g.y.sum()),
"rate": p, "lo": lo, "hi": hi})
return pd.DataFrame(rows).sort_values("rate", ascending=False).reset_index(drop=True)
for dim in ["design_model", "campaign", "generator", "sequence_design_method"]:
t = rate_table(ev, dim)
print(f"\n--- hit rate by {dim} ---")
print(t.to_string(index=False,
formatters={"rate": "{:.3f}".format, "lo": "{:.3f}".format, "hi": "{:.3f}".format}))
tt = rate_table(ev, "target")
fig, ax = plt.subplots(figsize=(9, 4.2))
ax.bar(tt.target, tt.rate, color="#4C72B0")
ax.errorbar(tt.target, tt.rate,
yerr=[(tt.rate - tt.lo).clip(lower=0), (tt.hi - tt.rate).clip(lower=0)],
fmt="none", ecolor="0.25", capsize=3, lw=1)
ax.axhline(ev.y.mean(), ls="--", c="crimson", lw=1, label=f"pooled {ev.y.mean():.2f}")
ax.set_ylabel("experimental hit rate"); ax.set_title("Hit rate by target (Wilson 95% CI)")
ax.tick_params(axis="x", rotation=55); ax.legend(); plt.tight_layout(); plt.show()
print("\nRead this plot as the dominant effect size in the dataset: target choice "
"swamps generator choice. Any model comparison that does not stratify by "
"target is mostly measuring which targets that model was pointed at.")We define the evaluable set by filtering on actual vendor calls rather than on binder_final, because that column is a bool and so records the 120 never-measured designs as False rather than missing. From there we compute hit rates by model, campaign, generator, and target, wrapping each in a Wilson interval since several subgroups sit in the small-n regime where the normal approximation misbehaves. The target plot is the one to read first: it shows antigen choice swamping every other factor we compare.
我们通过过滤实际的供应商调用（vendor calls）来定义可评估集，而不是依赖binder_final列，因为该列是布尔值，因此将120个从未测量的设计记录为False而非缺失。从那里，我们按模型、活动、生成器和靶标计算命中率，并将每个结果包裹在Wilson区间中，因为几个子组处于小样本量范围，此时正态近似表现不佳。靶标图是首先需要查看的：它显示抗原选择淹没了我们比较的所有其他因素。
更进一步：量化金融体系
看懂新闻只是起点——沿量化金融路径，把它变成能交付的工程能力