用 NVIDIA NeMo Retriever 构建多模态 RAG 流水线

In this tutorial, we build an advanced multimodal retrieval-augmented generation pipeline with NVIDIA NeMo Retriever. We begin by configuring a Python 3.12 environment, installing the required packages, and performing offline PDF text extraction without relying on a GPU or external API key. We then extend the workflow with hosted NVIDIA NIM endpoints to detect page elements, extract tables, charts, and infographics, generate dense vector embeddings, and store the processed content in LanceDB. Finally, we implement dense retrieval, vision-language reranking, metadata-filtered search, grounded response generation with inline citations, and a lightweight recall-at-k evaluation to validate retrieval quality across multimodal document content.

import sys, os, subprocess, textwrap, json, time, warnings
warnings.filterwarnings("ignore")
assert sys.version_info[:2] == (3, 12), (
   f"nemo-retriever requires Python 3.12.x (found {sys.version.split()[0]}). "
   "Colab's default runtime is 3.12; if you changed it, switch back."
)
def sh(cmd):
   print(f"$ {cmd}")
   subprocess.run(cmd, shell=True, check=False)
try:
   import nemo_retriever
   print("nemo-retriever already installed")
except ImportError:
   sh("pip install -q --ignore-installed PyJWT nemo-retriever openai")
import nemo_retriever
print("nemo-retriever version:", nemo_retriever.__version__)
from nemo_retriever import create_ingestor
try:
   from nemo_retriever.io import to_markdown, to_markdown_by_page
except ImportError:
   from nemo_retriever.common.io import to_markdown, to_markdown_by_page
try:
   from nemo_retriever.retriever import Retriever
except ImportError:
   from nemo_retriever.graph.retriever import Retriever
import pandas as pd
pd.set_option("display.max_colwidth", 160)
DOC = "multimodal_test.pdf"
if not os.path.exists(DOC):
   sh(f"curl -sL -o {DOC} "
      "https://raw.githubusercontent.com/NVIDIA/NeMo-Retriever/main/data/multimodal_test.pdf")
print("document:", DOC, os.path.getsize(DOC), "bytes")
DOCS = [DOC]
print("\n=== STAGE 1: offline text extraction (no API key) ===")
offline = (
   create_ingestor(run_mode="inprocess", allow_no_gpu=True)
   .files(DOCS)
   .extract(
       extract_text=True,
       extract_tables=False, extract_charts=False,
       extract_images=False, extract_infographics=False,
       use_page_elements=False,
       extract_page_as_image=False,
       method="pdfium",
   )
)
df_offline = offline.ingest()
print("rows:", df_offline.shape, "\ncolumns:", list(df_offline.columns))
print("\npage 1 text preview:\n", df_offline.iloc[0]["text"][:400])

We configure the Python 3.12 environment, install NVIDIA NeMo Retriever, and import the required ingestion and retrieval components. We download the sample multimodal PDF and define it as the input document for the pipeline. We then perform CPU-based offline text extraction with PDFium and inspect the extracted rows, columns, and page content.

from getpass import getpass
if not os.environ.get("NVIDIA_API_KEY"):
   try:
       from google.colab import userdata
       os.environ["NVIDIA_API_KEY"] = userdata.get("NVIDIA_API_KEY")
   except Exception:
       os.environ["NVIDIA_API_KEY"] = getpass("NVIDIA_API_KEY (nvapi-...): ").strip()
API_KEY = os.environ.get("NVIDIA_API_KEY", "").strip()
HAVE_KEY = API_KEY.startswith("nvapi-")
print("API key present:", HAVE_KEY)
PAGE_ELEMENTS_URL   = "https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-page-elements-v3"
OCR_URL             = "https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-ocr-v1"
TABLE_STRUCT_URL    = "https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-table-structure-v1"
GRAPHIC_ELEM_URL    = "https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-graphic-elements-v1"
EMBED_URL           = "https://integrate.api.nvidia.com/v1/embeddings"
RERANK_URL          = "https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-vl-1b-v2/reranking"
CHAT_URL            = "https://integrate.api.nvidia.com/v1"
EMBED_MODEL  = "nvidia/llama-nemotron-embed-1b-v2"
RERANK_MODEL = "nvidia/llama-nemotron-rerank-vl-1b-v2"
LLM_MODEL    = "nvidia/llama-3.3-nemotron-super-49b-v1.5"
LANCEDB_URI, TABLE = "./lancedb", "colab_demo"
df = df_offline
if HAVE_KEY:
   print("\n=== STAGE 2: multimodal ingest via hosted NIMs ===")
   ing = (
       create_ingestor(
           run_mode="inprocess",
           allow_no_gpu=True,
           error_policy="collect",
       )
       .files(DOCS)
       .extract(
           extract_text=True,
           extract_tables=True,
           extract_charts=True,
           extract_infographics=True,
           extract_images=False,
           method="pdfium",
           dpi=200,
           table_output_format="markdown",
           page_elements_invoke_url=PAGE_ELEMENTS_URL,
           ocr_invoke_url=OCR_URL,
           table_structure_invoke_url=TABLE_STRUCT_URL,
           graphic_elements_invoke_url=GRAPHIC_ELEM_URL,
           api_key=API_KEY,
           request_timeout_s=120.0,
           split_config={"text": {"max_tokens": 512, "overlap_tokens": 64}},
       )
       .dedup(content_hash=True, bbox_iou=True, iou_threshold=0.45)
       .embed(
           embedding_endpoint=EMBED_URL,
           model_name=EMBED_MODEL,
           embed_model_name=EMBED_MODEL,
           api_key=API_KEY,
           input_type="passage",
           inference_batch_size=16,
           nim_http_max_concurrent=8,
       )
       .vdb_upload(
           vdb_op="lancedb",
           vdb_kwargs={
               "uri": LANCEDB_URI,
               "table_name": TABLE,
               "overwrite": True,
               "create_index": True,
               "index_type": "IVF_HNSW_SQ",
               "metric": "l2",
           },
       )
   )
   t0 = time.time()
   df = ing.ingest(show_progress=True)
   print(f"ingested in {time.time()-t0:.1f}s -> {df.shape}")

We securely load the NVIDIA API key and define the hosted NIM endpoints for layout detection, OCR, table extraction, graphic analysis, embedding, reranking, and generation. We create a multimodal ingestion pipeline that extracts text, tables, charts, and infographics while applying token-aware chunking and content deduplication. We generate embeddings for the extracted content and upload the resulting vectors and metadata to a LanceDB table.