多模态RAG架构解析：当文本检索遇上图像/音频编码

prompt = f"基于以下上下文：\n{context}\n\n请回答：{query}"

def rewrite_query(query):
    prompt = f"将用户查询改写为3个不同表述：{query}"
    return call_llm(prompt).split("\n")

# 多跳查询处理示例
sub_queries = [
    "马斯克担任CEO的公司", 
    "这些公司的总部所在地"
]

from sentence_transformers import CrossEncoder
reranker = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2")
scores = reranker.predict([(query, doc) for doc in candidates])

agent = RAGAgent(
    tools=[WebSearch(), Calculator(), DB_Query()]
)
agent.run("特斯拉2025年Q1营收是多少？同比增长百分比？")

from langchain.text_splitter import RecursiveCharacterTextSplitter

# 自适应分块策略
text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=512,
    chunk_overlap=128,
    length_function=len,
    separators=["\n\n", "。", "！", "？", "；"]
)

# 添加元数据增强
chunks = text_splitter.create_documents(
    [text],
    metadatas=[{"source": "annual_report_2024.pdf", "page": 17}]
)

# 图像向量化检索
from torchvision.models import resnet50
img_encoder = resnet50(pretrained=True)
img_vector = img_encoder(preprocess(image)):cite[3]

from ragas import evaluate
from datasets import Dataset

dataset = Dataset.from_dict({
    "question": ["特斯拉的CEO是谁？"],
    "answer": ["埃隆·马斯克"],
    "contexts": [["埃隆·马斯克是特斯拉的CEO。"]] 
})

metrics = ["faithfulness", "answer_relevancy"]
results = evaluate(dataset, metrics=metrics)
print(results)

# 通过Docker安装
wget https://github.com/milvus-io/milvus/releases/download/v2.3.10/milvus-standalone-docker-compose.yml
docker-compose up -d:cite[3]

# 连接与操作
from pymilvus import MilvusClient
client = MilvusClient(uri="http://localhost:19530")

# 创建集合
schema = MilvusClient.create_schema(
    auto_id=True,
    enable_dynamic_field=True
)
schema.add_field("id", DataType.INT64, is_primary=True)
schema.add_field("vector", DataType.FLOAT_VECTOR, dim=768)
client.create_collection("docs", schema)

# 插入向量
data = [
    {"id": 1, "vector": [0.1, 0.2, ...], "text": "RAG详解...", "source": "web"},
    {"id": 2, "vector": [0.2, 0.3, ...], "text": "LLM优化...", "source": "pdf"}
]
client.insert("docs", data)

from langchain_community.vectorstores import FAISS
from langchain_core.output_parsers import StrOutputParser
from langchain_core.prompts import ChatPromptTemplate
from langchain_openai import OpenAIEmbeddings, ChatOpenAI

# 构建链
retriever = FAISS.load_local("vector_db").as_retriever()
prompt = ChatPromptTemplate.from_template("基于{context}回答：{question}")
model = ChatOpenAI(model="gpt-4-turbo")

rag_chain = (
    {"context": retriever, "question": RunnablePassthrough()} 
    | prompt 
    | model 
    | StrOutputParser()
)

# 执行查询
rag_chain.invoke("RAG的核心组件有哪些？")