多模态与视觉大模型深度实践：从架构解析到微调部署

class ImageProjectorMLP(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.mlp = nn.Sequential(
            nn.Linear(config.vision_hidden_size, config.hidden_size * 2),
            nn.GELU(),
            nn.Linear(config.hidden_size * 2, config.hidden_size)
        )
    def forward(self, image_features):
        return self.mlp(image_features)

from transformers import Qwen2_5_VLForConditionalGeneration, AutoProcessor
from qwen_vl_utils import process_vision_info
from PIL import Image
import torch

# 加载模型与处理器
model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
    "Qwen/Qwen2.5-VL-7B-Instruct",
    torch_dtype=torch.bfloat16,
    device_map="auto"
)
processor = AutoProcessor.from_pretrained("Qwen/Qwen2.5-VL-7B-Instruct")

# 构建多模态对话
messages = [
    {
        "role": "user",
        "content": [
            {"type": "image", "image": "example.jpg"},
            {"type": "text", "text": "请详细描述这张图片的内容。"}
        ]
    }
]

# 处理输入并生成
text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
image_inputs, video_inputs = process_vision_info(messages)
inputs = processor(
    text=[text],
    images=image_inputs,
    videos=video_inputs,
    padding=True,
    return_tensors="pt"
).to(model.device)

generated_ids = model.generate(**inputs, max_new_tokens=512)
response = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
print(response)

batch_messages = [
    [{"role": "user", "content": [{"type": "image", "image": f"img{i}.jpg"},
                                   {"type": "text", "text": "Describe this image."}]}]
    for i in range(4)
]
texts = [processor.apply_chat_template(msg, tokenize=False, add_generation_prompt=True)
         for msg in batch_messages]
images_batch = [Image.open(f"img{i}.jpg") for i in range(4)]
inputs = processor(text=texts, images=images_batch, return_tensors="pt", padding=True).to("cuda")
output_ids = model.generate(**inputs, max_new_tokens=512)
responses = processor.batch_decode(output_ids, skip_special_tokens=True)

{"image": "images/cat.jpg", "question": "这只猫在做什么？", "answer": "它正趴在床上休息。"}
{"image": "images/dog.jpg", "question": "描述画面。", "answer": "画面里是一只狗在草地上奔跑。"}

from transformers import Qwen2VLForConditionalGeneration, AutoProcessor, BitsAndBytesConfig
from peft import LoraConfig, get_peft_model
from transformers import TrainingArguments, Trainer
from datasets import load_dataset

# 1. 4bit量化加载（显存占用降低75%）
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16
)
model = Qwen2VLForConditionalGeneration.from_pretrained(
    "Qwen/Qwen2-VL-7B-Instruct",
    quantization_config=bnb_config,
    device_map="auto"
)
processor = AutoProcessor.from_pretrained("Qwen/Qwen2-VL-7B-Instruct")

# 2. 配置LoRA
lora_config = LoraConfig(
    r=16,                      # 低秩矩阵维度
    lora_alpha=32,
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj"],
    lora_dropout=0.05
)
model = get_peft_model(model, lora_config)
model.print_trainable_parameters()  # 约0.1%参数可训练

# 3. 加载数据集与训练
dataset = load_dataset("json", data_files="train.jsonl")
training_args = TrainingArguments(
    output_dir="./lora_checkpoint",
    per_device_train_batch_size=1,
    gradient_accumulation_steps=8,
    num_train_epochs=3,
    learning_rate=2e-4,
    fp16=True
)
trainer = Trainer(model=model, args=training_args, train_dataset=dataset["train"])
trainer.train()
model.save_pretrained("./lora_finetuned")

from lmdeploy import pipeline
from lmdeploy.vl import load_image

pipe = pipeline('openbmb/MiniCPM-V-2_6')
image = load_image('https://example.com/image.jpg')
response = pipe(('describe this image', image))
print(response)