4. 图像描述生成LoRA微调

任务说明

本案例使用LoRA对图像描述生成模型（基于CLIP ViT-L/14视觉编码器 + 文本解码器）进行参数高效微调，使模型对室内家居场景的图像生成更具体、更符合场景习惯的描述（如包含家具名称、空间关系、材质等），而非通用模型的泛泛描述。

环境依赖

pip install torch torchvision transformers datasets peft pillow

数据集准备

使用自定义小型数据集（20个训练样本），每条样本包含：image_path（本地图片路径）、caption（目标描述）。以下代码内置了3条示例数据用于演示，完整训练请替换为你的数据。

完整实现代码

import torch
from torch.utils.data import Dataset, DataLoader
from transformers import (
    BlipProcessor, BlipForConditionalGeneration,
    AdamW, get_scheduler
)
from peft import LoraConfig, get_peft_model, TaskType
from PIL import Image
import os

# ========== 1. 加载基础模型 ==========
model_name = "Salesforce/blip-image-captioning-base"
processor = BlipProcessor.from_pretrained(model_name)
model = BlipForConditionalGeneration.from_pretrained(model_name)

# ========== 2. 配置LoRA ==========
lora_config = LoraConfig(
    task_type=TaskType.SEQ_2_SEQ_LM,
    r=8,               # 低秩矩阵维度
    lora_alpha=16,     # 缩放参数
    target_modules=["q_proj", "v_proj"],  # 仅微调注意力层的Q和V矩阵
    lora_dropout=0.1,
    bias="none"
)
model = get_peft_model(model, lora_config)
model.print_trainable_parameters()  # 应显示可训练参数约0.5%

# ========== 3. 自定义数据集 ==========
class ImageCaptionDataset(Dataset):
    def __init__(self, data, processor, image_dir):
        self.data = data
        self.processor = processor
        self.image_dir = image_dir

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        item = self.data[idx]
        image = Image.open(os.path.join(self.image_dir, item["image"])).convert("RGB")
        encoding = self.processor(
            images=image,
            text=item["caption"],
            padding="max_length",
            max_length=32,
            return_tensors="pt"
        )
        # 移除batch维度
        return {k: v.squeeze(0) for k, v in encoding.items()}

# 示例数据（实际使用请替换为你的数据）
train_data = [
    {"image": "living_room_1.jpg", "caption": "a gray sofa with two cushions in a bright living room"},
    {"image": "bedroom_1.jpg", "caption": "a wooden bed with white bedding next to a nightstand"},
    {"image": "kitchen_1.jpg", "caption": "a modern kitchen with stainless steel appliances and marble countertop"},
]
image_dir = "./images"  # 图片存放目录

dataset = ImageCaptionDataset(train_data, processor, image_dir)
dataloader = DataLoader(dataset, batch_size=2, shuffle=True)

# ========== 4. 训练配置 ==========
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(device)

optimizer = AdamW(model.parameters(), lr=5e-5)
num_epochs = 10
num_training_steps = num_epochs * len(dataloader)
lr_scheduler = get_scheduler(
    "linear", optimizer=optimizer,
    num_warmup_steps=0, num_training_steps=num_training_steps
)

# ========== 5. 训练循环 ==========
model.train()
for epoch in range(num_epochs):
    total_loss = 0
    for batch in dataloader:
        batch = {k: v.to(device) for k, v in batch.items()}
        outputs = model(
            pixel_values=batch["pixel_values"],
            input_ids=batch["input_ids"],
            attention_mask=batch["attention_mask"],
            labels=batch["input_ids"]  # 语言建模任务
        )
        loss = outputs.loss
        loss.backward()
        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()
        total_loss += loss.item()
    print(f"Epoch {epoch+1}/{num_epochs}, Loss: {total_loss/len(dataloader):.4f}")

# ========== 6. 保存LoRA权重 ==========
model.save_pretrained("./lora_image_captioning")
processor.save_pretrained("./lora_image_captioning")
print("LoRA weights saved to ./lora_image_captioning")

推理验证

# 加载LoRA模型进行推理
from peft import PeftModel

base_model = BlipForConditionalGeneration.from_pretrained(model_name)
lora_model = PeftModel.from_pretrained(base_model, "./lora_image_captioning")
lora_model.to(device)
lora_model.eval()

# 测试图片
test_image = Image.open("./images/test_room.jpg").convert("RGB")
inputs = processor(images=test_image, return_tensors="pt").to(device)

with torch.no_grad():
    generated_ids = lora_model.generate(
        **inputs,
        max_length=30,
        num_beams=3,
        do_sample=False
    )
caption = processor.decode(generated_ids[0], skip_special_tokens=True)
print(f"Generated caption: {caption}")

验证方法与通过标准

• 训练收敛：运行训练代码后，loss应随epoch增加而下降（初始约5.0，最终应低于2.0）。
• 描述质量：推理代码输出的描述应包含场景特有名词（如"sofa"、"bed"、"kitchen"等），而非通用描述如"a room with furniture"。
• 参数效率：print_trainable_parameters()应显示可训练参数占比约0.5%（约55万参数），验证LoRA只微调了少量参数。
• 对比测试：用同一张测试图分别运行基础模型和LoRA模型，LoRA模型的输出应更具体、更贴近训练数据中的描述风格。

常见问题

• 图片路径错误：确保image_dir指向的目录存在且包含对应图片文件。
• 显存不足：若batch_size=2导致OOM，可降为1或使用gradient_accumulation_steps=2。
• LoRA不生效：确认target_modules中的模块名与模型实际层名匹配（BLIP的注意力层使用q_proj/v_proj命名）。