1. 简介

BLIP（Bootstrapped Language-Image Pre-training）是一种先进的多模态大模型，专门用于处理图像和文本之间的联合理解任务。它能够同时理解视觉内容和语言信息，并在这两种模态之间建立深层关联。

2. 应用场景

• 图像搜索：根据文本描述检索相关图像，或根据图像内容生成匹配的文本描述
• 自动图像标注：为图像自动生成准确、多样化的文字说明
• 视觉问答系统：基于图像内容回答用户的自然语言问题
• 多模态内容理解：在社交媒体、电商平台等场景中理解图文混合内容

3. 环境搭建

3.1 创建conda环境

conda create -n blip python==3.10
conda activate blip

3.2 下载源码和模型

源码下载地址
模型下载
从modelscope下载bert-base-uncased/模型放到BLIP-main目录下即可

3.3 安装依赖

将requirements.txt中的transformers版本有4.15.0改为4.20.0

timm==0.4.12
transformers==4.20.0
fairscale==0.4.4
pycocoevalcap

pip install -r requirements.txt

4. 运行demo

可以使用colab notepad执行demo.ipynb
也可以从demo.ipynb中截取代码放到py文件中执行

from PIL import Image
import requests
import torch
from torchvision import transforms
from torchvision.transforms.functional import InterpolationMode
import time
import argparse

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

def load_demo_image(img_path, image_size, device):
    raw_image = Image.open(img_path).convert('RGB')

    w,h = raw_image.size
    #display(raw_image.resize((w//5,h//5)))

    transform = transforms.Compose([
        transforms.Resize((image_size,image_size),interpolation=InterpolationMode.BICUBIC),
        transforms.ToTensor(),
        transforms.Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711))
        ])
    image = transform(raw_image).unsqueeze(0).to(device)
    return image

from models.blip import blip_decoder


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='GroundingDINO ONNX Inference')
    parser.add_argument('--model_path', type=str, default='model_base_capfilt_large.pth', help='Path to the ONNX model')
    parser.add_argument('--img_path', type=str, default='images/car_1.jpg', help='Path to the input image')
    args = parser.parse_args()
    
    image_size = 384
    image = load_demo_image(img_path=args.img_path, image_size=image_size, device=device)

    start_time = time.time()    
    model = blip_decoder(pretrained=args.model_path, image_size=image_size, vit='base')
    model.eval()
    model = model.to(device)
    load_time = time.time() - start_time
    print(f"模型加载耗时: {load_time:.2f} 秒\n")

    with torch.no_grad():
        # beam search
        start_time = time.time()
        caption = model.generate(image, sample=False, num_beams=3, max_length=20, min_length=5) 
        beam_time = time.time() - start_time
        print(f"Beam Search 耗时: {beam_time:.2f} 秒\n")
        # nucleus sampling
        # caption = model.generate(image, sample=True, top_p=0.9, max_length=20, min_length=5) 
        print('caption: '+caption[0])

python test.py --img_path="images/demo.jpg"

输出结果如下：