1.数据格式转换

模版格式：

[
  {
    "id": "identity_0",
    "conversations": [
      {
        "from": "user",
        "value": "你好"
      },
      {
        "from": "assistant",
        "value": "我是Qwen-VL,一个支持视觉输入的大模型。"
      }
    ]
  },
  {
    "id": "identity_1",
    "conversations": [
      {
        "from": "user",
        "value": "Picture 1: <img>https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg</img>\n图中的狗是什么品种？"
      },
      {
        "from": "assistant",
        "value": "图中是一只拉布拉多犬。"
      },
      {
        "from": "user",
        "value": "框出图中的格子衬衫"
      },
      {
        "from": "assistant",
        "value": "<ref>格子衬衫</ref><box>(588,499),(725,789)</box>"
      }
    ]
  },
  { 
    "id": "identity_2",
    "conversations": [
      {
        "from": "user",
        "value": "Picture 1: <img>assets/mm_tutorial/Chongqing.jpeg</img>\nPicture 2: <img>assets/mm_tutorial/Beijing.jpeg</img>\n图中都是哪"
      },
      {
        "from": "assistant",
        "value": "第一张图片是重庆的城市天际线，第二张图片是北京的天际线。"
      }
    ]
  }
]

import json

# 打开并加载.json文件
with open(r"E:\comprehensive_library\e_commerce_lmm\data\openi-zh-prompt.json", 'r', encoding='utf-8') as f:
    data = json.load(f)

# 数据格式转换
new_data = []
for i, d in enumerate(data):
    new_conversations = dict()
    new_conversations["id"] = f"identity_{i}"
    new_conversations["conversations"] = [
        {
            "from": "user",
            "value": "Picture 1: <img>" + d["img"] + "</img>\n" + d["prompt"]
        },
        {
            "from": "assistant",
            "value": d["label"]
        }
    ]
    new_data.append(new_conversations)

# 将新的数据写入新的.json文件中
with open("../data/openai-zh-qwenvl-prompt.json", 'w', encoding='utf-8') as f:
    json.dump(new_data, f, ensure_ascii=False, indent=4)

2.微调

直接pip install -r requirments.txt

注意gcc要升级到9.3

yum -y install centos-release-scl
yum -y install devtoolset-9-gcc devtoolset-9-gcc-c++ devtoolset-9-binutils
scl enable devtoolset-9 bash

echo "source /opt/rh/devtoolset-9/enable" >>/etc/profile

lora在V100上显存不够，微调不起来，在a800上可以。用swift库可以在V100上微调Qwen-vl。

#!/bin/bash
export CUDA_DEVICE_MAX_CONNECTIONS=1
DIR=$(pwd)

GPUS_PER_NODE=$(python -c 'import torch; print(torch.cuda.device_count())')
NNODES=1
NODE_RANK=0
MASTER_ADDR=localhost
MASTER_PORT=6001

MODEL="/home/image_team/image_team_docker_home/lgd/e_commerce_lmm/weights/qwen-vl-caht/" #"Qwen/Qwen-VL-Chat"/"Qwen/Qwen-VL"  Set the path if you do not want to load from huggingface directly
# ATTENTION: specify the path to your training data, which should be a json file consisting of a list of conversations.
# See the section for finetuning in README for more information.
DATA="/home/image_team/image_team_docker_home/lgd/e_commerce_lmm/data/openai-zh-qwenvl-prompt.json"

DISTRIBUTED_ARGS="
    --nproc_per_node $GPUS_PER_NODE \
    --nnodes $NNODES \
    --node_rank $NODE_RANK \
    --master_addr $MASTER_ADDR \
    --master_port $MASTER_PORT
"

torchrun $DISTRIBUTED_ARGS finetune.py \
  --model_name_or_path $MODEL \
  --data_path $DATA \
  --bf16 False \
  --fp16 True \
  --fix_vit True \
  --output_dir output_qwen \
  --num_train_epochs 5 \
  --per_device_train_batch_size 1 \
  --per_device_eval_batch_size 1 \
  --gradient_accumulation_steps 1 \
  --evaluation_strategy "no" \
  --save_strategy "steps" \
  --save_steps 1000 \
  --save_total_limit 10 \
  --learning_rate 1e-5 \
  --weight_decay 0.1 \
  --adam_beta2 0.95 \
  --warmup_ratio 0.01 \
  --lr_scheduler_type "cosine" \
  --logging_steps 1 \
  --report_to "none" \
  --model_max_length 1024 \
  --lazy_preprocess True \
  --use_lora \
  --gradient_checkpointing \
  --deepspeed finetune/ds_config_zero2.json