LLaMA-Factory 源码信息

register_model_group(
    models={
        "InternVL2.5-1B-MPO": {
            DownloadSource.DEFAULT: "kingsley01/InternVL2_5-1B-MPO-hf",
            DownloadSource.MODELSCOPE: "llamafactory/InternVL2_5-1B-MPO-hf",
        },
        "InternVL2.5-2B-MPO": {
            DownloadSource.DEFAULT: "kingsley01/InternVL2_5-2B-MPO-hf",
            DownloadSource.MODELSCOPE: "llamafactory/InternVL2_5-2B-MPO-hf",
        },
        "InternVL2.5-4B-MPO": {
            DownloadSource.DEFAULT: "kingsley01/InternVL2_5-4B-MPO-hf",
            DownloadSource.MODELSCOPE: "llamafactory/InternVL2_5-4B-MPO-hf",
        },
        "InternVL2.5-8B-MPO": {
            DownloadSource.DEFAULT: "kingsley01/InternVL2_5-8B-MPO-hf",
            DownloadSource.MODELSCOPE: "llamafactory/InternVL2_5-8B-MPO-hf",
        },
        "InternVL3-1B-hf": {
            DownloadSource.DEFAULT: "kingsley01/InternVL3-1B-hf",
            DownloadSource.MODELSCOPE: "llamafactory/InternVL3-1B-hf",
        },
        "InternVL3-2B-hf": {
            DownloadSource.DEFAULT: "kingsley01/InternVL3-2B-hf",
            DownloadSource.MODELSCOPE: "llamafactory/InternVL3-2B-hf",
        },
        "InternVL3-8B-hf": {
            DownloadSource.DEFAULT: "kingsley01/InternVL3-8B-hf",
            DownloadSource.MODELSCOPE: "llamafactory/InternVL3-8B-hf",
        },
    },
    template="intern_vl",
    multimodal=True,
)

模型下载

bash
./hfd.sh kingsley01/InternVL3-8B-hf --local-dir kingsley01/InternVL3-8B-hf

镜像

官网：https://github.com/hiyouga/LLaMA-Factory/pull/7258

要安装这个：pip install git+https://github.com/Kuangdd01/transformers.git@hf-internvl

为了匹配这个特定版本的Transformer，模型要转换，可以用官方转好的，不然这么转换：

# Find your transformers package dir
package_path=$(pip show transformers | grep "Location:" | cut -d':' -f2 | tr -d ' ')
# convert model arch name so it can be recognized by transformers
python $package_path/transformers/models/internvl/convert_internvl_weights_to_hf.py \
    --input_dir ./local_origin_internvl \
    --output_dir ./local_dir-hf

docker run -it kevinchina/deeplearning:llamafactory20250418 bash

pip install git+https://github.com/Kuangdd01/transformers.git@hf-internvl

指令

cd /app && FORCE_TORCHRUN=1 \
NNODES=3 \
NODE_RANK=${RANK} \
MASTER_ADDR=${MASTER_ADDR} \
MASTER_PORT=${MASTER_PORT} \
llamafactory-cli train \
  --model_name_or_path /InternVL3-8B-hf \
  --image_max_pixels 1048576 \
  --video_max_pixels 16384 \
  --trust_remote_code true \
  --stage sft \
  --do_train true \
  --finetuning_type full \
  --freeze_vision_tower true \
  --freeze_multi_modal_projector true \
  --freeze_language_model false \
  --deepspeed examples/deepspeed/ds_z3_config.json \
  --dataset xdx_b \
  --template intern_vl \
  --cutoff_len 4096 \
  --max_samples 10000 \
  --overwrite_cache true \
  --preprocessing_num_workers 32 \
  --dataloader_num_workers 8 \
  --output_dir output/saves/sft \
  --logging_steps 1 \
  --save_steps 100 \
  --plot_loss true \
  --overwrite_output_dir true \
  --save_only_model false \
  --per_device_train_batch_size 6 \
  --gradient_accumulation_steps 2 \
  --learning_rate 1.0e-5 \
  --num_train_epochs 1000.0 \
  --lr_scheduler_type cosine \
  --warmup_ratio 0.1 \
  --bf16 true \
  --ddp_timeout 180000000

训练报错：

ImportError: transformers>=4.45.0,<=4.51.3,!=4.46.0,!=4.46.1,!=4.46.2,!=4.46.3,!=4.47.0,!=4.47.1,!=4.48.0 is required for a normal functioning of this module, but found transformers==4.52.0.dev0.
To fix: run `pip install transformers>=4.45.0,<=4.51.3,!=4.46.0,!=4.46.1,!=4.46.2,!=4.46.3,!=4.47.0,!=4.47.1,!=4.48.0` or set `DISABLE_VERSION_CHECK=1` to skip this check.

等待LLaMA-Factory对InternVL3-8B进一步支持，我将使用InternVL3-8B的官方代码进行训练尝试。