Deepseek V3 Inference 报错 header too large

小乐快乐

956人浏览 · 2025-10-29 10:25:27

小乐快乐 · 2025-10-29 10:25:27 发布

问题描述

运行脚本

import argparse

import mindspore as ms
from mindspore import Model, Tensor
from mindspore.common import initializer

from mindformers import MindFormerConfig
from mindformers import build_context
from mindformers.tools.logger import logger
from mindformers.trainer.utils import transform_and_load_checkpoint
from mindformers.core.parallel_config import build_parallel_config
from mindformers.models.llama.llama_tokenizer_fast import LlamaTokenizerFast

from research.deepseek3.deepseek3_model_infer import InferenceDeepseekV3ForCausalLM
from research.deepseek3.deepseek3_config import DeepseekV3Config


def run_predict(args):
    """Deepseek-V3/R1 predict"""
    # inputs
    input_questions = [args.input]

    # set model config
    yaml_file = args.config
    config = MindFormerConfig(yaml_file)
    build_context(config)
    build_parallel_config(config)
    model_config = config.model.model_config
    model_config.parallel_config = config.parallel_config
    model_config.moe_config = config.moe_config
    model_config = DeepseekV3Config(**model_config)

    # build tokenizer
    tokenizer = LlamaTokenizerFast(config.processor.tokenizer.vocab_file,
                                   config.processor.tokenizer.tokenizer_file,
                                   unk_token=config.processor.tokenizer.unk_token,
                                   bos_token=config.processor.tokenizer.bos_token,
                                   eos_token=config.processor.tokenizer.eos_token,
                                   fast_tokenizer=True)
    tokenizer.pad_token = tokenizer.eos_token

    # build model from config
    network = InferenceDeepseekV3ForCausalLM(model_config)
    ms_model = Model(network)
    if config.load_checkpoint:
        logger.info("----------------Transform and load checkpoint----------------")
        seq_length = model_config.seq_length
        input_ids = Tensor(shape=(model_config.batch_size, seq_length), dtype=ms.int32, init=initializer.One())
        infer_data = network.prepare_inputs_for_predict_layout(input_ids)
        transform_and_load_checkpoint(config, ms_model, network, infer_data, do_predict=True)

    inputs = tokenizer(input_questions, max_length=64, padding="max_length")["input_ids"]
    outputs = network.generate(inputs,
                               max_length=1024,
                               do_sample=False,
                               top_k=5,
                               top_p=1,
                               max_new_tokens=128)
    answer = tokenizer.decode(outputs)
    print("answer: ", answer)


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--config',
        type=str,
        required=False,
        help='YAML config files, such as'
        '/home/ma-user/work/vllm-mindspore/install_depend_pkgs/mindformers-br_infer_boom/research/deepseek3/deepseek3_671b/predict_deepseek3_671b.yaml',
        
        default="/home/ma-user/work/vllm-mindspore/install_depend_pkgs/mindformers-br_infer_boom/research/deepseek3/deepseek3_671b/predict_deepseek3_671b.yaml")
    parser.add_argument(
        '--input',
        type=str,
        default="生抽和老抽的区别是什么？")
    args_ = parser.parse_args()

    run_predict(args_)

配置文件

seed: 0
output_dir: './output' # path to save checkpoint/strategy
run_mode: 'predict'
use_parallel: False

load_checkpoint: "/home/ma-user/work/models/deepseekv3_bf16/tokenizer.json"
load_ckpt_format: "safetensors"
auto_trans_ckpt: False # If true, auto transform load_checkpoint to load in distributed model

# trainer config
trainer:
  type: CausalLanguageModelingTrainer
  model_name: 'DeepSeekV3'

# default parallel of device num = 32 for Atlas 800T A2
parallel_config:
  model_parallel: 1
  pipeline_stage: 1
  expert_parallel: 1
  vocab_emb_dp: False

# mindspore context init config
context:
  mode: 0 # 0--Graph Mode; 1--Pynative Mode
  max_device_memory: "50GB"
  device_id: 0
  affinity_cpu_list: None

# parallel context config
parallel:
  parallel_mode: "STAND_ALONE" # use 'STAND_ALONE' mode for inference with parallelism in frontend
  full_batch: False
  strategy_ckpt_save_file: "./ckpt_strategy.ckpt"

# model config
model:
  model_config:
    type: DeepseekV3Config
    auto_register: deepseek3_config.DeepseekV3Config
    batch_size: 1 # add for incre predict
    seq_length: 4096
    hidden_size: 7168
    num_layers: 10
    num_heads: 128
    max_position_embeddings: 163840
    intermediate_size: 18432
    kv_lora_rank:  512
    q_lora_rank: 1536
    qk_rope_head_dim: 64
    v_head_dim: 128
    qk_nope_head_dim: 128
    vocab_size: 129280
    multiple_of: 256
    rms_norm_eps: 1.0e-6
    bos_token_id: 0
    eos_token_id: 1
    pad_token_id: 1
    ignore_token_id: -100
    compute_dtype: "bfloat16"
    layernorm_compute_type: "bfloat16"
    softmax_compute_type: "bfloat16"
    rotary_dtype: "bfloat16"
    router_dense_type: "bfloat16"
    param_init_type: "bfloat16"
    scaling_factor:
      beta_fast: 32.0
      beta_slow: 1.0
      factor: 40.0
      mscale: 1.0
      mscale_all_dim: 1.0
      original_max_position_embeddings: 4096
    use_past: True
    extend_method: "YARN"
    use_flash_attention: True
    block_size: 16
    num_blocks: 512
    offset: 0
    checkpoint_name_or_path: ""
    repetition_penalty: 1
    max_decode_length: 1024
    top_k: 1
    top_p: 1
    theta: 10000.0
    do_sample: False
    is_dynamic: True
    qkv_concat: False
    ffn_concat: False
    auto_map:
      AutoConfig: deepseek3_config.DeepseekV3Config
      AutoModel: deepseek3.DeepseekV3ForCausalLM
  arch:
    type: DeepseekV3ForCausalLM
    auto_register: deepseek3.DeepseekV3ForCausalLM

moe_config:
  expert_num: 256
  num_experts_chosen: 8
  routing_policy: "TopkRouterV2"
  shared_expert_num: 1
  routed_scaling_factor: 2.5
  first_k_dense_replace: 0
  moe_intermediate_size: 2048
  topk_group: 4
  n_group: 8

processor:
  return_tensors: ms
  tokenizer:
    unk_token: '<unk>'
    bos_token: '<｜begin▁of▁sentence｜>'
    eos_token: '<｜end▁of▁sentence｜>'
    pad_token: '<｜end▁of▁sentence｜>'
    type: LlamaTokenizerFast
    vocab_file: '/home/ma-user/work/models/deepseekv3_bf16/tokenizer.json'
    tokenizer_file: '/home/ma-user/work/models/deepseekv3_bf16/tokenizer.json'

报错信息

2025-09-09 22:59:06,762 - mindformers./output/log[mindformers/utils/load_checkpoint_utils.py:191] - INFO - ......Start load checkpoint from safetensors......
2025-09-09 22:59:06,762 - mindformers./output/log[mindformers/utils/load_checkpoint_utils.py:196] - INFO - Load checkpoint from /home/ma-user/work/models/deepseekv3_bf16/tokenizer.json.
2025-09-09 22:59:06,763 - mindformers./output/log[mindformers/utils/load_checkpoint_utils.py:205] - INFO - Set network.set_train=False, reduce compile time in prediction.
2025-09-09 22:59:06,775 - mindformers./output/log[mindformers/utils/load_checkpoint_utils.py:211] - INFO - ......Use single checkpoint file mode......
2025-09-09 22:59:06,775 - mindformers./output/log[mindformers/models/modeling_utils.py:1531] - INFO - InferenceDeepseekV3ForCausalLM does not support qkv concat check, skipping...
2025-09-09 22:59:06,776 - mindformers./output/log[mindformers/utils/load_checkpoint_utils.py:378] - INFO - ......Start load checkpoint to model......
Traceback (most recent call last):
File "/home/ma-user/work/mindarmour/examples/model_protection/deepseekv3/infer/run_deepseekv3_predict.py", line 79, in <module>
run_predict(args_)
File "/home/ma-user/work/mindarmour/examples/model_protection/deepseekv3/infer/run_deepseekv3_predict.py", line 50, in run_predict
transform_and_load_checkpoint(config, ms_model, network, infer_data, do_predict=True)
File "/home/ma-user/work/vllm-mindspore/install_depend_pkgs/mindformers-br_infer_boom/mindformers/trainer/utils.py", line 400, in transform_and_load_checkpoint
load_checkpoint_with_safetensors(config, model, network, dataset, do_eval=do_eval,
File "/home/ma-user/work/vllm-mindspore/install_depend_pkgs/mindformers-br_infer_boom/mindformers/utils/load_checkpoint_utils.py", line 275, in load_checkpoint_with_safetensors
load_safetensors_checkpoint(config, load_checkpoint_files, network, strategy_path, load_checkpoint, optimizer)
File "/home/ma-user/work/vllm-mindspore/install_depend_pkgs/mindformers-br_infer_boom/mindformers/utils/load_checkpoint_utils.py", line 382, in load_safetensors_checkpoint
with safe_open(checkpoint_file, framework='np') as f:
safetensors_rust.SafetensorError: Error while deserializing header: header too large