google/gemma-3-27b-it · Fail to Load Gemme3 27B

Code:

    config = AutoConfig.from_pretrained(base_model_name)
    for key, value in vars(config.text_config).items():
        setattr(config, key, value)
    config.use_cache=False

    #print(f"config: {config}")

    device_map = (
        {"": f"xpu:{Accelerator().local_process_index}"}
        if is_xpu_available()
        else {"": Accelerator().local_process_index}
    )

    base_model = AutoModelForCausalLM.from_pretrained(
        base_model_name,
        config=config,
        quantization_config=quantization_config,
        use_flash_attention_2=True,
        torch_dtype=torch.bfloat16,
        device_map = "cpu"
    )

Error:

Traceback (most recent call last):
  File "/home/ec2-user/SageMaker/zhichao/Gemma3-27B/UNA_trainer_score_MSE_1e-4_0.01.py", line 582, in <module>
    main()
  File "/home/ec2-user/SageMaker/zhichao/Gemma3-27B/UNA_trainer_score_MSE_1e-4_0.01.py", line 529, in main
    tokenizer, policy_model = initialize_tokenizer_and_model()
  File "/home/ec2-user/SageMaker/zhichao/Gemma3-27B/UNA_trainer_score_MSE_1e-4_0.01.py", line 508, in initialize_tokenizer_and_model
    base_model = AutoModelForCausalLM.from_pretrained(
  File "/home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py", line 564, in from_pretrained
    return model_class.from_pretrained(
  File "/home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages/transformers/modeling_utils.py", line 273, in _wrapper
    return func(*args, **kwargs)
  File "/home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages/transformers/modeling_utils.py", line 4531, in from_pretrained
    dispatch_model(model, **device_map_kwargs)
  File "/home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages/accelerate/big_modeling.py", line 501, in dispatch_model
    model.to(device)
  File "/home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages/transformers/modeling_utils.py", line 3263, in to
    return super().to(*args, **kwargs)
  File "/home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1152, in to
    return self._apply(convert)
  File "/home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages/torch/nn/modules/module.py", line 802, in _apply
    module._apply(fn)
  File "/home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages/torch/nn/modules/module.py", line 802, in _apply
    module._apply(fn)
  File "/home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages/torch/nn/modules/module.py", line 825, in _apply
    param_applied = fn(param)
  File "/home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1150, in convert
    return t.to(device, dtype if t.is_floating_point() or t.is_complex() else None, non_blocking)
NotImplementedError: Cannot copy out of meta tensor; no data!
Traceback (most recent call last):
  File "/home/ec2-user/SageMaker/zhichao/Gemma3-27B/UNA_trainer_score_MSE_1e-4_0.01.py", line 582, in <module>
    main()
  File "/home/ec2-user/SageMaker/zhichao/Gemma3-27B/UNA_trainer_score_MSE_1e-4_0.01.py", line 529, in main
    tokenizer, policy_model = initialize_tokenizer_and_model()
  File "/home/ec2-user/SageMaker/zhichao/Gemma3-27B/UNA_trainer_score_MSE_1e-4_0.01.py", line 508, in initialize_tokenizer_and_model
    base_model = AutoModelForCausalLM.from_pretrained(
  File "/home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py", line 564, in from_pretrained
    return model_class.from_pretrained(
  File "/home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages/transformers/modeling_utils.py", line 273, in _wrapper
    return func(*args, **kwargs)
  File "/home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages/transformers/modeling_utils.py", line 4531, in from_pretrained
    dispatch_model(model, **device_map_kwargs)
  File "/home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages/accelerate/big_modeling.py", line 501, in dispatch_model
    model.to(device)
  File "/home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages/transformers/modeling_utils.py", line 3263, in to
    return super().to(*args, **kwargs)
  File "/home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1152, in to
    return self._apply(convert)
  File "/home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages/torch/nn/modules/module.py", line 802, in _apply
    module._apply(fn)
  File "/home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages/torch/nn/modules/module.py", line 802, in _apply
    module._apply(fn)
  File "/home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages/torch/nn/modules/module.py", line 825, in _apply
    param_applied = fn(param)
  File "/home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1150, in convert
    return t.to(device, dtype if t.is_floating_point() or t.is_complex() else None, non_blocking)
NotImplementedError: Cannot copy out of meta tensor; no data!