# Fail to load from custom pretrained model **URL:** **Author:** ![jason-kim](https://avatars.discourse-cdn.com/v4/letter/j/4491bb/32.png) [@jason-kim](/static-proxy?url=https%3A%2F%2Fdiscuss.huggingface.co%2Fu%2Fjason-kim)%5C **Post date:** [March 11, 2024, 6:22am UTC](/static-proxy?url=https%3A%2F%2Fdiscuss.huggingface.co%2Ft%2Ffail-to-load-from-custom-pretrained-model%2F76837%2F1 "2024-03-11T06:22:53Z") Hi, I have pretrained a model using [transformers example](https://github.com/huggingface/transformers/blob/v4.36.2/examples/pytorch/language-modeling/run_mlm_no_trainer.py). And I got a custom model like below. ```auto $ CUDA_VISIBLE_DEVICES="0,1,2" accelerate launch run_mlm_no_trainer.py \ --model_type bert, \ --tokenizer_name ./tokenizer/wordpiece/ \ --train_file ./test/ \ --checkpointing_steps 1000 \ --validation_split_percentage 10 \ --max_seq_length 512 \ --per_device_train_batch_size 64 \ --per_device_eval_batch_size 64 \ --output_dir ./result/ \ --preprocessing_num_workers 32 \ --seed 42 \ --num_train_epochs 2 ``` > $ ls -l saved\_model > -rw-r–r–. 1 ps ps 34 Mar 11 14:35 all\_results.json > -rw-r–r–. 1 ps ps 589 Mar 11 14:35 config.json > -rw-r–r–. 1 ps ps 90 Mar 11 14:35 generation\_config.json > -rw-r–r–. 1 ps ps 374369196 Mar 11 14:35 model.safetensors > -rw-r–r–. 1 ps ps 695 Mar 11 14:35 special\_tokens\_map.json > -rw-r–r–. 1 ps ps 714913 Mar 11 14:35 tokenizer.json > -rw-r–r–. 1 ps ps 1264 Mar 11 14:35 tokenizer\_config.json > -rw-r–r–. 1 ps ps 211389 Mar 11 14:35 vocab.txt Now I want to load the model, but RuntimeError occurs. > from transformers import AutoModel, AutoConfig > config = AutoConfig.from\_pretrained(‘/opt/patbert/bert/test\_result2/’) > model = AutoModel.from\_pretrained(‘/opt/patbert/bert/test\_result2/’, config) > > RuntimeError Traceback (most recent call last) > Cell In[4], line 4 > 1 from transformers import AutoTokenizer, AutoModel, AutoConfig > 3 config = AutoConfig.from\_pretrained(‘saved\_model’) > ----\> 4 model = AutoModel.from\_pretrained(‘saved\_model’, config) > > File ~/.conda/envs/pyenv38/lib/python3.8/site-packages/transformers/models/auto/auto\_factory.py:566, in \_BaseAutoModelClass.from\_pretrained(cls, pretrained\_model\_name\_or\_path, \*model\_args, \*\*kwargs) > 564 elif type(config) in cls.\_model\_mapping.keys(): > 565 model\_class = \_get\_model\_class(config, cls.\_model\_mapping) > → 566 return model\_class.from\_pretrained( > 567 pretrained\_model\_name\_or\_path, \*model\_args, config=config, \*\*hub\_kwargs, \*\*kwargs > 568 ) > 569 raise ValueError( > 570 f"Unrecognized configuration class {config. **class** } for this kind of AutoModel: {cls. **name** }.\n" > 571 f"Model type should be one of {', '.join(c. **name** for c in cls.\_model\_mapping.keys())}." > 572 ) > > File ~/.conda/envs/pyenv38/lib/python3.8/site-packages/transformers/modeling\_utils.py:3706, in PreTrainedModel.from\_pretrained(cls, pretrained\_model\_name\_or\_path, config, cache\_dir, ignore\_mismatched\_sizes, force\_download, local\_files\_only, token, revision, use\_safetensors, \*model\_args, \*\*kwargs) > 3697 if dtype\_orig is not None: > 3698 torch.set\_default\_dtype(dtype\_orig) > 3699 ( > 3700 model, > 3701 missing\_keys, > 3702 unexpected\_keys, > 3703 mismatched\_keys, > 3704 offload\_index, > 3705 error\_msgs, > → 3706 ) = cls.\_load\_pretrained\_model( > 3707 model, > 3708 state\_dict, > 3709 loaded\_state\_dict\_keys, # XXX: rename? > 3710 resolved\_archive\_file, > 3711 pretrained\_model\_name\_or\_path, > 3712 ignore\_mismatched\_sizes=ignore\_mismatched\_sizes, > 3713 sharded\_metadata=sharded\_metadata, > 3714 \_fast\_init=\_fast\_init, > 3715 low\_cpu\_mem\_usage=low\_cpu\_mem\_usage, > 3716 device\_map=device\_map, > 3717 offload\_folder=offload\_folder, > 3718 offload\_state\_dict=offload\_state\_dict, > 3719 dtype=torch\_dtype, > 3720 is\_quantized=(getattr(model, “quantization\_method”, None) == QuantizationMethod.BITS\_AND\_BYTES), > 3721 keep\_in\_fp32\_modules=keep\_in\_fp32\_modules, > 3722 ) > 3724 model.is\_loaded\_in\_4bit = load\_in\_4bit > 3725 model.is\_loaded\_in\_8bit = load\_in\_8bit > > File ~/.conda/envs/pyenv38/lib/python3.8/site-packages/transformers/modeling\_utils.py:4166, in PreTrainedModel.\_load\_pretrained\_model(cls, model, state\_dict, loaded\_keys, resolved\_archive\_file, pretrained\_model\_name\_or\_path, ignore\_mismatched\_sizes, sharded\_metadata, \_fast\_init, low\_cpu\_mem\_usage, device\_map, offload\_folder, offload\_state\_dict, dtype, is\_quantized, keep\_in\_fp32\_modules) > 4162 if “size mismatch” in error\_msg: > 4163 error\_msg += ( > 4164 “\n\tYou may consider adding `ignore_mismatched_sizes=True` in the model `from_pretrained` method.” > 4165 ) > → 4166 raise RuntimeError(f"Error(s) in loading state\_dict for {model. **class**. **name** }:\n\t{error\_msg}") > 4168 if is\_quantized: > 4169 unexpected\_keys = [elem for elem in unexpected\_keys if “SCB” not in elem] > > RuntimeError: Error(s) in loading state\_dict for BertModel: > size mismatch for bert.embeddings.word\_embeddings.weight: copying a param with shape torch.Size([8532139]) from checkpoint, the shape in current model is torch.Size([32000, 768]). > size mismatch for bert.embeddings.position\_embeddings.weight: copying a param with shape torch.Size([0]) from checkpoint, the shape in current model is torch.Size([512, 768]). > You may consider adding `ignore_mismatched_sizes=True` in the model `from_pretrained` method. Of course, if i use ignore\_mismatched\_sizes=True, the model can be loaded. Is it common to use like this?