LLM-微调-方案（一）：Lora【案例：chatGLM-Lora】【在chatGLM原有结构中间插入新的网络层】【微调时冻结原有结构参数，只微调新加入的网络层参数】
Lora主要在模型中注入可训练模块，大模型在预训练完收敛之后模型包含许多进行矩阵乘法的稠密层，这些层通常是满秩的，在微调过程中其实改变量是比较小的，在矩阵乘法中表现为低秩的改变，注入可训练层的目的是想下游微调的低秩改变由可训练层来学习，冻结模型其他部分，大大减少模型训练参数。
这种方法有点类似于矩阵分解，可训练层维度和预训练模型层维度一致为d，先将维度d通过全连接层降维至r，再从r通过全连接层映射回d维度，r< 推理计算的时候，因为没有改变预训练权重，所以换不同的下游任务时，lora模型保存的权重也是可以相应加载进来的，通过矩阵分解的方法参数量减少了很多，且推理时可以并行，对于推理性能并没有增加多少负担，算是比较好的低资源微调方法。 Lora方法包实现： GitHub - huggingface/peft: PEFT: State-of-the-art Parameter-Efficient Fine-Tuning p-tuning-v2和Lora两者对于低资源微调大模型的共同点都是冻结大模型参数，通过小模块来学习微调产生的低秩改变。但目前存在的一些问题就是这两种训练方式很容易参数灾难性遗忘，因为模型在微调的时候整个模型层参数未改变，而少参数的学习模块微调时却是改变量巨大，容易给模型在推理时产生较大偏置，使得以前的回答能力被可学习模块带偏，在微调的时候也必须注意可学习模块不能过于拟合微调数据，否则会丧失原本的预训练知识能力，产生灾难性遗忘。 最好能够在微调语料中也加入通用学习语料一起微调，避免产生对微调语料极大的偏向，在instruct gpt论文中也提到在强化学习ppo的时候模型也会很容易对于ppo数据拟合，降低模型通用自然语言任务能力，所以在ppo loss中加入了SFT梯度和预训练梯度来缓解这种遗忘问题。 ChatGLM-6B模型微调实战（以 ADGEN (广告生成) 数据集为例，序列长度达 2048）_桂花很香,旭很美的博客-CSDN博客 https://songshanhu.csdn.net/64425c1dae650e245cfead85.html https://devpress.csdn.net/chuangye/6438f5c2986c660f3cf93bbb.html LLM大模型低资源微调p tuning v2和lora区别 - 知乎 【自然语言处理】【大模型】极低资源微调大模型方法LoRA以及BLOOM-LORA实现代码 - 知乎 from transformers.integrations import TensorBoardCallback
from torch.utils.tensorboard import SummaryWriter
from transformers import TrainingArguments
from transformers import Trainer, HfArgumentParser
from transformers import AutoTokenizer, AutoModel
import torch
import torch.nn as nn
from peft import get_peft_model, LoraConfig, TaskType
from dataclasses import dataclass, field
import datasets
import os
tokenizer = AutoTokenizer.from_pretrained("/data/pretrained_models/chatglm-6b", trust_remote_code=True)
@dataclass
class FinetuneArguments:
dataset_path: str = field(default="data/alpaca")
model_path: str = field(default="output")
lora_rank: int = field(default=8)
class CastOutputToFloat(nn.Sequential):
def forward(self, x):
return super().forward(x).to(torch.float32)
def data_collator(features: list) -> dict:
len_ids = [len(feature["input_ids"]) for feature in features]
longest = max(len_ids)
input_ids = []
labels_list = []
for ids_l, feature in sorted(zip(len_ids, features), key=lambda x: -x[0]):
ids = feature["input_ids"] # [37010, 12, 3461, 100, 294, 102, 104, 3539, 2549, 101, 104, 306, 101, 433, ...]
seq_len = feature["seq_len"]
labels = ([-100] * (seq_len - 1) + ids[(seq_len - 1) :] + [-100] * (longest - ids_l))
ids = ids + [tokenizer.pad_token_id] * (longest - ids_l) # [37010, 12, 3461, 100, 294, 102, 104, 3539, 2549, 101, 104, 306, 101, 433, ...]
_ids = torch.LongTensor(ids)
labels_list.append(torch.LongTensor(labels))
input_ids.append(_ids)
input_ids = torch.stack(input_ids) # torch.Size([6, 118])
labels = torch.stack(labels_list) # torch.Size([6, 118])
return {"input_ids": input_ids, "labels": labels}
class ModifiedTrainer(Trainer):
def compute_loss(self, model, inputs, return_outputs=False):
loss = model(input_ids=inputs["input_ids"], labels=inputs["labels"]).loss
return loss
def save_model(self, output_dir=None, _internal_call=False):
from transformers.trainer import TRAINING_ARGS_NAME
os.makedirs(output_dir, exist_ok=True)
torch.save(self.args, os.path.join(output_dir, TRAINING_ARGS_NAME))
saved_params = {k: v.to("cpu") for k, v in self.model.named_parameters() if v.requires_grad}
torch.save(saved_params, os.path.join(output_dir, "adapter_model.bin"))
def main():
writer = SummaryWriter()
finetune_args, training_args = HfArgumentParser((FinetuneArguments, TrainingArguments)).parse_args_into_dataclasses()
# init model
model = AutoModel.from_pretrained("/data/pretrained_models/chatglm-6b", load_in_8bit=False, trust_remote_code=True, device_map="auto")
model.gradient_checkpointing_enable()
model.enable_input_require_grads()
model.is_parallelizable = True
model.model_parallel = True
model.lm_head = CastOutputToFloat(model.lm_head)
model.config.use_cache = (False) # silence the warnings. Please re-enable for inference!
print("原始模型结构：\n", model)
print("-" * 100)
for param_tuple in model.named_parameters():
name, param = param_tuple
print("layer name = ", name)
print("-" * 100)
# setup peft
peft_config = LoraConfig(task_type=TaskType.CAUSAL_LM, inference_mode=False, r=finetune_args.lora_rank, lora_alpha=32, lora_dropout=0.1)
model = get_peft_model(model, peft_config)
print("=" * 200)
print("添加Lora之后的模型结构：\n", model)
print("-" * 100)
for param_tuple in model.named_parameters():
name, param = param_tuple
print("layer name = ", name)
print("-" * 100)
# load dataset
dataset = datasets.load_from_disk(finetune_args.dataset_path) # 'data/alpaca'
print(f"\n{len(dataset)=}\n")
# start train
trainer = ModifiedTrainer(model=model, train_dataset=dataset, args=training_args, callbacks=[TensorBoardCallback(writer)], data_collator=data_collator)
trainer.train()
writer.close()
# save model
model.save_pretrained(training_args.output_dir)
if __name__ == "__main__":
main()
ChatGLMForConditionalGeneration(
(transformer): ChatGLMModel(
(word_embeddings): Embedding(130528, 4096)
(layers): ModuleList(
(0-27): 28 x GLMBlock(
(input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)
(attention): SelfAttention(
(rotary_emb): RotaryEmbedding()
(query_key_value): Linear(in_features=4096, out_features=12288, bias=True)
(dense): Linear(in_features=4096, out_features=4096, bias=True)
)
(post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)
(mlp): GLU(
(dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)
(dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)
)
)
)
(final_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)
)
(lm_head): CastOutputToFloat(
(0): Linear(in_features=4096, out_features=130528, bias=False)
)
)layer name = transformer.word_embeddings.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.0.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.0.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.0.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.0.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.0.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.0.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.0.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.0.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.0.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.0.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.0.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.0.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.1.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.1.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.1.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.1.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.1.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.1.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.1.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.1.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.1.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.1.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.1.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.1.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.2.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.2.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.2.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.2.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.2.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.2.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.2.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.2.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.2.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.2.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.2.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.2.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.3.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.3.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.3.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.3.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.3.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.3.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.3.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.3.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.3.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.3.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.3.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.3.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.4.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.4.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.4.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.4.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.4.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.4.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.4.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.4.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.4.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.4.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.4.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.4.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.5.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.5.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.5.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.5.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.5.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.5.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.5.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.5.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.5.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.5.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.5.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.5.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.6.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.6.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.6.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.6.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.6.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.6.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.6.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.6.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.6.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.6.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.6.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.6.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.7.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.7.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.7.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.7.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.7.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.7.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.7.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.7.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.7.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.7.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.7.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.7.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.8.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.8.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.8.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.8.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.8.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.8.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.8.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.8.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.8.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.8.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.8.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.8.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.9.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.9.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.9.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.9.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.9.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.9.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.9.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.9.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.9.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.9.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.9.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.9.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.10.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.10.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.10.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.10.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.10.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.10.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.10.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.10.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.10.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.10.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.10.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.10.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.11.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.11.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.11.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.11.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.11.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.11.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.11.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.11.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.11.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.11.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.11.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.11.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.12.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.12.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.12.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.12.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.12.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.12.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.12.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.12.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.12.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.12.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.12.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.12.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.13.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.13.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.13.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.13.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.13.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.13.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.13.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.13.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.13.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.13.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.13.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.13.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.14.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.14.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.14.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.14.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.14.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.14.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.14.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.14.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.14.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.14.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.14.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.14.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.15.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.15.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.15.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.15.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.15.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.15.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.15.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.15.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.15.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.15.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.15.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.15.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.16.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.16.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.16.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.16.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.16.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.16.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.16.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.16.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.16.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.16.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.16.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.16.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.17.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.17.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.17.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.17.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.17.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.17.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.17.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.17.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.17.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.17.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.17.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.17.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.18.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.18.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.18.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.18.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.18.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.18.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.18.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.18.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.18.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.18.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.18.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.18.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.19.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.19.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.19.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.19.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.19.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.19.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.19.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.19.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.19.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.19.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.19.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.19.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.20.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.20.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.20.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.20.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.20.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.20.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.20.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.20.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.20.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.20.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.20.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.20.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.21.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.21.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.21.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.21.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.21.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.21.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.21.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.21.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.21.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.21.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.21.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.21.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.22.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.22.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.22.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.22.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.22.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.22.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.22.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.22.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.22.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.22.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.22.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.22.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.23.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.23.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.23.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.23.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.23.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.23.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.23.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.23.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.23.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.23.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.23.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.23.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.24.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.24.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.24.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.24.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.24.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.24.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.24.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.24.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.24.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.24.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.24.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.24.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.25.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.25.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.25.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.25.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.25.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.25.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.25.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.25.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.25.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.25.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.25.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.25.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.26.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.26.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.26.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.26.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.26.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.26.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.26.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.26.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.26.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.26.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.26.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.26.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.27.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.27.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.27.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.27.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.27.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.27.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.27.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.27.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.27.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.27.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.27.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.layers.27.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = transformer.final_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = transformer.final_layernorm.bias
----------------------------------------------------------------------------------------------------添加Lora层之后：
PeftModelForCausalLM(
(base_model): LoraModel(
(model): ChatGLMForConditionalGeneration(
(transformer): ChatGLMModel(
(word_embeddings): Embedding(130528, 4096)
(layers): ModuleList(
(0-27): 28 x GLMBlock(
(input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)
(attention): SelfAttention(
(rotary_emb): RotaryEmbedding()
(query_key_value): Linear(
in_features=4096, out_features=12288, bias=True
(lora_dropout): ModuleDict(
(default): Dropout(p=0.1, inplace=False)
)
(lora_A): ModuleDict(
(default): Linear(in_features=4096, out_features=8, bias=False)
)
(lora_B): ModuleDict(
(default): Linear(in_features=8, out_features=12288, bias=False)
)
(lora_embedding_A): ParameterDict()
(lora_embedding_B): ParameterDict()
)
(dense): Linear(in_features=4096, out_features=4096, bias=True)
)
(post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)
(mlp): GLU(
(dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)
(dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)
)
)
)
(final_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)
)
(lm_head): CastOutputToFloat(
(0): Linear(in_features=4096, out_features=130528, bias=False)
)
)
)
)layer name = base_model.model.transformer.word_embeddings.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.0.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.0.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.0.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.0.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.0.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.0.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.0.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.0.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.0.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.0.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.0.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.0.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.0.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.0.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.1.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.1.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.1.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.1.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.1.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.1.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.1.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.1.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.1.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.1.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.1.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.1.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.1.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.1.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.2.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.2.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.2.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.2.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.2.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.2.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.2.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.2.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.2.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.2.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.2.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.2.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.2.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.2.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.3.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.3.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.3.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.3.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.3.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.3.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.3.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.3.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.3.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.3.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.3.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.3.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.3.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.3.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.4.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.4.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.4.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.4.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.4.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.4.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.4.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.4.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.4.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.4.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.4.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.4.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.4.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.4.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.5.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.5.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.5.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.5.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.5.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.5.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.5.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.5.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.5.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.5.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.5.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.5.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.5.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.5.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.6.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.6.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.6.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.6.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.6.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.6.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.6.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.6.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.6.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.6.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.6.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.6.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.6.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.6.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.7.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.7.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.7.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.7.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.7.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.7.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.7.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.7.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.7.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.7.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.7.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.7.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.7.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.7.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.8.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.8.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.8.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.8.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.8.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.8.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.8.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.8.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.8.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.8.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.8.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.8.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.8.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.8.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.9.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.9.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.9.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.9.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.9.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.9.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.9.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.9.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.9.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.9.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.9.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.9.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.9.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.9.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.10.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.10.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.10.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.10.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.10.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.10.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.10.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.10.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.10.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.10.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.10.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.10.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.10.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.10.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.11.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.11.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.11.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.11.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.11.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.11.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.11.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.11.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.11.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.11.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.11.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.11.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.11.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.11.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.12.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.12.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.12.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.12.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.12.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.12.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.12.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.12.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.12.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.12.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.12.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.12.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.12.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.12.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.13.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.13.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.13.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.13.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.13.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.13.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.13.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.13.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.13.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.13.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.13.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.13.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.13.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.13.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.14.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.14.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.14.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.14.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.14.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.14.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.14.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.14.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.14.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.14.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.14.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.14.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.14.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.14.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.15.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.15.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.15.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.15.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.15.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.15.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.15.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.15.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.15.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.15.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.15.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.15.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.15.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.15.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.16.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.16.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.16.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.16.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.16.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.16.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.16.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.16.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.16.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.16.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.16.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.16.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.16.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.16.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.17.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.17.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.17.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.17.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.17.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.17.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.17.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.17.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.17.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.17.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.17.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.17.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.17.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.17.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.18.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.18.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.18.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.18.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.18.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.18.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.18.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.18.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.18.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.18.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.18.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.18.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.18.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.18.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.19.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.19.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.19.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.19.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.19.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.19.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.19.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.19.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.19.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.19.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.19.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.19.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.19.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.19.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.20.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.20.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.20.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.20.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.20.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.20.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.20.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.20.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.20.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.20.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.20.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.20.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.20.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.20.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.21.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.21.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.21.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.21.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.21.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.21.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.21.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.21.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.21.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.21.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.21.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.21.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.21.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.21.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.22.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.22.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.22.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.22.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.22.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.22.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.22.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.22.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.22.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.22.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.22.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.22.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.22.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.22.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.23.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.23.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.23.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.23.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.23.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.23.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.23.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.23.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.23.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.23.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.23.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.23.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.23.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.23.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.24.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.24.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.24.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.24.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.24.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.24.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.24.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.24.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.24.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.24.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.24.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.24.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.24.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.24.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.25.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.25.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.25.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.25.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.25.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.25.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.25.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.25.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.25.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.25.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.25.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.25.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.25.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.25.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.26.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.26.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.26.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.26.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.26.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.26.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.26.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.26.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.26.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.26.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.26.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.26.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.26.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.26.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.27.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.27.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.27.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.27.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.27.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.27.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.27.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.27.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.27.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.27.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.27.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.27.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.27.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.layers.27.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.final_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name = base_model.model.transformer.final_layernorm.bias
----------------------------------------------------------------------------------------------------