pip install -q datasets trl peft bitsandbytes sentencepiece wandb

import os
import gc
import torch

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments, BitsAndBytesConfig
from datasets import load_dataset
from peft import LoraConfig, PeftModel, get_peft_model, prepare_model_for_kbit_training
from trl import DPOTrainer
import bitsandbytes as bnb
from google.colab import userdata
import wandb

# 在Google Colab秘密的標簽中定義
hf_token = userdata.get('huggingface')
wb_token = userdata.get('wandb')
wandb.login(key=wb_token)

model_name = "teknium/OpenHermes-2.5-Mistral-7B"
new_model = "NeuralHermes-2.5-Mistral-7B"

<|im_start|>system
You are a helpful chatbot assistant.<|im_end|>
<|im_start|>user
Hi<|im_end|>
<|im_start|>assistant
Hi, how can I help you?<|im_end|>

def chatml_format(example):
 # 格式化系統(tǒng)列
 if len(example['system']) > 0:
 message = {"role": "system", "content": example['system']}
 system = tokenizer.apply_chat_template([message], tokenize=False)
 else:
 system = ""

 # 格式化指令
 message = {"role": "user", "content": example['question']}
 prompt = tokenizer.apply_chat_template([message], tokenize=False, add_generation_prompt=True)

 # 設置所選答案的格式
 chosen = example['chosen'] + "<|im_end|>\n"

 # 格式化拒絕的答案
 rejected = example['rejected'] + "<|im_end|>\n"

 return {
 "prompt": system + prompt,
 "chosen": chosen,
 "rejected": rejected,
 }

# 加載數(shù)據(jù)集
dataset = load_dataset("Intel/orca_dpo_pairs")['train']

# 保存列數(shù)據(jù)
original_columns = dataset.column_names

# 分詞器
tokenizer = AutoTokenizer.from_pretrained(model_name)
tokenizer.pad_token = tokenizer.eos_token
tokenizer.padding_side = "left"

# 格式化數(shù)據(jù)集
dataset = dataset.map(
 chatml_format,
 remove_columns=original_columns
)

{'prompt': '<|im_start|>system\nYou are an AI assistant. You will be given a task. You must generate a detailed and long answer.<|im_end|>\n<|im_start|>user\nGenerate an approximately fifteen-word sentence that describes all this data: Midsummer House eatType restaurant; Midsummer House food Chinese; Midsummer House priceRange moderate; Midsummer House customer rating 3 out of 5; Midsummer House near All Bar One<|im_end|>\n<|im_start|>assistant\n',
'chosen': 'Midsummer House is a moderately priced Chinese restaurant with a 3/5 customer rating, located near All Bar One.<|im_end|>\n',
'rejected': ' Sure! Here\'s a sentence that describes all the data you provided:\n\n"Midsummer House is a moderately priced Chinese restaurant with a customer rating of 3 out of 5, located near All Bar One, offering a variety of delicious dishes."<|im_end|>\n'}

# LoRA配置
peft_config = LoraConfig(
 r=16,
 lora_alpha=16,
 lora_dropout=0.05,
 bias="none",
 task_type="CAUSAL_LM",
 target_modules=['k_proj', 'gate_proj', 'v_proj', 'up_proj', 'q_proj', 'o_proj', 'down_proj']
)

# 要微調的模型
model = AutoModelForCausalLM.from_pretrained(
 model_name,
 torch_dtype=torch.float16,
 load_in_4bit=True
)
model.config.use_cache = False

# 參考模型
ref_model = AutoModelForCausalLM.from_pretrained(
 model_name,
 torch_dtype=torch.float16,
 load_in_4bit=True
)

# 訓練參數(shù)
training_args = TrainingArguments(
 per_device_train_batch_size=4,
 gradient_accumulation_steps=4,
 gradient_checkpointing=True,
 learning_rate=5e-5,
 lr_scheduler_type="cosine",
 max_steps=200,
 save_strategy="no",
 logging_steps=1,
 output_dir=new_model,
 optim="paged_adamw_32bit",
 warmup_steps=100,
 bf16=True,
 report_to="wandb",
)

# 創(chuàng)建DPO訓練器
dpo_trainer = DPOTrainer(
 model,
 ref_model,
 args=training_args,
 train_dataset=dataset,
 tokenizer=tokenizer,
 peft_cnotallow=peft_config,
 beta=0.1,
 max_prompt_length=1024,
 max_length=1536,
)

# 使用DPO微調模型
dpo_trainer.train()

#保存所有工作
dpo_trainer.model.save_pretrained("final_checkpoint")
tokenizer.save_pretrained("final_checkpoint")

# 刷新內存
del dpo_trainer, model, ref_model
gc.collect()
torch.cuda.empty_cache()

# 在FP16（而不是NF4）中重新加載模型
base_model = AutoModelForCausalLM.from_pretrained(
 model_name,
 return_dict=True,
 torch_dtype=torch.float16,
)
tokenizer = AutoTokenizer.from_pretrained(model_name)

# 將基本模型與適配器合并
model = PeftModel.from_pretrained(base_model, "final_checkpoint")
model = model.merge_and_unload()

# 保存模型和分詞器
model.save_pretrained(new_model)
tokenizer.save_pretrained(new_model)

# 然后將其推送到Hugging Face中心
model.push_to_hub(new_model, use_temp_dir=False, token=hf_token)
tokenizer.push_to_hub(new_model, use_temp_dir=False, token=hf_token)

#格式化提示
message = [
 {"role": "system", "content": "You are a helpful assistant chatbot."},
 {"role": "user", "content": "What is a Large Language Model?"}
]
tokenizer = AutoTokenizer.from_pretrained(new_model)
prompt = tokenizer.apply_chat_template(message, add_generation_prompt=True, tokenize=False)

# 創(chuàng)建管道
pipeline = transformers.pipeline(
 "text-generation",
 model=new_model,
 tokenizer=tokenizer
)

# 生成文本
sequences = pipeline(
 prompt,
 do_sample=True,
 temperature=0.7,
 top_p=0.9,
 num_return_sequences=1,
 max_length=200,
)
print(sequences[0]['generated_text'])

A large language model is a type of artificial intelligence (AI) system that has been trained on vast amounts of text data. These models are designed to understand and generate human language, allowing them to perform various natural language processing tasks, such as text generation, language translation, and question answering. Large language models typically use deep learning techniques, like recurrent neural networks (RNNs) or transformers, to learn patterns and relationships in the data, enabling them to generate coherent and contextually relevant responses. The size of these models, in terms of the number of parameters and the volume of data they are trained on, plays a significant role in their ability to comprehend and produce complex language structures.