from transformers import AutoModelForCausalLM, AutoTokenizer
# 加載原始LLaMA模型
model_path = "original_llama_7b"
tokenizer = AutoTokenizer.from_pretrained(model_path)
model = AutoModelForCausalLM.from_pretrained(model_path)
# 保存為Hugging Face格式
hf_path = "llama_7b_hf"
model.save_pretrained(hf_path)
tokenizer.save_pretrained(hf_path)

from transformers import AddedToken
# 加載原始tokenizer
tokenizer = AutoTokenizer.from_pretrained("llama_7b_hf")
# 添加中文字符
new_tokens = ["中文", "中國(guó)", "人工智能"]  # 實(shí)際應(yīng)用中需要更全面的詞表
new_tokens = [AddedToken(token) for token in new_tokens]
# 擴(kuò)充詞表
tokenizer.add_tokens(new_tokens)
model.resize_token_embeddings(len(tokenizer))
# 保存擴(kuò)充后的tokenizer
tokenizer.save_pretrained("llama_7b_zh_extended")

def clean_text(text):
    # 去除HTML標(biāo)簽
    text = re.sub(r'<[^>]+>', '', text)
    # 去除特殊字符和多余空格
    text = re.sub(r'\s+', ' ', text).strip()
    # 其他自定義清洗規(guī)則
    return text
# 假設(shè)我們有一個(gè)文本文件列表
text_files = ["data/wiki_zh.txt", "data/news_zh.txt"]
cleaned_data = []
for file in text_files:
    with open(file, 'r', encoding='utf-8') as f:
        for line in tqdm(f):
            cleaned = clean_text(line)
            if len(cleaned) > 10:  # 過(guò)濾過(guò)短文本
                cleaned_data.append(cleaned)
# 保存清洗后的數(shù)據(jù)
with open("cleaned_zh_data.txt", 'w', encoding='utf-8') as f:
    f.write('\n'.join(cleaned_data))

from transformers import Trainer, TrainingArguments
from peft import LoraConfig, get_peft_model
# 配置LoRA
lora_config = LoraConfig(
    r=8,  # 低秩矩陣的維度
    lora_alpha=32,
    target_modules=["q_proj", "v_proj"],  # 對(duì)query和value矩陣進(jìn)行適配
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM"
)
# 應(yīng)用LoRA到模型
model = get_peft_model(model, lora_config)
model.print_trainable_parameters()  # 查看可訓(xùn)練參數(shù)數(shù)量
# 準(zhǔn)備訓(xùn)練參數(shù)
training_args = TrainingArguments(
    output_dir="./output",
    overwrite_output_dir=True,
    num_train_epochs=3,
    per_device_train_batch_size=4,
    save_steps=10_000,
    save_total_limit=2,
    learning_rate=1e-4,
    fp16=True,
    logging_steps=100,
    evaluation_strategy="steps",
    eval_steps=5_000,
    load_best_model_at_end=True
)
# 創(chuàng)建數(shù)據(jù)集
from datasets import load_dataset
dataset = load_dataset("text", data_files={"train": "cleaned_zh_data.txt"})
# 開始訓(xùn)練
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=dataset["train"],
    tokenizer=tokenizer
)
trainer.train()

from transformers import DataCollatorForLanguageModeling
# 加載預(yù)訓(xùn)練模型
model = AutoModelForCausalLM.from_pretrained("./output/checkpoint-final")
tokenizer = AutoTokenizer.from_pretrained("llama_7b_zh_extended")
# 準(zhǔn)備數(shù)據(jù)集
def preprocess_function(examples):
    texts = []
    for inst in examples["text"]:
        try:
            data = json.loads(inst)
            text = f"指令：{data['instruction']}\n輸入：{data['input']}\n輸出：{data['output']}"
            texts.append(text)
        except:
            continue
    return tokenizer(texts, truncatinotallow=True, max_length=512)
dataset = load_dataset("json", data_files={"train": "instruction_data.json"})
tokenized_dataset = dataset.map(preprocess_function, batched=True)
# 數(shù)據(jù)收集器
data_collator = DataCollatorForLanguageModeling(
    tokenizer=tokenizer, 
    mlm=False
)
# 微調(diào)參數(shù)
training_args = TrainingArguments(
    output_dir="./sft_output",
    per_device_train_batch_size=4,
    num_train_epochs=5,
    learning_rate=5e-5,
    fp16=True,
    save_steps=10_000,
    logging_steps=100,
    evaluation_strategy="no",
)
# 開始微調(diào)
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_dataset["train"],
    data_collator=data_collator,
    tokenizer=tokenizer
)
trainer.train()

from transformers import BitsAndBytesConfig
quantization_config = BitsAndBytesConfig(
    load_in_8bit=True,
    llm_int8_threshold=6.0
)
model = AutoModelForCausalLM.from_pretrained(
    "llama_7b_hf",
    quantization_cnotallow=quantization_config
)