%%capture
# 該魔法命令用于在 Jupyter Notebook中隱藏輸出
# 避免輸出干擾執(zhí)行結果，例如 `pip install` 過程中的信息


# 安裝 Unsloth 和 vLLM 這兩個庫
!pip install unsloth vllm


# 升級 Pillow（Python Imaging Library），確保使用最新版本
!pip install --upgrade pillow

from unsloth import FastLanguageModel, PatchFastRL  
# 從 unsloth 庫中導入 FastLanguageModel（加速的語言模型）和 PatchFastRL（加速補?。? 


PatchFastRL("GRPO", FastLanguageModel)  
# 對 FastLanguageModel 應用 "GRPO" 方案的補丁優(yōu)化，以提高推理效率

from unsloth import is_bfloat16_supported  # 檢測是否支持 bfloat16（更節(jié)省顯存的數(shù)值格式）
import torch  # 導入 PyTorch 進行深度學習計算


# 設置模型參數(shù)
max_seq_length = 1024  # 最大序列長度，可以增大以支持更長的推理
lora_rank = 64  # LoRA 低秩適配矩陣的秩，較大值提高智能性但降低推理速度


# 加載 FastLanguageModel 預訓練模型
model, tokenizer = FastLanguageModel.from_pretrained(
    model_name="Qwen/Qwen2.5-3B-Instruct",  # 使用 Qwen2.5-3B-Instruct 作為基礎模型
    max_seq_length=max_seq_length,  # 設置最大序列長度
    load_in_4bit=True,  # 使用 4-bit 量化（減少顯存消耗），若為 False 則使用 16-bit LoRA
    fast_inference=True,  # 啟用 vLLM 加速推理（優(yōu)化并行計算）
    max_lora_rank=lora_rank,  # 設置 LoRA 適配的秩
    gpu_memory_utilization=0.5,  # 限制 GPU 內(nèi)存占用，降低該值可減少顯存溢出
)


# 為模型應用 PEFT（參數(shù)高效微調(diào)）和 LoRA（低秩適配）
model = FastLanguageModel.get_peft_model(
    model,
    r=lora_rank,  # LoRA 低秩矩陣的秩，可選 8, 16, 32, 64, 128
    target_modules=[
        "q_proj", "k_proj", "v_proj", "o_proj",  # 量化關鍵模塊：QKV 投影和輸出投影
        "gate_proj", "up_proj", "down_proj",  # MLP 相關模塊
    ],  # 若顯存不足，可以移除 QKVO 相關模塊
    lora_alpha=lora_rank,  # LoRA 適配器的 alpha 參數(shù)，通常設為與 r 相同
    use_gradient_checkpointing="unsloth",  # 啟用梯度檢查點，減少顯存占用，適用于長上下文微調(diào)
    random_state=3407,  # 設置隨機種子，保證實驗可復現(xiàn)
)

# 輸出 
==((====))==  Unsloth 2025.2.12: Fast Qwen2 patching. Transformers: 4.48.3.
   \\   /|    GPU: Tesla T4. Max memory: 14.741 GB. Platform: Linux.
O^O/ \_/ \    Torch: 2.5.1+cu124. CUDA: 7.5. CUDA Toolkit: 12.4. Triton: 3.1.0
\        /    Bfloat16 = FALSE. FA [Xformers = 0.0.28.post3. FA2 = False]
 "-____-"     Free Apache license: http://github.com/unslothai/unsloth
Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!
Unsloth: vLLM loading unsloth/qwen2.5-3b-instruct-unsloth-bnb-4bit with actual GPU utilization = 49.66%
Unsloth: Your GPU has CUDA compute capability 7.5 with VRAM = 14.74 GB.
Unsloth: Using conservativeness = 1.0. Chunked prefill tokens = 1024. Num Sequences = 192.
Unsloth: vLLM's KV Cache can use up to 4.9 GB. Also swap space = 2 GB.
WARNING 02-17 07:17:06 config.py:2386] Casting torch.bfloat16 to torch.float16.
INFO 02-17 07:17:19 config.py:542] This model supports multiple tasks: {'classify', 'generate', 'embed', 'reward', 'score'}. Defaulting to 'generate'.
Unsloth: vLLM Bitsandbytes config using kwargs = {'load_in_8bit': False, 'load_in_4bit': True, 'bnb_4bit_compute_dtype': 'float16', 'bnb_4bit_quant_storage': 'uint8', 'bnb_4bit_quant_type': 'nf4', 'bnb_4bit_use_double_quant': True, 'llm_int8_enable_fp32_cpu_offload': False, 'llm_int8_has_fp16_weight': False, 'llm_int8_skip_modules': ['lm_head', 'multi_modal_projector', 'merger', 'modality_projection', 'model.layers.2.mlp', 'model.layers.3.mlp', 'model.layers.30.mlp'], 'llm_int8_threshold': 6.0}
INFO 02-17 07:17:19 llm_engine.py:234] Initializing a V0 LLM engine (v0.7.2) with config: model='unsloth/qwen2.5-3b-instruct-unsloth-bnb-4bit', speculative_cnotallow=None, tokenizer='unsloth/qwen2.5-3b-instruct-unsloth-bnb-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revisinotallow=None, override_neuron_cnotallow=None, tokenizer_revisinotallow=None, trust_remote_code=False, dtype=torch.float16, max_seq_len=1024, download_dir=None, load_format=LoadFormat.BITSANDBYTES, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantizatinotallow=bitsandbytes, enforce_eager=False, kv_cache_dtype=auto,  device_cnotallow=cuda:0, decoding_cnotallow=DecodingConfig(guided_decoding_backend='xgrammar'), observability_cnotallow=ObservabilityConfig(otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=0, served_model_name=unsloth/qwen2.5-3b-instruct-unsloth-bnb-4bit, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=True, chunked_prefill_enabled=False, use_async_output_proc=True, disable_mm_preprocessor_cache=False, mm_processor_kwargs=None, pooler_cnotallow=None, compilation_cnotallow={"level":0,"splitting_ops":[],"compile_sizes":[],"cudagraph_capture_sizes":[192,184,176,168,160,152,144,136,128,120,112,104,96,88,80,72,64,56,48,40,32,24,16,8,4,2,1],"max_capture_size":192}, use_cached_outputs=False, 
tokenizer_config.json:?100%
?7.36k/7.36k?[00:00<00:00,?451kB/s]
vocab.json:?100%
?2.78M/2.78M?[00:00<00:00,?8.77MB/s]
merges.txt:?100%
?1.67M/1.67M?[00:00<00:00,?6.92MB/s]
tokenizer.json:?100%
?11.4M/11.4M?[00:00<00:00,?42.1MB/s]
added_tokens.json:?100%
?605/605?[00:00<00:00,?37.9kB/s]
special_tokens_map.json:?100%
?614/614?[00:00<00:00,?54.5kB/s]
generation_config.json:?100%
?271/271?[00:00<00:00,?17.2kB/s]
INFO 02-17 07:17:23 cuda.py:179] Cannot use FlashAttention-2 backend for Volta and Turing GPUs.
INFO 02-17 07:17:23 cuda.py:227] Using XFormers backend.
INFO 02-17 07:17:24 model_runner.py:1110] Starting to load model unsloth/qwen2.5-3b-instruct-unsloth-bnb-4bit...
INFO 02-17 07:17:24 loader.py:1102] Loading weights with BitsAndBytes quantization.  May take a while ...
INFO 02-17 07:17:25 weight_utils.py:252] Using model weights format ['*.safetensors']
model.safetensors:?100%
?2.36G/2.36G?[00:15<00:00,?165MB/s]
Loading?safetensors?checkpoint?shards:?100%?Completed?|?1/1?[00:04<00:00,??4.67s/it]
Loading?safetensors?checkpoint?shards:?100%?Completed?|?1/1?[00:01<00:00,??1.17s/it]
INFO 02-17 07:17:48 model_runner.py:1115] Loading model weights took 2.2160 GB
INFO 02-17 07:17:48 punica_selector.py:18] Using PunicaWrapperGPU.
INFO 02-17 07:17:58 worker.py:267] Memory profiling takes 9.77 seconds
INFO 02-17 07:17:58 worker.py:267] the current vLLM instance can use total_gpu_memory (14.74GiB) x gpu_memory_utilization (0.50) = 7.32GiB
INFO 02-17 07:17:58 worker.py:267] model weights take 2.22GiB; non_torch_memory takes 0.05GiB; PyTorch activation peak memory takes 1.05GiB; the rest of the memory reserved for KV Cache is 4.01GiB.
INFO 02-17 07:17:58 executor_base.py:110] # CUDA blocks: 7300, # CPU blocks: 3640
INFO 02-17 07:17:58 executor_base.py:115] Maximum concurrency for 1024 tokens per request: 114.06x
INFO 02-17 07:18:01 model_runner.py:1434] Capturing cudagraphs for decoding. This may lead to unexpected consequences if the model is not static. To run the model in eager mode, set 'enforce_eager=True' or use '--enforce-eager' in the CLI. If out-of-memory error occurs during cudagraph capture, consider decreasing `gpu_memory_utilization` or switching to eager mode. You can also reduce the `max_num_seqs` as needed to decrease memory usage.
Capturing CUDA graph shapes: 100%|██████████| 27/27 [00:43<00:00,  1.60s/it]
INFO 02-17 07:18:44 model_runner.py:1562] Graph capturing finished in 43 secs, took 0.62 GiB
INFO 02-17 07:18:44 llm_engine.py:431] init engine (profile, create kv cache, warmup model) took 56.33 seconds
tokenizer_config.json:?100%
?7.36k/7.36k?[00:00<00:00,?650kB/s]
vocab.json:?100%
?2.78M/2.78M?[00:00<00:00,?14.2MB/s]
merges.txt:?100%
?1.67M/1.67M?[00:00<00:00,?26.5MB/s]
added_tokens.json:?100%
?605/605?[00:00<00:00,?45.0kB/s]
special_tokens_map.json:?100%
?614/614?[00:00<00:00,?32.3kB/s]
tokenizer.json:?100%
?11.4M/11.4M?[00:00<00:00,?40.2MB/s]
Unsloth 2025.2.12 patched 36 layers with 36 QKV layers, 36 O layers and 36 MLP layers.

import re
from datasets import load_dataset, Dataset
# Load and prep dataset
SYSTEM_PROMPT = """
Respond in the following format:
<reasoning>
...
</reasoning>
<answer>
...
</answer>
"""
def extract_xml_answer(text: str) -> str:
    """提取 <answer> 標簽內(nèi)的內(nèi)容"""
    match = re.search(r"<answer>(.*?)</answer>", text, re.DOTALL)
    return match.group(1).strip() if match else text.strip()
def get_medical_questions(split="train") -> Dataset:
    """加載 FreedomIntelligence/medical-o1-reasoning-SFT 數(shù)據(jù)集并格式化"""
    data = load_dataset("FreedomIntelligence/medical-o1-reasoning-SFT", 'zh')[split]  
    def format_example(x):
        xml_answer = f"""\
<reasoning>
{x['Complex_CoT'].strip()}
</reasoning>
<answer>
{x['Response'].strip()}
</answer>"""
        
        return {
            'prompt': [
                {'role': 'system', 'content': SYSTEM_PROMPT},
                {'role': 'user', 'content': x['Question']}
            ],
            'answer': extract_xml_answer(xml_answer)  # 確保解析正確答案
        }
    data = data.map(format_example)
    return data
# 加載數(shù)據(jù)集
dataset = get_medical_questions()
# 獎勵函數(shù)
def correctness_reward_func(prompts, completions, answer, **kwargs) -> list[float]:
    """檢查模型輸出的答案是否正確"""
    responses = [completion[0]['content'] for completion in completions]
    extracted_responses = [extract_xml_answer(r) for r in responses]
    return [2.0 if r == a else 0.0 for r, a in zip(extracted_responses, answer)]
def int_reward_func(completions, **kwargs) -> list[float]:
    """檢查答案是否為整數(shù)"""
    responses = [completion[0]['content'] for completion in completions]
    extracted_responses = [extract_xml_answer(r) for r in responses]
    return [0.5 if r.isdigit() else 0.0 for r in extracted_responses]
def strict_format_reward_func(completions, **kwargs) -> list[float]:
    """嚴格格式檢查：必須有換行符"""
    pattern = r"^<reasoning>\n.*?\n</reasoning>\n<answer>\n.*?\n</answer>\n$"
    responses = [completion[0]["content"] for completion in completions]
    return [0.5 if re.match(pattern, r) else 0.0 for r in responses]
def soft_format_reward_func(completions, **kwargs) -> list[float]:
    """寬松格式檢查：允許不嚴格換行"""
    pattern = r"<reasoning>.*?</reasoning>\s*<answer>.*?</answer>"
    responses = [completion[0]["content"] for completion in completions]
    return [0.5 if re.match(pattern, r) else 0.0 for r in responses]
def count_xml(text) -> float:
    """檢查 XML 結構完整性"""
    count = 0.0
    if text.count("<reasoning>\n") == 1:
        count += 0.125
    if text.count("\n</reasoning>\n") == 1:
        count += 0.125
    if text.count("\n<answer>\n") == 1:
        count += 0.125
        count -= len(text.split("\n</answer>\n")[-1]) * 0.001
    if text.count("\n</answer>") == 1:
        count += 0.125
        count -= (len(text.split("\n</answer>")[-1]) - 1) * 0.001
    return count
def xmlcount_reward_func(completions, **kwargs) -> list[float]:
    """計算 XML 結構完整性分數(shù)"""
    contents = [completion[0]["content"] for completion in completions]
    return [count_xml(c) for c in contents]

#輸出
medical_o1_sft_Chinese.json:?100%
?64.8M/64.8M?[00:00<00:00,?120MB/s]
Generating?train?split:?100%
?24772/24772?[00:01<00:00,?12662.63?examples/s]
Map:?100%
?24772/24772?[00:04<00:00,?5735.65?examples/s]

from trl import GRPOConfig, GRPOTrainer  # 導入 GRPO 訓練配置和訓練器


# 設置 GRPO（General Reinforcement Preference Optimization）的訓練參數(shù)
training_args = GRPOConfig(
    use_vllm=True,  # 啟用 vLLM 進行推理加速，提高訓練效率
    learning_rate=5e-6,  # 設置學習率，較小值防止梯度爆炸
    adam_beta1=0.9,  # AdamW 優(yōu)化器的 beta1 參數(shù)（動量項）
    adam_beta2=0.99,  # AdamW 優(yōu)化器的 beta2 參數(shù)（平方梯度平滑項）
    weight_decay=0.1,  # 權重衰減，防止過擬合
    warmup_ratio=0.1,  # 預熱步數(shù)比例，避免初始學習率過高導致的不穩(wěn)定
    lr_scheduler_type="cosine",  # 采用余弦退火學習率調(diào)度
    optim="adamw_8bit",  # 使用 `8-bit AdamW` 優(yōu)化器，節(jié)省顯存
    logging_steps=1,  # 每 1 步記錄日志
    bf16=is_bfloat16_supported(),  # 如果支持 bfloat16，則使用 bfloat16 精度
    fp16=not is_bfloat16_supported(),  # 如果不支持 bfloat16，則使用 fp16 精度
    per_device_train_batch_size=1,  # 每個設備的批量大小，顯存不足時設為 1
    gradient_accumulation_steps=1,  # 梯度累積步數(shù)，增加到 4 可以平滑訓練
    num_generations=8,  # 生成的樣本數(shù)，減少該值可降低顯存占用
    max_prompt_length=256,  # 最大輸入提示長度
    max_completion_length=200,  # 最大模型輸出長度
    # num_train_epochs=1,  # 訓練 1 輪，實際使用時可以調(diào)整
    max_steps=10,  # 訓練的最大步數(shù)（短時間測試用）正常為250
    save_steps=10,  # 每 10 步保存一次模型 正常為250
    max_grad_norm=0.1,  # 梯度裁剪，防止梯度爆炸
    report_to="none",  # 訓練日志不上傳到 W&B（Weights & Biases）
    output_dir="outputs",  # 訓練結果的保存目錄
)

trainer = GRPOTrainer(
    model=model,  # 使用 FastLanguageModel 加載的 Qwen2.5-3B-Instruct
    processing_class=tokenizer,  # 令牌化工具（Tokenizer）
    reward_funcs=[  # 定義獎勵函數(shù)
        xmlcount_reward_func,  # XML 結構完整性評分
        soft_format_reward_func,  # 寬松格式檢查
        strict_format_reward_func,  # 嚴格格式檢查
        int_reward_func,  # 評估答案是否為整數(shù)
        correctness_reward_func,  # 檢查答案是否正確
    ],
    args=training_args,  # 訓練參數(shù)配置
    train_dataset=dataset,  # 訓練數(shù)據(jù)集（醫(yī)學問答數(shù)據(jù)）
)


trainer.train()  # 運行 GRPO 強化學習訓練

==((====))==  Unsloth - 2x faster free finetuning | Num GPUs = 1
   \\   /|    Num examples = 24,772 | Num Epochs = 1
O^O/ \_/ \    Batch size per device = 8 | Gradient Accumulation steps = 1
\        /    Total batch size = 8 | Total steps = 10
 "-____-"     Number of trainable parameters = 119,734,272
 [10/10 03:46, Epoch 0/1]
Step
Training Loss
reward
reward_std
completion_length
kl
rewards / xmlcount_reward_func
rewards / soft_format_reward_func
rewards / strict_format_reward_func
rewards / int_reward_func
rewards / correctness_reward_func
....因輸出太長，此處省略
TrainOutput(global_step=10, training_loss=2.103237093251664e-05, metrics={'train_runtime': 254.9268, 'train_samples_per_second': 0.314, 'train_steps_per_second': 0.039, 'total_flos': 0.0, 'train_loss': 2.103237093251664e-05})

text = tokenizer.apply_chat_template([
    {"role" : "user", "content" : "一個10歲男孩在患膿皮癥后出現(xiàn)眼瞼水腫、尿少和高血壓等癥狀，作為醫(yī)生，你認為首選的治療藥物是什么?"
},
], tokenize = False, add_generation_prompt = True)
from vllm import SamplingParams
sampling_params = SamplingParams(
    temperature = 0.8,
    top_p = 0.95,
    max_tokens = 1024,
)
output = model.fast_generate(
    [text],
    sampling_params = sampling_params,
    lora_request = None,
)[0].outputs[0].text
output

# 回答
Processed prompts: 100%|██████████| 1/1 [00:08<00:00,  8.94s/it, est. speed input: 6.94 toks/s, output: 37.73 toks/s]
'
膿皮癥（也稱為膿皰瘡或膿皰?。┦且环N由細菌感染引起的皮膚感染，雖然膿皮癥與眼瞼水腫、尿少和高血壓等癥狀聯(lián)系不大，但是這些癥狀可能提示存在其他問題，如膿皮癥并發(fā)了其他并發(fā)癥，或者出現(xiàn)了膿皮癥以外的其他疾病。因此，首先需要通過詳細的病史詢問、體檢、實驗室檢查（如血液檢查、尿液檢查等）和影像學檢查（如必要時進行）來明確診斷。\n\n如果確診為膿皮癥，治療通常依賴抗生素。對于10歲男孩的膿皮癥，首選的治療藥物通常是：\n\n1. **青霉素類**：對于大多數(shù)膿皮癥患者，首選青霉素類抗生素治療（如阿莫西林），尤其是對于兒童而言。\n2. **大環(huán)內(nèi)酯類**：如果患者對青霉素過敏，可以考慮使用大環(huán)內(nèi)酯類抗生素（如紅霉素）。\n\n如果出現(xiàn)眼瞼水腫、尿少和高血壓等癥狀，可能需要更進一步的評估和治療，這可能包括：\n\n- **利尿劑**：用于減少體內(nèi)液體積聚，減輕水腫。\n- **血管緊張素轉換酶抑制劑（ACEI）或血管緊張素受體拮抗劑（ARB）**：用于降低血壓，尤其是如果是因為腎臟受損導致的高血壓時。\n\n需要注意的是，這些癥狀可能提示存在并發(fā)癥或其它系統(tǒng)性疾病，因此，必須首先由專業(yè)的醫(yī)療人員進行評估和治療。請務必咨詢專業(yè)醫(yī)生，獲得針對具體情況的專業(yè)治療建議。在沒有得到專業(yè)醫(yī)生指導的情況下，不要自行給孩子使用任何藥物。

model.save_lora("grpo_saved_lora")

text = tokenizer.apply_chat_template([
    {"role" : "system", "content" : SYSTEM_PROMPT},
    {"role" : "user", "content" : "一個10歲男孩在患膿皮癥后出現(xiàn)眼瞼水腫、尿少和高血壓等癥狀，作為醫(yī)生，你認為首選的治療藥物是什么?"},
], tokenize = False, add_generation_prompt = True)
from vllm import SamplingParams
sampling_params = SamplingParams(
    temperature = 0.8,
    top_p = 0.95,
    max_tokens = 1024,
)
output = model.fast_generate(
    text,
    sampling_params = sampling_params,
    lora_request = model.load_lora("grpo_saved_lora"),
)[0].outputs[0].text
output

#回答
Processed prompts: 100%|██████████| 1/1 [00:09<00:00,  9.17s/it, est. speed input: 7.42 toks/s, output: 34.04 toks/s]
'
<reasoning>\n根據(jù)病史描述，這個10男孩出現(xiàn)眼瞼水腫、
尿少和高血壓的癥狀，而膿皮癥是引起這些癥狀的一個可能的原因，
但是最應該考慮的是水腫和高血壓可能與感染相關的系統(tǒng)性炎癥反應有關。
在這種情況下，首選的治療藥物需要考慮控制感染（膿皮癥）、
控制炎癥反應以及處理高血壓。考慮到患兒的年齡，
首選的藥物應該是安全的并且可以兼顧上述三個問題。
\n\n1. 抗生素：膿皮癥是一種細菌感染，需要使用抗生素進行治療。對于10歲的小男孩，青霉素類藥物通常是首選，因為這類抗生素對兒童來說副作用較小。
\n\n2. 非甾體抗炎藥（NSAIDs）：這類藥物可以減輕炎癥反應，幫助控制膿皮癥。例如布洛芬，它相對安全，副作用較少，適用于兒童。
\n\n3. 抗高血壓藥物：如果伴有高血壓，可能需要使用抗高血壓藥物，例如ACE抑制劑或者鈣通道阻滯劑。然而，考慮到10歲兒童的特殊性，這些藥物的使用需要謹慎，并且需要兒科醫(yī)生的指導。
\n\n綜上所述，治療首選藥物應該是青霉素類抗生素來控制膿皮癥，并且根據(jù)病情可能還需要使用NSAIDs以及兒科醫(yī)生指導下可能需要的抗高血壓藥物。
\n...\n<answer>\n首選治療藥物是青霉素類抗生素來控制膿皮癥。其他可能需要的藥物包括NSAIDs和抗高血壓藥物，但是需要兒科醫(yī)生指導。