import json
import time
import random
from zhipuai import ZhipuAI
# 初始化模型
client = ZhipuAI(api_key="48b393b3474e43f3b7eafceb662db81a.xxxxxx")
style_config = {
    "小白": {
        "system_prompt": "你是一位區(qū)塊鏈科普專家，需要用最簡單易懂的方式向完全不了解技術(shù)的普通人解釋區(qū)塊鏈概念。要求：\n1. 使用生活化比喻（如'像大家一起記賬本'）\n2. 避免專業(yè)術(shù)語，必須使用時立即解釋\n3. 回答控制在3句話以內(nèi)\n4. 可適當使用??????等表情符號",
        "examples": [
            {"role": "user", "content": "區(qū)塊鏈到底是什么？"},
            {"role": "assistant",
             "content": "想象一群人在共同維護一個公開賬本??，每筆交易都記錄在所有人的本子上，想作弊就得改掉大部分人的記錄，這幾乎不可能??！"},
            {"role": "user", "content": "比特幣為什么值錢？"},
            {"role": "assistant", "content": "就像限量版球星卡??，總量固定(2100萬枚)，越來越多人想要時價格就上漲啦！"}
        ],
        "temperature": 0.2  # 低溫度值保證回答穩(wěn)定性
    },
    "資深": {
        "system_prompt": "你是區(qū)塊鏈核心技術(shù)研究員，需滿足：\n1. 使用專業(yè)術(shù)語（如'橢圓曲線數(shù)字簽名''梅克爾帕特里夏樹'）\n2. 包含技術(shù)實現(xiàn)細節(jié)\n3. 必要時給出公式/代碼片段\n4. 引用最新論文成果（如2023 IEEE S&P會議）",
        "examples": [
            {"role": "user", "content": "請解釋zk-SNARKs的工作原理"},
            {"role": "assistant",
             "content": "zk-SNARKs依賴QAP(二次算術(shù)程序)將計算轉(zhuǎn)化為多項式關(guān)系：\n1) Setup生成CRS(公共參考串)\n2) Prover構(gòu)造π=(A,B,C)滿足A*B=C+H*Z\n3) Verifier通過雙線性配對驗證e(A,B)=e(C,g)?e(H,g^z)\n參考：Groth16方案優(yōu)化后驗證僅需3個配對運算。"},
            {"role": "user", "content": "分析EIP-4844對L2的影響"},
            {"role": "assistant",
             "content": "Proto-danksharding通過引入blob攜帶交易(每個blob≈125KB)，使Rollups數(shù)據(jù)可用性成本降低10-100倍。實測顯示：\n- Optimism交易費從$0.3→$0.007\n- zkSync證明生成時間減少40%"}
        ],
        "temperature": 0.5  # 中等溫度值允許一定創(chuàng)造性
    }
}
def generate_style_data(style_name, num_samples=50):
    config = style_config[style_name]
    data = []
    # 構(gòu)建對話上下文 = 系統(tǒng)提示 + 示例對話
    messages = [
        {"role": "system", "content": config["system_prompt"]},
        *config["examples"]
    ]
    # 預(yù)定義的用戶問題集（可擴展）
    user_inputs = [
        "比特幣和區(qū)塊鏈是什么關(guān)系？",
        "智能合約真的安全嗎？",
        "普通人怎么參與DeFi？",
        "解釋一下零知識證明",
        "為什么NFT能賣那么貴？",
        "區(qū)塊鏈能防止數(shù)據(jù)造假嗎？"
    ]
    for _ in range(num_samples):
        try:
            # 隨機選擇問題
            user_msg = random.choice(user_inputs)
            # 當前對話 = 歷史上下文 + 新問題
            current_messages = messages + [
                {"role": "user", "content": user_msg}
            ]
            # 調(diào)用API（修正模型名稱）
            response = client.chat.completions.create(
                model="glm-4-plus",
                messages=current_messages,
                temperature=config["temperature"],
                max_tokens=150
            )
            # 獲取回復(fù)內(nèi)容
            reply = response.choices[0].message.content
            data.append({
                "user": user_msg,
                "assistant": reply,
                "style": style_name
            })
            print("獲取最新回復(fù)內(nèi)容: ", data[-1])
            time.sleep(0.5)  # 限流
        except Exception as e:
            print(f"生成失?。簕str(e)}")
    return data
if __name__ == '__main__':
    all_data = []
    try:
        print("開始生成小白風(fēng)格數(shù)據(jù)...")
        gentle_data = generate_style_data("小白", 10)
        all_data.extend(gentle_data)
        print("開始生成資深風(fēng)格數(shù)據(jù)...")
        gentle_data = generate_style_data("資深", 10)
        all_data.extend(gentle_data)
    except KeyboardInterrupt:
        print("\n用戶中斷，保存已生成數(shù)據(jù)...")
    finally:
        with open("style_chat_data.json", "w", encoding="utf-8") as f:
            json.dump(all_data, f, ensure_ascii=False, indent=2)
        print(f"數(shù)據(jù)已保存，有效樣本數(shù)：{len(all_data)}")

import json
from typing import List, Dict


def is_meaningless(text: str) -> bool:
    """檢查是否是無意義內(nèi)容"""
    # 空白或超短內(nèi)容
    if len(text.strip()) < 5:
        return True

    # 常見無意義模式
    meaningless_phrases = [
        "我不知道", "無法回答",
        "這個問題", "請重新提問"
    ]
    return any(phrase in text for phrase in meaningless_phrases)


def simple_clean(data: List[Dict]) -> List[Dict]:
    """基礎(chǔ)清洗流程"""
    seen = set()  # 用于去重
    cleaned_data = []

    for item in data:
        text = item["assistant"].strip()

        # 檢查空白/重復(fù)/無意義
        if (not text or
                text in seen or
                is_meaningless(text)):
            continue

        seen.add(text)
        cleaned_data.append({
            "user": item["user"],
            "assistant": text,
            "style": item["style"]
        })

    return cleaned_data


if __name__ == '__main__':
    # 加載原始數(shù)據(jù)
    with open('style_chat_data.json', 'r', encoding='utf-8') as f:
        raw_data = json.load(f)

    # 執(zhí)行清洗
    cleaned_data = simple_clean(raw_data)

    # 保存結(jié)果
    with open('cleaned_data.json', 'w', encoding='utf-8') as f:
        json.dump(cleaned_data, f, ensure_ascii=False, indent=2)

    # 打印簡單報告
    print(f"原始數(shù)據(jù)量: {len(raw_data)}")
    print(f"清洗后保留: {len(cleaned_data)}")
    print(f"過濾數(shù)量: {len(raw_data) - len(cleaned_data)}")

import json
from sentence_transformers import SentenceTransformer
import numpy as np
def deduplicate_data(data_list, threshold=0.9):
    
"""
    語義去重：對 assistant 回復(fù)去重
    :param data_list: List[dict], 每個 dict 包含 user / assistant / style
    :param threshold: float, 相似度閾值
    :return: 去重后的 List[dict]
    """
    
model = SentenceTransformer("/root/autodl-tmp/models/thomas/text2vec-base-chinese")
    replies = [item["assistant"] for item in data_list]
    embeddings = model.encode(replies)
    kept = []
    seen = set()
    for i, emb_i in enumerate(embeddings):
        if i in seen:
            continue
        keep = True
        for j, emb_j in enumerate(embeddings):
            if j == i or j in seen:
                continue
            sim = np.dot(emb_i, emb_j) / (np.linalg.norm(emb_i) * np.linalg.norm(emb_j))
            print(f"sim({i}, {j}) = {sim:.4f}")
            if sim > threshold:
                seen.add(j)
        kept.append(data_list[i])
    return kept
if __name__ == '__main__':
    with open("style_chat_data.json", "r", encoding="utf-8") as f:
        all_data = json.loads(f.read())
        cleaned_data = deduplicate_data(all_data, 0.5)