pip install trulens_eval

# 導(dǎo)入LangChain和TruLens
from langchain.chains import LLMChain
from langchain.llms import OpenAI
from langchain.prompts import PromptTemplate
from langchain.prompts.chat import ChatPromptTemplate,HumanMessagePromptTemplate
from trulens_eval import TruChain,Feedback, Huggingface, Tru, OpenAI as TruOpenAI
from trulens_eval.feedback.provider.langchain import Langchain

tru = Tru()

# 定義一個(gè)問(wèn)答應(yīng)用的提示模板
full_prompt = HumanMessagePromptTemplate(
    prompt=PromptTemplate(
        template=
        "Provide a helpful response with relevant background information for the following: {prompt}",
        input_variables=["prompt"],
    )
)

chat_prompt_template = ChatPromptTemplate.from_messages([full_prompt])



# 創(chuàng)建一個(gè)LLMChain對(duì)象，使用llm和chat_prompt_template作為參數(shù)
llm = OpenAI()
chain = LLMChain(llm=llm, prompt=chat_prompt_template, verbose=True)



# Initialize Huggingface-based feedback function collection class:
# Define a language match feedback function using HuggingFace.
hugs = Huggingface()
f_lang_match = Feedback(hugs.language_match).on_input_output()
# Question/answer relevance between overall question and answer.
provider = TruOpenAI()
f_qa_relevance = Feedback(provider.relevance).on_input_output()

# 使用TruChain類來(lái)包裝chain對(duì)象，指定反饋函數(shù)和應(yīng)用ID
tru_recorder = TruChain(
    chain,
    app_id='Chain1_QAApplication',
    feedbacks=[f_lang_match,f_qa_relevance])


# 使用with語(yǔ)句來(lái)運(yùn)行chain對(duì)象，并記錄反饋數(shù)據(jù)
with tru_recorder as recording:
    # 輸入一個(gè)問(wèn)題，得到一個(gè)回答
    chain("What is langchain?")
    # 查看反饋數(shù)據(jù)
    tru_record = recording.records[0]
    # 打印反饋數(shù)據(jù)
    print("tru_record:",tru_record)
# 啟動(dòng)tru展示控制臺(tái)
tru.run_dashboard()

# 導(dǎo)入LangChain和TruLens
from IPython.display import JSON

# Imports main tools:
from trulens_eval import TruChain, Feedback, Huggingface, Tru
from trulens_eval.schema import FeedbackResult
tru = Tru()
tru.reset_database()

# Imports from langchain to build app
import bs4
from langchain import hub
from langchain.chat_models import ChatOpenAI
from langchain.document_loaders import WebBaseLoader
from langchain.embeddings import OpenAIEmbeddings
from langchain.schema import StrOutputParser
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.vectorstores import Chroma
from langchain_core.runnables import RunnablePassthrough
from trulens_eval.feedback.provider import OpenAI
import numpy as np
from trulens_eval.app import App
from trulens_eval.feedback import Groundedness


# 加載文件
loader = WebBaseLoader(
    web_paths=("https://lilianweng.github.io/posts/2023-06-23-agent/",),
    bs_kwargs=dict(
        parse_notallow=bs4.SoupStrainer(
            class_=("post-content", "post-title", "post-header")
        )
    ),
)
docs = loader.load()
# 分詞
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
splits = text_splitter.split_documents(docs)
# 存入到向量數(shù)據(jù)庫(kù)
vectorstore = Chroma.from_documents(documents=splits, embedding=OpenAIEmbeddings(
))
# 定義一個(gè)RAG Chain

retriever = vectorstore.as_retriever()

prompt = hub.pull("rlm/rag-prompt")
llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)

def format_docs(docs):
    return "\n\n".join(doc.page_content for doc in docs)

rag_chain = (
    {"context": retriever | format_docs, "question": RunnablePassthrough()}
    | prompt
    | llm
    | StrOutputParser()
)
# 使用TruChain類來(lái)包裝rag對(duì)象，指定反饋函數(shù)和應(yīng)用ID
# Initialize provider class
provider = OpenAI()
# select context to be used in feedback. the location of context is app specific.
context = App.select_context(rag_chain)
grounded = Groundedness(groundedness_provider=provider)
# f_context_relevance, f_groundness, f_answer_relevance 定義反饋函數(shù)
# Define a groundedness feedback function
f_groundedness = (
    Feedback(grounded.groundedness_measure_with_cot_reasons)
    .on(context.collect()) # collect context chunks into a list
    .on_output()
    .aggregate(grounded.grounded_statements_aggregator)
)

# Question/answer relevance between overall question and answer.
f_qa_relevance = Feedback(provider.relevance).on_input_output()
# Question/statement relevance between question and each context chunk.
f_context_relevance = (
    Feedback(provider.qs_relevance)
    .on_input()
    .on(context)
    .aggregate(np.mean)
    )
# 使用with語(yǔ)句來(lái)運(yùn)行rag對(duì)象，并記錄反饋數(shù)據(jù)
tru_recorder = TruChain(rag_chain,
    app_id='Chain1_ChatApplication',
    feedbacks=[f_qa_relevance, f_context_relevance, f_groundedness])

with tru_recorder as recording:
    # 輸入一個(gè)問(wèn)題，得到一個(gè)回答，以及檢索出的文檔
    llm_response = rag_chain.invoke("What is Task Decomposition?")
    # 查看反饋數(shù)據(jù)
    rec = recording.get() # use .get if only one record
    # 打印反饋數(shù)據(jù)
    print(rec)
# 啟動(dòng)tru展示控制臺(tái)
tru.run_dashboard()