import faiss
import numpy as np
# Set up HNSW parameters
d = 128  # Size of the vectors
M = 32   # Number of neighbors for each nodel

# Initialize the HNSW index
index = faiss.IndexHNSWFlat(d, M)

efConstruction = 200  # Example value for efConstruction
index.hnsw.efConstruction = efConstruction

# Generate random dataset of vectors
n = 10000  # Number of vectors to index
xb = np.random.random((n, d)).astype('float32')
# Add data to the index
index.add(xb)  # Build the index

efSearch = 100  # Example value for efSearch
index.hnsw.efSearch = efSearch

# Generate random query vectors
nq = 5  # Number of query vectors
xq = np.random.random((nq, d)).astype('float32')
# Perform a search for the top k nearest neighbors
k = 5  # Number of nearest neighbors to retrieve
distances, indices = index.search(xq, k)
# Output the results
print("Query Vectors:\n", xq)
print("\nNearest Neighbors Indices:\n", indices)
print("\nNearest Neighbors Distances:\n", distances)

查詢向量：

 [[0.12345678 0.23456789 ... 0.98765432] 
 [0.23456789 0.34567890 ... 0.87654321] 
 [0.34567890 0.45678901 ... 0.76543210] 
 [0.45678901 0.56789012 ... 0.65432109] 
 [0.56789012 0.67890123 ... 0.54321098]]

最近鄰索引：

 [[123 456 789 101 112] 
 [234 567 890 123 134] 
 [345 678 901 234 245] 
 [ 456 789 012 345 356] 
 [ 567 890 123 456 467]]

最近鄰距離：

 [[0.123 0.234 0.345 0.456 0.567] 
 [0.234 0.345 0.456 0.567 0.678] 
 [0.345 0.456 0.567 0.678 0.789] 
 [0.456 0.567 0.678 0.789 0.890] 
 [0.567 0.678 0.789 0.890 0.901]]

from langchain_experimental.text_splitter import SemanticChunker
from langchain_openai.embeddings import OpenAIEmbeddings
text_splitter = SemanticChunker(OpenAIEmbeddings())
docs = text_splitter.create_documents([document])
print(docs[0].page_content)

async def generate_contexts(document, chunks):
   asyncdef process_chunk(chunk):
       response = await client.chat.completions.create(
           model="gpt-4o",
           messages=[
               {"role": "system", "content": "Generate a brief context explaining how this chunk relates to the full document."},
               {"role": "user", "content": f"<document> \n{document} \n</document> \nHere is the chunk we want to situate within the whole document \n<chunk> \n{chunk} \n</chunk> \nPlease give a short succinct context to situate this chunk within the overall document for the purposes of improving search retrieval of the chunk. Answer only with the succinct context and nothing else."}
           ],
           temperature=0.3,
           max_tokens=100
       )
       context = response.choices[0].message.content
       returnf"{context} {chunk}"
   # Process all chunks concurrently
   contextual_chunks = await asyncio.gather(
       *[process_chunk(chunk) for chunk in chunks]
   )
   return contextual_chunks

Source Metadata:  {'id': 'doc:1c6f3e3f7ee14027bc856822871572dc:26e9aac7d5494208a56ff0c6cbbfda20', 'source': 'https://plato.stanford.edu/entries/goedel/'}

2.2.1 The First Incompleteness Theorem
In his Logical Journey (Wang 1996) Hao Wang published the
full text of material G?del had written (at Wang’s request)
about his discovery of the incompleteness theorems. This material had
formed the basis of Wang’s “Some Facts about Kurt
G?del,” and was read and approved by G?del:

Source Metadata:  {'id': 'doc:1c6f3e3f7ee14027bc856822871572dc:d15f62c453c64072b768e136080cb5ba', 'source': 'https://plato.stanford.edu/entries/goedel/'}

The First Incompleteness Theorem provides a counterexample to
completeness by exhibiting an arithmetic statement which is neither
provable nor refutable in Peano arithmetic, though true in the
standard model. The Second Incompleteness Theorem shows that the
consistency of arithmetic cannot be proved in arithmetic itself. Thus
G?del’s theorems demonstrated the infeasibility of the
Hilbert program, if it is to be characterized by those particular
desiderata, consistency and completeness.

from langchain_community.retrievers import WeaviateHybridSearchRetriever
from langchain_core.documents import Document
retriever = WeaviateHybridSearchRetriever(
   client=client,
   index_name="LangChain",
   text_key="text",
   attributes=[],
   create_schema_if_missing=True,
)
retriever.invoke("the ethical implications of AI")

from langchain_openai import ChatOpenAI
chatgpt = ChatOpenAI(model_name="gpt-4o", temperature=0)
from langchain.retrievers.multi_query import MultiQueryRetriever
# Set logging for the queries
import logging
similarity_retriever3 = chroma_db3.as_retriever(search_type="similarity",
                                               search_kwargs={"k": 2})
mq_retriever = MultiQueryRetriever.from_llm(
   retriever=similarity_retriever3, llm=chatgpt,
   include_original=True
)
logging.basicConfig()
# so we can see what queries are generated by the LLM
logging.getLogger("langchain.retrievers.multi_query").setLevel(logging.INFO)
query = "what is the capital of India?"
docs = mq_retriever.invoke(query)
docs

[Document(metadata={'article_id': '5117', 'title': 'New Delhi'},
 page_cnotallow='New Delhi () is the capital of India and a union territory of
 the megacity of Delhi. It has a very old history and is home to several
 monuments where the city is expensive to live in. In traditional Indian
 geography it falls under the North Indian zone. The city has an area of
 about 42.7\xa0km. New Delhi has a population of about 9.4 Million people.'),

 Document(metadata={'article_id': '4062', 'title': 'Kolkata'},
 page_cnotallow="Kolkata (spelled Calcutta before 1 January 2001) is the
 capital city of the Indian state of West Bengal. It is the second largest
 city in India after Mumbai. It is on the east bank of the River Hooghly.
 When it is called Calcutta, it includes the suburbs. This makes it the third
 largest city of India. This also makes it the world's 8th largest
 metropolitan area as defined by the United Nations. Kolkata served as the
 capital of India during the British Raj until 1911. Kolkata was once the
 center of industry and education. However, it has witnessed political
 violence and economic problems since 1954. Since 2000, Kolkata has grown due
 to economic growth. Like other metropolitan cities in India, Kolkata
 struggles with poverty, pollution and traffic congestion."),

 Document(metadata={'article_id': '22215', 'title': 'States and union
 territories of India'}, page_cnotallow='The Republic of India is divided into
 twenty-eight States,and eight union territories including the National
 Capital Territory.')]

from langchain.retrievers import ContextualCompressionRetriever
from langchain.retrievers.document_compressors import LLMChainExtractor
from langchain_openai import ChatOpenAI
# Initialize the language model
chatgpt = ChatOpenAI(model_name="gpt-4o", temperature=0)

# Set up a similarity retriever
similarity_retriever = chroma_db3.as_retriever(search_type="similarity", search_kwargs={"k": 3})

# Create the extractor to get relevant content
compressor = LLMChainExtractor.from_llm(llm=chatgpt)

# Combine the retriever and the extractor
compression_retriever = ContextualCompressionRetriever(base_compressor=compressor, base_retriever=similarity_retriever)

# Example query
query = "What is the capital of India?"
docs = compression_retriever.invoke(query)
print(docs)

[Document(metadata={'article_id': '5117', 'title': 'New Delhi'},
 page_cnotallow='New Delhi is the capital of India and a union territory of the 
megacity of Delhi.')]

query = "What is the old capital of India?"
docs = compression_retriever.invoke(query)
print(docs)

[Document(metadata={'article_id': '4062', 'title': 'Kolkata'},
 page_cnotallow='Kolkata served as the capital of India during the British Raj
 until 1911.')]

from langchain.retrievers.document_compressors import LLMChainFilter
# Set up the filter
_filter = LLMChainFilter.from_llm(llm=chatgpt)
# Combine the retriever and the filter
compression_retriever = ContextualCompressionRetriever(base_compressor=_filter, base_retriever=similarity_retriever)

# Example query
query = "What is the capital of India?"
docs = compression_retriever.invoke(query)
print(docs)

[Document(metadata={'article_id': '5117', 'title': 'New Delhi'},
 page_cnotallow='New Delhi is the capital of India and a union territory of the
 megacity of Delhi.')]

query = "What is the old capital of India?"
docs = compression_retriever.invoke(query)
print(docs)

[Document(metadata={'article_id': '4062', 'title': 'Kolkata'},
 page_cnotallow='Kolkata served as the capital of India during the British Raj
 until 1911.')]

from langchain_openai import OpenAIEmbeddings
from langchain_pinecone import PineconeVectorStore
from typing import List
from langchain_core.documents import Document
from langchain_core.runnables import chain
vectorstore = PineconeVectorStore.from_documents(
   docs, index_name="sample", embedding=OpenAIEmbeddings()
)
@chain
def retriever(query: str):
   docs, scores = zip(*vectorstore.similarity_search_with_score(query))
   for doc, score in zip(docs, scores):
       doc.metadata["score"] = score
   return docs
 result = retriever.invoke("dinosaur")
result

[Document(page_cnotallow='In her second book, Dr. Simmons delves deeper into
 the ethical considerations surrounding AI development and deployment. It is
 an eye-opening examination of the dilemmas faced by developers,
 policymakers, and society at large.', metadata={}),

 Document(page_cnotallow='A comprehensive analysis of the evolution of
 artificial intelligence, from its inception to its future prospects. Dr.
 Simmons covers ethical considerations, potentials, and threats posed by
 AI.', metadata={}),

 Document(page_cnotallow="In his follow-up to 'Symbiosis', Prof. Sterling takes
 a look at the subtle, unnoticed presence and influence of AI in our everyday
 lives. It reveals how AI has become woven into our routines, often without
 our explicit realization.", metadata={}),

 Document(page_cnotallow='Prof. Sterling explores the potential for harmonious
coexistence between humans and artificial intelligence. The book discusses
 how AI can be integrated into society in a beneficial and non-disruptive
manner.', metadata={})]

from langchain.retrievers import ContextualCompressionRetriever
from langchain.retrievers.document_compressors import FlashrankRerank
from langchain_openai import ChatOpenAI

llm = ChatOpenAI(temperature=0)

compressor = FlashrankRerank()
compression_retriever = ContextualCompressionRetriever(
   base_compressor=compressor, base_retriever=retriever
)
compressed_docs = compression_retriever.invoke(
   "What did the president say about Ketanji Jackson Brown"
)
print([doc.metadata["id"] for doc in compressed_docs])
pretty_print_docs(compressed_docs)

[0, 5, 3]

Document 1:

One of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court.

And I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence.

----------------------------------------------------------------------------------------------------

Document 2:

He met the Ukrainian people.

From President Zelenskyy to every Ukrainian, their fearlessness, their courage, their determination, inspires the world.

Groups of citizens blocking tanks with their bodies. Everyone from students to retirees teachers turned soldiers defending their homeland.

In this struggle as President Zelenskyy said in his speech to the European Parliament “Light will win over darkness.” The Ukrainian Ambassador to the United States is here tonight.

----------------------------------------------------------------------------------------------------

Document 3:

And tonight, I’m announcing that the Justice Department will name a chief prosecutor for pandemic fraud.

By the end of this year, the deficit will be down to less than half what it was before I took office.  

The only president ever to cut the deficit by more than one trillion dollars in a single year.

Lowering your costs also means demanding more competition.

I’m a capitalist, but capitalism without competition isn’t capitalism.

It’s exploitation—and it drives up prices.