from sklearn.feature_extraction.text import CountVectorizer
import pandas as pd

# 示例文本
documents = ["cat sits here", "dog barks loud", "cat barks loud"]

# 初始化 CountVectorizer
vectorizer = CountVectorizer(binary=True)

# 擬合并轉(zhuǎn)換文本數(shù)據(jù)
X = vectorizer.fit_transform(documents)

# 獲取特征名稱（唯一單詞）
feature_names = vectorizer.get_feature_names_out()

# 轉(zhuǎn)換為 DataFrame 以便更好地可視化
df = pd.DataFrame(X.toarray(), columns=feature_names)

# 打印獨(dú)熱編碼矩陣
print(df)

cat  barks  dog  here  loud  sits
0    1      0    0     1     0     1
1    0      1    1     0     1     0
2    1      1    0     0     1     0

from sklearn.preprocessing import OneHotEncoder

# 示例單詞
words = ["cat", "dog", "barks", "loud"]

# 初始化獨(dú)熱編碼器
encoder = OneHotEncoder(sparse=False)

# 擬合并轉(zhuǎn)換
encoded_words = encoder.fit_transform([[word] for word in words])

# 轉(zhuǎn)換為 DataFrame 以便更好地可視化
df_onehot = pd.DataFrame(encoded_words, columns=encoder.get_feature_names_out(['word']))

# 打印獨(dú)熱編碼矩陣
print(df_onehot)

word_cat  word_dog  word_barks  word_loud
0       1.0       0.0         0.0        0.0
1       0.0       1.0         0.0        0.0
2       0.0       0.0         1.0        0.0
3       0.0       0.0         0.0        1.0

from sklearn.feature_extraction.text import TfidfVectorizer

# 示例文本
documents = ["cat sits here", "dog barks loud", "cat barks loud"]

# 初始化 TfidfVectorizer
vectorizer = TfidfVectorizer()

# 擬合并轉(zhuǎn)換文本數(shù)據(jù)
X = vectorizer.fit_transform(documents)

# 獲取特征名稱（唯一單詞）
feature_names = vectorizer.get_feature_names_out()

# 轉(zhuǎn)換為 DataFrame 以便更好地可視化
df_tfidf = pd.DataFrame(X.toarray(), columns=feature_names)

# 打印 TF-IDF 矩陣
print(df_tfidf)

cat  barks  dog  here  loud  sits
0  0.5  0.0   0.0  0.5   0.0   0.5
1  0.0  0.7   0.7  0.0   0.7   0.0
2  0.5  0.7   0.0  0.0   0.7   0.0

import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer

# 示例文檔
documents = ["cat sits here", "dog barks loud", "cat barks loud"]

# 計(jì)算詞頻（TF）
vectorizer = CountVectorizer()
X = vectorizer.fit_transform(documents)
tf_matrix = X.toarray()
feature_names = vectorizer.get_feature_names_out()

# 計(jì)算逆文檔頻率（IDF）
N = len(documents)  # 文檔總數(shù)
df = np.sum(tf_matrix > 0, axis=0)  # 每個(gè)詞的文檔頻率
idf = np.log((N - df + 0.5) / (df + 0.5) + 1)  # BM25 IDF 公式

# 計(jì)算 BM25 分?jǐn)?shù)
k1 = 1.5  # 平滑參數(shù)
b = 0.75  # 長(zhǎng)度歸一化參數(shù)
avgdl = np.mean([len(doc.split()) for doc in documents])  # 平均文檔長(zhǎng)度
doc_lengths = np.array([len(doc.split()) for doc in documents])
bm25_matrix = np.zeros_like(tf_matrix, dtype=np.float64)

for i in range(N):  # 遍歷每個(gè)文檔
    for j in range(len(feature_names)):  # 遍歷每個(gè)詞
        term_freq = tf_matrix[i, j]
        num = term_freq * (k1 + 1)
        denom = term_freq + k1 * (1 - b + b * (doc_lengths[i] / avgdl))
        bm25_matrix[i, j] = idf[j] * (num / denom)

# 轉(zhuǎn)換為 DataFrame 以便更好地可視化
df_bm25 = pd.DataFrame(bm25_matrix, columns=feature_names)

# 打印 BM25 分?jǐn)?shù)矩陣
print(df_bm25)

cat  barks  dog  here  loud  sits
0  0.6  0.0   0.0  0.6   0.0   0.6
1  0.0  0.8   0.8  0.0   0.8   0.0
2  0.6  0.8   0.0  0.0   0.8   0.0

from gensim.models import Word2Vec

# 示例語(yǔ)料庫(kù)
sentences = [
    ["I", "love", "deep", "learning"],
    ["Natural", "language", "processing", "is", "fun"],
    ["Word2Vec", "is", "a", "great", "tool"],
    ["AI", "is", "the", "future"],
]

# 訓(xùn)練 CBOW 模型
cbow_model = Word2Vec(sentences, vector_size=10, window=2, min_count=1, sg=0)  # CBOW

# 訓(xùn)練 Skip-gram 模型
skipgram_model = Word2Vec(sentences, vector_size=10, window=2, min_count=1, sg=1)  # Skip-gram

# 獲取單詞向量
word = "is"
print(f"CBOW Vector for '{word}':\n", cbow_model.wv[word])
print(f"\nSkip-gram Vector for '{word}':\n", skipgram_model.wv[word])

# 獲取最相似的單詞
print("\nCBOW Most Similar Words:", cbow_model.wv.most_similar(word))
print("\nSkip-gram Most Similar Words:", skipgram_model.wv.most_similar(word))

CBOW Vector for 'is':
 [0.123, 0.456, 0.789, ...]

Skip-gram Vector for 'is':
 [0.987, 0.654, 0.321, ...]

CBOW Most Similar Words: [('learning', 0.85), ('fun', 0.80)]
Skip-gram Most Similar Words: [('tool', 0.90), ('future', 0.88)]

import gensim.downloader as api

# 加載預(yù)訓(xùn)練的 GloVe 模型
glove_model = api.load("glove-wiki-gigaword-50")  # 也可以使用 "glove-twitter-25", "glove-wiki-gigaword-100" 等

# 示例單詞
word = "king"
print(f"Vector representation for '{word}':\n", glove_model[word])

# 查找相似單詞
similar_words = glove_model.most_similar(word, topn=5)
print("\nWords similar to 'king':", similar_words)

# 計(jì)算單詞相似度
word1 = "king"
word2 = "queen"
similarity = glove_model.similarity(word1, word2)
print(f"Similarity between '{word1}' and '{word2}': {similarity:.4f}")

Vector representation for 'king':
 [0.123, 0.456, 0.789, ...]

Words similar to 'king': [('queen', 0.85), ('prince', 0.80), ('monarch', 0.78), ...]

Similarity between 'king' and 'queen': 0.85

import gensim.downloader as api

# 加載預(yù)訓(xùn)練的 FastText 模型
fasttext_model = api.load("fasttext-wiki-news-subwords-300")

# 示例單詞
word = "king"
print(f"Vector representation for '{word}':\n", fasttext_model[word])

# 查找相似單詞
similar_words = fasttext_model.most_similar(word, topn=5)
print("\nWords similar to 'king':", similar_words)

# 計(jì)算單詞相似度
word1 = "king"
word2 = "queen"
similarity = fasttext_model.similarity(word1, word2)
print(f"Similarity between '{word1}' and '{word2}': {similarity:.4f}")

Vector representation for 'king':
 [0.123, 0.456, 0.789, ...]

Words similar to 'king': [('queen', 0.85), ('prince', 0.80), ('monarch', 0.78), ...]

Similarity between 'king' and 'queen': 0.85

import gensim
from gensim.models.doc2vec import Doc2Vec, TaggedDocument
import nltk

nltk.download('punkt')

# 示例文檔
documents = [
    "Machine learning is amazing",
    "Natural language processing enables AI to understand text",
    "Deep learning advances artificial intelligence",
    "Word embeddings improve NLP tasks",
    "Doc2Vec is an extension of Word2Vec"
]

# 分詞并標(biāo)記文檔
tagged_data = [TaggedDocument(words=nltk.word_tokenize(doc.lower()), tags=[str(i)]) for i, doc in enumerate(documents)]

# 打印標(biāo)記后的數(shù)據(jù)
print(tagged_data)

# 定義模型參數(shù)
model = Doc2Vec(vector_size=50, window=2, min_count=1, workers=4, epochs=100)

# 構(gòu)建詞匯表
model.build_vocab(tagged_data)

# 訓(xùn)練模型
model.train(tagged_data, total_examples=model.corpus_count, epochs=model.epochs)

# 測(cè)試文檔的向量表示
test_doc = "Artificial intelligence uses machine learning"
test_vector = model.infer_vector(nltk.word_tokenize(test_doc.lower()))
print(f"Vector representation of test document:\n{test_vector}")

# 查找與測(cè)試文檔最相似的文檔
similar_docs = model.dv.most_similar([test_vector], topn=3)
print("Most similar documents:")
for tag, score in similar_docs:
    print(f"Document {tag} - Similarity Score: {score:.4f}")

Most similar documents:
Document 0 - Similarity Score: 0.85
Document 1 - Similarity Score: 0.80
Document 2 - Similarity Score: 0.78

# 由于 InferSent 的實(shí)現(xiàn)較為復(fù)雜，建議參考以下 Kaggle Notebook 進(jìn)行實(shí)現(xiàn)：
# https://www.kaggle.com/code/jeffd23/infer-sent-implementation

import tensorflow_hub as hub
import tensorflow as tf
import numpy as np
from sklearn.metrics.pairwise import cosine_similarity
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA

# 加載模型
embed = hub.load("https://tfhub.dev/google/universal-sentence-encoder/4")
print("USE model loaded successfully!")

# 示例句子
sentences = [
    "Machine learning is fun.",
    "Artificial intelligence and machine learning are related.",
    "I love playing football.",
    "Deep learning is a subset of machine learning."
]

# 獲取句子嵌入
embeddings = embed(sentences)
embeddings_np = embeddings.numpy()

# 打印嵌入形狀和第一個(gè)句子的嵌入（截?cái)啵?print(f"Embedding shape: {embeddings_np.shape}")
print(f"First sentence embedding (truncated):\n{embeddings_np[0][:10]} ...")

# 計(jì)算兩兩余弦相似度
similarity_matrix = cosine_similarity(embeddings_np)
similarity_df = pd.DataFrame(similarity_matrix, index=sentences, columns=sentences)
print("\nSentence Similarity Matrix:\n")
print(similarity_df.round(2))

# 可視化句子嵌入（PCA 降維）
pca = PCA(n_compnotallow=2)
reduced = pca.fit_transform(embeddings_np)

plt.figure(figsize=(8, 6))
plt.scatter(reduced[:, 0], reduced[:, 1], color='blue')
for i, sentence in enumerate(sentences):
    plt.annotate(f"Sentence {i+1}", (reduced[i, 0]+0.01, reduced[i, 1]+0.01))
plt.title("Sentence Embeddings (PCA projection)")
plt.xlabel("PCA 1")
plt.ylabel("PCA 2")
plt.grid(True)
plt.show()

Embedding shape: (4, 512)
First sentence embedding (truncated):
[0.123, 0.456, 0.789, ...] ...

Sentence Similarity Matrix:
Machine learning is fun.                     1.00  0.85  0.20  0.30
Artificial intelligence and machine learning are related.  0.85  1.00  0.25  0.35
I love playing football.                     0.20  0.25  1.00  0.15
Deep learning is a subset of machine learning.  0.30  0.35  0.15  1.00

import networkx as nx
import numpy as np
from node2vec import Node2Vec
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA

# 創(chuàng)建一個(gè)簡(jiǎn)單的圖
G = nx.karate_club_graph()  # 一個(gè)著名的測(cè)試圖，包含 34 個(gè)節(jié)點(diǎn)

# 可視化原始圖
plt.figure(figsize=(6, 6))
nx.draw(G, with_labels=True, node_color='skyblue', edge_color='gray', node_size=500)
plt.title("Original Karate Club Graph")
plt.show()

# 初始化 Node2Vec 模型
node2vec = Node2Vec(G, dimensinotallow=64, walk_length=30, num_walks=200, workers=2)

# 訓(xùn)練模型（底層使用 Word2Vec）
model = node2vec.fit(window=10, min_count=1, batch_words=4)

# 獲取某個(gè)節(jié)點(diǎn)的嵌入
node_id = 0
vector = model.wv[str(node_id)]  # 注意：節(jié)點(diǎn) ID 以字符串形式存儲(chǔ)
print(f"Embedding for node {node_id}:\n{vector[:10]}...")  # 截?cái)囡@示

# 獲取所有嵌入
node_ids = model.wv.index_to_key
embeddings = np.array([model.wv[node] for node in node_ids])

# 降維到 2D
pca = PCA(n_compnotallow=2)
reduced = pca.fit_transform(embeddings)

# 可視化嵌入
plt.figure(figsize=(8, 6))
plt.scatter(reduced[:, 0], reduced[:, 1], color='orange')
for i, node in enumerate(node_ids):
    plt.annotate(node, (reduced[i, 0] + 0.05, reduced[i, 1] + 0.05))
plt.title("Node2Vec Embeddings (PCA Projection)")
plt.xlabel("PCA 1")
plt.ylabel("PCA 2")
plt.grid(True)
plt.show()

# 查找與節(jié)點(diǎn) 0 最相似的節(jié)點(diǎn)
similar_nodes = model.wv.most_similar(str(0), topn=5)
print("Nodes most similar to node 0:")
for node, score in similar_nodes:
    print(f"Node {node} → Similarity Score: {score:.4f}")

Embedding for node 0:
[0.123, 0.456, 0.789, ...]

Nodes most similar to node 0:
Node 1 → Similarity Score: 0.85
Node 2 → Similarity Score: 0.80
Node 3 → Similarity Score: 0.78

# ELMo 的實(shí)現(xiàn)較為復(fù)雜，建議參考以下文章進(jìn)行實(shí)現(xiàn)：
# https://www.geeksforgeeks.org/elmo-embeddings-in-python/

from transformers import AutoTokenizer, AutoModel
import torch

# 輸入句子
sentence = "Natural Language Processing is transforming how machines understand humans."

# 選擇設(shè)備（如果有 GPU 則使用 GPU）
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

# 加載 BERT 模型
model_name = "bert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModel.from_pretrained(model_name).to(device)
model.eval()

# 分詞
inputs = tokenizer(sentence, return_tensors='pt', truncatinotallow=True, padding=True).to(device)

# 前向傳播獲取嵌入
with torch.no_grad():
    outputs = model(**inputs)

# 獲取單詞嵌入
token_embeddings = outputs.last_hidden_state  # (batch_size, seq_len, hidden_size)

# 通過(guò)平均池化獲取句子嵌入
sentence_embedding = torch.mean(token_embeddings, dim=1)
print(f"Sentence embedding from {model_name}:")
print(sentence_embedding)

Sentence embedding from bert-base-uncased:
tensor([[ 0.1234,  0.5678, -0.9012,  ...,  0.3456, -0.7890,  0.1111]])

from transformers import CLIPProcessor, CLIPModel
from PIL import Image
import torch
import requests

# 加載模型和處理器
clip_model_name = "openai/clip-vit-base-patch32"
clip_model = CLIPModel.from_pretrained(clip_model_name).to(device)
clip_processor = CLIPProcessor.from_pretrained(clip_model_name)

# 加載示例圖像和文本
image_url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/cat_style_layout.png"
image = Image.open(requests.get(image_url, stream=True).raw).convert("RGB")
text = "a cute puppy"

# 預(yù)處理輸入
inputs = clip_processor(text=[text], images=image, return_tensors="pt", padding=True).to(device)

# 獲取文本和圖像嵌入
with torch.no_grad():
    text_embeddings = clip_model.get_text_features(input_ids=inputs["input_ids"])
    image_embeddings = clip_model.get_image_features(pixel_values=inputs["pixel_values"])

# 歸一化嵌入（可選）
text_embeddings = text_embeddings / text_embeddings.norm(dim=-1, keepdim=True)
image_embeddings = image_embeddings / image_embeddings.norm(dim=-1, keepdim=True)

print("Text Embedding Shape (CLIP):", text_embeddings.shape)
print("Image Embedding Shape (CLIP):", image_embeddings.shape)

Text Embedding Shape (CLIP): torch.Size([1, 512])
Image Embedding Shape (CLIP): torch.Size([1, 512])