# env_utils.py
import os

# BGE-large-zh-v1.5 本地路径
LOCAL_MODEL_PATH_BGE = "/path/to/your/bge-large-zh-v1.5"
# Qwen3 嵌入模型本地路径
LOCAL_MODEL_PATH_QWEN3 = "/path/to/your/qwen3-embedding"
# OpenAI API配置（可选，对比测试用）
OPENAI_API_KEY = "your-api-key"
OPENAI_BASE_URL = "your-api-base-url"

from langchain_huggingface import HuggingFaceEmbeddings
from env_utils import LOCAL_MODEL_PATH_BGE

def init_local_bge_embeddings():
    """加载本地BGE-large-zh-v1.5模型，适配LangChain接口"""
    model_kwargs = {
        "device": "cpu",
        "trust_remote_code": True
    }
    # 关键参数：BGE必须开启归一化，否则相似度计算偏差大
    encode_kwargs = {
        "normalize_embeddings": True
    }

    embeddings = HuggingFaceEmbeddings(
        model_name=LOCAL_MODEL_PATH_BGE,
        model_kwargs=model_kwargs,
        encode_kwargs=encode_kwargs,
        cache_folder=None  # 禁用HuggingFace缓存，强制使用本地
    )
    return embeddings

# 测试验证
if __name__ == "__main__":
    bge_embeddings = init_local_bge_embeddings()
    test_text = "这是测试本地BGE大模型的中文句子"
    embedding = bge_embeddings.embed_query(test_text)
    
    print(f"测试文本：{test_text}")
    print(f"嵌入向量长度：{len(embedding)}")  # large版输出1024
    print(f"向量前5个值：{embedding[:5]}")
    
    # 验证归一化
    import numpy as np
    norm = np.linalg.norm(embedding)
    print(f"向量模长：{norm:.6f}（应接近1.0）")

import torch
from langchain_core.embeddings import Embeddings
from sentence_transformers import SentenceTransformer
from env_utils import LOCAL_MODEL_PATH_QWEN3

class CustomQwen3Embeddings(Embeddings):
    """自定义Qwen3嵌入模型类，适配LangChain标准接口"""
    
    def __init__(self, device="cpu"):
        self.model = SentenceTransformer(
            LOCAL_MODEL_PATH_QWEN3,
            model_kwargs={
                "torch_dtype": torch.float32,
                "trust_remote_code": True
            },
            tokenizer_kwargs={
                "padding_side": "left"  # Qwen官方推荐配置
            }
        )

    def embed_query(self, text: str) -> list[float]:
        """单查询文本向量化"""
        return self.embed_documents([text])[0]

    def embed_documents(self, texts: list[str]) -> list[list[float]]:
        """多文档批量向量化"""
        return self.model.encode(texts).tolist()

# 测试验证
if __name__ == "__main__":
    qwen3 = CustomQwen3Embeddings()
    resp = qwen3.embed_documents(['i love you', '你好'])
    print(f"单个嵌入向量维度：{len(resp[0])}")  # Qwen3输出1536

from langchain_openai import OpenAIEmbeddings
from env_utils import OPENAI_API_KEY, OPENAI_BASE_URL

openai_embedding = OpenAIEmbeddings(
    api_key=OPENAI_API_KEY,
    base_url=OPENAI_BASE_URL,
    model='text-embedding-3-large',
    dimensions=256,
)

resp = openai_embedding.embed_documents(['I like studying LLM!', '今天天气很好'])
print(f"向量长度：{len(resp[0])}")  # 输出256

import pandas as pd
import numpy as np
from env_utils import LOCAL_MODEL_PATH_BGE
from langchain_huggingface import HuggingFaceEmbeddings

# 初始化BGE模型
model_kwargs = {"device": "cpu", "trust_remote_code": True}
encode_kwargs = {"normalize_embeddings": True}
bge_embeddings = HuggingFaceEmbeddings(
    model_name=LOCAL_MODEL_PATH_BGE,
    model_kwargs=model_kwargs,
    encode_kwargs=encode_kwargs,
    cache_folder=None
)

def text_to_embedding(text):
    """单条文本转向量"""
    return bge_embeddings.embed_documents([text])[0]

def embedding_to_file(source_file, output_file):
    """批量处理评论数据并保存"""
    df = pd.read_csv(source_file, index_col=0)
    df = df[['Time','ProductId','UserId','Summary','Text']]
    df['text_content'] = 'Summary:' + df.Summary.fillna('').str.strip() + '; Text:' + df.Text.fillna('').str.strip()
    
    # 批量生成嵌入向量
    print(f"开始编码 {len(df)} 条文本...")
    df['embedding'] = df.text_content.apply(lambda x: text_to_embedding(x))
    df.to_csv(output_file)
    print(f"已保存至 {output_file}")

if __name__ == "__main__":
    embedding_to_file('data/fine_food_reviews_1k.csv', 'data/output_embedding.csv')

import ast
import pandas as pd
import numpy as np

def cosine_similarity(a, b):
    """余弦相似度计算（向量需归一化）"""
    return np.dot(a, b)

def search_text(input_text, embedding_file, top_n=3):
    """根据输入查询，返回最相似的top_n条评论"""
    df_data = pd.read_csv(embedding_file)
    
    # 字符串转数组
    df_data['embedding_vector'] = df_data['embedding'].apply(ast.literal_eval)
    
    # 生成查询向量
    input_vector = text_to_embedding(input_text)
    
    # 计算相似度
    df_data['similarity'] = df_data.embedding_vector.apply(
        lambda x: cosine_similarity(input_vector, x)
    )
    
    # 返回TopN结果
    resp = (
        df_data.sort_values('similarity', ascending=False)
        .head(top_n)
        .text_content
        .str.replace('Summary:', '')
        .str.replace('; Text:', ' ')
    )
    
    # 输出结果
    print(f"查询：{input_text}\n")
    for idx, r in enumerate(resp, 1):
        print(f"【相似结果{idx} 相似度：{df_data.sort_values('similarity', ascending=False).iloc[idx-1].similarity:.4f}】")
        print(r[:200] + "..." if len(r) > 200 else r)
        print('-' * 50)

if __name__ == "__main__":
    search_text('delicious beans', 'data/output_embedding.csv')