最强中文Embedding模子m3e和向量数据库Qdrant联动 - ToB企服应用市场:ToB评测及商务社交产业平台

# -*- coding: utf-8 -*-
from qdrant_client import QdrantClient
from qdrant_client.models import Distance, VectorParams, PointStruct
import requests
import json
import pandas as pd
import sys
arguments = sys.argv
print('以下是你的需要存储的信息：')
print('需要存入向量数据库的文件为：', arguments[1])
print('存入该文件中的字段为：', arguments[2])
print('向量数据库的地址为：' + 'http://' + arguments[3] + ':' + arguments[4] + '/dashboard#')
print('存储的向量数据库名为：', arguments[5])
def m3e_emdedding(data_input):
url = '模型地址'
headers = {
'Authorization': '模型参数',
'Content-Type': '模型参数',
}
data = {
'model': 'm3e',
'input': data_input
}
response = requests.post(url, headers=headers, json=data)
result = json.loads(response.text)
return result
# 读取指定文件路径
try:
df = pd.read_csv(arguments[1])
except:
df = pd.read_excel(arguments[1])
# 指定列列名
sentences = df[arguments[2]].tolist()
print('所需存入向量数据库的数量为：', len(sentences))
print('数据字段转向量中。。。')
try:
embedding = m3e_emdedding(sentences)
except:
embedding = []
print('me3模型地址失效')
print('数据字段转向量完成！')
embedding = embedding.get('data')
embeddings = [item['embedding'] for item in embedding]
client = QdrantClient(arguments[3], port=arguments[4])
# 创建向量数据库第一次执行后续千万别再执行这一条！！！！
client.recreate_collection(
collection_name=arguments[5],
vectors_config=VectorParams(size=1536, distance=Distance.COSINE),
)
print('向量数据库创建集合完成')
print('开始存入。。。')
# 插入数据
for i in range(len(sentences)):
str = sentences[i]
vectors = embeddings[i]
# print(f"str={str}, len(vectors)={len(vectors)}")
client.upsert(
collection_name=arguments[5],
points=[
PointStruct(
id=i,
vector=vectors,
payload={"info": str}
)
]
)
print('向量数据库存入数据完成')

复制代码

from qdrant_client import QdrantClient
import requests
import json
def m3e_emdedding(data_input):
url = '模型地址'
headers = {
'Authorization': '模型参数',
'Content-Type': '模型参数',
}
data = {
'model': 'm3e',
'input': data_input
}
response = requests.post(url, headers=headers, json=data)
result = json.loads(response.text)
return result
# 需要匹配的词
field = ['恩替卡韦', '血红蛋白']
sentence_embeddings = m3e_emdedding(field).get('data')
sentence_embeddings = [item['embedding'] for item in sentence_embeddings]
client = QdrantClient('向量数据库ip', port=端口)
current_matches = []
# 循环遍历输入的每个值
for i, embedding in enumerate(sentence_embeddings):
hits = client.search(
collection_name="fields",
query_vector=embedding,
limit=3 #搜寻条数
)
# 创建一个空列表，用于存储当前句子的匹配结果
current_matches = []
# 对每一次匹配的3个值循环判断
for j, item in enumerate(hits):
print(f'第{j + 1}次，{field[i]}，匹配后中的得分为：{hits[j].score}, 匹配值为：{hits[j].payload.get("info")}')
if hits[j].score > 0.5:
# 将符合条件的数据添加到当前句子的匹配结果列表中
match_data = {
"info": hits[j].payload.get("info"),
"score": hits[j].score,
'old': field[i]
}
current_matches.append(match_data)
print(current_matches)

复制代码