机器图纸文章标题搜索增强实现过程 - qidao123.com技术社区-IT企服评测·应用市场

from FlagEmbedding import FlagModel
import pandas as pd
import numpy as np
from datasets import Dataset
from scipy.spatial import distance
import datetime
import configparser
import pymysql
model = None
def getModel():
global model
if model is None:
model = FlagModel("./model",
query_instruction_for_retrieval="Represent this sentence for searching relevant passages:",
use_fp16=True)
return model
#获取向量
def getFlagEmbedding(title):
global model
model = getModel()
embedding = model.encode(title)
return embedding

复制代码

/// <summary>
/// 获取向量
/// </summary>
/// <param name="keyword"></param>
/// <returns></returns>
public async Task<double[]> GetFlagEmbedding(string keyword)
{
var vector = new double[] { };
vector = null;
try
{
var req = new
{
action = "getFlagEmbedding",
keyword
};
var content = new StringContent(
JsonSerializer.Serialize(req)
, Encoding.UTF8, "application/json");
var response = await _client.PostAsync(ConfigHelp.FlagSerachUrl, content);
if (response.IsSuccessStatusCode)
{
var result = await response.Content.ReadAsStringAsync();
var data = JsonSerializer.Deserialize<GetFlagEmbeddingRoot>(result);
if (data.op)
{
vector = data.msg.Split(',').Select(double.Parse).ToArray();
}
}
}
catch (Exception ex)
{
LogUtils.Error("GetFlagEmbedding ", ex);
}
return vector;
}

复制代码

public class RedisVectorHelp
{
private readonly IDatabase _db;
private string _freefix;
private string _indexName;
private SearchCommands ft;
public RedisVectorHelp(string freefix,string redisConnectionString,int dbNum=0)
{
var redis = ConnectionMultiplexer.Connect(redisConnectionString);
_db = redis.GetDatabase(dbNum);
_freefix = freefix;
_indexName = _freefix + "_index";
ft = new SearchCommands(_db, null);
}
/// <summary>
/// 创建索引
/// </summary>
public void CreateFt()
{
var list = ft._List();
var indexList = list.Select(result => result.ToString()).ToArray();
//判断是否存在索引
if (indexList.Contains(_indexName))
{
Console.WriteLine("Index already exists.");
return;
}
ft.Create(_indexName,
new FTCreateParams()
.On(IndexDataType.HASH)
.Prefix(_freefix + ":"),
new Schema()
.AddTextField("id")
.AddVectorField("vector",
VectorField.VectorAlgo.FLAT,
new Dictionary<string, object>
{
["TYPE"] = "FLOAT32",
["DIM"] = 1024,
["DISTANCE_METRIC"] = "COSINE"
})
);
}
/// <summary>
/// 存储向量
/// </summary>
/// <param name="id"></param>
/// <param name="vector"></param>
public void StoreVectorData(string id, float[] vector)
{
// 构造键名
var key = $"{_freefix}:{id}";
VectorDom dom = new VectorDom
{
id = id,
vector = vector
};
byte[] vectorBinary = vector.SelectMany(f => BitConverter.GetBytes(f)).ToArray();
_db.HashSet(key, "id", dom.id);
_db.HashSet(key, "vector", vectorBinary);
}
/// <summary>
/// 向量搜索
/// </summary>
/// <param name="queryVector"></param>
/// <param name="topK"></param>
public List<string> SearchSimilarVectors(float[] queryVector, int topK = 50)
{
byte[] vectorQueryBinary = queryVector.SelectMany(f => BitConverter.GetBytes(f)).ToArray();
//十六进制字符串
//string vectorQueryBinaryStr = BitConverter.ToString(vectorQueryBinary).Replace("-", "");
Query q = new Query($"*=>[KNN {topK} @vector $vec as score]");
q.SortBy = "score";
q.AddParam("vec", vectorQueryBinary);
q.ReturnFields("id", "vector");
q.Limit(0, topK);
q.Dialect(2);
var obj = ft.Search(_indexName, q);
var docList = obj.Documents;
var list = new List<string>();
foreach (var doc in docList)
{
list.Add(doc["id"]);
}
return list;
}
}
public class VectorDom
{
public string id { get; set; }
public float[] vector { get; set; }
}

复制代码

var searchVector = await GetFlagEmbedding(keyword);
if (searchVector != null)
{
var queryVector = Array.ConvertAll(searchVector, x => (float)x);
var temp = bykcsjRVHelp.SearchSimilarVectors(queryVector, 30);
foreach (var id in temp)
{
if (!ids.Contains(id))
{
ids.Add(id);
}
}
}

复制代码