余弦文本相似度匹配
python连接sqlserver对数据进行文本相似度匹配
from sqlalchemy import create_engine,Table,Column,Date,Integer,String,ForeignKey
from fuzzywuzzy import process
import os
import pymssql
import pymysql
import datetime
import time
import sys
import numpy
#import text2vec
from text2vec import Similarity
#from longconv import *
from gensim import similarities
import paramiko
from smb.SMBConnection import *
import csv
import re
import sqlalchemy
import pandas as pd
from sqlalchemy import MetaData,create_engine,Table,Integer
import socket
import pandas
#import gradio as gr
import datetime
from scipy.spatial.distance import pdist
from text2vec import Similarity
t1=time.time()
# 中文句向量模型(CoSENT)
sim_model = Similarity(model_name_or_path='shibing624/text2vec-base-chinese',
similarity_type='cosine', embedding_type='sbert')
def ai_text(sentence1, sentence2):
#scores=[]
#dict={}
score = sim_model.get_score(sentence1, sentence2)
# print("{} \t\t {} \t\t Score: {:.4f}".format(sentence1, sentence2, score))
return score
#dict[sentence2] = score
#print(dict)
def db_conn(db_host,db_user,db_passwd,db_database,db_port):
conn=pymssql.connect(server=db_host,user=db_user,password=db_passwd,database=db_database,port=db_port)
cur=conn.cursor()
if not cur:
raise Exception('数据库连接失败')
else:print('succeed')
cur.execute("""
select * from ODS_Legal_WorldCheck_Name""")
wc = cur.fetchall()
cur2 =conn.cursor()
# cur2.execute('''select top 50 * from ODS_Legal_Ariba_VendorData_New ''')
cur2.execute(''' select * from
(select *, ROW_NUMBER() OVER (order by SMVendorID) AS ROWNUM
from ODS_Legal_Ariba_VendorData_New) t
where ROWNUM between 1 and 10 ''')
av = cur2.fetchall()
for row1 in av:
max_rs = 0
for row2 in wc:
rs = ai_text(row1[1],row2[1])
if max_rs < rs :
max_rs = rs
max_rs = round(max_rs,2)
max_id = row2[0]
max_name=row2[1]
print(row1[0],row1[1],max_id,max_name,max_rs)
t2=time.time()
print(t2-t1)
if __name__=='__main__':
db_host='10.111.*.****'
db_user='***dmin'
db_passwd = '*******'
db_port='20001'
db_database='ODS_***'
db_conn(db_host,db_user,db_passwd,db_database,db_port)