文本相似度之潜在语义索引实现
问题导读：
1.什么是空间向量模型VSM？2.什么是潜在语义索引？3.怎样使用潜在语义索引计算文本相似度？
最新经典文章，欢迎关注公众号
http://www.aboutyun.com/data/attachment/forum/201406/15/084659qcxzzg8n59b6zejp.jpg
1.VSM简介
空间向量模型VSM，是将文本表示成数值表示的向量。在使用VSM做文本相似度计算时，其基本步骤是：
1）将文本分词，提取特征词s:（t1,t2,t3,t4）
2）将特征词用权重表示，从而将文本表示成数值向量s:(w1,w2,w3,w4)，权重表示的方式一般使用tfidf
3）计算文本向量间的余弦值，判断文本间的相似度
缺点：空间向量模型以词袋为基础，没有考虑词与词间的关系，近义词等。
2.LSI介绍
潜在语义索引(Latent Semantic Indexing,以下简称LSI)，有的文章也叫Latent SemanticAnalysis（LSA）。是一种简单实用的主题模型。LSI是基于奇异值分解（SVD）的方法来得到文本的主题的。其推导过程如下图：(参考：http://www.cnblogs.com/pinard/p/6251584.html ,https://www.cnblogs.com/pinard/p/6805861.html)
使用LSI计算文本相似度的基本过程：
1).和VSM一样，得到文本的tfidf值的向量表示
2).做SVD分解，得到Uk词和词义之间的相关性。Vk文本和主题的相关性。
3).利用文本主题矩阵计算文本的相似度(通过余弦值)
3.应用
(1)准备数据
数据分为两部分（Idx2ID,questionList）
idx2ID:保存数据库中的ID及ID在文件中的下标，例如（{0:100,1:101,2:102} 其中0,1,2为下标；100,102,103为数据库中的ID值）
questionList：保存的是数据库中问题切词后的二维数组，例如[['我','跑路','了']，['你'，‘很’，‘不开心’]]
其中questionList中的数组下标与idx2ID中的下标一 一对应，及数据库中的保存记录为：（id:100,question:‘我跑路了’）
import re
from tqdm import tqdm
import os
import pickle
import jieba
import pymysql
import pandas as pd
'''
从数据库获取训练数据
'''
#清洗数据，去掉停用词并进行jieba分词
stwlist = pickle.load(open('stop_word.pkl','rb'))
def getTrainData(sentence):
sentence=re.sub(r'+','', sentence)
wordList = jieba.cut(sentence.strip(), cut_all=False)
new_wordList = []
for word in wordList:
if word not in stwlist:
new_wordList.append(word)
return new_wordList
def get_ID_Question():
aldb = pymysql.connect(host='xx.xx.xx.xx', port=3306, user='user1', passwd='123456', db="testdb", charset='utf8')
IdList = []
QuestionList = []
with pymysql.cursors.SSCursor(aldb) as cursor:
cursor.execute('select id,question from test')
while True:
rs = cursor.fetchone()
if not rs:
break
id= rs
question = rs
IdList.append(id)
QuestionList.append(getTrainData(question))
#保存数据库中id及id在文件中对应的下标
Idx2ID = {}
for index, ID in enumerate(IdList):
Idx2ID = ID
return Idx2ID,QuestionList
def save_ID_Question():
Idx2ID_path = './cache/Idx2ID.pkl'
QuesttionData_path='./cache/traindata.pkl'
Idx2ID, QuestionList= get_ID_Question()
#保存数据
pickle.dump(Idx2ID, open(Idx2ID_path,'wb'))
pickle.dump(QuestionList, open(QuesttionData_path, 'wb'))
print('save data result!')
if __name__=='__main__':
save_ID_Question()
(2)训练模型
#建立lsi模型，并将生成的LSI模型保存
def lsi_model(num_topics):
lsi_model_dictionary = './cache/lsi_dictionary.pkl'
lsi_model_lsi = './cache/model.lsi'
lsi_model_index = './cache/lsimodel.index'
cut_question_path = r'./cache/traindata.pkl'
if os.path.exists(lsi_model_dictionary):
dictionary = pickle.load(open(lsi_model_dictionary,'rb'))
lsi = models.LsiModel.load(lsi_model_lsi)
index = similarities.MatrixSimilarity.load(lsi_model_index)
else:
textdata1 = pickle.load(open(cut_question_path, 'rb'))
#生成词典
dictionary = corpora.Dictionary(textdata1)
print('dictionary prepared!')
corpus =
print('corpus prepared!')
#计算tfidf
tfidf = models.TfidfModel(corpus)
print('tfidf prepared!')
corpus_tfidf = tfidf
#训练LSI
lsi = models.LsiModel(corpus_tfidf, id2word=dictionary, num_topics = num_topics)
print('lsi model prepared!')
corpus_lsi = lsi
#生成引用
index = similarities.MatrixSimilarity(corpus_lsi)
print('index prepared!')
pickle.dump(dictionary, open(lsi_model_dictionary,'wb'))
lsi.save(lsi_model_lsi)
index.save(lsi_model_index)
return dictionary, lsi, index
(3)测试
class test(object):
def __init__(self):
#加载模型数据
self.dictionary = pickle.load(open('./cache/lsi_dictionary.pkl','rb'))
self.lsi = models.LsiModel.load('./cache/model.lsi')
self.index = similarities.MatrixSimilarity.load('./cache/lsimodel.index')
self.stop_word = pickle.load(open('./stop_word.pkl','rb'))
self.idx2ID = pickle.load(open('./cache/idx2ID.pkl','rb'))
#清洗输入问题，并切词
def deal_query(self,query):
query = re.sub(r'+','',query)
query_ques = jieba.cut(query.strip(), cut_all=False)
new_wordList = []
for word in query_ques:
if word not in self.stop_word:
new_wordList.append(word)
return new_wordList
#清洗匹配到的问题
def clean_question(self, question):
question = re.sub('律师', '', question)
ques = pseg.cut(question)
for word in ques:
if word.flag == 'nr1' or word.flag == 'nr2' or word.flag == 'nr':
keys = word.word
question = re.sub(keys, '', question)
return question
#从数据库获取匹配到的问答对
def get_answer(self,idx_set):
question_set = []
answer_set = []
db = pymysql.connect(host='xx.xx.xx.x', port=3306, user='user1', passwd='123456', db="testdb", charset='utf8')
for i in idx_set:
ID = self.idx2ID
# print(ID)
cursor = db.cursor()
cursor.execute('select question,answer from test where id=%d;' %ID)
db.commit()
result = cursor.fetchone()
if result:
question = self.clean_question(result)
question_set.append(question)
answer = self.clean_answer(result)
answer_set.append(answer)
db.close()
return question_set,answer_set
#匹配
def match(self,query):
query_ques = self.deal_query(query)
query_bow = self.dictionary.doc2bow(query_ques)
query_lsi = self.lsi
#计算余弦相似度，获取前3个最匹配的值
sims = self.index
sort_sims = sorted(enumerate(sims), key=lambda item: -item)
idx_set = for i in sort_sims[:3]]
sim_value = for i in sort_sims[:3]]
question_set,answer_set = self.get_answer(idx_set)
return question_set,answer_set,sim_value
def request(self,query):
import json
question_set,answer_set,sim_value = self.match(query)
#相似度小于0.65的问题丢弃
if sim_value < 0.65:
print("we have no match answer")
return None
i=0
question_answer={}
while i<len(question_set):
q=question_set
a=answer_set
if len(q) > 2 and len(a) > 2:
question_answer=a
if len(question_answer)>3:
break
i+=1
# print_result2 = {k: v for k, v in question_answer.items() }
print(len(question_answer))
print(question_answer)
return json.dumps(question_answer, ensure_ascii=False)
#清洗答案数据
def clean_answer(self,answer):
# 将如果有多个回答，取长度最长的一个回答
answer_sent = answer.strip().split('NEXT_ANS')
if len(answer_sent) > 1:
answer_len =
inx = answer_len.index(max(answer_len))
answer = answer_sent
answer = re.sub('[，]+', '，', answer)
answer = re.sub('[。]+', '。', answer)
answer = re.sub('[.]+', '', answer)
answer = re.sub('[,]+', '，', answer)
from zhon.hanzi import punctuation
# 去除 '\nNEXT_ANS'
answer = answer.replace('NEXT_ANS', '')
duels = [x + y for x in punctuation
for y in punctuation]
pattern1 = re.compile('^[%s]' % punctuation)
pattern2 = re.compile('[%s]$' % punctuation)
pattern3 = re.compile('\[.*?\]')
# 去除[]中的内容
answer = pattern3.sub('', answer)
# 去除空格
answer = answer.split()
answer = ''.join(answer)
# 去除【】符号
answer = answer.replace('【', '')
answer = answer.replace('】', '')
# 去除《》符号不全的句子中的《》
if (answer.count('《') != answer.count('》')):
answer = answer.replace('《', '')
answer = answer.replace('》', '')
# 去除‘去),张三你好’类型的错误
delCStr = '《》（）&%￥#@！{}【】'
if (answer in delCStr):
answer = answer
# 去除\n \\n \u3000\u30005 \xa0 \r "
answer = re.sub(r'\\u3\d{3,4}', '', answer)
answer = re.sub(r'\\n', '', answer)
answer = re.sub(r'\\xa0', '', answer)
answer = re.sub(r'\\r', '', answer)
answer = re.sub(r'"', '', answer)
#去除电话
answer = re.sub(r'电话：', '', answer)
answer = re.sub(r'电话\d{11}', '', answer)# 电话13466352858
answer = re.sub(r'电话\d{3}\-\d{8}', '', answer)# 电话010-68812000
answer = re.sub(r'\d{3}\-\d{8}', '', answer)# 010-68812000
answer = re.sub(r'0\d{10}', '', answer)# 01065083250
answer = re.sub(r'1\d{9}', '', answer)# 手机号码
answer = re.sub(r'[一二三四五六七八九零]{11,}', '', answer)# 一三八二零二八八一二六
# 去除QQ
answer = re.sub(r'QQ：\d{4,10}', '', answer)
answer = re.sub(r'qq：\d{4,10}', '', answer)
answer = re.sub(r'QQ\d{4,10}', '', answer)
# 邮箱
answer = re.sub(r'*@.*.com', '', answer)
# answer = re.sub(r'\d{4,10}', '', answer)
# 去除邮箱
answer = re.sub(r'E-mail:', '', answer)
answer = re.sub(r'email:', '', answer)
answer = re.sub(r'电子邮箱', '', answer)
answer = re.sub(r'邮箱', '', answer)
answer = re.sub(r'[\w\.-]+@([\w]+\.)+{2,3}', '', answer)
# 去除网址
answer = re.sub(r'个人网址：', '', answer)
answer = re.sub(r'网址：', '', answer)
answer = re.sub('.*', '', answer)
# http://www51wfcom/zxzx/ZixunMainViewjsp?ID=bd84d23a2ec1fd73012ec2a422e40044
answer = re.sub(r'*', '', answer)
# 特殊符号
answer = re.sub(r'&.t', '', answer)
answer = re.sub(r'×', '', answer)
# 去除人名
address = ''
ans_list = pseg.cut(answer)
for word in ans_list:
# 去除不含国字的地名
'''
if (word.flag == 'ns' and not (word.word.find('国') > -1)):
keys = word.word
answer = re.sub(keys, '', answer)
continue
'''
# 去除姓名：nr 人名，nr1 汉语姓氏，nr2 汉语名字
if word.flag == 'nr1' or word.flag == 'nr2' or word.flag == 'nr':
keys = word.word
answer = re.sub(keys, '', answer)
# 去除重复标点符号
for d in duels:
while d in answer:
answer = answer.replace(d, d)
# 去除开头标点符号
if answer!='《':
answer = pattern1.sub('', answer)
# 结尾加句号
answer = pattern2.sub('。', answer)
if pattern2.search(answer) is None:
answer = answer + '。'
else:
answer = pattern2.sub('。', answer)
return answer
if __name__=='__main__':
# lsi_model(800)
tst=test()
while True:
text=input('输入问题：')
st=time.time()
tst.request(text)
end=time.time()
print(end-st)
来源： iteye作者： 孤狼18
原文链接：文本相似度之LSIhttp://985359995.iteye.com/blog/2403936
感谢分享
页:
[1]