句子相似度计算的几种方法

雅可比相似度，余弦相似度，带tfidf的余弦相似度

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# created by fhqplzj on 2017/07/22 上午12:23
import numpy as np
from scipy.linalg import norm
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer


def jaccard_similarity(s1, s2):
    """
    计算两个句子的雅可比相似度
    :param s1: 
    :param s2: 
    :return: 
    """
    vectorizer = CountVectorizer(tokenizer=lambda s: s.split())
    corpus = [s1, s2]
    vectors = vectorizer.fit_transform(corpus).toarray()
    numerator = np.sum(np.min(vectors, axis=0))
    denominator = np.sum(np.max(vectors, axis=0))
    return 1.0 * numerator / denominator


def cosine_similarity_tf(s1, s2):
    """
    计算两个句子的TF余弦相似度
    :param s1: 
    :param s2: 
    :return: 
    """
    vectorizer = CountVectorizer(tokenizer=lambda s: s.split())
    corpus = [s1, s2]
    vectors = vectorizer.fit_transform(corpus).toarray()
    return np.dot(vectors[0], vectors[1]) / (norm(vectors[0]) * norm(vectors[1]))


def cosine_similarity_tfidf(s1, s2):
    """
    计算两个句子的TFIDF余弦相似度
    :param s1: 
    :param s2: 
    :return: 
    """
    vectorizer = TfidfVectorizer(tokenizer=lambda s: s.split())
    corpus = [s1, s2]
    vectors = vectorizer.fit_transform(corpus).toarray()
    return np.dot(vectors[0], vectors[1]) / (norm(vectors[0]) * norm(vectors[1]))


params = ['a b b c c c', 'b c c d d d']
print jaccard_similarity(*params)
print cosine_similarity_tf(*params)
print cosine_similarity_tfidf(*params)