code example websites:
https://github.com/samlobel/RaCT_CF/blob/master/utils/evaluation_functions.py
https://github.com/dawenl/cofactor/blob/master/src/rec_eval.py
https://python.hotexamples.com/examples/bottleneck/-/argpartsort/python-argpartsort-function-examples.html
import numpy as np
import bottleneck as bn
import sys,math
hit rate implementations:
https://medium.com/@rishabhbhatia315/recommendation-system-evaluation-metrics-3f6739288870
def HR_at_k(X_pred, X_true, k=10):
batch_users = X_pred.shape[0]
idx = bn.argpartition(-X_pred, k, axis=1)
X_pred_binary = np.zeros_like(X_pred, dtype=bool)
X_pred_binary[np.arange(batch_users)[:, np.newaxis], idx[:, :k]] = True
X_true_binary = (X_true > 0)
hits_num = np.logical_and(X_true_binary, X_pred_binary).sum(axis=1)
return np.mean(hits_num/k)
def hit_rate1(X_pred, X_true, topk=5):
num_users = len(X_pred)
actual = [[] for _ in range(num_users)]
where = np.where(X_true!=0)
for idx in range(len(where[0])):
actual[where[0][idx]].append(where[1][idx])
#
rank = np.argsort(-X_pred)
predicted = rank[:,:topk]
#
hits = 0
num_users = len(predicted)
for i in range(num_users):
act_set = set(actual[i])
pred_set = set(predicted[i][:topk])
for item in pred_set:
if item in act_set:
hits += 1
return hits/topk/num_users
precision & recall:
def precision_recall_at_k(X_pred, X_true, k=10):
num_users = len(X_pred)
actual = [[] for _ in range(num_users)]
where = np.where(X_true!=0)
for idx in range(len(where[0])):
actual[where[0][idx]].append(where[1][idx])
#
rank = np.argsort(-X_pred)
predicted = rank[:,:k]
sum_recall = 0.0
sum_precision = 0.0
true_users = 0
for i in range(num_users):
act_set = set(actual[i])
pred_set = set(predicted[i])
if len(act_set) != 0:
sum_precision += len(act_set & pred_set) / float(k)
sum_recall += len(act_set & pred_set) / float(len(act_set))
true_users += 1
return sum_precision / true_users, sum_recall / true_users
#下面这个结果非常小
def precision_recall(x_pred, x_true, k = 10):
epsilon = 1e-10
pred_idx = bn.argpartition(-x_pred, k, axis=1)
x_pred_binary = np.zeros_like(x_pred)
x_pred_binary[pred_idx < k] = 1
x_true_binary = (x_true>0).astype(np.int)
tp = np.sum(x_pred_binary*x_true_binary, axis=1)
fp = np.sum((1-x_pred_binary)*x_true_binary, axis=1)
fn = np.sum(x_pred_binary*(1-x_true_binary), axis=1)
p = tp/(tp+fp+epsilon)#epsilon的意义在于防止分母为0
r = tp/(tp+fn+epsilon)
# print(tp,fp,fn)
# f1 = 2*p*r/(p+r+epsilon)
# f1 = np.where(np.isnan(f1), np.zeros_like(f1), f1)
# f1 = np.mean(f1)
return p,r
recall的另外一个实现
def Recall_at_k(X_pred, X_true, k=10):
batch_users = X_pred.shape[0]
idx = bn.argpartition(-X_pred, k, axis=1)
X_pred_binary = np.zeros_like(X_pred, dtype=bool)
X_pred_binary[np.arange(batch_users)[:, np.newaxis], idx[:, :k]] = True
X_true_binary = (X_true > 0)
tmp = (np.logical_and(X_true_binary, X_pred_binary).sum(axis=1)).astype(
np.float32)
recall = tmp / np.minimum(k, X_true_binary.sum(axis=1))
return np.nan_to_num(recall)
NDCG
注意：以下代码要把范围调到0-1之间
def NDCG_binary_at_k_batch(X_pred, heldout_batch, k=10):
"""
normalized discounted cumulative gain@k for binary relevance
ASSUMPTIONS: all the 0's in heldout_data indicate 0 relevance
"""
batch_users = X_pred.shape[0]
# x_pred_binary = (X_pred>0)*1
idx_topk_part = bn.argpartition(-X_pred, k, axis=1)
#
topk_part = X_pred[np.arange(batch_users)[:, np.newaxis], idx_topk_part[:, :k]]
idx_part = np.argsort(-topk_part, axis=1)
# X_pred[np.arange(batch_users)[:, np.newaxis], idx_topk] is the sorted
# topk predicted score
idx_topk = idx_topk_part[np.arange(batch_users)[:, np.newaxis], idx_part]
# build the discount template
tp = 1.0 / np.log2(np.arange(2, k + 2))
#
DCG = (heldout_batch[np.arange(batch_users)[:, np.newaxis], idx_topk] * tp).sum(axis=1)
IDCG = np.array([(tp[: min(n, k)]).sum() for n in np.sum(heldout_batch!=0,axis=1)])
return np.mean(DCG / IDCG)
#以下的代码初步结果一样
def NDCG_binary_at_k_batch1(X_pred, heldout_batch, k=10, input_batch=None, normalize=True):
'''
normalized discounted cumulative gain@k for binary relevance
ASSUMPTIONS: all the 0's in heldout_data indicate 0 relevance
If normalize is set to False, then we actually return DCG, not NDCG.
'''
if input_batch is not None:
X_pred[input_batch.nonzero()] = -np.inf
batch_users = X_pred.shape[0]
# Get the indexes of the top K predictions.
idx_topk_part = bn.argpartition(-X_pred, k, axis=1)
# Get only the top k predictions.
topk_part = X_pred[np.arange(batch_users)[:, np.newaxis],
idx_topk_part[:, :k]]
# Get sorted index...
idx_part = np.argsort(-topk_part, axis=1)
# X_pred[np.arange(batch_users)[:, np.newaxis], idx_topk] is the sorted
# topk predicted score
# Get sorted index...
idx_topk = idx_topk_part[np.arange(batch_users)[:, np.newaxis], idx_part]
# build the discount template
tp = 1. / np.log2(np.arange(2, k + 2))
# You add up the ones you've seen, scaled by their discount...
# top_k_results = heldout_batch[np.arange()]
maybe_sparse_top_results = heldout_batch[np.arange(batch_users)[:, np.newaxis], idx_topk]
try:
top_results = maybe_sparse_top_results.toarray()
except:
top_results = maybe_sparse_top_results
#
try:
number_non_zero = heldout_batch.getnnz(axis=1)
except:
number_non_zero = ((heldout_batch > 0) * 1).sum(axis=1)
#
DCG = (top_results * tp).sum(axis=1)
# DCG = (heldout_batch[np.arange(batch_users)[:, np.newaxis],
# idx_topk].toarray() * tp).sum(axis=1)
# Gets denominator, could be the whole sum, could be only part of it if there's not many.
IDCG = np.array([(tp[:min(n, k)]).sum()
for n in number_non_zero])
#
IDCG = np.maximum(0.1, IDCG) #Necessary, because sometimes you're not given ANY heldout things to work with. Crazy...
# IDCG = np.array([(tp[:min(n, k)]).sum()
# for n in heldout_batch.getnnz(axis=1)])
# to_return = DCG / IDCG
# if np.any(np.isnan(to_return)):
# print("bad?!")
# import ipdb; ipdb.set_trace()
# print("dab!?")
if normalize:
result = (DCG / IDCG)
else:
result = DCG
result = result.astype(np.float32)
return result
sklearn的实现，会比以上实现偏小
from sklearn.metrics import precision_score, recall_score, f1_score
def metrics_sklearn(X_pred, X_true, k=10):
import bottleneck as bn
from sklearn.metrics import precision_score, recall_score, f1_score,ndcg_score
pred_idx = bn.argpartition(-X_pred, k, axis=1)
x_pred_binary = np.zeros_like(X_pred)
x_pred_binary[pred_idx < k] = 1
p = np.array([])
r = np.array([])
for idx in range(X_pred.shape[0]):
p = np.append(p,precision_score(np.int8(X_true[idx]>0), x_pred_binary[idx]) )
r = np.append(r, recall_score(np.int8(X_true[idx]>0), x_pred_binary[idx],'macro'))
return ndcg_score(np.int8(X_true>0), x_pred_binary), np.mean(p), np.mean(r)