In machine learning we can define a ROC (Receiver Operating Characteristic) Curve. This plots the performance of the classifier for all possible thresholds [article]:
ROC (Receiver Operating Characteristic) CurveCodingThe coding is: import sys import numpy as np from sklearn import metrics from sklearn.metrics import roc_auc_score import matplotlib.pyplot as plt from sklearn.ensemble import RandomForestClassifier def show_roc(FPR, TPR, AUC): plt.plot(FPR, TPR, color='blue', label='ROC') plt.plot([0, 1], [0, 1], color='darkblue', linestyle='--') plt.xlabel('FPR') plt.ylabel('TPR') plt.title('Receiver Operating Characteristic (ROC) Curve') plt.legend(["AUC=%.3f" % AUC]) plt.show() y = ['Eve', 'Eve', 'Eve', 'Eve','Eve','Bob','Bob', 'Bob','Bob','Bob'] #y=[0,0,0,0,0,1,1,1,1,1] scores = [20,25,16,42,22,50,41,60,54,39] if (len(sys.argv)>1): file=str(sys.argv[1]) if (len(sys.argv)>2): y=str(sys.argv[2]).split() if (len(sys.argv)>3): s=str(sys.argv[3]) scores=map(int, s.split(" ")) print y print scores if len(y) != len(scores) : print ("Mismatch in y and scores") sys.exit(0) positive_label = 'Bob' fpr, tpr, thresholds = metrics.roc_curve(y, scores, pos_label=positive_label) auc=metrics.auc(fpr, tpr) print "FPR:",fpr print "TPR:",tpr print "Thresholds:",thresholds show_roc(fpr, tpr,auc) And a sample run is: ['Eve', 'Eve', 'Eve', 'Eve', 'Eve', 'Bob', 'Bob', 'Bob', 'Bob', 'Bob'] [30, 25, 16, 42, 22, 50, 41, 60, 54, 80] FPR: [0. 0. 0. 0.2 0.2 1. ] TPR: [0. 0.2 0.8 0.8 1. 1. ] Thresholds: [81 80 50 42 41 16] And the chart: |