Commit b40979c3 authored by Nishtha Jain's avatar Nishtha Jain
Browse files

eval_scores saving added

parent 3536edb7
This diff is collapsed.
......@@ -137,11 +137,37 @@ PREDICTED_DATASET = {
}
EVALUATION_SCORES = {
('svm','cv','trial','random',0.2,'bio') : 'eval_scores/eval_sv_cv_tri_ran_0.2_b',
('svm','cv','trial','balanced',0.2,'bio') : 'eval_scores/eval_sv_cv_tri_bal_0.2_b',
('svm','cv','medical','random',0.2,'raw') : 'eval_scores/eval_sv_cv_med_ran_0.2_r',
('svm','cv','medical','random',0.2,'bio') : 'eval_scores/eval_sv_cv_med_ran_0.2_b',
('svm','w2v','medical','random',0.2,'raw') : 'eval_scores/eval_sv_wv_med_ran_0.2_r',
('svm','w2v','medical','random',0.2,'bio') : 'eval_scores/eval_sv_wv_med_ran_0.2_b',
('svm','self_w2v','medical','random',0.2,'raw') : 'eval_scores/eval_sv_tv_med_ran_0.2_r',
('svm','self_w2v','medical','random',0.2,'bio') : 'eval_scores/eval_sv_tv_med_ran_0.2_b',
('svm','elmo','medical','random',0.2,'raw') : 'eval_scores/eval_sv_ev_med_ran_0.2_r',
('svm','elmo','medical','random',0.2,'bio') : 'eval_scores/eval_sv_ev_med_ran_0.2_b',
('svm','cv','medical','balanced',0.2,'raw') : 'eval_scores/eval_sv_cv_med_bal_0.2_r',
('svm','cv','medical','balanced',0.2,'bio') : 'eval_scores/eval_sv_cv_med_bal_0.2_b',
('svm','w2v','medical','balanced',0.2,'raw') : 'eval_scores/eval_sv_wv_med_bal_0.2_r',
('svm','w2v','medical','balanced',0.2,'bio') : 'eval_scores/eval_sv_wv_med_bal_0.2_b',
('svm','self_w2v','medical','balanced',0.2,'raw') : 'eval_scores/eval_sv_tv_med_bal_0.2_r',
('svm','self_w2v','medical','balanced',0.2,'bio') : 'eval_scores/eval_sv_tv_med_bal_0.2_b',
('svm','elmo','medical','balanced',0.2,'raw') : 'eval_scores/eval_sv_ev_med_bal_0.2_r',
('svm','elmo','medical','balanced',0.2,'bio') : 'eval_scores/eval_sv_ev_med_bal_0.2_b'
}
PLOT_NAMES = {
# Naming convention : plot_[tgp|aod]_[M|F]_[sv|rf|nn]_[cv|wv|tv]_[tri|med]_[ran|bal]_[test_spit]_[b|r]
('tgp','svm','cv','trial','balanced',0.2,'bio') : 'plots_and_graphs/plot_tgp_sv_cv_tri_bal_0.2_b',
('aod','svm','cv','trial','balanced',0.2,'bio') : 'plots_and_graphs/plot_aod_sv_cv_tri_bal_0.2_b',
('tgp','svm','self_w2v','trial','balanced',0.2,'bio') : 'plots_and_graphs/plot_tgp_sv_tv_tri_bal_0.2_b',
('aod','svm','self_w2v','trial','balanced',0.2,'bio') : 'plots_and_graphs/plot_aod_sv_tv_tri_bal_0.2_b',
('tgp','svm','elmo','trial','balanced',0.2,'bio') : 'plots_and_graphs/plot_tgp_sv_ev_tri_bal_0.2_b',
......
......@@ -7,116 +7,116 @@ import matplotlib.pyplot as plt
# res.class.gender.tp|fp
def tpr_fpr(test, model, embedding, class_group, sampling, test_size, masking):
print("processing evaluation.tpr_fpr ...")
res = {}
for c in CLASS_GROUP[class_group]:
res[c] = {}
for gender in ['M','F']:
mini_set = test.loc[(test['gender']==gender) & (test[TITLE]==c)]
X_test = embedding_transform(mini_set[MASKED[masking]], embedding, class_group, sampling, test_size, masking)
pred, acc = model_prediction(X_test=X_test, Y_test=mini_set[TITLE], model=model, embedding=embedding, class_group=class_group, sampling=sampling, test_size=test_size, masking=masking)
tp = np.sum(pred == mini_set[TITLE])
fp = np.sum(pred != mini_set[TITLE])
res[c][gender] = {
'count':len(pred),
'tp':tp,
'tpr':float(tp)/len(pred),
'fp':fp, 'count':len(pred),
'fpr':float(fp)/len(pred)}
scores = { 'count_males' :[],
'count_females' : [],
'tpr_males' : [],
'tpr_females' : [],
'tp_males' : [],
'tp_females' : [],
'fpr_males' : [],
'fpr_females' : [],
'fp_males' : [],
'fp_females' : []
}
for c in res:
scores['count_males'].append(res[c]['M']['count'])
scores['count_females'].append(res[c]['F']['count'])
scores['tpr_males'].append(res[c]['M']['tpr'])
scores['tpr_females'].append(res[c]['F']['tpr'])
scores['tp_males'].append(res[c]['M']['tp'])
scores['tp_females'].append(res[c]['F']['tp'])
scores['fpr_males'].append(res[c]['M']['fpr'])
scores['fpr_females'].append(res[c]['F']['fpr'])
scores['fp_males'].append(res[c]['M']['fp'])
scores['fp_females'].append(res[c]['F']['fp'])
return(scores)
print("processing evaluation.tpr_fpr ...")
res = {}
for c in CLASS_GROUP[class_group]:
res[c] = {}
for gender in ['M','F']:
mini_set = test.loc[(test['gender']==gender) & (test[TITLE]==c)]
X_test = embedding_transform(mini_set[MASKED[masking]], embedding, class_group, sampling, test_size, masking)
pred, acc = model_prediction(X_test=X_test, Y_test=mini_set[TITLE], model=model, embedding=embedding, class_group=class_group, sampling=sampling, test_size=test_size, masking=masking)
tp = np.sum(pred == mini_set[TITLE])
fp = np.sum(pred != mini_set[TITLE])
res[c][gender] = {
'count':len(pred),
'tp':tp,
'tpr':float(tp)/len(pred),
'fp':fp, 'count':len(pred),
'fpr':float(fp)/len(pred)}
scores = { 'count_males' :[],
'count_females' : [],
'tpr_males' : [],
'tpr_females' : [],
'tp_males' : [],
'tp_females' : [],
'fpr_males' : [],
'fpr_females' : [],
'fp_males' : [],
'fp_females' : []
}
for c in res:
scores['count_males'].append(res[c]['M']['count'])
scores['count_females'].append(res[c]['F']['count'])
scores['tpr_males'].append(res[c]['M']['tpr'])
scores['tpr_females'].append(res[c]['F']['tpr'])
scores['tp_males'].append(res[c]['M']['tp'])
scores['tp_females'].append(res[c]['F']['tp'])
scores['fpr_males'].append(res[c]['M']['fpr'])
scores['fpr_females'].append(res[c]['F']['fpr'])
scores['fp_males'].append(res[c]['M']['fp'])
scores['fp_females'].append(res[c]['F']['fp'])
return(scores)
def tpr_gender_gap(scores ,test, model, embedding, class_group, sampling, test_size, masking):
print("processing evaluation.tpr_gender_gap ...")
if not scores:
scores = tpr_fpr(test, model, embedding, class_group, sampling, test_size, masking)
x_males = [scores['count_males'][i]/(scores['count_males'][i]+scores['count_females'][i]) for i in range(len(CLASS_GROUP[class_group]))]
y_males = [scores['tpr_males'][i]-scores['tpr_females'][i] for i in range(len(CLASS_GROUP[class_group]))]
x_females = [scores['count_females'][i]/(scores['count_males'][i]+scores['count_females'][i]) for i in range(len(CLASS_GROUP[class_group]))]
y_females = [scores['tpr_females'][i]-scores['tpr_males'][i] for i in range(len(CLASS_GROUP[class_group]))]
fig, (axs_males,axs_females) = plt.subplots(1, 2, figsize=(20, 8))
axs_males.scatter(x_males, y_males)
axs_males.set_xlabel("% Male")
axs_males.set_ylabel("TPR Gender Gap Male")
for i, txt in enumerate(CLASS_GROUP[class_group]):
axs_males.annotate(txt, (x_males[i], y_males[i]))
axs_females.scatter(x_females, y_females)
axs_females.set_xlabel("% Female")
axs_females.set_ylabel("TPR Gender Gap Female")
for i, txt in enumerate(CLASS_GROUP[class_group]):
axs_females.annotate(txt, (x_females[i], y_females[i]))
plt.savefig(PLOT_NAMES['tgp',model,embedding,class_group,sampling,test_size,MASKED[masking]] + '.png')
plt.show()
return (scores,x_males,y_males,x_females,y_females)
print("processing evaluation.tpr_gender_gap ...")
if not scores:
scores = tpr_fpr(test, model, embedding, class_group, sampling, test_size, masking)
x_males = [scores['count_males'][i]/(scores['count_males'][i]+scores['count_females'][i]) for i in range(len(CLASS_GROUP[class_group]))]
y_males = [scores['tpr_males'][i]-scores['tpr_females'][i] for i in range(len(CLASS_GROUP[class_group]))]
x_females = [scores['count_females'][i]/(scores['count_males'][i]+scores['count_females'][i]) for i in range(len(CLASS_GROUP[class_group]))]
y_females = [scores['tpr_females'][i]-scores['tpr_males'][i] for i in range(len(CLASS_GROUP[class_group]))]
fig, (axs_males,axs_females) = plt.subplots(1, 2, figsize=(20, 8))
axs_males.scatter(x_males, y_males)
axs_males.set_xlabel("% Male")
axs_males.set_ylabel("TPR Gender Gap Male")
for i, txt in enumerate(CLASS_GROUP[class_group]):
axs_males.annotate(txt, (x_males[i], y_males[i]))
axs_females.scatter(x_females, y_females)
axs_females.set_xlabel("% Female")
axs_females.set_ylabel("TPR Gender Gap Female")
for i, txt in enumerate(CLASS_GROUP[class_group]):
axs_females.annotate(txt, (x_females[i], y_females[i]))
plt.savefig(PLOT_NAMES['tgp',model,embedding,class_group,sampling,test_size,MASKED[masking]] + '.png')
plt.show()
return (scores,x_males,y_males,x_females,y_females)
def average_odds_difference(scores, test, model, embedding, class_group, sampling, test_size, masking):
print("processing evaluation.average_odds_difference ...")
print("processing evaluation.average_odds_difference ...")
if not scores:
scores = tpr_fpr(test, model, embedding, class_group, sampling, test_size, masking)
if not scores:
scores = tpr_fpr(test, model, embedding, class_group, sampling, test_size, masking)
x_males = [scores['count_males'][i]/(scores['count_males'][i]+scores['count_females'][i]) for i in range(len(CLASS_GROUP[class_group]))]
y_males = [(scores['fpr_males'][i]-scores['fpr_females'][i] + scores['tpr_males'][i]-scores['tpr_females'][i])/2 for i in range(len(CLASS_GROUP[class_group]))]
x_females = [scores['count_females'][i]/(scores['count_males'][i]+scores['count_females'][i]) for i in range(len(CLASS_GROUP[class_group]))]
y_females = [(scores['fpr_females'][i]-scores['fpr_males'][i] + scores['tpr_females'][i]-scores['tpr_males'][i])/2 for i in range(len(CLASS_GROUP[class_group]))]
x_males = [scores['count_males'][i]/(scores['count_males'][i]+scores['count_females'][i]) for i in range(len(CLASS_GROUP[class_group]))]
y_males = [(scores['fpr_males'][i]-scores['fpr_females'][i] + scores['tpr_males'][i]-scores['tpr_females'][i])/2 for i in range(len(CLASS_GROUP[class_group]))]
x_females = [scores['count_females'][i]/(scores['count_males'][i]+scores['count_females'][i]) for i in range(len(CLASS_GROUP[class_group]))]
y_females = [(scores['fpr_females'][i]-scores['fpr_males'][i] + scores['tpr_females'][i]-scores['tpr_males'][i])/2 for i in range(len(CLASS_GROUP[class_group]))]
fig, (axs_males,axs_females) = plt.subplots(1, 2, figsize=(20, 8))
fig, (axs_males,axs_females) = plt.subplots(1, 2, figsize=(20, 8))
axs_males.scatter(x_males, y_males)
axs_males.set_xlabel("% Male")
axs_males.set_ylabel("Avg odds difference Male")
for i, txt in enumerate(CLASS_GROUP[class_group]):
axs_males.annotate(txt, (x_males[i], y_males[i]))
axs_males.scatter(x_males, y_males)
axs_males.set_xlabel("% Male")
axs_males.set_ylabel("Avg odds difference Male")
for i, txt in enumerate(CLASS_GROUP[class_group]):
axs_males.annotate(txt, (x_males[i], y_males[i]))
axs_females.scatter(x_females, y_females)
axs_females.set_xlabel("% Female")
axs_females.set_ylabel("Avg odds difference Male")
for i, txt in enumerate(CLASS_GROUP[class_group]):
axs_females.annotate(txt, (x_females[i], y_females[i]))
axs_females.scatter(x_females, y_females)
axs_females.set_xlabel("% Female")
axs_females.set_ylabel("Avg odds difference Male")
for i, txt in enumerate(CLASS_GROUP[class_group]):
axs_females.annotate(txt, (x_females[i], y_females[i]))
plt.savefig(PLOT_NAMES['aod',model,embedding,class_group,sampling,test_size,MASKED[masking]] + '.png')
plt.show()
plt.savefig(PLOT_NAMES['aod',model,embedding,class_group,sampling,test_size,MASKED[masking]] + '.png')
plt.show()
return (scores,x_males,y_males,x_females,y_females)
return (scores,x_males,y_males,x_females,y_females)
from config import CLASS_GROUP, TITLE, MASKED, SEED, DATASET_NAMES, PREDICTED_DATASET
from config import CLASS_GROUP, TITLE, MASKED, SEED, DATASET_NAMES, PREDICTED_DATASET, EVALUATION_SCORES
from preprocessing import embedding_fit_transform, embedding_transform
from sampling import data_selection, load_data
from model import model_training, model_prediction
......@@ -15,6 +15,9 @@ def main(load_data_from_saved, embedding_train, model_train, predict, evaluate,
train_set,test_set = data_selection(data, class_group, sampling, test_size, masking)
# TO-DO:
# no-model_train but embedding_train true
# training
if model_train:
if embedding_train:
......@@ -33,7 +36,7 @@ def main(load_data_from_saved, embedding_train, model_train, predict, evaluate,
print("processing model.model_prediction ...")
pred, acc = model_prediction(X_test, Y_test, model, embedding, class_group, sampling, test_size, masking)
predicted_dataset = pd.DataFrame({MASKED[masking]:test_set[MASKED[masking]], TITLE:test_set[TITLE], 'predicted':pred})
predicted_dataset = pd.DataFrame({MASKED[masking]:test_set[MASKED[masking]], TITLE:test_set[TITLE], 'predicted':pred, 'gender': test_set['gender']})
print("\t saving file :",PREDICTED_DATASET[model, embedding, class_group, sampling, test_size, MASKED[masking]])
dump(predicted_dataset, PREDICTED_DATASET[model, embedding, class_group, sampling, test_size, MASKED[masking]] + '.joblib')
......@@ -47,6 +50,8 @@ def main(load_data_from_saved, embedding_train, model_train, predict, evaluate,
if evaluate:
scores,x_males,y_males,x_females,y_females = tpr_gender_gap(None,test_set, model, embedding, class_group, sampling, test_size, masking)
scores,x_males,y_males,x_females,y_females = average_odds_difference(scores,test_set, model, embedding, class_group, sampling, test_size, masking)
print("\t saving file :",EVALUATION_SCORES[model, embedding, class_group, sampling, test_size, MASKED[masking]])
dump(scores,EVALUATION_SCORES[model, embedding, class_group, sampling, test_size, MASKED[masking]] + '.joblib')
print(scores)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment