Commit 8afd5def authored by Verena Praher's avatar Verena Praher
Browse files

fix computation of PR-AUC

parent d8360832
......@@ -5,6 +5,7 @@ from torch.utils.data import DataLoader
from sklearn import metrics
import os
from datasets.mtgjamendo import df_get_mtg_set
import numpy as np
def my_loss(y_hat, y):
return F.binary_cross_entropy(y_hat, y)
......@@ -23,28 +24,52 @@ def validation_step(model, data_batch, batch_nb):
x, _, y = data_batch
# print("x", x)
# print("y", y)
y_hat = model.forward(x)
y_hat = model.forward(x[:, :, :, :512]) # TODO: why is this necessary?
y = y.float()
y_hat = y_hat.float()
rocauc = metrics.roc_auc_score(y.t().cpu(), y_hat.t().cpu())
prauc = metrics.average_precision_score(y.t().cpu(), y_hat.t().cpu())
#print("y", y)
#print("y_hat", y_hat)
#rocauc = metrics.roc_auc_score(y.t().cpu(), y_hat.t().cpu(), average='macro')
#prauc = metrics.average_precision_score(y.t().cpu(), y_hat.t().cpu(), average='macro')
# _, _, fscore, _ = metrics.precision_recall_fscore_support(y.t().cpu(), y_hat.t().cpu())
fscore = 0.
#fscore = 0.
return {'val_loss': model.my_loss(y_hat, y),
'val_rocauc': rocauc,
'val_prauc': prauc,
'val_fscore': fscore}
'y': y.cpu().numpy(),
'y_hat': y_hat.cpu().numpy(),
#'val_rocauc': rocauc,
#'val_prauc': prauc,
#'val_fscore': fscore
}
def validation_end(outputs):
avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()
avg_auc = torch.stack([torch.tensor([x['val_rocauc']]) for x in outputs]).mean()
avg_prauc = torch.stack([torch.tensor([x['val_prauc']]) for x in outputs]).mean()
avg_fscore = torch.stack([torch.tensor([x['val_fscore']]) for x in outputs]).mean()
y = []
y_hat = []
for output in outputs:
y.append(output['y'])
y_hat.append(output['y_hat'])
y = np.concatenate(y)
y_hat = np.concatenate(y_hat)
#print(y[0:10])
#print(y_hat[0:10])
rocauc = metrics.roc_auc_score(y, y_hat, average='macro')
prauc = metrics.average_precision_score(y, y_hat, average='macro')
#_, _, fscore, _ = metrics.precision_recall_fscore_support(y, y_hat, average='macro')
fscore = 0.
#print('metrics', rocauc, prauc, fscore)
#avg_auc = torch.stack([torch.tensor([x['val_rocauc']]) for x in outputs]).mean()
#avg_prauc = torch.stack([torch.tensor([x['val_prauc']]) for x in outputs]).mean()
#avg_fscore = torch.stack([torch.tensor([x['val_fscore']]) for x in outputs]).mean()
return {'val_loss': avg_loss,
'val_rocauc': avg_auc,
'val_prauc': avg_prauc,
'val_fscore': avg_fscore}
'val_rocauc': rocauc,
'val_prauc': prauc,
'val_fscore': fscore}
def test_step(model, data_batch, batch_nb):
......@@ -52,27 +77,45 @@ def test_step(model, data_batch, batch_nb):
x, _, y = data_batch
# print("x", x)
# print("y", y)
y_hat = model.forward(x)
y_hat = model.forward(x[:, :, :, :512]) # TODO why is this necessary?
y = y.float()
y_hat = y_hat.float()
rocauc = metrics.roc_auc_score(y.t().cpu(), y_hat.t().cpu())
prauc = metrics.average_precision_score(y.t().cpu(), y_hat.t().cpu())
rocauc = metrics.roc_auc_score(y.t().cpu(), y_hat.t().cpu(), average='macro')
prauc = metrics.average_precision_score(y.t().cpu(), y_hat.t().cpu(), average='macro')
# _, _, fscore, _ = metrics.precision_recall_fscore_support(y.t().cpu(), y_hat.t().cpu())
fscore = 0.
return {'test_loss': model.my_loss(y_hat, y),
'test_rocauc': rocauc,
'test_prauc': prauc,
'test_fscore': fscore}
'y': y.cpu(),
'y_hat': y_hat.cpu(),
#'test_rocauc': rocauc,
#'test_prauc': prauc,
#'test_fscore': fscore
}
def test_end(outputs):
avg_loss = torch.stack([x['test_loss'] for x in outputs]).mean()
avg_auc = torch.stack([torch.tensor([x['test_rocauc']]) for x in outputs]).mean()
avg_prauc = torch.stack([torch.tensor([x['test_prauc']]) for x in outputs]).mean()
avg_fscore = torch.stack([torch.tensor([x['test_fscore']]) for x in outputs]).mean()
y = []
y_hat = []
for output in outputs:
y.append(output['y'])
y_hat.append(output['y_hat'])
y = np.concatenate(y)
y_hat = np.concatenate(y_hat)
#print(y[0:10])
#print(y_hat[0:10])
rocauc = metrics.roc_auc_score(y, y_hat, average='macro')
prauc = metrics.average_precision_score(y, y_hat, average='macro')
#_, _, fscore, _ = metrics.precision_recall_fscore_support(y, y_hat, average='macro')
fscore = 0.
return {'test_loss': avg_loss,
'test_rocauc': avg_auc,
'test_prauc': avg_prauc,
'test_fscore': avg_fscore}
'test_rocauc': rocauc,
'test_prauc': prauc,
'test_fscore': fscore}
def tng_dataloader():
train_csv = os.path.join(PATH_ANNOTATIONS, 'train_processed.tsv')
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment