Skip to content
GitLab
Commit 483f75aa authored by Paul Primus's avatar Paul Primus
Browse files

add final submission package

parent dfebc6ca
Expand all Hide whitespace changes
Inline Side-by-side
notebooks/create_bar_plot.ipynb
View file @ 483f75aa
%% Cell type:code id: tags:
``` python
from pymongo import MongoClient
from matplotlib import pyplot as plt
import numpy as np
from dcase2020_task2.data_sets.mcm_dataset import INVERSE_CLASS_MAP, TRAINING_ID_MAP, CLASS_MAP
from scipy.stats import rankdata
baseline_auc = {
'name': 'baseline',
0: {0: 0.5441, 2: 0.7340, 4: 0.6161, 6: 0.7392},
1: {0: 0.6715, 2: 0.6153, 4: 0.8833, 6: 0.7455},
2: {0: 0.9619, 2: 0.7897, 4: 0.9430, 6: 0.6959},
3: {1: 0.8136, 2: 0.8597, 3: 0.6330, 4: 0.8445},
4: {1: 0.7807, 2: 0.6416, 3: 0.7535},
5: {0: 0.6876, 2: 0.6818, 4: 0.7430, 6: 0.5390}
}
baseline_pauc = {
'name': 'baseline',
0: {0: 0.4937, 2: 0.5481, 4: 0.5326, 6: 0.5235},
1: {0: 0.5674, 2: 0.5810, 4: 0.6710, 6: 0.5802},
2: {0: 0.8144, 2: 0.6368, 4: 0.7198, 6: 0.4902},
3: {1: 0.6840, 2: 0.7772, 3: 0.5521, 4: 0.6897},
4: {1: 0.6425, 2: 0.5601, 3: 0.6103},
5: {0: 0.5170, 2: 0.5183, 4: 0.5197, 6: 0.4843}
}
baseline_both = {}
for t in baseline_auc:
if t == 'name':
baseline_both[t] = 'baseline'
continue
else:
baseline_both[t] = {}
for i in baseline_auc[t]:
baseline_both[t][i] = np.array([baseline_auc[t][i], baseline_pauc[t][i]])
def get_experiment(runs, name):
experiment_dict = dict()
for i in range(6):
experiment_dict[i] = dict()
experiment_dict['name'] = name
for experiment in runs:
if experiment['config'].get('id') == name:
machine_dict = experiment_dict.get(experiment['config']['machine_type'])
result = experiment.get('result')
machine_type = INVERSE_CLASS_MAP[experiment['config']['machine_type']]
machine_id = experiment['config']['machine_id']
if result:
machine_dict[experiment['config']['machine_id']] = result.get(
machine_type, {}
).get(
f'json://{machine_id}', -1
).get('py/tuple', [0, 0])[:2]
else:
machine_dict[experiment['config']['machine_id']] = np.array([0, 0])
return experiment_dict
def get_record(experiment):
record = []
for i in range(6):
for j in TRAINING_ID_MAP[i]:
v = experiment.get(i)
if v:
v = v.get(j, [0, 0])
else:
v = np.array([0, 0])
record.append(np.array(v))
assert len(record) == 23
return experiment['name'], record
```
%% Cell type:code id: tags:
``` python
client = MongoClient('mongodb://student2.cp.jku.at:27017/')
experiments = [r for r in client.resnet_gridsearch.runs.find({"experiment.name": "dcase2020_task2_ClassificationExperiment"})]
print(f'Loaded {len(experiments)} runs.')
```
%% Output
Loaded 563 runs.
Loaded 572 runs.
%% Cell type:code id: tags:
``` python
descriptors = set()
for experiment in experiments:
descriptors = descriptors.union(set([experiment['config']['id']]))
descriptors = list(descriptors)
descriptors = [d for d in descriptors if d.split('_')[-1] != 'rerun']
print(f'Loaded {len(descriptors)} distinct experiments.')
```
%% Output
Loaded 13 distinct experiments.
%% Cell type:code id: tags:
``` python
# Extract Results
# Concatenate Baseline Results
n, m = get_record(baseline_both)
names = [n]
metrics = [np.array(m)]
for descriptor in descriptors:
n, m = get_record(
get_experiment(
experiments,
descriptor
)
)
names.append(n)
metrics.append(np.array(m))
```
%% Cell type:code id: tags:
``` python
data = np.array(metrics)
auc_ranks = []
pauc_ranks = []
idxes = [0, 4, 8, 12, 16, 19, 23]
for type_, (i, j) in enumerate(zip(idxes[:-1], idxes[1:])):
average_auc = data[:, i:j, 0].mean(axis=1)
average_pauc = data[:, i:j, 1].mean(axis=1)
print(f'Best Model for Machine Type {type_}: {np.argsort(average_auc + average_pauc)[::-1]}')
auc_ranks.append(rankdata(-average_auc))
pauc_ranks.append(rankdata(-average_pauc))
```
%% Output
Best Model for Machine Type 0: [ 1 6 5 12 11 9 2 10 3 7 13 4 8 0]
Best Model for Machine Type 1: [ 1 10 6 9 3 5 13 2 11 12 7 8 4 0]
Best Model for Machine Type 2: [10 5 12 1 11 3 2 13 9 6 4 8 7 0]
Best Model for Machine Type 3: [ 2 12 4 5 11 6 10 3 1 13 9 8 7 0]
Best Model for Machine Type 4: [13 3 12 11 10 9 7 6 8 4 5 2 0 1]
Best Model for Machine Type 5: [ 4 8 2 7 10 5 9 6 13 11 12 3 0 1]
Best Model for Machine Type 4: [13 3 12 11 10 9 7 6 8 4 5 2 1 0]
Best Model for Machine Type 5: [ 1 4 8 2 7 10 5 9 6 13 11 12 3 0]
%% Cell type:code id: tags:
``` python
ranks = np.stack([np.array(list(zip(*auc_ranks))), np.array(list(zip(*pauc_ranks)))], axis=-1).mean(axis=-1).mean(axis=-1)
indices = list(np.argsort(ranks))
names = np.array(names)
for i, (n, r, j) in enumerate(zip(names[indices], ranks[indices], indices)):
print(f'{i}: ID-{j} {n}')
```
%% Output
0: ID-10 resnet_gridsearch_a_bit_larger_loose_1e-4_100_BCE
1: ID-5 resnet_gridsearch_a_bit_larger_loose_1e-4_100_AUC
2: ID-12 resnet_gridsearch_a_bit_larger_loose_1e-5_100_BCE
3: ID-2 resnet_gridsearch_a_bit_smaller_loose_1e-4_100_BCE
4: ID-11 resnet_gridsearch_a_bit_larger_loose_1e-5_100_AUC
5: ID-6 resnet_gridsearch_normal_loose_1e-4_100_BCE
6: ID-1 resnet_gridsearch_2_a_bit_larger_loose_1e-4_0.99_100_BCE
0: ID-1 resnet_gridsearch_2_a_bit_larger_loose_1e-4_0.99_100_BCE
1: ID-10 resnet_gridsearch_a_bit_larger_loose_1e-4_100_BCE
2: ID-5 resnet_gridsearch_a_bit_larger_loose_1e-4_100_AUC
3: ID-12 resnet_gridsearch_a_bit_larger_loose_1e-5_100_BCE
4: ID-2 resnet_gridsearch_a_bit_smaller_loose_1e-4_100_BCE
5: ID-11 resnet_gridsearch_a_bit_larger_loose_1e-5_100_AUC
6: ID-6 resnet_gridsearch_normal_loose_1e-4_100_BCE
7: ID-3 resnet_gridsearch_normal_loose_1e-5_100_BCE
8: ID-9 resnet_gridsearch_normal_loose_1e-4_100_AUC
9: ID-4 resnet_gridsearch_a_bit_smaller_loose_1e-4_100_AUC
10: ID-13 resnet_gridsearch_normal_loose_1e-5_100_AUC
11: ID-7 resnet_gridsearch_a_bit_smaller_loose_1e-5_100_AUC
12: ID-8 resnet_gridsearch_a_bit_smaller_loose_1e-5_100_BCE
13: ID-0 baseline
%% Cell type:code id: tags:
``` python
indices
```
%% Output
[10, 5, 12, 2, 11, 6, 1, 3, 9, 4, 13, 7, 8, 0]
[1, 10, 5, 12, 2, 11, 6, 3, 9, 4, 13, 7, 8, 0]
%% Cell type:code id: tags:
``` python
metric = 'auroc_mean'
bar_width = 0.6
bar_spacing=0.00
top = 10
top_k_data = data[[10, 1], :, 0 if metric == 'auroc_mean' else 1][:top]
baseline_data = data[0:1, :, 0 if metric == 'auroc_mean' else 1]
to_visualize = np.concatenate([baseline_data, top_k_data])
plt.figure(figsize=(20,10))
plt.rcParams.update({'font.size': 22})
plt.title(f'{metric}')
labels = []
for i in range(6):
for j in TRAINING_ID_MAP[i]:
labels.append("{}, {}".format(INVERSE_CLASS_MAP[i][:6], j))
for i, d in enumerate(to_visualize):
plt.bar(
np.arange(len(labels)) + i * (bar_width / len(to_visualize) + bar_spacing),
d,
bar_width/ len(to_visualize),
)
plt.xticks(np.arange(len(labels)), labels, rotation='vertical')
plt.ylim(0.5, 1.)
plt.yticks(np.arange(0.5, 1., 0.1))
plt.grid()
plt.savefig(f'top_{top}_auc.png')
plt.show()
data.shape
```
%% Output
(14, 23, 2)
%% Cell type:code id: tags:
``` python
metric = 'pauroc_mean'
bar_width = 0.6
bar_spacing=0.00
top = 10
top_k_data = data[indices, :, 0 if metric == 'auroc_mean' else 1][:top]
baseline_data = data[0:1, :, 0 if metric == 'auroc_mean' else 1]
to_visualize = np.concatenate([baseline_data, top_k_data])
plt.figure(figsize=(20,10))
plt.rcParams.update({'font.size': 22})
plt.title(f'{metric}')
labels = []
for i in range(6):
for j in TRAINING_ID_MAP[i]:
labels.append("{}, {}".format(INVERSE_CLASS_MAP[i][:6], j))
for i, d in enumerate(to_visualize):
plt.bar(
np.arange(len(labels)) + i * (bar_width / len(to_visualize) + bar_spacing),
d,
bar_width/ len(to_visualize),
)
plt.xticks(np.arange(len(labels)), labels, rotation='vertical')
plt.ylim(0.45, 1.)
plt.yticks(np.arange(0.5, 1., 0.1))
plt.grid()
plt.savefig(f'top_{top}_auc.png')
plt.show()
data.shape
```
%% Output
(11, 23, 2)
%% Cell type:code id: tags:
``` python
```
......
notebooks/create_results_table.ipynb deleted 100644 → 0
View file @ dfebc6ca
This diff is collapsed.
notebooks/submission.ipynb
View file @ 483f75aa
%% Cell type:code id: tags:
``` python
from pymongo import MongoClient
from matplotlib import pyplot as plt
import numpy as np
from dcase2020_task2.data_sets.mcm_dataset import INVERSE_CLASS_MAP, TRAINING_ID_MAP, EVALUATION_ID_MAP, CLASS_MAP
from scipy.stats import rankdata
import os
from shutil import copyfile
import pandas as pd
baseline_auc = {
'name': 'baseline',
0: {0: 0.5441, 2: 0.7340, 4: 0.6161, 6: 0.7392},
1: {0: 0.6715, 2: 0.6153, 4: 0.8833, 6: 0.7455},
2: {0: 0.9619, 2: 0.7897, 4: 0.9430, 6: 0.6959},
3: {1: 0.8136, 2: 0.8597, 3: 0.6330, 4: 0.8445},
4: {1: 0.7807, 2: 0.6416, 3: 0.7535},
5: {0: 0.6876, 2: 0.6818, 4: 0.7430, 6: 0.5390}
}
baseline_pauc = {
'name': 'baseline',
0: {0: 0.4937, 2: 0.5481, 4: 0.5326, 6: 0.5235},
1: {0: 0.5674, 2: 0.5810, 4: 0.6710, 6: 0.5802},
2: {0: 0.8144, 2: 0.6368, 4: 0.7198, 6: 0.4902},
3: {1: 0.6840, 2: 0.7772, 3: 0.5521, 4: 0.6897},
4: {1: 0.6425, 2: 0.5601, 3: 0.6103},
5: {0: 0.5170, 2: 0.5183, 4: 0.5197, 6: 0.4843}
}
baseline_both = {}
for t in baseline_auc:
if t == 'name':
baseline_both[t] = 'baseline'
continue
else:
baseline_both[t] = {}
for i in baseline_auc[t]:
baseline_both[t][i] = np.array([baseline_auc[t][i], baseline_pauc[t][i]])
def get_experiment(runs, name):
experiment_dict = dict()
for i in range(6):
experiment_dict[i] = dict()
experiment_dict['name'] = name
for experiment in runs:
if experiment['config'].get('id') == name:
machine_dict = experiment_dict.get(experiment['config']['machine_type'])
result = experiment.get('result')
machine_type = INVERSE_CLASS_MAP[experiment['config']['machine_type']]
machine_id = experiment['config']['machine_id']
if result:
machine_dict[experiment['config']['machine_id']] = result.get(
machine_type, {}
).get(
f'json://{machine_id}', -1
).get('py/tuple', [0, 0])[:2]
else:
machine_dict[experiment['config']['machine_id']] = np.array([0, 0])
return experiment_dict
def get_record(experiment):
record = []
for i in range(6):
for j in TRAINING_ID_MAP[i]:
v = experiment.get(i)
if v:
v = v.get(j, [0, 0])
else:
v = np.array([0, 0])
record.append(np.array(v))
assert len(record) == 23
return experiment['name'], record
```
%% Cell type:code id: tags:
``` python
client = MongoClient('mongodb://student2.cp.jku.at:27017/')
experiments = [r for r in client.resnet_gridsearch.runs.find({"experiment.name": "dcase2020_task2_ClassificationExperiment"})]
print(f'Loaded {len(experiments)} runs.')
```
%% Output
Loaded 563 runs.
Loaded 572 runs.
%% Cell type:code id: tags:
``` python
descriptors = set()
for experiment in experiments:
descriptors = descriptors.union(set([experiment['config']['id']]))
descriptors